t4_main.c revision 308154
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 308154 2016-10-31 22:45:11Z jhb $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/systm.h> 40#include <sys/counter.h> 41#include <sys/module.h> 42#include <sys/malloc.h> 43#include <sys/queue.h> 44#include <sys/taskqueue.h> 45#include <sys/pciio.h> 46#include <dev/pci/pcireg.h> 47#include <dev/pci/pcivar.h> 48#include <dev/pci/pci_private.h> 49#include <sys/firmware.h> 50#include <sys/sbuf.h> 51#include <sys/smp.h> 52#include <sys/socket.h> 53#include <sys/sockio.h> 54#include <sys/sysctl.h> 55#include <net/ethernet.h> 56#include <net/if.h> 57#include <net/if_types.h> 58#include <net/if_dl.h> 59#include <net/if_vlan_var.h> 60#ifdef RSS 61#include <net/rss_config.h> 62#endif 63#if defined(__i386__) || defined(__amd64__) 64#include <vm/vm.h> 65#include <vm/pmap.h> 66#endif 67 68#include "common/common.h" 69#include "common/t4_msg.h" 70#include "common/t4_regs.h" 71#include "common/t4_regs_values.h" 72#include "t4_ioctl.h" 73#include "t4_l2t.h" 74#include "t4_mp_ring.h" 75 76/* T4 bus driver interface */ 77static int t4_probe(device_t); 78static int t4_attach(device_t); 79static int t4_detach(device_t); 80static device_method_t t4_methods[] = { 81 DEVMETHOD(device_probe, t4_probe), 82 DEVMETHOD(device_attach, t4_attach), 83 DEVMETHOD(device_detach, t4_detach), 84 85 DEVMETHOD_END 86}; 87static driver_t t4_driver = { 88 "t4nex", 89 t4_methods, 90 sizeof(struct adapter) 91}; 92 93 94/* T4 port (cxgbe) interface */ 95static int cxgbe_probe(device_t); 96static int cxgbe_attach(device_t); 97static int cxgbe_detach(device_t); 98static device_method_t cxgbe_methods[] = { 99 DEVMETHOD(device_probe, cxgbe_probe), 100 DEVMETHOD(device_attach, cxgbe_attach), 101 DEVMETHOD(device_detach, cxgbe_detach), 102 { 0, 0 } 103}; 104static driver_t cxgbe_driver = { 105 "cxgbe", 106 cxgbe_methods, 107 sizeof(struct port_info) 108}; 109 110/* T4 VI (vcxgbe) interface */ 111static int vcxgbe_probe(device_t); 112static int vcxgbe_attach(device_t); 113static int vcxgbe_detach(device_t); 114static device_method_t vcxgbe_methods[] = { 115 DEVMETHOD(device_probe, vcxgbe_probe), 116 DEVMETHOD(device_attach, vcxgbe_attach), 117 DEVMETHOD(device_detach, vcxgbe_detach), 118 { 0, 0 } 119}; 120static driver_t vcxgbe_driver = { 121 "vcxgbe", 122 vcxgbe_methods, 123 sizeof(struct vi_info) 124}; 125 126static d_ioctl_t t4_ioctl; 127static d_open_t t4_open; 128static d_close_t t4_close; 129 130static struct cdevsw t4_cdevsw = { 131 .d_version = D_VERSION, 132 .d_flags = 0, 133 .d_open = t4_open, 134 .d_close = t4_close, 135 .d_ioctl = t4_ioctl, 136 .d_name = "t4nex", 137}; 138 139/* T5 bus driver interface */ 140static int t5_probe(device_t); 141static device_method_t t5_methods[] = { 142 DEVMETHOD(device_probe, t5_probe), 143 DEVMETHOD(device_attach, t4_attach), 144 DEVMETHOD(device_detach, t4_detach), 145 146 DEVMETHOD_END 147}; 148static driver_t t5_driver = { 149 "t5nex", 150 t5_methods, 151 sizeof(struct adapter) 152}; 153 154 155/* T5 port (cxl) interface */ 156static driver_t cxl_driver = { 157 "cxl", 158 cxgbe_methods, 159 sizeof(struct port_info) 160}; 161 162/* T5 VI (vcxl) interface */ 163static driver_t vcxl_driver = { 164 "vcxl", 165 vcxgbe_methods, 166 sizeof(struct vi_info) 167}; 168 169static struct cdevsw t5_cdevsw = { 170 .d_version = D_VERSION, 171 .d_flags = 0, 172 .d_open = t4_open, 173 .d_close = t4_close, 174 .d_ioctl = t4_ioctl, 175 .d_name = "t5nex", 176}; 177 178/* ifnet + media interface */ 179static void cxgbe_init(void *); 180static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 181static int cxgbe_transmit(struct ifnet *, struct mbuf *); 182static void cxgbe_qflush(struct ifnet *); 183static int cxgbe_media_change(struct ifnet *); 184static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 185 186MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 187 188/* 189 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 190 * then ADAPTER_LOCK, then t4_uld_list_lock. 191 */ 192static struct sx t4_list_lock; 193SLIST_HEAD(, adapter) t4_list; 194#ifdef TCP_OFFLOAD 195static struct sx t4_uld_list_lock; 196SLIST_HEAD(, uld_info) t4_uld_list; 197#endif 198 199/* 200 * Tunables. See tweak_tunables() too. 201 * 202 * Each tunable is set to a default value here if it's known at compile-time. 203 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 204 * provide a reasonable default when the driver is loaded. 205 * 206 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 207 * T5 are under hw.cxl. 208 */ 209 210/* 211 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 212 */ 213#define NTXQ_10G 16 214static int t4_ntxq10g = -1; 215TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 216 217#define NRXQ_10G 8 218static int t4_nrxq10g = -1; 219TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 220 221#define NTXQ_1G 4 222static int t4_ntxq1g = -1; 223TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 224 225#define NRXQ_1G 2 226static int t4_nrxq1g = -1; 227TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 228 229#define NTXQ_VI 1 230static int t4_ntxq_vi = -1; 231TUNABLE_INT("hw.cxgbe.ntxq_vi", &t4_ntxq_vi); 232 233#define NRXQ_VI 1 234static int t4_nrxq_vi = -1; 235TUNABLE_INT("hw.cxgbe.nrxq_vi", &t4_nrxq_vi); 236 237static int t4_rsrv_noflowq = 0; 238TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 239 240#ifdef TCP_OFFLOAD 241#define NOFLDTXQ_10G 8 242static int t4_nofldtxq10g = -1; 243TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 244 245#define NOFLDRXQ_10G 2 246static int t4_nofldrxq10g = -1; 247TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 248 249#define NOFLDTXQ_1G 2 250static int t4_nofldtxq1g = -1; 251TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 252 253#define NOFLDRXQ_1G 1 254static int t4_nofldrxq1g = -1; 255TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 256 257#define NOFLDTXQ_VI 1 258static int t4_nofldtxq_vi = -1; 259TUNABLE_INT("hw.cxgbe.nofldtxq_vi", &t4_nofldtxq_vi); 260 261#define NOFLDRXQ_VI 1 262static int t4_nofldrxq_vi = -1; 263TUNABLE_INT("hw.cxgbe.nofldrxq_vi", &t4_nofldrxq_vi); 264#endif 265 266#ifdef DEV_NETMAP 267#define NNMTXQ_VI 2 268static int t4_nnmtxq_vi = -1; 269TUNABLE_INT("hw.cxgbe.nnmtxq_vi", &t4_nnmtxq_vi); 270 271#define NNMRXQ_VI 2 272static int t4_nnmrxq_vi = -1; 273TUNABLE_INT("hw.cxgbe.nnmrxq_vi", &t4_nnmrxq_vi); 274#endif 275 276/* 277 * Holdoff parameters for 10G and 1G ports. 278 */ 279#define TMR_IDX_10G 1 280static int t4_tmr_idx_10g = TMR_IDX_10G; 281TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 282 283#define PKTC_IDX_10G (-1) 284static int t4_pktc_idx_10g = PKTC_IDX_10G; 285TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 286 287#define TMR_IDX_1G 1 288static int t4_tmr_idx_1g = TMR_IDX_1G; 289TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 290 291#define PKTC_IDX_1G (-1) 292static int t4_pktc_idx_1g = PKTC_IDX_1G; 293TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 294 295/* 296 * Size (# of entries) of each tx and rx queue. 297 */ 298static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 299TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 300 301static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 302TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 303 304/* 305 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 306 */ 307static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 308TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 309 310/* 311 * Configuration file. 312 */ 313#define DEFAULT_CF "default" 314#define FLASH_CF "flash" 315#define UWIRE_CF "uwire" 316#define FPGA_CF "fpga" 317static char t4_cfg_file[32] = DEFAULT_CF; 318TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 319 320/* 321 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 322 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 323 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 324 * mark or when signalled to do so, 0 to never emit PAUSE. 325 */ 326static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 327TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 328 329/* 330 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 331 * encouraged respectively). 332 */ 333static unsigned int t4_fw_install = 1; 334TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 335 336/* 337 * ASIC features that will be used. Disable the ones you don't want so that the 338 * chip resources aren't wasted on features that will not be used. 339 */ 340static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 341TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 342 343static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 344TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 345 346static int t4_toecaps_allowed = -1; 347TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 348 349static int t4_rdmacaps_allowed = 0; 350TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 351 352static int t4_iscsicaps_allowed = 0; 353TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 354 355static int t4_fcoecaps_allowed = 0; 356TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 357 358static int t5_write_combine = 0; 359TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 360 361static int t4_num_vis = 1; 362TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis); 363 364/* Functions used by extra VIs to obtain unique MAC addresses for each VI. */ 365static int vi_mac_funcs[] = { 366 FW_VI_FUNC_OFLD, 367 FW_VI_FUNC_IWARP, 368 FW_VI_FUNC_OPENISCSI, 369 FW_VI_FUNC_OPENFCOE, 370 FW_VI_FUNC_FOISCSI, 371 FW_VI_FUNC_FOFCOE, 372}; 373 374struct intrs_and_queues { 375 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 376 uint16_t nirq; /* Total # of vectors */ 377 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 378 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 379 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 380 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 381 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 382 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 383 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 384 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 385 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 386 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 387 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 388 389 /* The vcxgbe/vcxl interfaces use these and not the ones above. */ 390 uint16_t ntxq_vi; /* # of NIC txq's */ 391 uint16_t nrxq_vi; /* # of NIC rxq's */ 392 uint16_t nofldtxq_vi; /* # of TOE txq's */ 393 uint16_t nofldrxq_vi; /* # of TOE rxq's */ 394 uint16_t nnmtxq_vi; /* # of netmap txq's */ 395 uint16_t nnmrxq_vi; /* # of netmap rxq's */ 396}; 397 398struct filter_entry { 399 uint32_t valid:1; /* filter allocated and valid */ 400 uint32_t locked:1; /* filter is administratively locked */ 401 uint32_t pending:1; /* filter action is pending firmware reply */ 402 uint32_t smtidx:8; /* Source MAC Table index for smac */ 403 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 404 405 struct t4_filter_specification fs; 406}; 407 408static int map_bars_0_and_4(struct adapter *); 409static int map_bar_2(struct adapter *); 410static void setup_memwin(struct adapter *); 411static int validate_mem_range(struct adapter *, uint32_t, int); 412static int fwmtype_to_hwmtype(int); 413static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 414 uint32_t *); 415static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 416static uint32_t position_memwin(struct adapter *, int, uint32_t); 417static int cfg_itype_and_nqueues(struct adapter *, int, int, int, 418 struct intrs_and_queues *); 419static int prep_firmware(struct adapter *); 420static int partition_resources(struct adapter *, const struct firmware *, 421 const char *); 422static int get_params__pre_init(struct adapter *); 423static int get_params__post_init(struct adapter *); 424static int set_params__post_init(struct adapter *); 425static void t4_set_desc(struct adapter *); 426static void build_medialist(struct port_info *, struct ifmedia *); 427static int cxgbe_init_synchronized(struct vi_info *); 428static int cxgbe_uninit_synchronized(struct vi_info *); 429static int setup_intr_handlers(struct adapter *); 430static void quiesce_txq(struct adapter *, struct sge_txq *); 431static void quiesce_wrq(struct adapter *, struct sge_wrq *); 432static void quiesce_iq(struct adapter *, struct sge_iq *); 433static void quiesce_fl(struct adapter *, struct sge_fl *); 434static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 435 driver_intr_t *, void *, char *); 436static int t4_free_irq(struct adapter *, struct irq *); 437static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 438 unsigned int); 439static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 440static void vi_refresh_stats(struct adapter *, struct vi_info *); 441static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 442static void cxgbe_tick(void *); 443static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 444static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 445 struct mbuf *); 446static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 447static int fw_msg_not_handled(struct adapter *, const __be64 *); 448static void t4_sysctls(struct adapter *); 449static void cxgbe_sysctls(struct port_info *); 450static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 451static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 452static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 453static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 454static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 455static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 456static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 457static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 458static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 459static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 460static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 461#ifdef SBUF_DRAIN 462static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 463static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 464static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 465static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 466static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 467static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 468static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 469static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 470static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 471static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 472static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 473static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 474static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 475static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 476static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 477static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 478static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 479static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 480static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 481static int sysctl_tids(SYSCTL_HANDLER_ARGS); 482static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 483static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 484static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 485static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 486static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 487#endif 488static uint32_t fconf_to_mode(uint32_t); 489static uint32_t mode_to_fconf(uint32_t); 490static uint32_t fspec_to_fconf(struct t4_filter_specification *); 491static int get_filter_mode(struct adapter *, uint32_t *); 492static int set_filter_mode(struct adapter *, uint32_t); 493static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 494static int get_filter(struct adapter *, struct t4_filter *); 495static int set_filter(struct adapter *, struct t4_filter *); 496static int del_filter(struct adapter *, struct t4_filter *); 497static void clear_filter(struct filter_entry *); 498static int set_filter_wr(struct adapter *, int); 499static int del_filter_wr(struct adapter *, int); 500static int get_sge_context(struct adapter *, struct t4_sge_context *); 501static int load_fw(struct adapter *, struct t4_data *); 502static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 503static int read_i2c(struct adapter *, struct t4_i2c_data *); 504static int set_sched_class(struct adapter *, struct t4_sched_params *); 505static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 506#ifdef TCP_OFFLOAD 507static int toe_capability(struct vi_info *, int); 508#endif 509static int mod_event(module_t, int, void *); 510 511struct { 512 uint16_t device; 513 char *desc; 514} t4_pciids[] = { 515 {0xa000, "Chelsio Terminator 4 FPGA"}, 516 {0x4400, "Chelsio T440-dbg"}, 517 {0x4401, "Chelsio T420-CR"}, 518 {0x4402, "Chelsio T422-CR"}, 519 {0x4403, "Chelsio T440-CR"}, 520 {0x4404, "Chelsio T420-BCH"}, 521 {0x4405, "Chelsio T440-BCH"}, 522 {0x4406, "Chelsio T440-CH"}, 523 {0x4407, "Chelsio T420-SO"}, 524 {0x4408, "Chelsio T420-CX"}, 525 {0x4409, "Chelsio T420-BT"}, 526 {0x440a, "Chelsio T404-BT"}, 527 {0x440e, "Chelsio T440-LP-CR"}, 528}, t5_pciids[] = { 529 {0xb000, "Chelsio Terminator 5 FPGA"}, 530 {0x5400, "Chelsio T580-dbg"}, 531 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 532 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 533 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 534 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 535 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 536 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 537 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 538 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 539 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 540 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 541 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 542 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 543 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 544#ifdef notyet 545 {0x5404, "Chelsio T520-BCH"}, 546 {0x5405, "Chelsio T540-BCH"}, 547 {0x5406, "Chelsio T540-CH"}, 548 {0x5408, "Chelsio T520-CX"}, 549 {0x540b, "Chelsio B520-SR"}, 550 {0x540c, "Chelsio B504-BT"}, 551 {0x540f, "Chelsio Amsterdam"}, 552 {0x5413, "Chelsio T580-CHR"}, 553#endif 554}; 555 556#ifdef TCP_OFFLOAD 557/* 558 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 559 * exactly the same for both rxq and ofld_rxq. 560 */ 561CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 562CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 563#endif 564 565/* No easy way to include t4_msg.h before adapter.h so we check this way */ 566CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 567CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 568 569CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 570 571static int 572t4_probe(device_t dev) 573{ 574 int i; 575 uint16_t v = pci_get_vendor(dev); 576 uint16_t d = pci_get_device(dev); 577 uint8_t f = pci_get_function(dev); 578 579 if (v != PCI_VENDOR_ID_CHELSIO) 580 return (ENXIO); 581 582 /* Attach only to PF0 of the FPGA */ 583 if (d == 0xa000 && f != 0) 584 return (ENXIO); 585 586 for (i = 0; i < nitems(t4_pciids); i++) { 587 if (d == t4_pciids[i].device) { 588 device_set_desc(dev, t4_pciids[i].desc); 589 return (BUS_PROBE_DEFAULT); 590 } 591 } 592 593 return (ENXIO); 594} 595 596static int 597t5_probe(device_t dev) 598{ 599 int i; 600 uint16_t v = pci_get_vendor(dev); 601 uint16_t d = pci_get_device(dev); 602 uint8_t f = pci_get_function(dev); 603 604 if (v != PCI_VENDOR_ID_CHELSIO) 605 return (ENXIO); 606 607 /* Attach only to PF0 of the FPGA */ 608 if (d == 0xb000 && f != 0) 609 return (ENXIO); 610 611 for (i = 0; i < nitems(t5_pciids); i++) { 612 if (d == t5_pciids[i].device) { 613 device_set_desc(dev, t5_pciids[i].desc); 614 return (BUS_PROBE_DEFAULT); 615 } 616 } 617 618 return (ENXIO); 619} 620 621static void 622t5_attribute_workaround(device_t dev) 623{ 624 device_t root_port; 625 uint32_t v; 626 627 /* 628 * The T5 chips do not properly echo the No Snoop and Relaxed 629 * Ordering attributes when replying to a TLP from a Root 630 * Port. As a workaround, find the parent Root Port and 631 * disable No Snoop and Relaxed Ordering. Note that this 632 * affects all devices under this root port. 633 */ 634 root_port = pci_find_pcie_root_port(dev); 635 if (root_port == NULL) { 636 device_printf(dev, "Unable to find parent root port\n"); 637 return; 638 } 639 640 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 641 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 642 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 643 0) 644 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 645 device_get_nameunit(root_port)); 646} 647 648static int 649t4_attach(device_t dev) 650{ 651 struct adapter *sc; 652 int rc = 0, i, j, n10g, n1g, rqidx, tqidx; 653 struct intrs_and_queues iaq; 654 struct sge *s; 655#ifdef TCP_OFFLOAD 656 int ofld_rqidx, ofld_tqidx; 657#endif 658#ifdef DEV_NETMAP 659 int nm_rqidx, nm_tqidx; 660#endif 661 int num_vis; 662 663 sc = device_get_softc(dev); 664 sc->dev = dev; 665 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 666 667 if ((pci_get_device(dev) & 0xff00) == 0x5400) 668 t5_attribute_workaround(dev); 669 pci_enable_busmaster(dev); 670 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 671 uint32_t v; 672 673 pci_set_max_read_req(dev, 4096); 674 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 675 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 676 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 677 678 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 679 } 680 681 sc->traceq = -1; 682 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 683 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 684 device_get_nameunit(dev)); 685 686 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 687 device_get_nameunit(dev)); 688 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 689 sx_xlock(&t4_list_lock); 690 SLIST_INSERT_HEAD(&t4_list, sc, link); 691 sx_xunlock(&t4_list_lock); 692 693 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 694 TAILQ_INIT(&sc->sfl); 695 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 696 697 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 698 699 rc = map_bars_0_and_4(sc); 700 if (rc != 0) 701 goto done; /* error message displayed already */ 702 703 /* 704 * This is the real PF# to which we're attaching. Works from within PCI 705 * passthrough environments too, where pci_get_function() could return a 706 * different PF# depending on the passthrough configuration. We need to 707 * use the real PF# in all our communication with the firmware. 708 */ 709 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 710 sc->mbox = sc->pf; 711 712 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 713 sc->an_handler = an_not_handled; 714 for (i = 0; i < nitems(sc->cpl_handler); i++) 715 sc->cpl_handler[i] = cpl_not_handled; 716 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 717 sc->fw_msg_handler[i] = fw_msg_not_handled; 718 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 719 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 720 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 721 t4_init_sge_cpl_handlers(sc); 722 723 /* Prepare the adapter for operation */ 724 rc = -t4_prep_adapter(sc); 725 if (rc != 0) { 726 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 727 goto done; 728 } 729 730 /* 731 * Do this really early, with the memory windows set up even before the 732 * character device. The userland tool's register i/o and mem read 733 * will work even in "recovery mode". 734 */ 735 setup_memwin(sc); 736 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 737 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 738 device_get_nameunit(dev)); 739 if (sc->cdev == NULL) 740 device_printf(dev, "failed to create nexus char device.\n"); 741 else 742 sc->cdev->si_drv1 = sc; 743 744 /* Go no further if recovery mode has been requested. */ 745 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 746 device_printf(dev, "recovery mode.\n"); 747 goto done; 748 } 749 750#if defined(__i386__) 751 if ((cpu_feature & CPUID_CX8) == 0) { 752 device_printf(dev, "64 bit atomics not available.\n"); 753 rc = ENOTSUP; 754 goto done; 755 } 756#endif 757 758 /* Prepare the firmware for operation */ 759 rc = prep_firmware(sc); 760 if (rc != 0) 761 goto done; /* error message displayed already */ 762 763 rc = get_params__post_init(sc); 764 if (rc != 0) 765 goto done; /* error message displayed already */ 766 767 rc = set_params__post_init(sc); 768 if (rc != 0) 769 goto done; /* error message displayed already */ 770 771 rc = map_bar_2(sc); 772 if (rc != 0) 773 goto done; /* error message displayed already */ 774 775 rc = t4_create_dma_tag(sc); 776 if (rc != 0) 777 goto done; /* error message displayed already */ 778 779 /* 780 * Number of VIs to create per-port. The first VI is the "main" regular 781 * VI for the port. The rest are additional virtual interfaces on the 782 * same physical port. Note that the main VI does not have native 783 * netmap support but the extra VIs do. 784 * 785 * Limit the number of VIs per port to the number of available 786 * MAC addresses per port. 787 */ 788 if (t4_num_vis >= 1) 789 num_vis = t4_num_vis; 790 else 791 num_vis = 1; 792 if (num_vis > nitems(vi_mac_funcs)) { 793 num_vis = nitems(vi_mac_funcs); 794 device_printf(dev, "Number of VIs limited to %d\n", num_vis); 795 } 796 797 /* 798 * First pass over all the ports - allocate VIs and initialize some 799 * basic parameters like mac address, port type, etc. We also figure 800 * out whether a port is 10G or 1G and use that information when 801 * calculating how many interrupts to attempt to allocate. 802 */ 803 n10g = n1g = 0; 804 for_each_port(sc, i) { 805 struct port_info *pi; 806 807 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 808 sc->port[i] = pi; 809 810 /* These must be set before t4_port_init */ 811 pi->adapter = sc; 812 pi->port_id = i; 813 /* 814 * XXX: vi[0] is special so we can't delay this allocation until 815 * pi->nvi's final value is known. 816 */ 817 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE, 818 M_ZERO | M_WAITOK); 819 820 /* 821 * Allocate the "main" VI and initialize parameters 822 * like mac addr. 823 */ 824 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 825 if (rc != 0) { 826 device_printf(dev, "unable to initialize port %d: %d\n", 827 i, rc); 828 free(pi->vi, M_CXGBE); 829 free(pi, M_CXGBE); 830 sc->port[i] = NULL; 831 goto done; 832 } 833 834 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 835 pi->link_cfg.requested_fc |= t4_pause_settings; 836 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 837 pi->link_cfg.fc |= t4_pause_settings; 838 839 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 840 if (rc != 0) { 841 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 842 free(pi->vi, M_CXGBE); 843 free(pi, M_CXGBE); 844 sc->port[i] = NULL; 845 goto done; 846 } 847 848 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 849 device_get_nameunit(dev), i); 850 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 851 sc->chan_map[pi->tx_chan] = i; 852 853 if (is_10G_port(pi) || is_40G_port(pi)) { 854 n10g++; 855 } else { 856 n1g++; 857 } 858 859 pi->linkdnrc = -1; 860 861 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 862 if (pi->dev == NULL) { 863 device_printf(dev, 864 "failed to add device for port %d.\n", i); 865 rc = ENXIO; 866 goto done; 867 } 868 pi->vi[0].dev = pi->dev; 869 device_set_softc(pi->dev, pi); 870 } 871 872 /* 873 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 874 */ 875 rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq); 876 if (rc != 0) 877 goto done; /* error message displayed already */ 878 if (iaq.nrxq_vi + iaq.nofldrxq_vi + iaq.nnmrxq_vi == 0) 879 num_vis = 1; 880 881 sc->intr_type = iaq.intr_type; 882 sc->intr_count = iaq.nirq; 883 884 s = &sc->sge; 885 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 886 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 887 if (num_vis > 1) { 888 s->nrxq += (n10g + n1g) * (num_vis - 1) * iaq.nrxq_vi; 889 s->ntxq += (n10g + n1g) * (num_vis - 1) * iaq.ntxq_vi; 890 } 891 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 892 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 893 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 894#ifdef TCP_OFFLOAD 895 if (is_offload(sc)) { 896 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 897 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 898 if (num_vis > 1) { 899 s->nofldrxq += (n10g + n1g) * (num_vis - 1) * 900 iaq.nofldrxq_vi; 901 s->nofldtxq += (n10g + n1g) * (num_vis - 1) * 902 iaq.nofldtxq_vi; 903 } 904 s->neq += s->nofldtxq + s->nofldrxq; 905 s->niq += s->nofldrxq; 906 907 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 908 M_CXGBE, M_ZERO | M_WAITOK); 909 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 910 M_CXGBE, M_ZERO | M_WAITOK); 911 } 912#endif 913#ifdef DEV_NETMAP 914 if (num_vis > 1) { 915 s->nnmrxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmrxq_vi; 916 s->nnmtxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmtxq_vi; 917 } 918 s->neq += s->nnmtxq + s->nnmrxq; 919 s->niq += s->nnmrxq; 920 921 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 922 M_CXGBE, M_ZERO | M_WAITOK); 923 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 924 M_CXGBE, M_ZERO | M_WAITOK); 925#endif 926 927 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 928 M_ZERO | M_WAITOK); 929 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 930 M_ZERO | M_WAITOK); 931 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 932 M_ZERO | M_WAITOK); 933 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 934 M_ZERO | M_WAITOK); 935 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 936 M_ZERO | M_WAITOK); 937 938 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 939 M_ZERO | M_WAITOK); 940 941 t4_init_l2t(sc, M_WAITOK); 942 943 /* 944 * Second pass over the ports. This time we know the number of rx and 945 * tx queues that each port should get. 946 */ 947 rqidx = tqidx = 0; 948#ifdef TCP_OFFLOAD 949 ofld_rqidx = ofld_tqidx = 0; 950#endif 951#ifdef DEV_NETMAP 952 nm_rqidx = nm_tqidx = 0; 953#endif 954 for_each_port(sc, i) { 955 struct port_info *pi = sc->port[i]; 956 struct vi_info *vi; 957 958 if (pi == NULL) 959 continue; 960 961 pi->nvi = num_vis; 962 for_each_vi(pi, j, vi) { 963 vi->pi = pi; 964 vi->qsize_rxq = t4_qsize_rxq; 965 vi->qsize_txq = t4_qsize_txq; 966 967 vi->first_rxq = rqidx; 968 vi->first_txq = tqidx; 969 if (is_10G_port(pi) || is_40G_port(pi)) { 970 vi->tmr_idx = t4_tmr_idx_10g; 971 vi->pktc_idx = t4_pktc_idx_10g; 972 vi->flags |= iaq.intr_flags_10g & INTR_RXQ; 973 vi->nrxq = j == 0 ? iaq.nrxq10g : iaq.nrxq_vi; 974 vi->ntxq = j == 0 ? iaq.ntxq10g : iaq.ntxq_vi; 975 } else { 976 vi->tmr_idx = t4_tmr_idx_1g; 977 vi->pktc_idx = t4_pktc_idx_1g; 978 vi->flags |= iaq.intr_flags_1g & INTR_RXQ; 979 vi->nrxq = j == 0 ? iaq.nrxq1g : iaq.nrxq_vi; 980 vi->ntxq = j == 0 ? iaq.ntxq1g : iaq.ntxq_vi; 981 } 982 rqidx += vi->nrxq; 983 tqidx += vi->ntxq; 984 985 if (j == 0 && vi->ntxq > 1) 986 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 987 else 988 vi->rsrv_noflowq = 0; 989 990#ifdef TCP_OFFLOAD 991 vi->first_ofld_rxq = ofld_rqidx; 992 vi->first_ofld_txq = ofld_tqidx; 993 if (is_10G_port(pi) || is_40G_port(pi)) { 994 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ; 995 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g : 996 iaq.nofldrxq_vi; 997 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g : 998 iaq.nofldtxq_vi; 999 } else { 1000 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ; 1001 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g : 1002 iaq.nofldrxq_vi; 1003 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g : 1004 iaq.nofldtxq_vi; 1005 } 1006 ofld_rqidx += vi->nofldrxq; 1007 ofld_tqidx += vi->nofldtxq; 1008#endif 1009#ifdef DEV_NETMAP 1010 if (j > 0) { 1011 vi->first_nm_rxq = nm_rqidx; 1012 vi->first_nm_txq = nm_tqidx; 1013 vi->nnmrxq = iaq.nnmrxq_vi; 1014 vi->nnmtxq = iaq.nnmtxq_vi; 1015 nm_rqidx += vi->nnmrxq; 1016 nm_tqidx += vi->nnmtxq; 1017 } 1018#endif 1019 } 1020 } 1021 1022 rc = setup_intr_handlers(sc); 1023 if (rc != 0) { 1024 device_printf(dev, 1025 "failed to setup interrupt handlers: %d\n", rc); 1026 goto done; 1027 } 1028 1029 rc = bus_generic_attach(dev); 1030 if (rc != 0) { 1031 device_printf(dev, 1032 "failed to attach all child ports: %d\n", rc); 1033 goto done; 1034 } 1035 1036 device_printf(dev, 1037 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 1038 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 1039 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 1040 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 1041 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 1042 1043 t4_set_desc(sc); 1044 1045done: 1046 if (rc != 0 && sc->cdev) { 1047 /* cdev was created and so cxgbetool works; recover that way. */ 1048 device_printf(dev, 1049 "error during attach, adapter is now in recovery mode.\n"); 1050 rc = 0; 1051 } 1052 1053 if (rc != 0) 1054 t4_detach(dev); 1055 else 1056 t4_sysctls(sc); 1057 1058 return (rc); 1059} 1060 1061/* 1062 * Idempotent 1063 */ 1064static int 1065t4_detach(device_t dev) 1066{ 1067 struct adapter *sc; 1068 struct port_info *pi; 1069 int i, rc; 1070 1071 sc = device_get_softc(dev); 1072 1073 if (sc->flags & FULL_INIT_DONE) 1074 t4_intr_disable(sc); 1075 1076 if (sc->cdev) { 1077 destroy_dev(sc->cdev); 1078 sc->cdev = NULL; 1079 } 1080 1081 rc = bus_generic_detach(dev); 1082 if (rc) { 1083 device_printf(dev, 1084 "failed to detach child devices: %d\n", rc); 1085 return (rc); 1086 } 1087 1088 for (i = 0; i < sc->intr_count; i++) 1089 t4_free_irq(sc, &sc->irq[i]); 1090 1091 for (i = 0; i < MAX_NPORTS; i++) { 1092 pi = sc->port[i]; 1093 if (pi) { 1094 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); 1095 if (pi->dev) 1096 device_delete_child(dev, pi->dev); 1097 1098 mtx_destroy(&pi->pi_lock); 1099 free(pi->vi, M_CXGBE); 1100 free(pi, M_CXGBE); 1101 } 1102 } 1103 1104 if (sc->flags & FULL_INIT_DONE) 1105 adapter_full_uninit(sc); 1106 1107 if (sc->flags & FW_OK) 1108 t4_fw_bye(sc, sc->mbox); 1109 1110 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1111 pci_release_msi(dev); 1112 1113 if (sc->regs_res) 1114 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1115 sc->regs_res); 1116 1117 if (sc->udbs_res) 1118 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1119 sc->udbs_res); 1120 1121 if (sc->msix_res) 1122 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1123 sc->msix_res); 1124 1125 if (sc->l2t) 1126 t4_free_l2t(sc->l2t); 1127 1128#ifdef TCP_OFFLOAD 1129 free(sc->sge.ofld_rxq, M_CXGBE); 1130 free(sc->sge.ofld_txq, M_CXGBE); 1131#endif 1132#ifdef DEV_NETMAP 1133 free(sc->sge.nm_rxq, M_CXGBE); 1134 free(sc->sge.nm_txq, M_CXGBE); 1135#endif 1136 free(sc->irq, M_CXGBE); 1137 free(sc->sge.rxq, M_CXGBE); 1138 free(sc->sge.txq, M_CXGBE); 1139 free(sc->sge.ctrlq, M_CXGBE); 1140 free(sc->sge.iqmap, M_CXGBE); 1141 free(sc->sge.eqmap, M_CXGBE); 1142 free(sc->tids.ftid_tab, M_CXGBE); 1143 t4_destroy_dma_tag(sc); 1144 if (mtx_initialized(&sc->sc_lock)) { 1145 sx_xlock(&t4_list_lock); 1146 SLIST_REMOVE(&t4_list, sc, adapter, link); 1147 sx_xunlock(&t4_list_lock); 1148 mtx_destroy(&sc->sc_lock); 1149 } 1150 1151 callout_drain(&sc->sfl_callout); 1152 if (mtx_initialized(&sc->tids.ftid_lock)) 1153 mtx_destroy(&sc->tids.ftid_lock); 1154 if (mtx_initialized(&sc->sfl_lock)) 1155 mtx_destroy(&sc->sfl_lock); 1156 if (mtx_initialized(&sc->ifp_lock)) 1157 mtx_destroy(&sc->ifp_lock); 1158 if (mtx_initialized(&sc->regwin_lock)) 1159 mtx_destroy(&sc->regwin_lock); 1160 1161 bzero(sc, sizeof(*sc)); 1162 1163 return (0); 1164} 1165 1166static int 1167cxgbe_probe(device_t dev) 1168{ 1169 char buf[128]; 1170 struct port_info *pi = device_get_softc(dev); 1171 1172 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1173 device_set_desc_copy(dev, buf); 1174 1175 return (BUS_PROBE_DEFAULT); 1176} 1177 1178#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1179 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1180 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1181#define T4_CAP_ENABLE (T4_CAP) 1182 1183static int 1184cxgbe_vi_attach(device_t dev, struct vi_info *vi) 1185{ 1186 struct ifnet *ifp; 1187 struct sbuf *sb; 1188 1189 vi->xact_addr_filt = -1; 1190 callout_init(&vi->tick, 1); 1191 1192 /* Allocate an ifnet and set it up */ 1193 ifp = if_alloc(IFT_ETHER); 1194 if (ifp == NULL) { 1195 device_printf(dev, "Cannot allocate ifnet\n"); 1196 return (ENOMEM); 1197 } 1198 vi->ifp = ifp; 1199 ifp->if_softc = vi; 1200 1201 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1202 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1203 1204 ifp->if_init = cxgbe_init; 1205 ifp->if_ioctl = cxgbe_ioctl; 1206 ifp->if_transmit = cxgbe_transmit; 1207 ifp->if_qflush = cxgbe_qflush; 1208 1209 ifp->if_capabilities = T4_CAP; 1210#ifdef TCP_OFFLOAD 1211 if (vi->nofldrxq != 0) 1212 ifp->if_capabilities |= IFCAP_TOE; 1213#endif 1214 ifp->if_capenable = T4_CAP_ENABLE; 1215 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1216 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1217 1218 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1219 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1220 ifp->if_hw_tsomaxsegsize = 65536; 1221 1222 /* Initialize ifmedia for this VI */ 1223 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1224 cxgbe_media_status); 1225 build_medialist(vi->pi, &vi->media); 1226 1227 vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1228 EVENTHANDLER_PRI_ANY); 1229 1230 ether_ifattach(ifp, vi->hw_addr); 1231#ifdef DEV_NETMAP 1232 if (vi->nnmrxq != 0) 1233 cxgbe_nm_attach(vi); 1234#endif 1235 sb = sbuf_new_auto(); 1236 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); 1237#ifdef TCP_OFFLOAD 1238 if (ifp->if_capabilities & IFCAP_TOE) 1239 sbuf_printf(sb, "; %d txq, %d rxq (TOE)", 1240 vi->nofldtxq, vi->nofldrxq); 1241#endif 1242#ifdef DEV_NETMAP 1243 if (ifp->if_capabilities & IFCAP_NETMAP) 1244 sbuf_printf(sb, "; %d txq, %d rxq (netmap)", 1245 vi->nnmtxq, vi->nnmrxq); 1246#endif 1247 sbuf_finish(sb); 1248 device_printf(dev, "%s\n", sbuf_data(sb)); 1249 sbuf_delete(sb); 1250 1251 vi_sysctls(vi); 1252 1253 return (0); 1254} 1255 1256static int 1257cxgbe_attach(device_t dev) 1258{ 1259 struct port_info *pi = device_get_softc(dev); 1260 struct vi_info *vi; 1261 int i, rc; 1262 1263 callout_init_mtx(&pi->tick, &pi->pi_lock, 0); 1264 1265 rc = cxgbe_vi_attach(dev, &pi->vi[0]); 1266 if (rc) 1267 return (rc); 1268 1269 for_each_vi(pi, i, vi) { 1270 if (i == 0) 1271 continue; 1272 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1273 "vcxgbe" : "vcxl", -1); 1274 if (vi->dev == NULL) { 1275 device_printf(dev, "failed to add VI %d\n", i); 1276 continue; 1277 } 1278 device_set_softc(vi->dev, vi); 1279 } 1280 1281 cxgbe_sysctls(pi); 1282 1283 bus_generic_attach(dev); 1284 1285 return (0); 1286} 1287 1288static void 1289cxgbe_vi_detach(struct vi_info *vi) 1290{ 1291 struct ifnet *ifp = vi->ifp; 1292 1293 ether_ifdetach(ifp); 1294 1295 if (vi->vlan_c) 1296 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c); 1297 1298 /* Let detach proceed even if these fail. */ 1299#ifdef DEV_NETMAP 1300 if (ifp->if_capabilities & IFCAP_NETMAP) 1301 cxgbe_nm_detach(vi); 1302#endif 1303 cxgbe_uninit_synchronized(vi); 1304 callout_drain(&vi->tick); 1305 vi_full_uninit(vi); 1306 1307 ifmedia_removeall(&vi->media); 1308 if_free(vi->ifp); 1309 vi->ifp = NULL; 1310} 1311 1312static int 1313cxgbe_detach(device_t dev) 1314{ 1315 struct port_info *pi = device_get_softc(dev); 1316 struct adapter *sc = pi->adapter; 1317 int rc; 1318 1319 /* Detach the extra VIs first. */ 1320 rc = bus_generic_detach(dev); 1321 if (rc) 1322 return (rc); 1323 device_delete_children(dev); 1324 1325 doom_vi(sc, &pi->vi[0]); 1326 1327 if (pi->flags & HAS_TRACEQ) { 1328 sc->traceq = -1; /* cloner should not create ifnet */ 1329 t4_tracer_port_detach(sc); 1330 } 1331 1332 cxgbe_vi_detach(&pi->vi[0]); 1333 callout_drain(&pi->tick); 1334 1335 end_synchronized_op(sc, 0); 1336 1337 return (0); 1338} 1339 1340static void 1341cxgbe_init(void *arg) 1342{ 1343 struct vi_info *vi = arg; 1344 struct adapter *sc = vi->pi->adapter; 1345 1346 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) 1347 return; 1348 cxgbe_init_synchronized(vi); 1349 end_synchronized_op(sc, 0); 1350} 1351 1352static int 1353cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1354{ 1355 int rc = 0, mtu, flags, can_sleep; 1356 struct vi_info *vi = ifp->if_softc; 1357 struct adapter *sc = vi->pi->adapter; 1358 struct ifreq *ifr = (struct ifreq *)data; 1359 uint32_t mask; 1360 1361 switch (cmd) { 1362 case SIOCSIFMTU: 1363 mtu = ifr->ifr_mtu; 1364 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1365 return (EINVAL); 1366 1367 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); 1368 if (rc) 1369 return (rc); 1370 ifp->if_mtu = mtu; 1371 if (vi->flags & VI_INIT_DONE) { 1372 t4_update_fl_bufsize(ifp); 1373 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1374 rc = update_mac_settings(ifp, XGMAC_MTU); 1375 } 1376 end_synchronized_op(sc, 0); 1377 break; 1378 1379 case SIOCSIFFLAGS: 1380 can_sleep = 0; 1381redo_sifflags: 1382 rc = begin_synchronized_op(sc, vi, 1383 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1384 if (rc) 1385 return (rc); 1386 1387 if (ifp->if_flags & IFF_UP) { 1388 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1389 flags = vi->if_flags; 1390 if ((ifp->if_flags ^ flags) & 1391 (IFF_PROMISC | IFF_ALLMULTI)) { 1392 if (can_sleep == 1) { 1393 end_synchronized_op(sc, 0); 1394 can_sleep = 0; 1395 goto redo_sifflags; 1396 } 1397 rc = update_mac_settings(ifp, 1398 XGMAC_PROMISC | XGMAC_ALLMULTI); 1399 } 1400 } else { 1401 if (can_sleep == 0) { 1402 end_synchronized_op(sc, LOCK_HELD); 1403 can_sleep = 1; 1404 goto redo_sifflags; 1405 } 1406 rc = cxgbe_init_synchronized(vi); 1407 } 1408 vi->if_flags = ifp->if_flags; 1409 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1410 if (can_sleep == 0) { 1411 end_synchronized_op(sc, LOCK_HELD); 1412 can_sleep = 1; 1413 goto redo_sifflags; 1414 } 1415 rc = cxgbe_uninit_synchronized(vi); 1416 } 1417 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1418 break; 1419 1420 case SIOCADDMULTI: 1421 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1422 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi"); 1423 if (rc) 1424 return (rc); 1425 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1426 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1427 end_synchronized_op(sc, LOCK_HELD); 1428 break; 1429 1430 case SIOCSIFCAP: 1431 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); 1432 if (rc) 1433 return (rc); 1434 1435 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1436 if (mask & IFCAP_TXCSUM) { 1437 ifp->if_capenable ^= IFCAP_TXCSUM; 1438 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1439 1440 if (IFCAP_TSO4 & ifp->if_capenable && 1441 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1442 ifp->if_capenable &= ~IFCAP_TSO4; 1443 if_printf(ifp, 1444 "tso4 disabled due to -txcsum.\n"); 1445 } 1446 } 1447 if (mask & IFCAP_TXCSUM_IPV6) { 1448 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1449 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1450 1451 if (IFCAP_TSO6 & ifp->if_capenable && 1452 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1453 ifp->if_capenable &= ~IFCAP_TSO6; 1454 if_printf(ifp, 1455 "tso6 disabled due to -txcsum6.\n"); 1456 } 1457 } 1458 if (mask & IFCAP_RXCSUM) 1459 ifp->if_capenable ^= IFCAP_RXCSUM; 1460 if (mask & IFCAP_RXCSUM_IPV6) 1461 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1462 1463 /* 1464 * Note that we leave CSUM_TSO alone (it is always set). The 1465 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1466 * sending a TSO request our way, so it's sufficient to toggle 1467 * IFCAP_TSOx only. 1468 */ 1469 if (mask & IFCAP_TSO4) { 1470 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1471 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1472 if_printf(ifp, "enable txcsum first.\n"); 1473 rc = EAGAIN; 1474 goto fail; 1475 } 1476 ifp->if_capenable ^= IFCAP_TSO4; 1477 } 1478 if (mask & IFCAP_TSO6) { 1479 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1480 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1481 if_printf(ifp, "enable txcsum6 first.\n"); 1482 rc = EAGAIN; 1483 goto fail; 1484 } 1485 ifp->if_capenable ^= IFCAP_TSO6; 1486 } 1487 if (mask & IFCAP_LRO) { 1488#if defined(INET) || defined(INET6) 1489 int i; 1490 struct sge_rxq *rxq; 1491 1492 ifp->if_capenable ^= IFCAP_LRO; 1493 for_each_rxq(vi, i, rxq) { 1494 if (ifp->if_capenable & IFCAP_LRO) 1495 rxq->iq.flags |= IQ_LRO_ENABLED; 1496 else 1497 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1498 } 1499#endif 1500 } 1501#ifdef TCP_OFFLOAD 1502 if (mask & IFCAP_TOE) { 1503 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1504 1505 rc = toe_capability(vi, enable); 1506 if (rc != 0) 1507 goto fail; 1508 1509 ifp->if_capenable ^= mask; 1510 } 1511#endif 1512 if (mask & IFCAP_VLAN_HWTAGGING) { 1513 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1514 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1515 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1516 } 1517 if (mask & IFCAP_VLAN_MTU) { 1518 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1519 1520 /* Need to find out how to disable auto-mtu-inflation */ 1521 } 1522 if (mask & IFCAP_VLAN_HWTSO) 1523 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1524 if (mask & IFCAP_VLAN_HWCSUM) 1525 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1526 1527#ifdef VLAN_CAPABILITIES 1528 VLAN_CAPABILITIES(ifp); 1529#endif 1530fail: 1531 end_synchronized_op(sc, 0); 1532 break; 1533 1534 case SIOCSIFMEDIA: 1535 case SIOCGIFMEDIA: 1536 ifmedia_ioctl(ifp, ifr, &vi->media, cmd); 1537 break; 1538 1539 case SIOCGI2C: { 1540 struct ifi2creq i2c; 1541 1542 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1543 if (rc != 0) 1544 break; 1545 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1546 rc = EPERM; 1547 break; 1548 } 1549 if (i2c.len > sizeof(i2c.data)) { 1550 rc = EINVAL; 1551 break; 1552 } 1553 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); 1554 if (rc) 1555 return (rc); 1556 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr, 1557 i2c.offset, i2c.len, &i2c.data[0]); 1558 end_synchronized_op(sc, 0); 1559 if (rc == 0) 1560 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1561 break; 1562 } 1563 1564 default: 1565 rc = ether_ioctl(ifp, cmd, data); 1566 } 1567 1568 return (rc); 1569} 1570 1571static int 1572cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1573{ 1574 struct vi_info *vi = ifp->if_softc; 1575 struct port_info *pi = vi->pi; 1576 struct adapter *sc = pi->adapter; 1577 struct sge_txq *txq; 1578 void *items[1]; 1579 int rc; 1580 1581 M_ASSERTPKTHDR(m); 1582 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1583 1584 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1585 m_freem(m); 1586 return (ENETDOWN); 1587 } 1588 1589 rc = parse_pkt(&m); 1590 if (__predict_false(rc != 0)) { 1591 MPASS(m == NULL); /* was freed already */ 1592 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1593 return (rc); 1594 } 1595 1596 /* Select a txq. */ 1597 txq = &sc->sge.txq[vi->first_txq]; 1598 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1599 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + 1600 vi->rsrv_noflowq); 1601 1602 items[0] = m; 1603 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1604 if (__predict_false(rc != 0)) 1605 m_freem(m); 1606 1607 return (rc); 1608} 1609 1610static void 1611cxgbe_qflush(struct ifnet *ifp) 1612{ 1613 struct vi_info *vi = ifp->if_softc; 1614 struct sge_txq *txq; 1615 int i; 1616 1617 /* queues do not exist if !VI_INIT_DONE. */ 1618 if (vi->flags & VI_INIT_DONE) { 1619 for_each_txq(vi, i, txq) { 1620 TXQ_LOCK(txq); 1621 txq->eq.flags &= ~EQ_ENABLED; 1622 TXQ_UNLOCK(txq); 1623 while (!mp_ring_is_idle(txq->r)) { 1624 mp_ring_check_drainage(txq->r, 0); 1625 pause("qflush", 1); 1626 } 1627 } 1628 } 1629 if_qflush(ifp); 1630} 1631 1632static int 1633cxgbe_media_change(struct ifnet *ifp) 1634{ 1635 struct vi_info *vi = ifp->if_softc; 1636 1637 device_printf(vi->dev, "%s unimplemented.\n", __func__); 1638 1639 return (EOPNOTSUPP); 1640} 1641 1642static void 1643cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1644{ 1645 struct vi_info *vi = ifp->if_softc; 1646 struct port_info *pi = vi->pi; 1647 struct ifmedia_entry *cur; 1648 int speed = pi->link_cfg.speed; 1649 1650 cur = vi->media.ifm_cur; 1651 1652 ifmr->ifm_status = IFM_AVALID; 1653 if (!pi->link_cfg.link_ok) 1654 return; 1655 1656 ifmr->ifm_status |= IFM_ACTIVE; 1657 1658 /* active and current will differ iff current media is autoselect. */ 1659 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1660 return; 1661 1662 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1663 if (speed == SPEED_10000) 1664 ifmr->ifm_active |= IFM_10G_T; 1665 else if (speed == SPEED_1000) 1666 ifmr->ifm_active |= IFM_1000_T; 1667 else if (speed == SPEED_100) 1668 ifmr->ifm_active |= IFM_100_TX; 1669 else if (speed == SPEED_10) 1670 ifmr->ifm_active |= IFM_10_T; 1671 else 1672 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1673 speed)); 1674} 1675 1676static int 1677vcxgbe_probe(device_t dev) 1678{ 1679 char buf[128]; 1680 struct vi_info *vi = device_get_softc(dev); 1681 1682 snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id, 1683 vi - vi->pi->vi); 1684 device_set_desc_copy(dev, buf); 1685 1686 return (BUS_PROBE_DEFAULT); 1687} 1688 1689static int 1690vcxgbe_attach(device_t dev) 1691{ 1692 struct vi_info *vi; 1693 struct port_info *pi; 1694 struct adapter *sc; 1695 int func, index, rc; 1696 u32 param, val; 1697 1698 vi = device_get_softc(dev); 1699 pi = vi->pi; 1700 sc = pi->adapter; 1701 1702 index = vi - pi->vi; 1703 KASSERT(index < nitems(vi_mac_funcs), 1704 ("%s: VI %s doesn't have a MAC func", __func__, 1705 device_get_nameunit(dev))); 1706 func = vi_mac_funcs[index]; 1707 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, 1708 vi->hw_addr, &vi->rss_size, func, 0); 1709 if (rc < 0) { 1710 device_printf(dev, "Failed to allocate virtual interface " 1711 "for port %d: %d\n", pi->port_id, -rc); 1712 return (-rc); 1713 } 1714 vi->viid = rc; 1715 1716 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 1717 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | 1718 V_FW_PARAMS_PARAM_YZ(vi->viid); 1719 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1720 if (rc) 1721 vi->rss_base = 0xffff; 1722 else { 1723 /* MPASS((val >> 16) == rss_size); */ 1724 vi->rss_base = val & 0xffff; 1725 } 1726 1727 rc = cxgbe_vi_attach(dev, vi); 1728 if (rc) { 1729 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1730 return (rc); 1731 } 1732 return (0); 1733} 1734 1735static int 1736vcxgbe_detach(device_t dev) 1737{ 1738 struct vi_info *vi; 1739 struct adapter *sc; 1740 1741 vi = device_get_softc(dev); 1742 sc = vi->pi->adapter; 1743 1744 doom_vi(sc, vi); 1745 1746 cxgbe_vi_detach(vi); 1747 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1748 1749 end_synchronized_op(sc, 0); 1750 1751 return (0); 1752} 1753 1754void 1755t4_fatal_err(struct adapter *sc) 1756{ 1757 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1758 t4_intr_disable(sc); 1759 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1760 device_get_nameunit(sc->dev)); 1761} 1762 1763static int 1764map_bars_0_and_4(struct adapter *sc) 1765{ 1766 sc->regs_rid = PCIR_BAR(0); 1767 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1768 &sc->regs_rid, RF_ACTIVE); 1769 if (sc->regs_res == NULL) { 1770 device_printf(sc->dev, "cannot map registers.\n"); 1771 return (ENXIO); 1772 } 1773 sc->bt = rman_get_bustag(sc->regs_res); 1774 sc->bh = rman_get_bushandle(sc->regs_res); 1775 sc->mmio_len = rman_get_size(sc->regs_res); 1776 setbit(&sc->doorbells, DOORBELL_KDB); 1777 1778 sc->msix_rid = PCIR_BAR(4); 1779 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1780 &sc->msix_rid, RF_ACTIVE); 1781 if (sc->msix_res == NULL) { 1782 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1783 return (ENXIO); 1784 } 1785 1786 return (0); 1787} 1788 1789static int 1790map_bar_2(struct adapter *sc) 1791{ 1792 1793 /* 1794 * T4: only iWARP driver uses the userspace doorbells. There is no need 1795 * to map it if RDMA is disabled. 1796 */ 1797 if (is_t4(sc) && sc->rdmacaps == 0) 1798 return (0); 1799 1800 sc->udbs_rid = PCIR_BAR(2); 1801 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1802 &sc->udbs_rid, RF_ACTIVE); 1803 if (sc->udbs_res == NULL) { 1804 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1805 return (ENXIO); 1806 } 1807 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1808 1809 if (is_t5(sc)) { 1810 setbit(&sc->doorbells, DOORBELL_UDB); 1811#if defined(__i386__) || defined(__amd64__) 1812 if (t5_write_combine) { 1813 int rc; 1814 1815 /* 1816 * Enable write combining on BAR2. This is the 1817 * userspace doorbell BAR and is split into 128B 1818 * (UDBS_SEG_SIZE) doorbell regions, each associated 1819 * with an egress queue. The first 64B has the doorbell 1820 * and the second 64B can be used to submit a tx work 1821 * request with an implicit doorbell. 1822 */ 1823 1824 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1825 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1826 if (rc == 0) { 1827 clrbit(&sc->doorbells, DOORBELL_UDB); 1828 setbit(&sc->doorbells, DOORBELL_WCWR); 1829 setbit(&sc->doorbells, DOORBELL_UDBWC); 1830 } else { 1831 device_printf(sc->dev, 1832 "couldn't enable write combining: %d\n", 1833 rc); 1834 } 1835 1836 t4_write_reg(sc, A_SGE_STAT_CFG, 1837 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1838 } 1839#endif 1840 } 1841 1842 return (0); 1843} 1844 1845static const struct memwin t4_memwin[] = { 1846 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1847 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1848 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1849}; 1850 1851static const struct memwin t5_memwin[] = { 1852 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1853 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1854 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1855}; 1856 1857static void 1858setup_memwin(struct adapter *sc) 1859{ 1860 const struct memwin *mw; 1861 int i, n; 1862 uint32_t bar0; 1863 1864 if (is_t4(sc)) { 1865 /* 1866 * Read low 32b of bar0 indirectly via the hardware backdoor 1867 * mechanism. Works from within PCI passthrough environments 1868 * too, where rman_get_start() can return a different value. We 1869 * need to program the T4 memory window decoders with the actual 1870 * addresses that will be coming across the PCIe link. 1871 */ 1872 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1873 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1874 1875 mw = &t4_memwin[0]; 1876 n = nitems(t4_memwin); 1877 } else { 1878 /* T5 uses the relative offset inside the PCIe BAR */ 1879 bar0 = 0; 1880 1881 mw = &t5_memwin[0]; 1882 n = nitems(t5_memwin); 1883 } 1884 1885 for (i = 0; i < n; i++, mw++) { 1886 t4_write_reg(sc, 1887 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1888 (mw->base + bar0) | V_BIR(0) | 1889 V_WINDOW(ilog2(mw->aperture) - 10)); 1890 } 1891 1892 /* flush */ 1893 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1894} 1895 1896/* 1897 * Verify that the memory range specified by the addr/len pair is valid and lies 1898 * entirely within a single region (EDCx or MCx). 1899 */ 1900static int 1901validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1902{ 1903 uint32_t em, addr_len, maddr, mlen; 1904 1905 /* Memory can only be accessed in naturally aligned 4 byte units */ 1906 if (addr & 3 || len & 3 || len == 0) 1907 return (EINVAL); 1908 1909 /* Enabled memories */ 1910 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1911 if (em & F_EDRAM0_ENABLE) { 1912 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1913 maddr = G_EDRAM0_BASE(addr_len) << 20; 1914 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1915 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1916 addr + len <= maddr + mlen) 1917 return (0); 1918 } 1919 if (em & F_EDRAM1_ENABLE) { 1920 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1921 maddr = G_EDRAM1_BASE(addr_len) << 20; 1922 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1923 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1924 addr + len <= maddr + mlen) 1925 return (0); 1926 } 1927 if (em & F_EXT_MEM_ENABLE) { 1928 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1929 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1930 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1931 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1932 addr + len <= maddr + mlen) 1933 return (0); 1934 } 1935 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1936 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1937 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1938 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1939 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1940 addr + len <= maddr + mlen) 1941 return (0); 1942 } 1943 1944 return (EFAULT); 1945} 1946 1947static int 1948fwmtype_to_hwmtype(int mtype) 1949{ 1950 1951 switch (mtype) { 1952 case FW_MEMTYPE_EDC0: 1953 return (MEM_EDC0); 1954 case FW_MEMTYPE_EDC1: 1955 return (MEM_EDC1); 1956 case FW_MEMTYPE_EXTMEM: 1957 return (MEM_MC0); 1958 case FW_MEMTYPE_EXTMEM1: 1959 return (MEM_MC1); 1960 default: 1961 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1962 } 1963} 1964 1965/* 1966 * Verify that the memory range specified by the memtype/offset/len pair is 1967 * valid and lies entirely within the memtype specified. The global address of 1968 * the start of the range is returned in addr. 1969 */ 1970static int 1971validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1972 uint32_t *addr) 1973{ 1974 uint32_t em, addr_len, maddr, mlen; 1975 1976 /* Memory can only be accessed in naturally aligned 4 byte units */ 1977 if (off & 3 || len & 3 || len == 0) 1978 return (EINVAL); 1979 1980 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1981 switch (fwmtype_to_hwmtype(mtype)) { 1982 case MEM_EDC0: 1983 if (!(em & F_EDRAM0_ENABLE)) 1984 return (EINVAL); 1985 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1986 maddr = G_EDRAM0_BASE(addr_len) << 20; 1987 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1988 break; 1989 case MEM_EDC1: 1990 if (!(em & F_EDRAM1_ENABLE)) 1991 return (EINVAL); 1992 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1993 maddr = G_EDRAM1_BASE(addr_len) << 20; 1994 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1995 break; 1996 case MEM_MC: 1997 if (!(em & F_EXT_MEM_ENABLE)) 1998 return (EINVAL); 1999 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2000 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2001 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 2002 break; 2003 case MEM_MC1: 2004 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 2005 return (EINVAL); 2006 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2007 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2008 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 2009 break; 2010 default: 2011 return (EINVAL); 2012 } 2013 2014 if (mlen > 0 && off < mlen && off + len <= mlen) { 2015 *addr = maddr + off; /* global address */ 2016 return (0); 2017 } 2018 2019 return (EFAULT); 2020} 2021 2022static void 2023memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 2024{ 2025 const struct memwin *mw; 2026 2027 if (is_t4(sc)) { 2028 KASSERT(win >= 0 && win < nitems(t4_memwin), 2029 ("%s: incorrect memwin# (%d)", __func__, win)); 2030 mw = &t4_memwin[win]; 2031 } else { 2032 KASSERT(win >= 0 && win < nitems(t5_memwin), 2033 ("%s: incorrect memwin# (%d)", __func__, win)); 2034 mw = &t5_memwin[win]; 2035 } 2036 2037 if (base != NULL) 2038 *base = mw->base; 2039 if (aperture != NULL) 2040 *aperture = mw->aperture; 2041} 2042 2043/* 2044 * Positions the memory window such that it can be used to access the specified 2045 * address in the chip's address space. The return value is the offset of addr 2046 * from the start of the window. 2047 */ 2048static uint32_t 2049position_memwin(struct adapter *sc, int n, uint32_t addr) 2050{ 2051 uint32_t start, pf; 2052 uint32_t reg; 2053 2054 KASSERT(n >= 0 && n <= 3, 2055 ("%s: invalid window %d.", __func__, n)); 2056 KASSERT((addr & 3) == 0, 2057 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 2058 2059 if (is_t4(sc)) { 2060 pf = 0; 2061 start = addr & ~0xf; /* start must be 16B aligned */ 2062 } else { 2063 pf = V_PFNUM(sc->pf); 2064 start = addr & ~0x7f; /* start must be 128B aligned */ 2065 } 2066 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 2067 2068 t4_write_reg(sc, reg, start | pf); 2069 t4_read_reg(sc, reg); 2070 2071 return (addr - start); 2072} 2073 2074static int 2075cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis, 2076 struct intrs_and_queues *iaq) 2077{ 2078 int rc, itype, navail, nrxq10g, nrxq1g, n; 2079 int nofldrxq10g = 0, nofldrxq1g = 0; 2080 2081 bzero(iaq, sizeof(*iaq)); 2082 2083 iaq->ntxq10g = t4_ntxq10g; 2084 iaq->ntxq1g = t4_ntxq1g; 2085 iaq->ntxq_vi = t4_ntxq_vi; 2086 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 2087 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 2088 iaq->nrxq_vi = t4_nrxq_vi; 2089 iaq->rsrv_noflowq = t4_rsrv_noflowq; 2090#ifdef TCP_OFFLOAD 2091 if (is_offload(sc)) { 2092 iaq->nofldtxq10g = t4_nofldtxq10g; 2093 iaq->nofldtxq1g = t4_nofldtxq1g; 2094 iaq->nofldtxq_vi = t4_nofldtxq_vi; 2095 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 2096 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 2097 iaq->nofldrxq_vi = t4_nofldrxq_vi; 2098 } 2099#endif 2100#ifdef DEV_NETMAP 2101 iaq->nnmtxq_vi = t4_nnmtxq_vi; 2102 iaq->nnmrxq_vi = t4_nnmrxq_vi; 2103#endif 2104 2105 for (itype = INTR_MSIX; itype; itype >>= 1) { 2106 2107 if ((itype & t4_intr_types) == 0) 2108 continue; /* not allowed */ 2109 2110 if (itype == INTR_MSIX) 2111 navail = pci_msix_count(sc->dev); 2112 else if (itype == INTR_MSI) 2113 navail = pci_msi_count(sc->dev); 2114 else 2115 navail = 1; 2116restart: 2117 if (navail == 0) 2118 continue; 2119 2120 iaq->intr_type = itype; 2121 iaq->intr_flags_10g = 0; 2122 iaq->intr_flags_1g = 0; 2123 2124 /* 2125 * Best option: an interrupt vector for errors, one for the 2126 * firmware event queue, and one for every rxq (NIC and TOE) of 2127 * every VI. The VIs that support netmap use the same 2128 * interrupts for the NIC rx queues and the netmap rx queues 2129 * because only one set of queues is active at a time. 2130 */ 2131 iaq->nirq = T4_EXTRA_INTR; 2132 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 2133 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 2134 iaq->nirq += (n10g + n1g) * (num_vis - 1) * 2135 max(iaq->nrxq_vi, iaq->nnmrxq_vi); /* See comment above. */ 2136 iaq->nirq += (n10g + n1g) * (num_vis - 1) * iaq->nofldrxq_vi; 2137 if (iaq->nirq <= navail && 2138 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2139 iaq->intr_flags_10g = INTR_ALL; 2140 iaq->intr_flags_1g = INTR_ALL; 2141 goto allocate; 2142 } 2143 2144 /* Disable the VIs (and netmap) if there aren't enough intrs */ 2145 if (num_vis > 1) { 2146 device_printf(sc->dev, "virtual interfaces disabled " 2147 "because num_vis=%u with current settings " 2148 "(nrxq10g=%u, nrxq1g=%u, nofldrxq10g=%u, " 2149 "nofldrxq1g=%u, nrxq_vi=%u nofldrxq_vi=%u, " 2150 "nnmrxq_vi=%u) would need %u interrupts but " 2151 "only %u are available.\n", num_vis, nrxq10g, 2152 nrxq1g, nofldrxq10g, nofldrxq1g, iaq->nrxq_vi, 2153 iaq->nofldrxq_vi, iaq->nnmrxq_vi, iaq->nirq, 2154 navail); 2155 num_vis = 1; 2156 iaq->ntxq_vi = iaq->nrxq_vi = 0; 2157 iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0; 2158 iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0; 2159 goto restart; 2160 } 2161 2162 /* 2163 * Second best option: a vector for errors, one for the firmware 2164 * event queue, and vectors for either all the NIC rx queues or 2165 * all the TOE rx queues. The queues that don't get vectors 2166 * will forward their interrupts to those that do. 2167 */ 2168 iaq->nirq = T4_EXTRA_INTR; 2169 if (nrxq10g >= nofldrxq10g) { 2170 iaq->intr_flags_10g = INTR_RXQ; 2171 iaq->nirq += n10g * nrxq10g; 2172 } else { 2173 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2174 iaq->nirq += n10g * nofldrxq10g; 2175 } 2176 if (nrxq1g >= nofldrxq1g) { 2177 iaq->intr_flags_1g = INTR_RXQ; 2178 iaq->nirq += n1g * nrxq1g; 2179 } else { 2180 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2181 iaq->nirq += n1g * nofldrxq1g; 2182 } 2183 if (iaq->nirq <= navail && 2184 (itype != INTR_MSI || powerof2(iaq->nirq))) 2185 goto allocate; 2186 2187 /* 2188 * Next best option: an interrupt vector for errors, one for the 2189 * firmware event queue, and at least one per main-VI. At this 2190 * point we know we'll have to downsize nrxq and/or nofldrxq to 2191 * fit what's available to us. 2192 */ 2193 iaq->nirq = T4_EXTRA_INTR; 2194 iaq->nirq += n10g + n1g; 2195 if (iaq->nirq <= navail) { 2196 int leftover = navail - iaq->nirq; 2197 2198 if (n10g > 0) { 2199 int target = max(nrxq10g, nofldrxq10g); 2200 2201 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2202 INTR_RXQ : INTR_OFLD_RXQ; 2203 2204 n = 1; 2205 while (n < target && leftover >= n10g) { 2206 leftover -= n10g; 2207 iaq->nirq += n10g; 2208 n++; 2209 } 2210 iaq->nrxq10g = min(n, nrxq10g); 2211#ifdef TCP_OFFLOAD 2212 iaq->nofldrxq10g = min(n, nofldrxq10g); 2213#endif 2214 } 2215 2216 if (n1g > 0) { 2217 int target = max(nrxq1g, nofldrxq1g); 2218 2219 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2220 INTR_RXQ : INTR_OFLD_RXQ; 2221 2222 n = 1; 2223 while (n < target && leftover >= n1g) { 2224 leftover -= n1g; 2225 iaq->nirq += n1g; 2226 n++; 2227 } 2228 iaq->nrxq1g = min(n, nrxq1g); 2229#ifdef TCP_OFFLOAD 2230 iaq->nofldrxq1g = min(n, nofldrxq1g); 2231#endif 2232 } 2233 2234 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2235 goto allocate; 2236 } 2237 2238 /* 2239 * Least desirable option: one interrupt vector for everything. 2240 */ 2241 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2242 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2243#ifdef TCP_OFFLOAD 2244 if (is_offload(sc)) 2245 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2246#endif 2247allocate: 2248 navail = iaq->nirq; 2249 rc = 0; 2250 if (itype == INTR_MSIX) 2251 rc = pci_alloc_msix(sc->dev, &navail); 2252 else if (itype == INTR_MSI) 2253 rc = pci_alloc_msi(sc->dev, &navail); 2254 2255 if (rc == 0) { 2256 if (navail == iaq->nirq) 2257 return (0); 2258 2259 /* 2260 * Didn't get the number requested. Use whatever number 2261 * the kernel is willing to allocate (it's in navail). 2262 */ 2263 device_printf(sc->dev, "fewer vectors than requested, " 2264 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2265 itype, iaq->nirq, navail); 2266 pci_release_msi(sc->dev); 2267 goto restart; 2268 } 2269 2270 device_printf(sc->dev, 2271 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2272 itype, rc, iaq->nirq, navail); 2273 } 2274 2275 device_printf(sc->dev, 2276 "failed to find a usable interrupt type. " 2277 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2278 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2279 2280 return (ENXIO); 2281} 2282 2283#define FW_VERSION(chip) ( \ 2284 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2285 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2286 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2287 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2288#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2289 2290struct fw_info { 2291 uint8_t chip; 2292 char *kld_name; 2293 char *fw_mod_name; 2294 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2295} fw_info[] = { 2296 { 2297 .chip = CHELSIO_T4, 2298 .kld_name = "t4fw_cfg", 2299 .fw_mod_name = "t4fw", 2300 .fw_hdr = { 2301 .chip = FW_HDR_CHIP_T4, 2302 .fw_ver = htobe32_const(FW_VERSION(T4)), 2303 .intfver_nic = FW_INTFVER(T4, NIC), 2304 .intfver_vnic = FW_INTFVER(T4, VNIC), 2305 .intfver_ofld = FW_INTFVER(T4, OFLD), 2306 .intfver_ri = FW_INTFVER(T4, RI), 2307 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2308 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2309 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2310 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2311 }, 2312 }, { 2313 .chip = CHELSIO_T5, 2314 .kld_name = "t5fw_cfg", 2315 .fw_mod_name = "t5fw", 2316 .fw_hdr = { 2317 .chip = FW_HDR_CHIP_T5, 2318 .fw_ver = htobe32_const(FW_VERSION(T5)), 2319 .intfver_nic = FW_INTFVER(T5, NIC), 2320 .intfver_vnic = FW_INTFVER(T5, VNIC), 2321 .intfver_ofld = FW_INTFVER(T5, OFLD), 2322 .intfver_ri = FW_INTFVER(T5, RI), 2323 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2324 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2325 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2326 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2327 }, 2328 } 2329}; 2330 2331static struct fw_info * 2332find_fw_info(int chip) 2333{ 2334 int i; 2335 2336 for (i = 0; i < nitems(fw_info); i++) { 2337 if (fw_info[i].chip == chip) 2338 return (&fw_info[i]); 2339 } 2340 return (NULL); 2341} 2342 2343/* 2344 * Is the given firmware API compatible with the one the driver was compiled 2345 * with? 2346 */ 2347static int 2348fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2349{ 2350 2351 /* short circuit if it's the exact same firmware version */ 2352 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2353 return (1); 2354 2355 /* 2356 * XXX: Is this too conservative? Perhaps I should limit this to the 2357 * features that are supported in the driver. 2358 */ 2359#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2360 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2361 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2362 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2363 return (1); 2364#undef SAME_INTF 2365 2366 return (0); 2367} 2368 2369/* 2370 * The firmware in the KLD is usable, but should it be installed? This routine 2371 * explains itself in detail if it indicates the KLD firmware should be 2372 * installed. 2373 */ 2374static int 2375should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2376{ 2377 const char *reason; 2378 2379 if (!card_fw_usable) { 2380 reason = "incompatible or unusable"; 2381 goto install; 2382 } 2383 2384 if (k > c) { 2385 reason = "older than the version bundled with this driver"; 2386 goto install; 2387 } 2388 2389 if (t4_fw_install == 2 && k != c) { 2390 reason = "different than the version bundled with this driver"; 2391 goto install; 2392 } 2393 2394 return (0); 2395 2396install: 2397 if (t4_fw_install == 0) { 2398 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2399 "but the driver is prohibited from installing a different " 2400 "firmware on the card.\n", 2401 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2402 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2403 2404 return (0); 2405 } 2406 2407 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2408 "installing firmware %u.%u.%u.%u on card.\n", 2409 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2410 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2411 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2412 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2413 2414 return (1); 2415} 2416/* 2417 * Establish contact with the firmware and determine if we are the master driver 2418 * or not, and whether we are responsible for chip initialization. 2419 */ 2420static int 2421prep_firmware(struct adapter *sc) 2422{ 2423 const struct firmware *fw = NULL, *default_cfg; 2424 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2425 enum dev_state state; 2426 struct fw_info *fw_info; 2427 struct fw_hdr *card_fw; /* fw on the card */ 2428 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2429 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2430 against */ 2431 2432 /* Contact firmware. */ 2433 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2434 if (rc < 0 || state == DEV_STATE_ERR) { 2435 rc = -rc; 2436 device_printf(sc->dev, 2437 "failed to connect to the firmware: %d, %d.\n", rc, state); 2438 return (rc); 2439 } 2440 pf = rc; 2441 if (pf == sc->mbox) 2442 sc->flags |= MASTER_PF; 2443 else if (state == DEV_STATE_UNINIT) { 2444 /* 2445 * We didn't get to be the master so we definitely won't be 2446 * configuring the chip. It's a bug if someone else hasn't 2447 * configured it already. 2448 */ 2449 device_printf(sc->dev, "couldn't be master(%d), " 2450 "device not already initialized either(%d).\n", rc, state); 2451 return (EDOOFUS); 2452 } 2453 2454 /* This is the firmware whose headers the driver was compiled against */ 2455 fw_info = find_fw_info(chip_id(sc)); 2456 if (fw_info == NULL) { 2457 device_printf(sc->dev, 2458 "unable to look up firmware information for chip %d.\n", 2459 chip_id(sc)); 2460 return (EINVAL); 2461 } 2462 drv_fw = &fw_info->fw_hdr; 2463 2464 /* 2465 * The firmware KLD contains many modules. The KLD name is also the 2466 * name of the module that contains the default config file. 2467 */ 2468 default_cfg = firmware_get(fw_info->kld_name); 2469 2470 /* Read the header of the firmware on the card */ 2471 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2472 rc = -t4_read_flash(sc, FLASH_FW_START, 2473 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2474 if (rc == 0) 2475 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2476 else { 2477 device_printf(sc->dev, 2478 "Unable to read card's firmware header: %d\n", rc); 2479 card_fw_usable = 0; 2480 } 2481 2482 /* This is the firmware in the KLD */ 2483 fw = firmware_get(fw_info->fw_mod_name); 2484 if (fw != NULL) { 2485 kld_fw = (const void *)fw->data; 2486 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2487 } else { 2488 kld_fw = NULL; 2489 kld_fw_usable = 0; 2490 } 2491 2492 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2493 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2494 /* 2495 * Common case: the firmware on the card is an exact match and 2496 * the KLD is an exact match too, or the KLD is 2497 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2498 * here -- use cxgbetool loadfw if you want to reinstall the 2499 * same firmware as the one on the card. 2500 */ 2501 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2502 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2503 be32toh(card_fw->fw_ver))) { 2504 2505 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2506 if (rc != 0) { 2507 device_printf(sc->dev, 2508 "failed to install firmware: %d\n", rc); 2509 goto done; 2510 } 2511 2512 /* Installed successfully, update the cached header too. */ 2513 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2514 card_fw_usable = 1; 2515 need_fw_reset = 0; /* already reset as part of load_fw */ 2516 } 2517 2518 if (!card_fw_usable) { 2519 uint32_t d, c, k; 2520 2521 d = ntohl(drv_fw->fw_ver); 2522 c = ntohl(card_fw->fw_ver); 2523 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2524 2525 device_printf(sc->dev, "Cannot find a usable firmware: " 2526 "fw_install %d, chip state %d, " 2527 "driver compiled with %d.%d.%d.%d, " 2528 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2529 t4_fw_install, state, 2530 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2531 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2532 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2533 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2534 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2535 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2536 rc = EINVAL; 2537 goto done; 2538 } 2539 2540 /* We're using whatever's on the card and it's known to be good. */ 2541 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2542 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2543 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2544 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2545 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2546 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2547 t4_get_tp_version(sc, &sc->params.tp_vers); 2548 2549 /* Reset device */ 2550 if (need_fw_reset && 2551 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2552 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2553 if (rc != ETIMEDOUT && rc != EIO) 2554 t4_fw_bye(sc, sc->mbox); 2555 goto done; 2556 } 2557 sc->flags |= FW_OK; 2558 2559 rc = get_params__pre_init(sc); 2560 if (rc != 0) 2561 goto done; /* error message displayed already */ 2562 2563 /* Partition adapter resources as specified in the config file. */ 2564 if (state == DEV_STATE_UNINIT) { 2565 2566 KASSERT(sc->flags & MASTER_PF, 2567 ("%s: trying to change chip settings when not master.", 2568 __func__)); 2569 2570 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2571 if (rc != 0) 2572 goto done; /* error message displayed already */ 2573 2574 t4_tweak_chip_settings(sc); 2575 2576 /* get basic stuff going */ 2577 rc = -t4_fw_initialize(sc, sc->mbox); 2578 if (rc != 0) { 2579 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2580 goto done; 2581 } 2582 } else { 2583 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2584 sc->cfcsum = 0; 2585 } 2586 2587done: 2588 free(card_fw, M_CXGBE); 2589 if (fw != NULL) 2590 firmware_put(fw, FIRMWARE_UNLOAD); 2591 if (default_cfg != NULL) 2592 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2593 2594 return (rc); 2595} 2596 2597#define FW_PARAM_DEV(param) \ 2598 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2599 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2600#define FW_PARAM_PFVF(param) \ 2601 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2602 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2603 2604/* 2605 * Partition chip resources for use between various PFs, VFs, etc. 2606 */ 2607static int 2608partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2609 const char *name_prefix) 2610{ 2611 const struct firmware *cfg = NULL; 2612 int rc = 0; 2613 struct fw_caps_config_cmd caps; 2614 uint32_t mtype, moff, finicsum, cfcsum; 2615 2616 /* 2617 * Figure out what configuration file to use. Pick the default config 2618 * file for the card if the user hasn't specified one explicitly. 2619 */ 2620 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2621 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2622 /* Card specific overrides go here. */ 2623 if (pci_get_device(sc->dev) == 0x440a) 2624 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2625 if (is_fpga(sc)) 2626 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2627 } 2628 2629 /* 2630 * We need to load another module if the profile is anything except 2631 * "default" or "flash". 2632 */ 2633 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2634 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2635 char s[32]; 2636 2637 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2638 cfg = firmware_get(s); 2639 if (cfg == NULL) { 2640 if (default_cfg != NULL) { 2641 device_printf(sc->dev, 2642 "unable to load module \"%s\" for " 2643 "configuration profile \"%s\", will use " 2644 "the default config file instead.\n", 2645 s, sc->cfg_file); 2646 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2647 "%s", DEFAULT_CF); 2648 } else { 2649 device_printf(sc->dev, 2650 "unable to load module \"%s\" for " 2651 "configuration profile \"%s\", will use " 2652 "the config file on the card's flash " 2653 "instead.\n", s, sc->cfg_file); 2654 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2655 "%s", FLASH_CF); 2656 } 2657 } 2658 } 2659 2660 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2661 default_cfg == NULL) { 2662 device_printf(sc->dev, 2663 "default config file not available, will use the config " 2664 "file on the card's flash instead.\n"); 2665 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2666 } 2667 2668 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2669 u_int cflen, i, n; 2670 const uint32_t *cfdata; 2671 uint32_t param, val, addr, off, mw_base, mw_aperture; 2672 2673 KASSERT(cfg != NULL || default_cfg != NULL, 2674 ("%s: no config to upload", __func__)); 2675 2676 /* 2677 * Ask the firmware where it wants us to upload the config file. 2678 */ 2679 param = FW_PARAM_DEV(CF); 2680 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2681 if (rc != 0) { 2682 /* No support for config file? Shouldn't happen. */ 2683 device_printf(sc->dev, 2684 "failed to query config file location: %d.\n", rc); 2685 goto done; 2686 } 2687 mtype = G_FW_PARAMS_PARAM_Y(val); 2688 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2689 2690 /* 2691 * XXX: sheer laziness. We deliberately added 4 bytes of 2692 * useless stuffing/comments at the end of the config file so 2693 * it's ok to simply throw away the last remaining bytes when 2694 * the config file is not an exact multiple of 4. This also 2695 * helps with the validate_mt_off_len check. 2696 */ 2697 if (cfg != NULL) { 2698 cflen = cfg->datasize & ~3; 2699 cfdata = cfg->data; 2700 } else { 2701 cflen = default_cfg->datasize & ~3; 2702 cfdata = default_cfg->data; 2703 } 2704 2705 if (cflen > FLASH_CFG_MAX_SIZE) { 2706 device_printf(sc->dev, 2707 "config file too long (%d, max allowed is %d). " 2708 "Will try to use the config on the card, if any.\n", 2709 cflen, FLASH_CFG_MAX_SIZE); 2710 goto use_config_on_flash; 2711 } 2712 2713 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2714 if (rc != 0) { 2715 device_printf(sc->dev, 2716 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2717 "Will try to use the config on the card, if any.\n", 2718 __func__, mtype, moff, cflen, rc); 2719 goto use_config_on_flash; 2720 } 2721 2722 memwin_info(sc, 2, &mw_base, &mw_aperture); 2723 while (cflen) { 2724 off = position_memwin(sc, 2, addr); 2725 n = min(cflen, mw_aperture - off); 2726 for (i = 0; i < n; i += 4) 2727 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2728 cflen -= n; 2729 addr += n; 2730 } 2731 } else { 2732use_config_on_flash: 2733 mtype = FW_MEMTYPE_FLASH; 2734 moff = t4_flash_cfg_addr(sc); 2735 } 2736 2737 bzero(&caps, sizeof(caps)); 2738 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2739 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2740 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2741 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2742 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2743 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2744 if (rc != 0) { 2745 device_printf(sc->dev, 2746 "failed to pre-process config file: %d " 2747 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2748 goto done; 2749 } 2750 2751 finicsum = be32toh(caps.finicsum); 2752 cfcsum = be32toh(caps.cfcsum); 2753 if (finicsum != cfcsum) { 2754 device_printf(sc->dev, 2755 "WARNING: config file checksum mismatch: %08x %08x\n", 2756 finicsum, cfcsum); 2757 } 2758 sc->cfcsum = cfcsum; 2759 2760#define LIMIT_CAPS(x) do { \ 2761 caps.x &= htobe16(t4_##x##_allowed); \ 2762} while (0) 2763 2764 /* 2765 * Let the firmware know what features will (not) be used so it can tune 2766 * things accordingly. 2767 */ 2768 LIMIT_CAPS(linkcaps); 2769 LIMIT_CAPS(niccaps); 2770 LIMIT_CAPS(toecaps); 2771 LIMIT_CAPS(rdmacaps); 2772 LIMIT_CAPS(iscsicaps); 2773 LIMIT_CAPS(fcoecaps); 2774#undef LIMIT_CAPS 2775 2776 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2777 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2778 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2779 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2780 if (rc != 0) { 2781 device_printf(sc->dev, 2782 "failed to process config file: %d.\n", rc); 2783 } 2784done: 2785 if (cfg != NULL) 2786 firmware_put(cfg, FIRMWARE_UNLOAD); 2787 return (rc); 2788} 2789 2790/* 2791 * Retrieve parameters that are needed (or nice to have) very early. 2792 */ 2793static int 2794get_params__pre_init(struct adapter *sc) 2795{ 2796 int rc; 2797 uint32_t param[2], val[2]; 2798 struct fw_devlog_cmd cmd; 2799 struct devlog_params *dlog = &sc->params.devlog; 2800 2801 param[0] = FW_PARAM_DEV(PORTVEC); 2802 param[1] = FW_PARAM_DEV(CCLK); 2803 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2804 if (rc != 0) { 2805 device_printf(sc->dev, 2806 "failed to query parameters (pre_init): %d.\n", rc); 2807 return (rc); 2808 } 2809 2810 sc->params.portvec = val[0]; 2811 sc->params.nports = bitcount32(val[0]); 2812 sc->params.vpd.cclk = val[1]; 2813 2814 /* Read device log parameters. */ 2815 bzero(&cmd, sizeof(cmd)); 2816 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2817 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2818 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2819 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2820 if (rc != 0) { 2821 device_printf(sc->dev, 2822 "failed to get devlog parameters: %d.\n", rc); 2823 bzero(dlog, sizeof (*dlog)); 2824 rc = 0; /* devlog isn't critical for device operation */ 2825 } else { 2826 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2827 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2828 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2829 dlog->size = be32toh(cmd.memsize_devlog); 2830 } 2831 2832 return (rc); 2833} 2834 2835/* 2836 * Retrieve various parameters that are of interest to the driver. The device 2837 * has been initialized by the firmware at this point. 2838 */ 2839static int 2840get_params__post_init(struct adapter *sc) 2841{ 2842 int rc; 2843 uint32_t param[7], val[7]; 2844 struct fw_caps_config_cmd caps; 2845 2846 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2847 param[1] = FW_PARAM_PFVF(EQ_START); 2848 param[2] = FW_PARAM_PFVF(FILTER_START); 2849 param[3] = FW_PARAM_PFVF(FILTER_END); 2850 param[4] = FW_PARAM_PFVF(L2T_START); 2851 param[5] = FW_PARAM_PFVF(L2T_END); 2852 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2853 if (rc != 0) { 2854 device_printf(sc->dev, 2855 "failed to query parameters (post_init): %d.\n", rc); 2856 return (rc); 2857 } 2858 2859 sc->sge.iq_start = val[0]; 2860 sc->sge.eq_start = val[1]; 2861 sc->tids.ftid_base = val[2]; 2862 sc->tids.nftids = val[3] - val[2] + 1; 2863 sc->params.ftid_min = val[2]; 2864 sc->params.ftid_max = val[3]; 2865 sc->vres.l2t.start = val[4]; 2866 sc->vres.l2t.size = val[5] - val[4] + 1; 2867 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2868 ("%s: L2 table size (%u) larger than expected (%u)", 2869 __func__, sc->vres.l2t.size, L2T_SIZE)); 2870 2871 /* get capabilites */ 2872 bzero(&caps, sizeof(caps)); 2873 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2874 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2875 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2876 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2877 if (rc != 0) { 2878 device_printf(sc->dev, 2879 "failed to get card capabilities: %d.\n", rc); 2880 return (rc); 2881 } 2882 2883#define READ_CAPS(x) do { \ 2884 sc->x = htobe16(caps.x); \ 2885} while (0) 2886 READ_CAPS(linkcaps); 2887 READ_CAPS(niccaps); 2888 READ_CAPS(toecaps); 2889 READ_CAPS(rdmacaps); 2890 READ_CAPS(iscsicaps); 2891 READ_CAPS(fcoecaps); 2892 2893 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2894 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2895 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2896 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2897 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2898 if (rc != 0) { 2899 device_printf(sc->dev, 2900 "failed to query NIC parameters: %d.\n", rc); 2901 return (rc); 2902 } 2903 sc->tids.etid_base = val[0]; 2904 sc->params.etid_min = val[0]; 2905 sc->tids.netids = val[1] - val[0] + 1; 2906 sc->params.netids = sc->tids.netids; 2907 sc->params.eo_wr_cred = val[2]; 2908 sc->params.ethoffload = 1; 2909 } 2910 2911 if (sc->toecaps) { 2912 /* query offload-related parameters */ 2913 param[0] = FW_PARAM_DEV(NTID); 2914 param[1] = FW_PARAM_PFVF(SERVER_START); 2915 param[2] = FW_PARAM_PFVF(SERVER_END); 2916 param[3] = FW_PARAM_PFVF(TDDP_START); 2917 param[4] = FW_PARAM_PFVF(TDDP_END); 2918 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2919 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2920 if (rc != 0) { 2921 device_printf(sc->dev, 2922 "failed to query TOE parameters: %d.\n", rc); 2923 return (rc); 2924 } 2925 sc->tids.ntids = val[0]; 2926 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2927 sc->tids.stid_base = val[1]; 2928 sc->tids.nstids = val[2] - val[1] + 1; 2929 sc->vres.ddp.start = val[3]; 2930 sc->vres.ddp.size = val[4] - val[3] + 1; 2931 sc->params.ofldq_wr_cred = val[5]; 2932 sc->params.offload = 1; 2933 } 2934 if (sc->rdmacaps) { 2935 param[0] = FW_PARAM_PFVF(STAG_START); 2936 param[1] = FW_PARAM_PFVF(STAG_END); 2937 param[2] = FW_PARAM_PFVF(RQ_START); 2938 param[3] = FW_PARAM_PFVF(RQ_END); 2939 param[4] = FW_PARAM_PFVF(PBL_START); 2940 param[5] = FW_PARAM_PFVF(PBL_END); 2941 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2942 if (rc != 0) { 2943 device_printf(sc->dev, 2944 "failed to query RDMA parameters(1): %d.\n", rc); 2945 return (rc); 2946 } 2947 sc->vres.stag.start = val[0]; 2948 sc->vres.stag.size = val[1] - val[0] + 1; 2949 sc->vres.rq.start = val[2]; 2950 sc->vres.rq.size = val[3] - val[2] + 1; 2951 sc->vres.pbl.start = val[4]; 2952 sc->vres.pbl.size = val[5] - val[4] + 1; 2953 2954 param[0] = FW_PARAM_PFVF(SQRQ_START); 2955 param[1] = FW_PARAM_PFVF(SQRQ_END); 2956 param[2] = FW_PARAM_PFVF(CQ_START); 2957 param[3] = FW_PARAM_PFVF(CQ_END); 2958 param[4] = FW_PARAM_PFVF(OCQ_START); 2959 param[5] = FW_PARAM_PFVF(OCQ_END); 2960 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2961 if (rc != 0) { 2962 device_printf(sc->dev, 2963 "failed to query RDMA parameters(2): %d.\n", rc); 2964 return (rc); 2965 } 2966 sc->vres.qp.start = val[0]; 2967 sc->vres.qp.size = val[1] - val[0] + 1; 2968 sc->vres.cq.start = val[2]; 2969 sc->vres.cq.size = val[3] - val[2] + 1; 2970 sc->vres.ocq.start = val[4]; 2971 sc->vres.ocq.size = val[5] - val[4] + 1; 2972 } 2973 if (sc->iscsicaps) { 2974 param[0] = FW_PARAM_PFVF(ISCSI_START); 2975 param[1] = FW_PARAM_PFVF(ISCSI_END); 2976 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2977 if (rc != 0) { 2978 device_printf(sc->dev, 2979 "failed to query iSCSI parameters: %d.\n", rc); 2980 return (rc); 2981 } 2982 sc->vres.iscsi.start = val[0]; 2983 sc->vres.iscsi.size = val[1] - val[0] + 1; 2984 } 2985 2986 /* 2987 * We've got the params we wanted to query via the firmware. Now grab 2988 * some others directly from the chip. 2989 */ 2990 rc = t4_read_chip_settings(sc); 2991 2992 return (rc); 2993} 2994 2995static int 2996set_params__post_init(struct adapter *sc) 2997{ 2998 uint32_t param, val; 2999 3000 /* ask for encapsulated CPLs */ 3001 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 3002 val = 1; 3003 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3004 3005 return (0); 3006} 3007 3008#undef FW_PARAM_PFVF 3009#undef FW_PARAM_DEV 3010 3011static void 3012t4_set_desc(struct adapter *sc) 3013{ 3014 char buf[128]; 3015 struct adapter_params *p = &sc->params; 3016 3017 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 3018 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 3019 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 3020 3021 device_set_desc_copy(sc->dev, buf); 3022} 3023 3024static void 3025build_medialist(struct port_info *pi, struct ifmedia *media) 3026{ 3027 int m; 3028 3029 PORT_LOCK(pi); 3030 3031 ifmedia_removeall(media); 3032 3033 m = IFM_ETHER | IFM_FDX; 3034 3035 switch(pi->port_type) { 3036 case FW_PORT_TYPE_BT_XFI: 3037 case FW_PORT_TYPE_BT_XAUI: 3038 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 3039 /* fall through */ 3040 3041 case FW_PORT_TYPE_BT_SGMII: 3042 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 3043 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 3044 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 3045 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 3046 break; 3047 3048 case FW_PORT_TYPE_CX4: 3049 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 3050 ifmedia_set(media, m | IFM_10G_CX4); 3051 break; 3052 3053 case FW_PORT_TYPE_QSFP_10G: 3054 case FW_PORT_TYPE_SFP: 3055 case FW_PORT_TYPE_FIBER_XFI: 3056 case FW_PORT_TYPE_FIBER_XAUI: 3057 switch (pi->mod_type) { 3058 3059 case FW_PORT_MOD_TYPE_LR: 3060 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 3061 ifmedia_set(media, m | IFM_10G_LR); 3062 break; 3063 3064 case FW_PORT_MOD_TYPE_SR: 3065 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 3066 ifmedia_set(media, m | IFM_10G_SR); 3067 break; 3068 3069 case FW_PORT_MOD_TYPE_LRM: 3070 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 3071 ifmedia_set(media, m | IFM_10G_LRM); 3072 break; 3073 3074 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3075 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3076 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 3077 ifmedia_set(media, m | IFM_10G_TWINAX); 3078 break; 3079 3080 case FW_PORT_MOD_TYPE_NONE: 3081 m &= ~IFM_FDX; 3082 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3083 ifmedia_set(media, m | IFM_NONE); 3084 break; 3085 3086 case FW_PORT_MOD_TYPE_NA: 3087 case FW_PORT_MOD_TYPE_ER: 3088 default: 3089 device_printf(pi->dev, 3090 "unknown port_type (%d), mod_type (%d)\n", 3091 pi->port_type, pi->mod_type); 3092 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3093 ifmedia_set(media, m | IFM_UNKNOWN); 3094 break; 3095 } 3096 break; 3097 3098 case FW_PORT_TYPE_QSFP: 3099 switch (pi->mod_type) { 3100 3101 case FW_PORT_MOD_TYPE_LR: 3102 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 3103 ifmedia_set(media, m | IFM_40G_LR4); 3104 break; 3105 3106 case FW_PORT_MOD_TYPE_SR: 3107 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 3108 ifmedia_set(media, m | IFM_40G_SR4); 3109 break; 3110 3111 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3112 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3113 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 3114 ifmedia_set(media, m | IFM_40G_CR4); 3115 break; 3116 3117 case FW_PORT_MOD_TYPE_NONE: 3118 m &= ~IFM_FDX; 3119 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3120 ifmedia_set(media, m | IFM_NONE); 3121 break; 3122 3123 default: 3124 device_printf(pi->dev, 3125 "unknown port_type (%d), mod_type (%d)\n", 3126 pi->port_type, pi->mod_type); 3127 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3128 ifmedia_set(media, m | IFM_UNKNOWN); 3129 break; 3130 } 3131 break; 3132 3133 default: 3134 device_printf(pi->dev, 3135 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3136 pi->mod_type); 3137 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3138 ifmedia_set(media, m | IFM_UNKNOWN); 3139 break; 3140 } 3141 3142 PORT_UNLOCK(pi); 3143} 3144 3145#define FW_MAC_EXACT_CHUNK 7 3146 3147/* 3148 * Program the port's XGMAC based on parameters in ifnet. The caller also 3149 * indicates which parameters should be programmed (the rest are left alone). 3150 */ 3151int 3152update_mac_settings(struct ifnet *ifp, int flags) 3153{ 3154 int rc = 0; 3155 struct vi_info *vi = ifp->if_softc; 3156 struct port_info *pi = vi->pi; 3157 struct adapter *sc = pi->adapter; 3158 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3159 3160 ASSERT_SYNCHRONIZED_OP(sc); 3161 KASSERT(flags, ("%s: not told what to update.", __func__)); 3162 3163 if (flags & XGMAC_MTU) 3164 mtu = ifp->if_mtu; 3165 3166 if (flags & XGMAC_PROMISC) 3167 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3168 3169 if (flags & XGMAC_ALLMULTI) 3170 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3171 3172 if (flags & XGMAC_VLANEX) 3173 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3174 3175 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3176 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, 3177 allmulti, 1, vlanex, false); 3178 if (rc) { 3179 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3180 rc); 3181 return (rc); 3182 } 3183 } 3184 3185 if (flags & XGMAC_UCADDR) { 3186 uint8_t ucaddr[ETHER_ADDR_LEN]; 3187 3188 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3189 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, 3190 ucaddr, true, true); 3191 if (rc < 0) { 3192 rc = -rc; 3193 if_printf(ifp, "change_mac failed: %d\n", rc); 3194 return (rc); 3195 } else { 3196 vi->xact_addr_filt = rc; 3197 rc = 0; 3198 } 3199 } 3200 3201 if (flags & XGMAC_MCADDRS) { 3202 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3203 int del = 1; 3204 uint64_t hash = 0; 3205 struct ifmultiaddr *ifma; 3206 int i = 0, j; 3207 3208 if_maddr_rlock(ifp); 3209 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3210 if (ifma->ifma_addr->sa_family != AF_LINK) 3211 continue; 3212 mcaddr[i] = 3213 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3214 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3215 i++; 3216 3217 if (i == FW_MAC_EXACT_CHUNK) { 3218 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, 3219 del, i, mcaddr, NULL, &hash, 0); 3220 if (rc < 0) { 3221 rc = -rc; 3222 for (j = 0; j < i; j++) { 3223 if_printf(ifp, 3224 "failed to add mc address" 3225 " %02x:%02x:%02x:" 3226 "%02x:%02x:%02x rc=%d\n", 3227 mcaddr[j][0], mcaddr[j][1], 3228 mcaddr[j][2], mcaddr[j][3], 3229 mcaddr[j][4], mcaddr[j][5], 3230 rc); 3231 } 3232 goto mcfail; 3233 } 3234 del = 0; 3235 i = 0; 3236 } 3237 } 3238 if (i > 0) { 3239 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i, 3240 mcaddr, NULL, &hash, 0); 3241 if (rc < 0) { 3242 rc = -rc; 3243 for (j = 0; j < i; j++) { 3244 if_printf(ifp, 3245 "failed to add mc address" 3246 " %02x:%02x:%02x:" 3247 "%02x:%02x:%02x rc=%d\n", 3248 mcaddr[j][0], mcaddr[j][1], 3249 mcaddr[j][2], mcaddr[j][3], 3250 mcaddr[j][4], mcaddr[j][5], 3251 rc); 3252 } 3253 goto mcfail; 3254 } 3255 } 3256 3257 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0); 3258 if (rc != 0) 3259 if_printf(ifp, "failed to set mc address hash: %d", rc); 3260mcfail: 3261 if_maddr_runlock(ifp); 3262 } 3263 3264 return (rc); 3265} 3266 3267/* 3268 * {begin|end}_synchronized_op must be called from the same thread. 3269 */ 3270int 3271begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, 3272 char *wmesg) 3273{ 3274 int rc, pri; 3275 3276#ifdef WITNESS 3277 /* the caller thinks it's ok to sleep, but is it really? */ 3278 if (flags & SLEEP_OK) 3279 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 3280 "begin_synchronized_op"); 3281#endif 3282 3283 if (INTR_OK) 3284 pri = PCATCH; 3285 else 3286 pri = 0; 3287 3288 ADAPTER_LOCK(sc); 3289 for (;;) { 3290 3291 if (vi && IS_DOOMED(vi)) { 3292 rc = ENXIO; 3293 goto done; 3294 } 3295 3296 if (!IS_BUSY(sc)) { 3297 rc = 0; 3298 break; 3299 } 3300 3301 if (!(flags & SLEEP_OK)) { 3302 rc = EBUSY; 3303 goto done; 3304 } 3305 3306 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3307 rc = EINTR; 3308 goto done; 3309 } 3310 } 3311 3312 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3313 SET_BUSY(sc); 3314#ifdef INVARIANTS 3315 sc->last_op = wmesg; 3316 sc->last_op_thr = curthread; 3317 sc->last_op_flags = flags; 3318#endif 3319 3320done: 3321 if (!(flags & HOLD_LOCK) || rc) 3322 ADAPTER_UNLOCK(sc); 3323 3324 return (rc); 3325} 3326 3327/* 3328 * Tell if_ioctl and if_init that the VI is going away. This is 3329 * special variant of begin_synchronized_op and must be paired with a 3330 * call to end_synchronized_op. 3331 */ 3332void 3333doom_vi(struct adapter *sc, struct vi_info *vi) 3334{ 3335 3336 ADAPTER_LOCK(sc); 3337 SET_DOOMED(vi); 3338 wakeup(&sc->flags); 3339 while (IS_BUSY(sc)) 3340 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 3341 SET_BUSY(sc); 3342#ifdef INVARIANTS 3343 sc->last_op = "t4detach"; 3344 sc->last_op_thr = curthread; 3345 sc->last_op_flags = 0; 3346#endif 3347 ADAPTER_UNLOCK(sc); 3348} 3349 3350/* 3351 * {begin|end}_synchronized_op must be called from the same thread. 3352 */ 3353void 3354end_synchronized_op(struct adapter *sc, int flags) 3355{ 3356 3357 if (flags & LOCK_HELD) 3358 ADAPTER_LOCK_ASSERT_OWNED(sc); 3359 else 3360 ADAPTER_LOCK(sc); 3361 3362 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3363 CLR_BUSY(sc); 3364 wakeup(&sc->flags); 3365 ADAPTER_UNLOCK(sc); 3366} 3367 3368static int 3369cxgbe_init_synchronized(struct vi_info *vi) 3370{ 3371 struct port_info *pi = vi->pi; 3372 struct adapter *sc = pi->adapter; 3373 struct ifnet *ifp = vi->ifp; 3374 int rc = 0, i; 3375 struct sge_txq *txq; 3376 3377 ASSERT_SYNCHRONIZED_OP(sc); 3378 3379 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3380 return (0); /* already running */ 3381 3382 if (!(sc->flags & FULL_INIT_DONE) && 3383 ((rc = adapter_full_init(sc)) != 0)) 3384 return (rc); /* error message displayed already */ 3385 3386 if (!(vi->flags & VI_INIT_DONE) && 3387 ((rc = vi_full_init(vi)) != 0)) 3388 return (rc); /* error message displayed already */ 3389 3390 rc = update_mac_settings(ifp, XGMAC_ALL); 3391 if (rc) 3392 goto done; /* error message displayed already */ 3393 3394 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); 3395 if (rc != 0) { 3396 if_printf(ifp, "enable_vi failed: %d\n", rc); 3397 goto done; 3398 } 3399 3400 /* 3401 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3402 * if this changes. 3403 */ 3404 3405 for_each_txq(vi, i, txq) { 3406 TXQ_LOCK(txq); 3407 txq->eq.flags |= EQ_ENABLED; 3408 TXQ_UNLOCK(txq); 3409 } 3410 3411 /* 3412 * The first iq of the first port to come up is used for tracing. 3413 */ 3414 if (sc->traceq < 0 && IS_MAIN_VI(vi)) { 3415 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; 3416 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3417 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3418 V_QUEUENUMBER(sc->traceq)); 3419 pi->flags |= HAS_TRACEQ; 3420 } 3421 3422 /* all ok */ 3423 PORT_LOCK(pi); 3424 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3425 pi->up_vis++; 3426 3427 if (pi->nvi > 1) 3428 callout_reset(&vi->tick, hz, vi_tick, vi); 3429 else 3430 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3431 PORT_UNLOCK(pi); 3432done: 3433 if (rc != 0) 3434 cxgbe_uninit_synchronized(vi); 3435 3436 return (rc); 3437} 3438 3439/* 3440 * Idempotent. 3441 */ 3442static int 3443cxgbe_uninit_synchronized(struct vi_info *vi) 3444{ 3445 struct port_info *pi = vi->pi; 3446 struct adapter *sc = pi->adapter; 3447 struct ifnet *ifp = vi->ifp; 3448 int rc, i; 3449 struct sge_txq *txq; 3450 3451 ASSERT_SYNCHRONIZED_OP(sc); 3452 3453 if (!(vi->flags & VI_INIT_DONE)) { 3454 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3455 ("uninited VI is running")); 3456 return (0); 3457 } 3458 3459 /* 3460 * Disable the VI so that all its data in either direction is discarded 3461 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3462 * tick) intact as the TP can deliver negative advice or data that it's 3463 * holding in its RAM (for an offloaded connection) even after the VI is 3464 * disabled. 3465 */ 3466 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); 3467 if (rc) { 3468 if_printf(ifp, "disable_vi failed: %d\n", rc); 3469 return (rc); 3470 } 3471 3472 for_each_txq(vi, i, txq) { 3473 TXQ_LOCK(txq); 3474 txq->eq.flags &= ~EQ_ENABLED; 3475 TXQ_UNLOCK(txq); 3476 } 3477 3478 PORT_LOCK(pi); 3479 if (pi->nvi == 1) 3480 callout_stop(&pi->tick); 3481 else 3482 callout_stop(&vi->tick); 3483 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3484 PORT_UNLOCK(pi); 3485 return (0); 3486 } 3487 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3488 pi->up_vis--; 3489 if (pi->up_vis > 0) { 3490 PORT_UNLOCK(pi); 3491 return (0); 3492 } 3493 PORT_UNLOCK(pi); 3494 3495 pi->link_cfg.link_ok = 0; 3496 pi->link_cfg.speed = 0; 3497 pi->linkdnrc = -1; 3498 t4_os_link_changed(sc, pi->port_id, 0, -1); 3499 3500 return (0); 3501} 3502 3503/* 3504 * It is ok for this function to fail midway and return right away. t4_detach 3505 * will walk the entire sc->irq list and clean up whatever is valid. 3506 */ 3507static int 3508setup_intr_handlers(struct adapter *sc) 3509{ 3510 int rc, rid, p, q, v; 3511 char s[8]; 3512 struct irq *irq; 3513 struct port_info *pi; 3514 struct vi_info *vi; 3515 struct sge *sge = &sc->sge; 3516 struct sge_rxq *rxq; 3517#ifdef TCP_OFFLOAD 3518 struct sge_ofld_rxq *ofld_rxq; 3519#endif 3520#ifdef DEV_NETMAP 3521 struct sge_nm_rxq *nm_rxq; 3522#endif 3523 3524 /* 3525 * Setup interrupts. 3526 */ 3527 irq = &sc->irq[0]; 3528 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3529 if (sc->intr_count == 1) 3530 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3531 3532 /* Multiple interrupts. */ 3533 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3534 ("%s: too few intr.", __func__)); 3535 3536 /* The first one is always error intr */ 3537 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3538 if (rc != 0) 3539 return (rc); 3540 irq++; 3541 rid++; 3542 3543 /* The second one is always the firmware event queue */ 3544 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt"); 3545 if (rc != 0) 3546 return (rc); 3547 irq++; 3548 rid++; 3549 3550 for_each_port(sc, p) { 3551 pi = sc->port[p]; 3552 for_each_vi(pi, v, vi) { 3553 vi->first_intr = rid - 1; 3554 3555 if (vi->nnmrxq > 0) { 3556 int n = max(vi->nrxq, vi->nnmrxq); 3557 3558 MPASS(vi->flags & INTR_RXQ); 3559 3560 rxq = &sge->rxq[vi->first_rxq]; 3561#ifdef DEV_NETMAP 3562 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq]; 3563#endif 3564 for (q = 0; q < n; q++) { 3565 snprintf(s, sizeof(s), "%x%c%x", p, 3566 'a' + v, q); 3567 if (q < vi->nrxq) 3568 irq->rxq = rxq++; 3569#ifdef DEV_NETMAP 3570 if (q < vi->nnmrxq) 3571 irq->nm_rxq = nm_rxq++; 3572#endif 3573 rc = t4_alloc_irq(sc, irq, rid, 3574 t4_vi_intr, irq, s); 3575 if (rc != 0) 3576 return (rc); 3577 irq++; 3578 rid++; 3579 vi->nintr++; 3580 } 3581 } else if (vi->flags & INTR_RXQ) { 3582 for_each_rxq(vi, q, rxq) { 3583 snprintf(s, sizeof(s), "%x%c%x", p, 3584 'a' + v, q); 3585 rc = t4_alloc_irq(sc, irq, rid, 3586 t4_intr, rxq, s); 3587 if (rc != 0) 3588 return (rc); 3589 irq++; 3590 rid++; 3591 vi->nintr++; 3592 } 3593 } 3594#ifdef TCP_OFFLOAD 3595 if (vi->flags & INTR_OFLD_RXQ) { 3596 for_each_ofld_rxq(vi, q, ofld_rxq) { 3597 snprintf(s, sizeof(s), "%x%c%x", p, 3598 'A' + v, q); 3599 rc = t4_alloc_irq(sc, irq, rid, 3600 t4_intr, ofld_rxq, s); 3601 if (rc != 0) 3602 return (rc); 3603 irq++; 3604 rid++; 3605 vi->nintr++; 3606 } 3607 } 3608#endif 3609 } 3610 } 3611 MPASS(irq == &sc->irq[sc->intr_count]); 3612 3613 return (0); 3614} 3615 3616int 3617adapter_full_init(struct adapter *sc) 3618{ 3619 int rc, i; 3620 3621 ASSERT_SYNCHRONIZED_OP(sc); 3622 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3623 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3624 ("%s: FULL_INIT_DONE already", __func__)); 3625 3626 /* 3627 * queues that belong to the adapter (not any particular port). 3628 */ 3629 rc = t4_setup_adapter_queues(sc); 3630 if (rc != 0) 3631 goto done; 3632 3633 for (i = 0; i < nitems(sc->tq); i++) { 3634 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3635 taskqueue_thread_enqueue, &sc->tq[i]); 3636 if (sc->tq[i] == NULL) { 3637 device_printf(sc->dev, 3638 "failed to allocate task queue %d\n", i); 3639 rc = ENOMEM; 3640 goto done; 3641 } 3642 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3643 device_get_nameunit(sc->dev), i); 3644 } 3645 3646 t4_intr_enable(sc); 3647 sc->flags |= FULL_INIT_DONE; 3648done: 3649 if (rc != 0) 3650 adapter_full_uninit(sc); 3651 3652 return (rc); 3653} 3654 3655int 3656adapter_full_uninit(struct adapter *sc) 3657{ 3658 int i; 3659 3660 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3661 3662 t4_teardown_adapter_queues(sc); 3663 3664 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3665 taskqueue_free(sc->tq[i]); 3666 sc->tq[i] = NULL; 3667 } 3668 3669 sc->flags &= ~FULL_INIT_DONE; 3670 3671 return (0); 3672} 3673 3674#ifdef RSS 3675#define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 3676 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 3677 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 3678 RSS_HASHTYPE_RSS_UDP_IPV6) 3679 3680/* Translates kernel hash types to hardware. */ 3681static int 3682hashconfig_to_hashen(int hashconfig) 3683{ 3684 int hashen = 0; 3685 3686 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 3687 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 3688 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 3689 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 3690 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 3691 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3692 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3693 } 3694 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 3695 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3696 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3697 } 3698 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 3699 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3700 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 3701 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3702 3703 return (hashen); 3704} 3705 3706/* Translates hardware hash types to kernel. */ 3707static int 3708hashen_to_hashconfig(int hashen) 3709{ 3710 int hashconfig = 0; 3711 3712 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 3713 /* 3714 * If UDP hashing was enabled it must have been enabled for 3715 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 3716 * enabling any 4-tuple hash is nonsense configuration. 3717 */ 3718 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3719 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 3720 3721 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3722 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 3723 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3724 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 3725 } 3726 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3727 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 3728 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3729 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 3730 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 3731 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 3732 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 3733 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 3734 3735 return (hashconfig); 3736} 3737#endif 3738 3739int 3740vi_full_init(struct vi_info *vi) 3741{ 3742 struct adapter *sc = vi->pi->adapter; 3743 struct ifnet *ifp = vi->ifp; 3744 uint16_t *rss; 3745 struct sge_rxq *rxq; 3746 int rc, i, j, hashen; 3747#ifdef RSS 3748 int nbuckets = rss_getnumbuckets(); 3749 int hashconfig = rss_gethashconfig(); 3750 int extra; 3751 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3752 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3753#endif 3754 3755 ASSERT_SYNCHRONIZED_OP(sc); 3756 KASSERT((vi->flags & VI_INIT_DONE) == 0, 3757 ("%s: VI_INIT_DONE already", __func__)); 3758 3759 sysctl_ctx_init(&vi->ctx); 3760 vi->flags |= VI_SYSCTL_CTX; 3761 3762 /* 3763 * Allocate tx/rx/fl queues for this VI. 3764 */ 3765 rc = t4_setup_vi_queues(vi); 3766 if (rc != 0) 3767 goto done; /* error message displayed already */ 3768 3769 /* 3770 * Setup RSS for this VI. Save a copy of the RSS table for later use. 3771 */ 3772 if (vi->nrxq > vi->rss_size) { 3773 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " 3774 "some queues will never receive traffic.\n", vi->nrxq, 3775 vi->rss_size); 3776 } else if (vi->rss_size % vi->nrxq) { 3777 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " 3778 "expect uneven traffic distribution.\n", vi->nrxq, 3779 vi->rss_size); 3780 } 3781#ifdef RSS 3782 MPASS(RSS_KEYSIZE == 40); 3783 if (vi->nrxq != nbuckets) { 3784 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" 3785 "performance will be impacted.\n", vi->nrxq, nbuckets); 3786 } 3787 3788 rss_getkey((void *)&raw_rss_key[0]); 3789 for (i = 0; i < nitems(rss_key); i++) { 3790 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 3791 } 3792 t4_write_rss_key(sc, (void *)&rss_key[0], -1); 3793#endif 3794 rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3795 for (i = 0; i < vi->rss_size;) { 3796#ifdef RSS 3797 j = rss_get_indirection_to_bucket(i); 3798 j %= vi->nrxq; 3799 rxq = &sc->sge.rxq[vi->first_rxq + j]; 3800 rss[i++] = rxq->iq.abs_id; 3801#else 3802 for_each_rxq(vi, j, rxq) { 3803 rss[i++] = rxq->iq.abs_id; 3804 if (i == vi->rss_size) 3805 break; 3806 } 3807#endif 3808 } 3809 3810 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss, 3811 vi->rss_size); 3812 if (rc != 0) { 3813 if_printf(ifp, "rss_config failed: %d\n", rc); 3814 goto done; 3815 } 3816 3817#ifdef RSS 3818 hashen = hashconfig_to_hashen(hashconfig); 3819 3820 /* 3821 * We may have had to enable some hashes even though the global config 3822 * wants them disabled. This is a potential problem that must be 3823 * reported to the user. 3824 */ 3825 extra = hashen_to_hashconfig(hashen) ^ hashconfig; 3826 3827 /* 3828 * If we consider only the supported hash types, then the enabled hashes 3829 * are a superset of the requested hashes. In other words, there cannot 3830 * be any supported hash that was requested but not enabled, but there 3831 * can be hashes that were not requested but had to be enabled. 3832 */ 3833 extra &= SUPPORTED_RSS_HASHTYPES; 3834 MPASS((extra & hashconfig) == 0); 3835 3836 if (extra) { 3837 if_printf(ifp, 3838 "global RSS config (0x%x) cannot be accomodated.\n", 3839 hashconfig); 3840 } 3841 if (extra & RSS_HASHTYPE_RSS_IPV4) 3842 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); 3843 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 3844 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); 3845 if (extra & RSS_HASHTYPE_RSS_IPV6) 3846 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); 3847 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 3848 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); 3849 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 3850 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); 3851 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 3852 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); 3853#else 3854 hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | 3855 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | 3856 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3857 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; 3858#endif 3859 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0]); 3860 if (rc != 0) { 3861 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); 3862 goto done; 3863 } 3864 3865 vi->rss = rss; 3866 vi->flags |= VI_INIT_DONE; 3867done: 3868 if (rc != 0) 3869 vi_full_uninit(vi); 3870 3871 return (rc); 3872} 3873 3874/* 3875 * Idempotent. 3876 */ 3877int 3878vi_full_uninit(struct vi_info *vi) 3879{ 3880 struct port_info *pi = vi->pi; 3881 struct adapter *sc = pi->adapter; 3882 int i; 3883 struct sge_rxq *rxq; 3884 struct sge_txq *txq; 3885#ifdef TCP_OFFLOAD 3886 struct sge_ofld_rxq *ofld_rxq; 3887 struct sge_wrq *ofld_txq; 3888#endif 3889 3890 if (vi->flags & VI_INIT_DONE) { 3891 3892 /* Need to quiesce queues. */ 3893 3894 /* XXX: Only for the first VI? */ 3895 if (IS_MAIN_VI(vi)) 3896 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3897 3898 for_each_txq(vi, i, txq) { 3899 quiesce_txq(sc, txq); 3900 } 3901 3902#ifdef TCP_OFFLOAD 3903 for_each_ofld_txq(vi, i, ofld_txq) { 3904 quiesce_wrq(sc, ofld_txq); 3905 } 3906#endif 3907 3908 for_each_rxq(vi, i, rxq) { 3909 quiesce_iq(sc, &rxq->iq); 3910 quiesce_fl(sc, &rxq->fl); 3911 } 3912 3913#ifdef TCP_OFFLOAD 3914 for_each_ofld_rxq(vi, i, ofld_rxq) { 3915 quiesce_iq(sc, &ofld_rxq->iq); 3916 quiesce_fl(sc, &ofld_rxq->fl); 3917 } 3918#endif 3919 free(vi->rss, M_CXGBE); 3920 free(vi->nm_rss, M_CXGBE); 3921 } 3922 3923 t4_teardown_vi_queues(vi); 3924 vi->flags &= ~VI_INIT_DONE; 3925 3926 return (0); 3927} 3928 3929static void 3930quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3931{ 3932 struct sge_eq *eq = &txq->eq; 3933 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3934 3935 (void) sc; /* unused */ 3936 3937#ifdef INVARIANTS 3938 TXQ_LOCK(txq); 3939 MPASS((eq->flags & EQ_ENABLED) == 0); 3940 TXQ_UNLOCK(txq); 3941#endif 3942 3943 /* Wait for the mp_ring to empty. */ 3944 while (!mp_ring_is_idle(txq->r)) { 3945 mp_ring_check_drainage(txq->r, 0); 3946 pause("rquiesce", 1); 3947 } 3948 3949 /* Then wait for the hardware to finish. */ 3950 while (spg->cidx != htobe16(eq->pidx)) 3951 pause("equiesce", 1); 3952 3953 /* Finally, wait for the driver to reclaim all descriptors. */ 3954 while (eq->cidx != eq->pidx) 3955 pause("dquiesce", 1); 3956} 3957 3958static void 3959quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3960{ 3961 3962 /* XXXTX */ 3963} 3964 3965static void 3966quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3967{ 3968 (void) sc; /* unused */ 3969 3970 /* Synchronize with the interrupt handler */ 3971 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3972 pause("iqfree", 1); 3973} 3974 3975static void 3976quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3977{ 3978 mtx_lock(&sc->sfl_lock); 3979 FL_LOCK(fl); 3980 fl->flags |= FL_DOOMED; 3981 FL_UNLOCK(fl); 3982 callout_stop(&sc->sfl_callout); 3983 mtx_unlock(&sc->sfl_lock); 3984 3985 KASSERT((fl->flags & FL_STARVING) == 0, 3986 ("%s: still starving", __func__)); 3987} 3988 3989static int 3990t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3991 driver_intr_t *handler, void *arg, char *name) 3992{ 3993 int rc; 3994 3995 irq->rid = rid; 3996 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3997 RF_SHAREABLE | RF_ACTIVE); 3998 if (irq->res == NULL) { 3999 device_printf(sc->dev, 4000 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 4001 return (ENOMEM); 4002 } 4003 4004 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 4005 NULL, handler, arg, &irq->tag); 4006 if (rc != 0) { 4007 device_printf(sc->dev, 4008 "failed to setup interrupt for rid %d, name %s: %d\n", 4009 rid, name, rc); 4010 } else if (name) 4011 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 4012 4013 return (rc); 4014} 4015 4016static int 4017t4_free_irq(struct adapter *sc, struct irq *irq) 4018{ 4019 if (irq->tag) 4020 bus_teardown_intr(sc->dev, irq->res, irq->tag); 4021 if (irq->res) 4022 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 4023 4024 bzero(irq, sizeof(*irq)); 4025 4026 return (0); 4027} 4028 4029static void 4030reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 4031 unsigned int end) 4032{ 4033 uint32_t *p = (uint32_t *)(buf + start); 4034 4035 for ( ; start <= end; start += sizeof(uint32_t)) 4036 *p++ = t4_read_reg(sc, start); 4037} 4038 4039static void 4040t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 4041{ 4042 int i, n; 4043 const unsigned int *reg_ranges; 4044 static const unsigned int t4_reg_ranges[] = { 4045 0x1008, 0x1108, 4046 0x1180, 0x11b4, 4047 0x11fc, 0x123c, 4048 0x1300, 0x173c, 4049 0x1800, 0x18fc, 4050 0x3000, 0x30d8, 4051 0x30e0, 0x5924, 4052 0x5960, 0x59d4, 4053 0x5a00, 0x5af8, 4054 0x6000, 0x6098, 4055 0x6100, 0x6150, 4056 0x6200, 0x6208, 4057 0x6240, 0x6248, 4058 0x6280, 0x6338, 4059 0x6370, 0x638c, 4060 0x6400, 0x643c, 4061 0x6500, 0x6524, 4062 0x6a00, 0x6a38, 4063 0x6a60, 0x6a78, 4064 0x6b00, 0x6b84, 4065 0x6bf0, 0x6c84, 4066 0x6cf0, 0x6d84, 4067 0x6df0, 0x6e84, 4068 0x6ef0, 0x6f84, 4069 0x6ff0, 0x7084, 4070 0x70f0, 0x7184, 4071 0x71f0, 0x7284, 4072 0x72f0, 0x7384, 4073 0x73f0, 0x7450, 4074 0x7500, 0x7530, 4075 0x7600, 0x761c, 4076 0x7680, 0x76cc, 4077 0x7700, 0x7798, 4078 0x77c0, 0x77fc, 4079 0x7900, 0x79fc, 4080 0x7b00, 0x7c38, 4081 0x7d00, 0x7efc, 4082 0x8dc0, 0x8e1c, 4083 0x8e30, 0x8e78, 4084 0x8ea0, 0x8f6c, 4085 0x8fc0, 0x9074, 4086 0x90fc, 0x90fc, 4087 0x9400, 0x9458, 4088 0x9600, 0x96bc, 4089 0x9800, 0x9808, 4090 0x9820, 0x983c, 4091 0x9850, 0x9864, 4092 0x9c00, 0x9c6c, 4093 0x9c80, 0x9cec, 4094 0x9d00, 0x9d6c, 4095 0x9d80, 0x9dec, 4096 0x9e00, 0x9e6c, 4097 0x9e80, 0x9eec, 4098 0x9f00, 0x9f6c, 4099 0x9f80, 0x9fec, 4100 0xd004, 0xd03c, 4101 0xdfc0, 0xdfe0, 4102 0xe000, 0xea7c, 4103 0xf000, 0x11110, 4104 0x11118, 0x11190, 4105 0x19040, 0x1906c, 4106 0x19078, 0x19080, 4107 0x1908c, 0x19124, 4108 0x19150, 0x191b0, 4109 0x191d0, 0x191e8, 4110 0x19238, 0x1924c, 4111 0x193f8, 0x19474, 4112 0x19490, 0x194f8, 4113 0x19800, 0x19f30, 4114 0x1a000, 0x1a06c, 4115 0x1a0b0, 0x1a120, 4116 0x1a128, 0x1a138, 4117 0x1a190, 0x1a1c4, 4118 0x1a1fc, 0x1a1fc, 4119 0x1e040, 0x1e04c, 4120 0x1e284, 0x1e28c, 4121 0x1e2c0, 0x1e2c0, 4122 0x1e2e0, 0x1e2e0, 4123 0x1e300, 0x1e384, 4124 0x1e3c0, 0x1e3c8, 4125 0x1e440, 0x1e44c, 4126 0x1e684, 0x1e68c, 4127 0x1e6c0, 0x1e6c0, 4128 0x1e6e0, 0x1e6e0, 4129 0x1e700, 0x1e784, 4130 0x1e7c0, 0x1e7c8, 4131 0x1e840, 0x1e84c, 4132 0x1ea84, 0x1ea8c, 4133 0x1eac0, 0x1eac0, 4134 0x1eae0, 0x1eae0, 4135 0x1eb00, 0x1eb84, 4136 0x1ebc0, 0x1ebc8, 4137 0x1ec40, 0x1ec4c, 4138 0x1ee84, 0x1ee8c, 4139 0x1eec0, 0x1eec0, 4140 0x1eee0, 0x1eee0, 4141 0x1ef00, 0x1ef84, 4142 0x1efc0, 0x1efc8, 4143 0x1f040, 0x1f04c, 4144 0x1f284, 0x1f28c, 4145 0x1f2c0, 0x1f2c0, 4146 0x1f2e0, 0x1f2e0, 4147 0x1f300, 0x1f384, 4148 0x1f3c0, 0x1f3c8, 4149 0x1f440, 0x1f44c, 4150 0x1f684, 0x1f68c, 4151 0x1f6c0, 0x1f6c0, 4152 0x1f6e0, 0x1f6e0, 4153 0x1f700, 0x1f784, 4154 0x1f7c0, 0x1f7c8, 4155 0x1f840, 0x1f84c, 4156 0x1fa84, 0x1fa8c, 4157 0x1fac0, 0x1fac0, 4158 0x1fae0, 0x1fae0, 4159 0x1fb00, 0x1fb84, 4160 0x1fbc0, 0x1fbc8, 4161 0x1fc40, 0x1fc4c, 4162 0x1fe84, 0x1fe8c, 4163 0x1fec0, 0x1fec0, 4164 0x1fee0, 0x1fee0, 4165 0x1ff00, 0x1ff84, 4166 0x1ffc0, 0x1ffc8, 4167 0x20000, 0x2002c, 4168 0x20100, 0x2013c, 4169 0x20190, 0x201c8, 4170 0x20200, 0x20318, 4171 0x20400, 0x20528, 4172 0x20540, 0x20614, 4173 0x21000, 0x21040, 4174 0x2104c, 0x21060, 4175 0x210c0, 0x210ec, 4176 0x21200, 0x21268, 4177 0x21270, 0x21284, 4178 0x212fc, 0x21388, 4179 0x21400, 0x21404, 4180 0x21500, 0x21518, 4181 0x2152c, 0x2153c, 4182 0x21550, 0x21554, 4183 0x21600, 0x21600, 4184 0x21608, 0x21628, 4185 0x21630, 0x2163c, 4186 0x21700, 0x2171c, 4187 0x21780, 0x2178c, 4188 0x21800, 0x21c38, 4189 0x21c80, 0x21d7c, 4190 0x21e00, 0x21e04, 4191 0x22000, 0x2202c, 4192 0x22100, 0x2213c, 4193 0x22190, 0x221c8, 4194 0x22200, 0x22318, 4195 0x22400, 0x22528, 4196 0x22540, 0x22614, 4197 0x23000, 0x23040, 4198 0x2304c, 0x23060, 4199 0x230c0, 0x230ec, 4200 0x23200, 0x23268, 4201 0x23270, 0x23284, 4202 0x232fc, 0x23388, 4203 0x23400, 0x23404, 4204 0x23500, 0x23518, 4205 0x2352c, 0x2353c, 4206 0x23550, 0x23554, 4207 0x23600, 0x23600, 4208 0x23608, 0x23628, 4209 0x23630, 0x2363c, 4210 0x23700, 0x2371c, 4211 0x23780, 0x2378c, 4212 0x23800, 0x23c38, 4213 0x23c80, 0x23d7c, 4214 0x23e00, 0x23e04, 4215 0x24000, 0x2402c, 4216 0x24100, 0x2413c, 4217 0x24190, 0x241c8, 4218 0x24200, 0x24318, 4219 0x24400, 0x24528, 4220 0x24540, 0x24614, 4221 0x25000, 0x25040, 4222 0x2504c, 0x25060, 4223 0x250c0, 0x250ec, 4224 0x25200, 0x25268, 4225 0x25270, 0x25284, 4226 0x252fc, 0x25388, 4227 0x25400, 0x25404, 4228 0x25500, 0x25518, 4229 0x2552c, 0x2553c, 4230 0x25550, 0x25554, 4231 0x25600, 0x25600, 4232 0x25608, 0x25628, 4233 0x25630, 0x2563c, 4234 0x25700, 0x2571c, 4235 0x25780, 0x2578c, 4236 0x25800, 0x25c38, 4237 0x25c80, 0x25d7c, 4238 0x25e00, 0x25e04, 4239 0x26000, 0x2602c, 4240 0x26100, 0x2613c, 4241 0x26190, 0x261c8, 4242 0x26200, 0x26318, 4243 0x26400, 0x26528, 4244 0x26540, 0x26614, 4245 0x27000, 0x27040, 4246 0x2704c, 0x27060, 4247 0x270c0, 0x270ec, 4248 0x27200, 0x27268, 4249 0x27270, 0x27284, 4250 0x272fc, 0x27388, 4251 0x27400, 0x27404, 4252 0x27500, 0x27518, 4253 0x2752c, 0x2753c, 4254 0x27550, 0x27554, 4255 0x27600, 0x27600, 4256 0x27608, 0x27628, 4257 0x27630, 0x2763c, 4258 0x27700, 0x2771c, 4259 0x27780, 0x2778c, 4260 0x27800, 0x27c38, 4261 0x27c80, 0x27d7c, 4262 0x27e00, 0x27e04 4263 }; 4264 static const unsigned int t5_reg_ranges[] = { 4265 0x1008, 0x1148, 4266 0x1180, 0x11b4, 4267 0x11fc, 0x123c, 4268 0x1280, 0x173c, 4269 0x1800, 0x18fc, 4270 0x3000, 0x3028, 4271 0x3060, 0x30d8, 4272 0x30e0, 0x30fc, 4273 0x3140, 0x357c, 4274 0x35a8, 0x35cc, 4275 0x35ec, 0x35ec, 4276 0x3600, 0x5624, 4277 0x56cc, 0x575c, 4278 0x580c, 0x5814, 4279 0x5890, 0x58bc, 4280 0x5940, 0x59dc, 4281 0x59fc, 0x5a18, 4282 0x5a60, 0x5a9c, 4283 0x5b94, 0x5bfc, 4284 0x6000, 0x6040, 4285 0x6058, 0x614c, 4286 0x7700, 0x7798, 4287 0x77c0, 0x78fc, 4288 0x7b00, 0x7c54, 4289 0x7d00, 0x7efc, 4290 0x8dc0, 0x8de0, 4291 0x8df8, 0x8e84, 4292 0x8ea0, 0x8f84, 4293 0x8fc0, 0x90f8, 4294 0x9400, 0x9470, 4295 0x9600, 0x96f4, 4296 0x9800, 0x9808, 4297 0x9820, 0x983c, 4298 0x9850, 0x9864, 4299 0x9c00, 0x9c6c, 4300 0x9c80, 0x9cec, 4301 0x9d00, 0x9d6c, 4302 0x9d80, 0x9dec, 4303 0x9e00, 0x9e6c, 4304 0x9e80, 0x9eec, 4305 0x9f00, 0x9f6c, 4306 0x9f80, 0xa020, 4307 0xd004, 0xd03c, 4308 0xdfc0, 0xdfe0, 4309 0xe000, 0x11088, 4310 0x1109c, 0x11110, 4311 0x11118, 0x1117c, 4312 0x11190, 0x11204, 4313 0x19040, 0x1906c, 4314 0x19078, 0x19080, 4315 0x1908c, 0x19124, 4316 0x19150, 0x191b0, 4317 0x191d0, 0x191e8, 4318 0x19238, 0x19290, 4319 0x193f8, 0x19474, 4320 0x19490, 0x194cc, 4321 0x194f0, 0x194f8, 4322 0x19c00, 0x19c60, 4323 0x19c94, 0x19e10, 4324 0x19e50, 0x19f34, 4325 0x19f40, 0x19f50, 4326 0x19f90, 0x19fe4, 4327 0x1a000, 0x1a06c, 4328 0x1a0b0, 0x1a120, 4329 0x1a128, 0x1a138, 4330 0x1a190, 0x1a1c4, 4331 0x1a1fc, 0x1a1fc, 4332 0x1e008, 0x1e00c, 4333 0x1e040, 0x1e04c, 4334 0x1e284, 0x1e290, 4335 0x1e2c0, 0x1e2c0, 4336 0x1e2e0, 0x1e2e0, 4337 0x1e300, 0x1e384, 4338 0x1e3c0, 0x1e3c8, 4339 0x1e408, 0x1e40c, 4340 0x1e440, 0x1e44c, 4341 0x1e684, 0x1e690, 4342 0x1e6c0, 0x1e6c0, 4343 0x1e6e0, 0x1e6e0, 4344 0x1e700, 0x1e784, 4345 0x1e7c0, 0x1e7c8, 4346 0x1e808, 0x1e80c, 4347 0x1e840, 0x1e84c, 4348 0x1ea84, 0x1ea90, 4349 0x1eac0, 0x1eac0, 4350 0x1eae0, 0x1eae0, 4351 0x1eb00, 0x1eb84, 4352 0x1ebc0, 0x1ebc8, 4353 0x1ec08, 0x1ec0c, 4354 0x1ec40, 0x1ec4c, 4355 0x1ee84, 0x1ee90, 4356 0x1eec0, 0x1eec0, 4357 0x1eee0, 0x1eee0, 4358 0x1ef00, 0x1ef84, 4359 0x1efc0, 0x1efc8, 4360 0x1f008, 0x1f00c, 4361 0x1f040, 0x1f04c, 4362 0x1f284, 0x1f290, 4363 0x1f2c0, 0x1f2c0, 4364 0x1f2e0, 0x1f2e0, 4365 0x1f300, 0x1f384, 4366 0x1f3c0, 0x1f3c8, 4367 0x1f408, 0x1f40c, 4368 0x1f440, 0x1f44c, 4369 0x1f684, 0x1f690, 4370 0x1f6c0, 0x1f6c0, 4371 0x1f6e0, 0x1f6e0, 4372 0x1f700, 0x1f784, 4373 0x1f7c0, 0x1f7c8, 4374 0x1f808, 0x1f80c, 4375 0x1f840, 0x1f84c, 4376 0x1fa84, 0x1fa90, 4377 0x1fac0, 0x1fac0, 4378 0x1fae0, 0x1fae0, 4379 0x1fb00, 0x1fb84, 4380 0x1fbc0, 0x1fbc8, 4381 0x1fc08, 0x1fc0c, 4382 0x1fc40, 0x1fc4c, 4383 0x1fe84, 0x1fe90, 4384 0x1fec0, 0x1fec0, 4385 0x1fee0, 0x1fee0, 4386 0x1ff00, 0x1ff84, 4387 0x1ffc0, 0x1ffc8, 4388 0x30000, 0x30030, 4389 0x30100, 0x30144, 4390 0x30190, 0x301d0, 4391 0x30200, 0x30318, 4392 0x30400, 0x3052c, 4393 0x30540, 0x3061c, 4394 0x30800, 0x30834, 4395 0x308c0, 0x30908, 4396 0x30910, 0x309ac, 4397 0x30a00, 0x30a2c, 4398 0x30a44, 0x30a50, 4399 0x30a74, 0x30c24, 4400 0x30d00, 0x30d00, 4401 0x30d08, 0x30d14, 4402 0x30d1c, 0x30d20, 4403 0x30d3c, 0x30d50, 4404 0x31200, 0x3120c, 4405 0x31220, 0x31220, 4406 0x31240, 0x31240, 4407 0x31600, 0x3160c, 4408 0x31a00, 0x31a1c, 4409 0x31e00, 0x31e20, 4410 0x31e38, 0x31e3c, 4411 0x31e80, 0x31e80, 4412 0x31e88, 0x31ea8, 4413 0x31eb0, 0x31eb4, 4414 0x31ec8, 0x31ed4, 4415 0x31fb8, 0x32004, 4416 0x32200, 0x32200, 4417 0x32208, 0x32240, 4418 0x32248, 0x32280, 4419 0x32288, 0x322c0, 4420 0x322c8, 0x322fc, 4421 0x32600, 0x32630, 4422 0x32a00, 0x32abc, 4423 0x32b00, 0x32b70, 4424 0x33000, 0x33048, 4425 0x33060, 0x3309c, 4426 0x330f0, 0x33148, 4427 0x33160, 0x3319c, 4428 0x331f0, 0x332e4, 4429 0x332f8, 0x333e4, 4430 0x333f8, 0x33448, 4431 0x33460, 0x3349c, 4432 0x334f0, 0x33548, 4433 0x33560, 0x3359c, 4434 0x335f0, 0x336e4, 4435 0x336f8, 0x337e4, 4436 0x337f8, 0x337fc, 4437 0x33814, 0x33814, 4438 0x3382c, 0x3382c, 4439 0x33880, 0x3388c, 4440 0x338e8, 0x338ec, 4441 0x33900, 0x33948, 4442 0x33960, 0x3399c, 4443 0x339f0, 0x33ae4, 4444 0x33af8, 0x33b10, 4445 0x33b28, 0x33b28, 4446 0x33b3c, 0x33b50, 4447 0x33bf0, 0x33c10, 4448 0x33c28, 0x33c28, 4449 0x33c3c, 0x33c50, 4450 0x33cf0, 0x33cfc, 4451 0x34000, 0x34030, 4452 0x34100, 0x34144, 4453 0x34190, 0x341d0, 4454 0x34200, 0x34318, 4455 0x34400, 0x3452c, 4456 0x34540, 0x3461c, 4457 0x34800, 0x34834, 4458 0x348c0, 0x34908, 4459 0x34910, 0x349ac, 4460 0x34a00, 0x34a2c, 4461 0x34a44, 0x34a50, 4462 0x34a74, 0x34c24, 4463 0x34d00, 0x34d00, 4464 0x34d08, 0x34d14, 4465 0x34d1c, 0x34d20, 4466 0x34d3c, 0x34d50, 4467 0x35200, 0x3520c, 4468 0x35220, 0x35220, 4469 0x35240, 0x35240, 4470 0x35600, 0x3560c, 4471 0x35a00, 0x35a1c, 4472 0x35e00, 0x35e20, 4473 0x35e38, 0x35e3c, 4474 0x35e80, 0x35e80, 4475 0x35e88, 0x35ea8, 4476 0x35eb0, 0x35eb4, 4477 0x35ec8, 0x35ed4, 4478 0x35fb8, 0x36004, 4479 0x36200, 0x36200, 4480 0x36208, 0x36240, 4481 0x36248, 0x36280, 4482 0x36288, 0x362c0, 4483 0x362c8, 0x362fc, 4484 0x36600, 0x36630, 4485 0x36a00, 0x36abc, 4486 0x36b00, 0x36b70, 4487 0x37000, 0x37048, 4488 0x37060, 0x3709c, 4489 0x370f0, 0x37148, 4490 0x37160, 0x3719c, 4491 0x371f0, 0x372e4, 4492 0x372f8, 0x373e4, 4493 0x373f8, 0x37448, 4494 0x37460, 0x3749c, 4495 0x374f0, 0x37548, 4496 0x37560, 0x3759c, 4497 0x375f0, 0x376e4, 4498 0x376f8, 0x377e4, 4499 0x377f8, 0x377fc, 4500 0x37814, 0x37814, 4501 0x3782c, 0x3782c, 4502 0x37880, 0x3788c, 4503 0x378e8, 0x378ec, 4504 0x37900, 0x37948, 4505 0x37960, 0x3799c, 4506 0x379f0, 0x37ae4, 4507 0x37af8, 0x37b10, 4508 0x37b28, 0x37b28, 4509 0x37b3c, 0x37b50, 4510 0x37bf0, 0x37c10, 4511 0x37c28, 0x37c28, 4512 0x37c3c, 0x37c50, 4513 0x37cf0, 0x37cfc, 4514 0x38000, 0x38030, 4515 0x38100, 0x38144, 4516 0x38190, 0x381d0, 4517 0x38200, 0x38318, 4518 0x38400, 0x3852c, 4519 0x38540, 0x3861c, 4520 0x38800, 0x38834, 4521 0x388c0, 0x38908, 4522 0x38910, 0x389ac, 4523 0x38a00, 0x38a2c, 4524 0x38a44, 0x38a50, 4525 0x38a74, 0x38c24, 4526 0x38d00, 0x38d00, 4527 0x38d08, 0x38d14, 4528 0x38d1c, 0x38d20, 4529 0x38d3c, 0x38d50, 4530 0x39200, 0x3920c, 4531 0x39220, 0x39220, 4532 0x39240, 0x39240, 4533 0x39600, 0x3960c, 4534 0x39a00, 0x39a1c, 4535 0x39e00, 0x39e20, 4536 0x39e38, 0x39e3c, 4537 0x39e80, 0x39e80, 4538 0x39e88, 0x39ea8, 4539 0x39eb0, 0x39eb4, 4540 0x39ec8, 0x39ed4, 4541 0x39fb8, 0x3a004, 4542 0x3a200, 0x3a200, 4543 0x3a208, 0x3a240, 4544 0x3a248, 0x3a280, 4545 0x3a288, 0x3a2c0, 4546 0x3a2c8, 0x3a2fc, 4547 0x3a600, 0x3a630, 4548 0x3aa00, 0x3aabc, 4549 0x3ab00, 0x3ab70, 4550 0x3b000, 0x3b048, 4551 0x3b060, 0x3b09c, 4552 0x3b0f0, 0x3b148, 4553 0x3b160, 0x3b19c, 4554 0x3b1f0, 0x3b2e4, 4555 0x3b2f8, 0x3b3e4, 4556 0x3b3f8, 0x3b448, 4557 0x3b460, 0x3b49c, 4558 0x3b4f0, 0x3b548, 4559 0x3b560, 0x3b59c, 4560 0x3b5f0, 0x3b6e4, 4561 0x3b6f8, 0x3b7e4, 4562 0x3b7f8, 0x3b7fc, 4563 0x3b814, 0x3b814, 4564 0x3b82c, 0x3b82c, 4565 0x3b880, 0x3b88c, 4566 0x3b8e8, 0x3b8ec, 4567 0x3b900, 0x3b948, 4568 0x3b960, 0x3b99c, 4569 0x3b9f0, 0x3bae4, 4570 0x3baf8, 0x3bb10, 4571 0x3bb28, 0x3bb28, 4572 0x3bb3c, 0x3bb50, 4573 0x3bbf0, 0x3bc10, 4574 0x3bc28, 0x3bc28, 4575 0x3bc3c, 0x3bc50, 4576 0x3bcf0, 0x3bcfc, 4577 0x3c000, 0x3c030, 4578 0x3c100, 0x3c144, 4579 0x3c190, 0x3c1d0, 4580 0x3c200, 0x3c318, 4581 0x3c400, 0x3c52c, 4582 0x3c540, 0x3c61c, 4583 0x3c800, 0x3c834, 4584 0x3c8c0, 0x3c908, 4585 0x3c910, 0x3c9ac, 4586 0x3ca00, 0x3ca2c, 4587 0x3ca44, 0x3ca50, 4588 0x3ca74, 0x3cc24, 4589 0x3cd00, 0x3cd00, 4590 0x3cd08, 0x3cd14, 4591 0x3cd1c, 0x3cd20, 4592 0x3cd3c, 0x3cd50, 4593 0x3d200, 0x3d20c, 4594 0x3d220, 0x3d220, 4595 0x3d240, 0x3d240, 4596 0x3d600, 0x3d60c, 4597 0x3da00, 0x3da1c, 4598 0x3de00, 0x3de20, 4599 0x3de38, 0x3de3c, 4600 0x3de80, 0x3de80, 4601 0x3de88, 0x3dea8, 4602 0x3deb0, 0x3deb4, 4603 0x3dec8, 0x3ded4, 4604 0x3dfb8, 0x3e004, 4605 0x3e200, 0x3e200, 4606 0x3e208, 0x3e240, 4607 0x3e248, 0x3e280, 4608 0x3e288, 0x3e2c0, 4609 0x3e2c8, 0x3e2fc, 4610 0x3e600, 0x3e630, 4611 0x3ea00, 0x3eabc, 4612 0x3eb00, 0x3eb70, 4613 0x3f000, 0x3f048, 4614 0x3f060, 0x3f09c, 4615 0x3f0f0, 0x3f148, 4616 0x3f160, 0x3f19c, 4617 0x3f1f0, 0x3f2e4, 4618 0x3f2f8, 0x3f3e4, 4619 0x3f3f8, 0x3f448, 4620 0x3f460, 0x3f49c, 4621 0x3f4f0, 0x3f548, 4622 0x3f560, 0x3f59c, 4623 0x3f5f0, 0x3f6e4, 4624 0x3f6f8, 0x3f7e4, 4625 0x3f7f8, 0x3f7fc, 4626 0x3f814, 0x3f814, 4627 0x3f82c, 0x3f82c, 4628 0x3f880, 0x3f88c, 4629 0x3f8e8, 0x3f8ec, 4630 0x3f900, 0x3f948, 4631 0x3f960, 0x3f99c, 4632 0x3f9f0, 0x3fae4, 4633 0x3faf8, 0x3fb10, 4634 0x3fb28, 0x3fb28, 4635 0x3fb3c, 0x3fb50, 4636 0x3fbf0, 0x3fc10, 4637 0x3fc28, 0x3fc28, 4638 0x3fc3c, 0x3fc50, 4639 0x3fcf0, 0x3fcfc, 4640 0x40000, 0x4000c, 4641 0x40040, 0x40068, 4642 0x4007c, 0x40144, 4643 0x40180, 0x4018c, 4644 0x40200, 0x40298, 4645 0x402ac, 0x4033c, 4646 0x403f8, 0x403fc, 4647 0x41304, 0x413c4, 4648 0x41400, 0x4141c, 4649 0x41480, 0x414d0, 4650 0x44000, 0x44078, 4651 0x440c0, 0x44278, 4652 0x442c0, 0x44478, 4653 0x444c0, 0x44678, 4654 0x446c0, 0x44878, 4655 0x448c0, 0x449fc, 4656 0x45000, 0x45068, 4657 0x45080, 0x45084, 4658 0x450a0, 0x450b0, 4659 0x45200, 0x45268, 4660 0x45280, 0x45284, 4661 0x452a0, 0x452b0, 4662 0x460c0, 0x460e4, 4663 0x47000, 0x4708c, 4664 0x47200, 0x47250, 4665 0x47400, 0x47420, 4666 0x47600, 0x47618, 4667 0x47800, 0x47814, 4668 0x48000, 0x4800c, 4669 0x48040, 0x48068, 4670 0x4807c, 0x48144, 4671 0x48180, 0x4818c, 4672 0x48200, 0x48298, 4673 0x482ac, 0x4833c, 4674 0x483f8, 0x483fc, 4675 0x49304, 0x493c4, 4676 0x49400, 0x4941c, 4677 0x49480, 0x494d0, 4678 0x4c000, 0x4c078, 4679 0x4c0c0, 0x4c278, 4680 0x4c2c0, 0x4c478, 4681 0x4c4c0, 0x4c678, 4682 0x4c6c0, 0x4c878, 4683 0x4c8c0, 0x4c9fc, 4684 0x4d000, 0x4d068, 4685 0x4d080, 0x4d084, 4686 0x4d0a0, 0x4d0b0, 4687 0x4d200, 0x4d268, 4688 0x4d280, 0x4d284, 4689 0x4d2a0, 0x4d2b0, 4690 0x4e0c0, 0x4e0e4, 4691 0x4f000, 0x4f08c, 4692 0x4f200, 0x4f250, 4693 0x4f400, 0x4f420, 4694 0x4f600, 0x4f618, 4695 0x4f800, 0x4f814, 4696 0x50000, 0x500cc, 4697 0x50400, 0x50400, 4698 0x50800, 0x508cc, 4699 0x50c00, 0x50c00, 4700 0x51000, 0x5101c, 4701 0x51300, 0x51308, 4702 }; 4703 4704 if (is_t4(sc)) { 4705 reg_ranges = &t4_reg_ranges[0]; 4706 n = nitems(t4_reg_ranges); 4707 } else { 4708 reg_ranges = &t5_reg_ranges[0]; 4709 n = nitems(t5_reg_ranges); 4710 } 4711 4712 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4713 for (i = 0; i < n; i += 2) 4714 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4715} 4716 4717#define A_PL_INDIR_CMD 0x1f8 4718 4719#define S_PL_AUTOINC 31 4720#define M_PL_AUTOINC 0x1U 4721#define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) 4722#define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) 4723 4724#define S_PL_VFID 20 4725#define M_PL_VFID 0xffU 4726#define V_PL_VFID(x) ((x) << S_PL_VFID) 4727#define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) 4728 4729#define S_PL_ADDR 0 4730#define M_PL_ADDR 0xfffffU 4731#define V_PL_ADDR(x) ((x) << S_PL_ADDR) 4732#define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) 4733 4734#define A_PL_INDIR_DATA 0x1fc 4735 4736static uint64_t 4737read_vf_stat(struct adapter *sc, unsigned int viid, int reg) 4738{ 4739 u32 stats[2]; 4740 4741 mtx_assert(&sc->regwin_lock, MA_OWNED); 4742 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4743 V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg))); 4744 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); 4745 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); 4746 return (((uint64_t)stats[1]) << 32 | stats[0]); 4747} 4748 4749static void 4750t4_get_vi_stats(struct adapter *sc, unsigned int viid, 4751 struct fw_vi_stats_vf *stats) 4752{ 4753 4754#define GET_STAT(name) \ 4755 read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L) 4756 4757 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); 4758 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); 4759 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); 4760 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); 4761 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); 4762 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); 4763 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); 4764 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); 4765 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); 4766 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); 4767 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); 4768 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); 4769 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); 4770 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); 4771 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); 4772 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); 4773 4774#undef GET_STAT 4775} 4776 4777static void 4778t4_clr_vi_stats(struct adapter *sc, unsigned int viid) 4779{ 4780 int reg; 4781 4782 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4783 V_PL_VFID(G_FW_VIID_VIN(viid)) | 4784 V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); 4785 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 4786 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 4787 t4_write_reg(sc, A_PL_INDIR_DATA, 0); 4788} 4789 4790static void 4791vi_refresh_stats(struct adapter *sc, struct vi_info *vi) 4792{ 4793 struct ifnet *ifp = vi->ifp; 4794 struct sge_txq *txq; 4795 int i, drops; 4796 struct fw_vi_stats_vf *s = &vi->stats; 4797 struct timeval tv; 4798 const struct timeval interval = {0, 250000}; /* 250ms */ 4799 4800 if (!(vi->flags & VI_INIT_DONE)) 4801 return; 4802 4803 getmicrotime(&tv); 4804 timevalsub(&tv, &interval); 4805 if (timevalcmp(&tv, &vi->last_refreshed, <)) 4806 return; 4807 4808 mtx_lock(&sc->regwin_lock); 4809 t4_get_vi_stats(sc, vi->viid, &vi->stats); 4810 4811 ifp->if_ipackets = s->rx_bcast_frames + s->rx_mcast_frames + 4812 s->rx_ucast_frames; 4813 ifp->if_ierrors = s->rx_err_frames; 4814 ifp->if_opackets = s->tx_bcast_frames + s->tx_mcast_frames + 4815 s->tx_ucast_frames + s->tx_offload_frames; 4816 ifp->if_oerrors = s->tx_drop_frames; 4817 ifp->if_ibytes = s->rx_bcast_bytes + s->rx_mcast_bytes + 4818 s->rx_ucast_bytes; 4819 ifp->if_obytes = s->tx_bcast_bytes + s->tx_mcast_bytes + 4820 s->tx_ucast_bytes + s->tx_offload_bytes; 4821 ifp->if_imcasts = s->rx_mcast_frames; 4822 ifp->if_omcasts = s->tx_mcast_frames; 4823 4824 drops = 0; 4825 for_each_txq(vi, i, txq) 4826 drops += counter_u64_fetch(txq->r->drops); 4827 ifp->if_snd.ifq_drops = drops; 4828 4829 getmicrotime(&vi->last_refreshed); 4830 mtx_unlock(&sc->regwin_lock); 4831} 4832 4833static void 4834cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4835{ 4836 struct vi_info *vi = &pi->vi[0]; 4837 struct ifnet *ifp = vi->ifp; 4838 struct sge_txq *txq; 4839 int i, drops; 4840 struct port_stats *s = &pi->stats; 4841 struct timeval tv; 4842 const struct timeval interval = {0, 250000}; /* 250ms */ 4843 4844 getmicrotime(&tv); 4845 timevalsub(&tv, &interval); 4846 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4847 return; 4848 4849 t4_get_port_stats(sc, pi->tx_chan, s); 4850 4851 ifp->if_opackets = s->tx_frames - s->tx_pause; 4852 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4853 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4854 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4855 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4856 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4857 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4858 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4859 s->rx_trunc3; 4860 for (i = 0; i < NCHAN; i++) { 4861 if (pi->rx_chan_map & (1 << i)) { 4862 uint32_t v; 4863 4864 mtx_lock(&sc->regwin_lock); 4865 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4866 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4867 mtx_unlock(&sc->regwin_lock); 4868 ifp->if_iqdrops += v; 4869 } 4870 } 4871 4872 drops = s->tx_drop; 4873 for_each_txq(vi, i, txq) 4874 drops += counter_u64_fetch(txq->r->drops); 4875 ifp->if_snd.ifq_drops = drops; 4876 4877 ifp->if_oerrors = s->tx_error_frames; 4878 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4879 s->rx_fcs_err + s->rx_len_err; 4880 4881 getmicrotime(&pi->last_refreshed); 4882} 4883 4884static void 4885cxgbe_tick(void *arg) 4886{ 4887 struct port_info *pi = arg; 4888 struct adapter *sc = pi->adapter; 4889 4890 PORT_LOCK_ASSERT_OWNED(pi); 4891 cxgbe_refresh_stats(sc, pi); 4892 4893 callout_schedule(&pi->tick, hz); 4894} 4895 4896void 4897vi_tick(void *arg) 4898{ 4899 struct vi_info *vi = arg; 4900 struct adapter *sc = vi->pi->adapter; 4901 4902 vi_refresh_stats(sc, vi); 4903 4904 callout_schedule(&vi->tick, hz); 4905} 4906 4907static void 4908cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4909{ 4910 struct ifnet *vlan; 4911 4912 if (arg != ifp || ifp->if_type != IFT_ETHER) 4913 return; 4914 4915 vlan = VLAN_DEVAT(ifp, vid); 4916 VLAN_SETCOOKIE(vlan, ifp); 4917} 4918 4919static int 4920cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4921{ 4922 4923#ifdef INVARIANTS 4924 panic("%s: opcode 0x%02x on iq %p with payload %p", 4925 __func__, rss->opcode, iq, m); 4926#else 4927 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4928 __func__, rss->opcode, iq, m); 4929 m_freem(m); 4930#endif 4931 return (EDOOFUS); 4932} 4933 4934int 4935t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4936{ 4937 uintptr_t *loc, new; 4938 4939 if (opcode >= nitems(sc->cpl_handler)) 4940 return (EINVAL); 4941 4942 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4943 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4944 atomic_store_rel_ptr(loc, new); 4945 4946 return (0); 4947} 4948 4949static int 4950an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4951{ 4952 4953#ifdef INVARIANTS 4954 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4955#else 4956 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4957 __func__, iq, ctrl); 4958#endif 4959 return (EDOOFUS); 4960} 4961 4962int 4963t4_register_an_handler(struct adapter *sc, an_handler_t h) 4964{ 4965 uintptr_t *loc, new; 4966 4967 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4968 loc = (uintptr_t *) &sc->an_handler; 4969 atomic_store_rel_ptr(loc, new); 4970 4971 return (0); 4972} 4973 4974static int 4975fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4976{ 4977 const struct cpl_fw6_msg *cpl = 4978 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4979 4980#ifdef INVARIANTS 4981 panic("%s: fw_msg type %d", __func__, cpl->type); 4982#else 4983 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4984#endif 4985 return (EDOOFUS); 4986} 4987 4988int 4989t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4990{ 4991 uintptr_t *loc, new; 4992 4993 if (type >= nitems(sc->fw_msg_handler)) 4994 return (EINVAL); 4995 4996 /* 4997 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4998 * handler dispatch table. Reject any attempt to install a handler for 4999 * this subtype. 5000 */ 5001 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 5002 return (EINVAL); 5003 5004 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 5005 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 5006 atomic_store_rel_ptr(loc, new); 5007 5008 return (0); 5009} 5010 5011static void 5012t4_sysctls(struct adapter *sc) 5013{ 5014 struct sysctl_ctx_list *ctx; 5015 struct sysctl_oid *oid; 5016 struct sysctl_oid_list *children, *c0; 5017 static char *caps[] = { 5018 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 5019 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 5020 "\6HASHFILTER\7ETHOFLD", 5021 "\20\1TOE", /* caps[2] toecaps */ 5022 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 5023 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 5024 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 5025 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 5026 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 5027 "\4PO_INITIAOR\5PO_TARGET" 5028 }; 5029 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 5030 5031 ctx = device_get_sysctl_ctx(sc->dev); 5032 5033 /* 5034 * dev.t4nex.X. 5035 */ 5036 oid = device_get_sysctl_tree(sc->dev); 5037 c0 = children = SYSCTL_CHILDREN(oid); 5038 5039 sc->sc_do_rxcopy = 1; 5040 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 5041 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 5042 5043 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 5044 sc->params.nports, "# of ports"); 5045 5046 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 5047 NULL, chip_rev(sc), "chip hardware revision"); 5048 5049 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 5050 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 5051 5052 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 5053 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 5054 5055 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 5056 sc->cfcsum, "config file checksum"); 5057 5058 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 5059 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 5060 sysctl_bitfield, "A", "available doorbells"); 5061 5062 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 5063 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 5064 sysctl_bitfield, "A", "available link capabilities"); 5065 5066 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 5067 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 5068 sysctl_bitfield, "A", "available NIC capabilities"); 5069 5070 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 5071 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 5072 sysctl_bitfield, "A", "available TCP offload capabilities"); 5073 5074 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 5075 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 5076 sysctl_bitfield, "A", "available RDMA capabilities"); 5077 5078 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 5079 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 5080 sysctl_bitfield, "A", "available iSCSI capabilities"); 5081 5082 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 5083 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 5084 sysctl_bitfield, "A", "available FCoE capabilities"); 5085 5086 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 5087 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 5088 5089 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 5090 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 5091 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 5092 "interrupt holdoff timer values (us)"); 5093 5094 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 5095 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 5096 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 5097 "interrupt holdoff packet counter values"); 5098 5099 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 5100 NULL, sc->tids.nftids, "number of filters"); 5101 5102 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 5103 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 5104 "chip temperature (in Celsius)"); 5105 5106 t4_sge_sysctls(sc, ctx, children); 5107 5108 sc->lro_timeout = 100; 5109 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 5110 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 5111 5112 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 5113 &sc->debug_flags, 0, "flags to enable runtime debugging"); 5114 5115#ifdef SBUF_DRAIN 5116 /* 5117 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 5118 */ 5119 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 5120 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 5121 "logs and miscellaneous information"); 5122 children = SYSCTL_CHILDREN(oid); 5123 5124 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 5125 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5126 sysctl_cctrl, "A", "congestion control"); 5127 5128 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 5129 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5130 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 5131 5132 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 5133 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 5134 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 5135 5136 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 5137 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 5138 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 5139 5140 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 5141 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 5142 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 5143 5144 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 5145 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 5146 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 5147 5148 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 5149 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 5150 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 5151 5152 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 5153 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5154 sysctl_cim_la, "A", "CIM logic analyzer"); 5155 5156 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 5157 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5158 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 5159 5160 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 5161 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 5162 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 5163 5164 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 5165 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 5166 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 5167 5168 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 5169 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 5170 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 5171 5172 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 5173 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 5174 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 5175 5176 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 5177 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 5178 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 5179 5180 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 5181 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 5182 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 5183 5184 if (is_t5(sc)) { 5185 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 5186 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 5187 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 5188 5189 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 5190 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 5191 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 5192 } 5193 5194 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 5195 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5196 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 5197 5198 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 5199 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5200 sysctl_cim_qcfg, "A", "CIM queue configuration"); 5201 5202 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 5203 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5204 sysctl_cpl_stats, "A", "CPL statistics"); 5205 5206 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 5207 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5208 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 5209 5210 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 5211 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5212 sysctl_devlog, "A", "firmware's device log"); 5213 5214 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 5215 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5216 sysctl_fcoe_stats, "A", "FCoE statistics"); 5217 5218 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 5219 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5220 sysctl_hw_sched, "A", "hardware scheduler "); 5221 5222 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 5223 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5224 sysctl_l2t, "A", "hardware L2 table"); 5225 5226 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 5227 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5228 sysctl_lb_stats, "A", "loopback statistics"); 5229 5230 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 5231 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5232 sysctl_meminfo, "A", "memory regions"); 5233 5234 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 5235 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5236 sysctl_mps_tcam, "A", "MPS TCAM entries"); 5237 5238 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 5239 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5240 sysctl_path_mtus, "A", "path MTUs"); 5241 5242 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 5243 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5244 sysctl_pm_stats, "A", "PM statistics"); 5245 5246 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 5247 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5248 sysctl_rdma_stats, "A", "RDMA statistics"); 5249 5250 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 5251 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5252 sysctl_tcp_stats, "A", "TCP statistics"); 5253 5254 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 5255 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5256 sysctl_tids, "A", "TID information"); 5257 5258 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 5259 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5260 sysctl_tp_err_stats, "A", "TP error statistics"); 5261 5262 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 5263 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5264 sysctl_tp_la, "A", "TP logic analyzer"); 5265 5266 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 5267 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5268 sysctl_tx_rate, "A", "Tx rate"); 5269 5270 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 5271 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5272 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 5273 5274 if (is_t5(sc)) { 5275 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 5276 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 5277 sysctl_wcwr_stats, "A", "write combined work requests"); 5278 } 5279#endif 5280 5281#ifdef TCP_OFFLOAD 5282 if (is_offload(sc)) { 5283 /* 5284 * dev.t4nex.X.toe. 5285 */ 5286 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 5287 NULL, "TOE parameters"); 5288 children = SYSCTL_CHILDREN(oid); 5289 5290 sc->tt.sndbuf = 256 * 1024; 5291 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 5292 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 5293 5294 sc->tt.ddp = 0; 5295 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 5296 &sc->tt.ddp, 0, "DDP allowed"); 5297 5298 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 5299 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 5300 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 5301 5302 sc->tt.ddp_thres = 5303 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 5304 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 5305 &sc->tt.ddp_thres, 0, "DDP threshold"); 5306 5307 sc->tt.rx_coalesce = 1; 5308 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 5309 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 5310 5311 sc->tt.tx_align = 1; 5312 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 5313 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 5314 } 5315#endif 5316} 5317 5318void 5319vi_sysctls(struct vi_info *vi) 5320{ 5321 struct sysctl_ctx_list *ctx; 5322 struct sysctl_oid *oid; 5323 struct sysctl_oid_list *children; 5324 5325 ctx = device_get_sysctl_ctx(vi->dev); 5326 5327 /* 5328 * dev.v?(cxgbe|cxl).X. 5329 */ 5330 oid = device_get_sysctl_tree(vi->dev); 5331 children = SYSCTL_CHILDREN(oid); 5332 5333 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, 5334 vi->viid, "VI identifer"); 5335 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 5336 &vi->nrxq, 0, "# of rx queues"); 5337 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 5338 &vi->ntxq, 0, "# of tx queues"); 5339 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 5340 &vi->first_rxq, 0, "index of first rx queue"); 5341 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 5342 &vi->first_txq, 0, "index of first tx queue"); 5343 5344 if (IS_MAIN_VI(vi)) { 5345 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", 5346 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU", 5347 "Reserve queue 0 for non-flowid packets"); 5348 } 5349 5350#ifdef TCP_OFFLOAD 5351 if (vi->nofldrxq != 0) { 5352 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 5353 &vi->nofldrxq, 0, 5354 "# of rx queues for offloaded TCP connections"); 5355 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 5356 &vi->nofldtxq, 0, 5357 "# of tx queues for offloaded TCP connections"); 5358 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 5359 CTLFLAG_RD, &vi->first_ofld_rxq, 0, 5360 "index of first TOE rx queue"); 5361 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 5362 CTLFLAG_RD, &vi->first_ofld_txq, 0, 5363 "index of first TOE tx queue"); 5364 } 5365#endif 5366#ifdef DEV_NETMAP 5367 if (vi->nnmrxq != 0) { 5368 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 5369 &vi->nnmrxq, 0, "# of netmap rx queues"); 5370 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 5371 &vi->nnmtxq, 0, "# of netmap tx queues"); 5372 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 5373 CTLFLAG_RD, &vi->first_nm_rxq, 0, 5374 "index of first netmap rx queue"); 5375 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 5376 CTLFLAG_RD, &vi->first_nm_txq, 0, 5377 "index of first netmap tx queue"); 5378 } 5379#endif 5380 5381 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 5382 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I", 5383 "holdoff timer index"); 5384 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 5385 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I", 5386 "holdoff packet counter index"); 5387 5388 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 5389 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I", 5390 "rx queue size"); 5391 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 5392 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I", 5393 "tx queue size"); 5394} 5395 5396static void 5397cxgbe_sysctls(struct port_info *pi) 5398{ 5399 struct sysctl_ctx_list *ctx; 5400 struct sysctl_oid *oid; 5401 struct sysctl_oid_list *children; 5402 struct adapter *sc = pi->adapter; 5403 5404 ctx = device_get_sysctl_ctx(pi->dev); 5405 5406 /* 5407 * dev.cxgbe.X. 5408 */ 5409 oid = device_get_sysctl_tree(pi->dev); 5410 children = SYSCTL_CHILDREN(oid); 5411 5412 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 5413 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 5414 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 5415 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 5416 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 5417 "PHY temperature (in Celsius)"); 5418 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 5419 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 5420 "PHY firmware version"); 5421 } 5422 5423 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 5424 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 5425 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 5426 5427 /* 5428 * dev.cxgbe.X.stats. 5429 */ 5430 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 5431 NULL, "port statistics"); 5432 children = SYSCTL_CHILDREN(oid); 5433 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 5434 &pi->tx_parse_error, 0, 5435 "# of tx packets with invalid length or # of segments"); 5436 5437#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 5438 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 5439 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 5440 sysctl_handle_t4_reg64, "QU", desc) 5441 5442 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 5443 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 5444 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 5445 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 5446 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 5447 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 5448 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 5449 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 5450 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 5451 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 5452 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 5453 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 5454 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 5455 "# of tx frames in this range", 5456 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 5457 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 5458 "# of tx frames in this range", 5459 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 5460 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 5461 "# of tx frames in this range", 5462 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 5463 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 5464 "# of tx frames in this range", 5465 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 5466 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 5467 "# of tx frames in this range", 5468 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 5469 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 5470 "# of tx frames in this range", 5471 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 5472 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 5473 "# of tx frames in this range", 5474 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 5475 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 5476 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 5477 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 5478 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 5479 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 5480 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 5481 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 5482 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 5483 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 5484 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 5485 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 5486 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 5487 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 5488 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 5489 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 5490 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 5491 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 5492 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 5493 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 5494 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 5495 5496 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 5497 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 5498 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 5499 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 5500 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 5501 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 5502 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 5503 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 5504 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 5505 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 5506 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 5507 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 5508 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 5509 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 5510 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 5511 "# of frames received with bad FCS", 5512 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 5513 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 5514 "# of frames received with length error", 5515 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 5516 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 5517 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 5518 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 5519 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 5520 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 5521 "# of rx frames in this range", 5522 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 5523 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5524 "# of rx frames in this range", 5525 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5526 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5527 "# of rx frames in this range", 5528 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5529 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5530 "# of rx frames in this range", 5531 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5532 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5533 "# of rx frames in this range", 5534 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5535 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5536 "# of rx frames in this range", 5537 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5538 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5539 "# of rx frames in this range", 5540 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5541 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5542 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5543 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5544 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5545 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5546 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5547 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5548 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5549 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5550 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5551 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5552 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5553 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5554 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5555 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5556 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5557 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5558 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5559 5560#undef SYSCTL_ADD_T4_REG64 5561 5562#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5563 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5564 &pi->stats.name, desc) 5565 5566 /* We get these from port_stats and they may be stale by upto 1s */ 5567 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5568 "# drops due to buffer-group 0 overflows"); 5569 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5570 "# drops due to buffer-group 1 overflows"); 5571 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5572 "# drops due to buffer-group 2 overflows"); 5573 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5574 "# drops due to buffer-group 3 overflows"); 5575 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5576 "# of buffer-group 0 truncated packets"); 5577 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5578 "# of buffer-group 1 truncated packets"); 5579 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5580 "# of buffer-group 2 truncated packets"); 5581 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5582 "# of buffer-group 3 truncated packets"); 5583 5584#undef SYSCTL_ADD_T4_PORTSTAT 5585} 5586 5587static int 5588sysctl_int_array(SYSCTL_HANDLER_ARGS) 5589{ 5590 int rc, *i, space = 0; 5591 struct sbuf sb; 5592 5593 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5594 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5595 if (space) 5596 sbuf_printf(&sb, " "); 5597 sbuf_printf(&sb, "%d", *i); 5598 space = 1; 5599 } 5600 sbuf_finish(&sb); 5601 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5602 sbuf_delete(&sb); 5603 return (rc); 5604} 5605 5606static int 5607sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5608{ 5609 int rc; 5610 struct sbuf *sb; 5611 5612 rc = sysctl_wire_old_buffer(req, 0); 5613 if (rc != 0) 5614 return(rc); 5615 5616 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5617 if (sb == NULL) 5618 return (ENOMEM); 5619 5620 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5621 rc = sbuf_finish(sb); 5622 sbuf_delete(sb); 5623 5624 return (rc); 5625} 5626 5627static int 5628sysctl_btphy(SYSCTL_HANDLER_ARGS) 5629{ 5630 struct port_info *pi = arg1; 5631 int op = arg2; 5632 struct adapter *sc = pi->adapter; 5633 u_int v; 5634 int rc; 5635 5636 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); 5637 if (rc) 5638 return (rc); 5639 /* XXX: magic numbers */ 5640 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5641 &v); 5642 end_synchronized_op(sc, 0); 5643 if (rc) 5644 return (rc); 5645 if (op == 0) 5646 v /= 256; 5647 5648 rc = sysctl_handle_int(oidp, &v, 0, req); 5649 return (rc); 5650} 5651 5652static int 5653sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5654{ 5655 struct vi_info *vi = arg1; 5656 int rc, val; 5657 5658 val = vi->rsrv_noflowq; 5659 rc = sysctl_handle_int(oidp, &val, 0, req); 5660 if (rc != 0 || req->newptr == NULL) 5661 return (rc); 5662 5663 if ((val >= 1) && (vi->ntxq > 1)) 5664 vi->rsrv_noflowq = 1; 5665 else 5666 vi->rsrv_noflowq = 0; 5667 5668 return (rc); 5669} 5670 5671static int 5672sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5673{ 5674 struct vi_info *vi = arg1; 5675 struct adapter *sc = vi->pi->adapter; 5676 int idx, rc, i; 5677 struct sge_rxq *rxq; 5678#ifdef TCP_OFFLOAD 5679 struct sge_ofld_rxq *ofld_rxq; 5680#endif 5681 uint8_t v; 5682 5683 idx = vi->tmr_idx; 5684 5685 rc = sysctl_handle_int(oidp, &idx, 0, req); 5686 if (rc != 0 || req->newptr == NULL) 5687 return (rc); 5688 5689 if (idx < 0 || idx >= SGE_NTIMERS) 5690 return (EINVAL); 5691 5692 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5693 "t4tmr"); 5694 if (rc) 5695 return (rc); 5696 5697 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); 5698 for_each_rxq(vi, i, rxq) { 5699#ifdef atomic_store_rel_8 5700 atomic_store_rel_8(&rxq->iq.intr_params, v); 5701#else 5702 rxq->iq.intr_params = v; 5703#endif 5704 } 5705#ifdef TCP_OFFLOAD 5706 for_each_ofld_rxq(vi, i, ofld_rxq) { 5707#ifdef atomic_store_rel_8 5708 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5709#else 5710 ofld_rxq->iq.intr_params = v; 5711#endif 5712 } 5713#endif 5714 vi->tmr_idx = idx; 5715 5716 end_synchronized_op(sc, LOCK_HELD); 5717 return (0); 5718} 5719 5720static int 5721sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5722{ 5723 struct vi_info *vi = arg1; 5724 struct adapter *sc = vi->pi->adapter; 5725 int idx, rc; 5726 5727 idx = vi->pktc_idx; 5728 5729 rc = sysctl_handle_int(oidp, &idx, 0, req); 5730 if (rc != 0 || req->newptr == NULL) 5731 return (rc); 5732 5733 if (idx < -1 || idx >= SGE_NCOUNTERS) 5734 return (EINVAL); 5735 5736 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5737 "t4pktc"); 5738 if (rc) 5739 return (rc); 5740 5741 if (vi->flags & VI_INIT_DONE) 5742 rc = EBUSY; /* cannot be changed once the queues are created */ 5743 else 5744 vi->pktc_idx = idx; 5745 5746 end_synchronized_op(sc, LOCK_HELD); 5747 return (rc); 5748} 5749 5750static int 5751sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5752{ 5753 struct vi_info *vi = arg1; 5754 struct adapter *sc = vi->pi->adapter; 5755 int qsize, rc; 5756 5757 qsize = vi->qsize_rxq; 5758 5759 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5760 if (rc != 0 || req->newptr == NULL) 5761 return (rc); 5762 5763 if (qsize < 128 || (qsize & 7)) 5764 return (EINVAL); 5765 5766 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5767 "t4rxqs"); 5768 if (rc) 5769 return (rc); 5770 5771 if (vi->flags & VI_INIT_DONE) 5772 rc = EBUSY; /* cannot be changed once the queues are created */ 5773 else 5774 vi->qsize_rxq = qsize; 5775 5776 end_synchronized_op(sc, LOCK_HELD); 5777 return (rc); 5778} 5779 5780static int 5781sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5782{ 5783 struct vi_info *vi = arg1; 5784 struct adapter *sc = vi->pi->adapter; 5785 int qsize, rc; 5786 5787 qsize = vi->qsize_txq; 5788 5789 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5790 if (rc != 0 || req->newptr == NULL) 5791 return (rc); 5792 5793 if (qsize < 128 || qsize > 65536) 5794 return (EINVAL); 5795 5796 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5797 "t4txqs"); 5798 if (rc) 5799 return (rc); 5800 5801 if (vi->flags & VI_INIT_DONE) 5802 rc = EBUSY; /* cannot be changed once the queues are created */ 5803 else 5804 vi->qsize_txq = qsize; 5805 5806 end_synchronized_op(sc, LOCK_HELD); 5807 return (rc); 5808} 5809 5810static int 5811sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5812{ 5813 struct port_info *pi = arg1; 5814 struct adapter *sc = pi->adapter; 5815 struct link_config *lc = &pi->link_cfg; 5816 int rc; 5817 5818 if (req->newptr == NULL) { 5819 struct sbuf *sb; 5820 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5821 5822 rc = sysctl_wire_old_buffer(req, 0); 5823 if (rc != 0) 5824 return(rc); 5825 5826 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5827 if (sb == NULL) 5828 return (ENOMEM); 5829 5830 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5831 rc = sbuf_finish(sb); 5832 sbuf_delete(sb); 5833 } else { 5834 char s[2]; 5835 int n; 5836 5837 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5838 s[1] = 0; 5839 5840 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5841 if (rc != 0) 5842 return(rc); 5843 5844 if (s[1] != 0) 5845 return (EINVAL); 5846 if (s[0] < '0' || s[0] > '9') 5847 return (EINVAL); /* not a number */ 5848 n = s[0] - '0'; 5849 if (n & ~(PAUSE_TX | PAUSE_RX)) 5850 return (EINVAL); /* some other bit is set too */ 5851 5852 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 5853 "t4PAUSE"); 5854 if (rc) 5855 return (rc); 5856 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5857 int link_ok = lc->link_ok; 5858 5859 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5860 lc->requested_fc |= n; 5861 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5862 lc->link_ok = link_ok; /* restore */ 5863 } 5864 end_synchronized_op(sc, 0); 5865 } 5866 5867 return (rc); 5868} 5869 5870static int 5871sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5872{ 5873 struct adapter *sc = arg1; 5874 int reg = arg2; 5875 uint64_t val; 5876 5877 val = t4_read_reg64(sc, reg); 5878 5879 return (sysctl_handle_64(oidp, &val, 0, req)); 5880} 5881 5882static int 5883sysctl_temperature(SYSCTL_HANDLER_ARGS) 5884{ 5885 struct adapter *sc = arg1; 5886 int rc, t; 5887 uint32_t param, val; 5888 5889 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5890 if (rc) 5891 return (rc); 5892 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5893 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5894 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5895 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5896 end_synchronized_op(sc, 0); 5897 if (rc) 5898 return (rc); 5899 5900 /* unknown is returned as 0 but we display -1 in that case */ 5901 t = val == 0 ? -1 : val; 5902 5903 rc = sysctl_handle_int(oidp, &t, 0, req); 5904 return (rc); 5905} 5906 5907#ifdef SBUF_DRAIN 5908static int 5909sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5910{ 5911 struct adapter *sc = arg1; 5912 struct sbuf *sb; 5913 int rc, i; 5914 uint16_t incr[NMTUS][NCCTRL_WIN]; 5915 static const char *dec_fac[] = { 5916 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5917 "0.9375" 5918 }; 5919 5920 rc = sysctl_wire_old_buffer(req, 0); 5921 if (rc != 0) 5922 return (rc); 5923 5924 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5925 if (sb == NULL) 5926 return (ENOMEM); 5927 5928 t4_read_cong_tbl(sc, incr); 5929 5930 for (i = 0; i < NCCTRL_WIN; ++i) { 5931 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5932 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5933 incr[5][i], incr[6][i], incr[7][i]); 5934 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5935 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5936 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5937 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5938 } 5939 5940 rc = sbuf_finish(sb); 5941 sbuf_delete(sb); 5942 5943 return (rc); 5944} 5945 5946static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5947 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5948 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5949 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5950}; 5951 5952static int 5953sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5954{ 5955 struct adapter *sc = arg1; 5956 struct sbuf *sb; 5957 int rc, i, n, qid = arg2; 5958 uint32_t *buf, *p; 5959 char *qtype; 5960 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5961 5962 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5963 ("%s: bad qid %d\n", __func__, qid)); 5964 5965 if (qid < CIM_NUM_IBQ) { 5966 /* inbound queue */ 5967 qtype = "IBQ"; 5968 n = 4 * CIM_IBQ_SIZE; 5969 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5970 rc = t4_read_cim_ibq(sc, qid, buf, n); 5971 } else { 5972 /* outbound queue */ 5973 qtype = "OBQ"; 5974 qid -= CIM_NUM_IBQ; 5975 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5976 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5977 rc = t4_read_cim_obq(sc, qid, buf, n); 5978 } 5979 5980 if (rc < 0) { 5981 rc = -rc; 5982 goto done; 5983 } 5984 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5985 5986 rc = sysctl_wire_old_buffer(req, 0); 5987 if (rc != 0) 5988 goto done; 5989 5990 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5991 if (sb == NULL) { 5992 rc = ENOMEM; 5993 goto done; 5994 } 5995 5996 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5997 for (i = 0, p = buf; i < n; i += 16, p += 4) 5998 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5999 p[2], p[3]); 6000 6001 rc = sbuf_finish(sb); 6002 sbuf_delete(sb); 6003done: 6004 free(buf, M_CXGBE); 6005 return (rc); 6006} 6007 6008static int 6009sysctl_cim_la(SYSCTL_HANDLER_ARGS) 6010{ 6011 struct adapter *sc = arg1; 6012 u_int cfg; 6013 struct sbuf *sb; 6014 uint32_t *buf, *p; 6015 int rc; 6016 6017 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 6018 if (rc != 0) 6019 return (rc); 6020 6021 rc = sysctl_wire_old_buffer(req, 0); 6022 if (rc != 0) 6023 return (rc); 6024 6025 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6026 if (sb == NULL) 6027 return (ENOMEM); 6028 6029 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 6030 M_ZERO | M_WAITOK); 6031 6032 rc = -t4_cim_read_la(sc, buf, NULL); 6033 if (rc != 0) 6034 goto done; 6035 6036 sbuf_printf(sb, "Status Data PC%s", 6037 cfg & F_UPDBGLACAPTPCONLY ? "" : 6038 " LS0Stat LS0Addr LS0Data"); 6039 6040 KASSERT((sc->params.cim_la_size & 7) == 0, 6041 ("%s: p will walk off the end of buf", __func__)); 6042 6043 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 6044 if (cfg & F_UPDBGLACAPTPCONLY) { 6045 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 6046 p[6], p[7]); 6047 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 6048 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 6049 p[4] & 0xff, p[5] >> 8); 6050 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 6051 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6052 p[1] & 0xf, p[2] >> 4); 6053 } else { 6054 sbuf_printf(sb, 6055 "\n %02x %x%07x %x%07x %08x %08x " 6056 "%08x%08x%08x%08x", 6057 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 6058 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 6059 p[6], p[7]); 6060 } 6061 } 6062 6063 rc = sbuf_finish(sb); 6064 sbuf_delete(sb); 6065done: 6066 free(buf, M_CXGBE); 6067 return (rc); 6068} 6069 6070static int 6071sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 6072{ 6073 struct adapter *sc = arg1; 6074 u_int i; 6075 struct sbuf *sb; 6076 uint32_t *buf, *p; 6077 int rc; 6078 6079 rc = sysctl_wire_old_buffer(req, 0); 6080 if (rc != 0) 6081 return (rc); 6082 6083 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6084 if (sb == NULL) 6085 return (ENOMEM); 6086 6087 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 6088 M_ZERO | M_WAITOK); 6089 6090 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 6091 p = buf; 6092 6093 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6094 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 6095 p[1], p[0]); 6096 } 6097 6098 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 6099 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 6100 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 6101 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 6102 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 6103 (p[1] >> 2) | ((p[2] & 3) << 30), 6104 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 6105 p[0] & 1); 6106 } 6107 6108 rc = sbuf_finish(sb); 6109 sbuf_delete(sb); 6110 free(buf, M_CXGBE); 6111 return (rc); 6112} 6113 6114static int 6115sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 6116{ 6117 struct adapter *sc = arg1; 6118 u_int i; 6119 struct sbuf *sb; 6120 uint32_t *buf, *p; 6121 int rc; 6122 6123 rc = sysctl_wire_old_buffer(req, 0); 6124 if (rc != 0) 6125 return (rc); 6126 6127 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6128 if (sb == NULL) 6129 return (ENOMEM); 6130 6131 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 6132 M_ZERO | M_WAITOK); 6133 6134 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 6135 p = buf; 6136 6137 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 6138 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 6139 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 6140 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 6141 p[4], p[3], p[2], p[1], p[0]); 6142 } 6143 6144 sbuf_printf(sb, "\n\nCntl ID Data"); 6145 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 6146 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 6147 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 6148 } 6149 6150 rc = sbuf_finish(sb); 6151 sbuf_delete(sb); 6152 free(buf, M_CXGBE); 6153 return (rc); 6154} 6155 6156static int 6157sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 6158{ 6159 struct adapter *sc = arg1; 6160 struct sbuf *sb; 6161 int rc, i; 6162 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6163 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 6164 uint16_t thres[CIM_NUM_IBQ]; 6165 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 6166 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 6167 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 6168 6169 if (is_t4(sc)) { 6170 cim_num_obq = CIM_NUM_OBQ; 6171 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 6172 obq_rdaddr = A_UP_OBQ_0_REALADDR; 6173 } else { 6174 cim_num_obq = CIM_NUM_OBQ_T5; 6175 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 6176 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 6177 } 6178 nq = CIM_NUM_IBQ + cim_num_obq; 6179 6180 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 6181 if (rc == 0) 6182 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 6183 if (rc != 0) 6184 return (rc); 6185 6186 t4_read_cimq_cfg(sc, base, size, thres); 6187 6188 rc = sysctl_wire_old_buffer(req, 0); 6189 if (rc != 0) 6190 return (rc); 6191 6192 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 6193 if (sb == NULL) 6194 return (ENOMEM); 6195 6196 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 6197 6198 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 6199 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 6200 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 6201 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6202 G_QUEREMFLITS(p[2]) * 16); 6203 for ( ; i < nq; i++, p += 4, wr += 2) 6204 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 6205 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 6206 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 6207 G_QUEREMFLITS(p[2]) * 16); 6208 6209 rc = sbuf_finish(sb); 6210 sbuf_delete(sb); 6211 6212 return (rc); 6213} 6214 6215static int 6216sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 6217{ 6218 struct adapter *sc = arg1; 6219 struct sbuf *sb; 6220 int rc; 6221 struct tp_cpl_stats stats; 6222 6223 rc = sysctl_wire_old_buffer(req, 0); 6224 if (rc != 0) 6225 return (rc); 6226 6227 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6228 if (sb == NULL) 6229 return (ENOMEM); 6230 6231 t4_tp_get_cpl_stats(sc, &stats); 6232 6233 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6234 "channel 3\n"); 6235 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 6236 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 6237 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 6238 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 6239 6240 rc = sbuf_finish(sb); 6241 sbuf_delete(sb); 6242 6243 return (rc); 6244} 6245 6246static int 6247sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 6248{ 6249 struct adapter *sc = arg1; 6250 struct sbuf *sb; 6251 int rc; 6252 struct tp_usm_stats stats; 6253 6254 rc = sysctl_wire_old_buffer(req, 0); 6255 if (rc != 0) 6256 return(rc); 6257 6258 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6259 if (sb == NULL) 6260 return (ENOMEM); 6261 6262 t4_get_usm_stats(sc, &stats); 6263 6264 sbuf_printf(sb, "Frames: %u\n", stats.frames); 6265 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 6266 sbuf_printf(sb, "Drops: %u", stats.drops); 6267 6268 rc = sbuf_finish(sb); 6269 sbuf_delete(sb); 6270 6271 return (rc); 6272} 6273 6274const char *devlog_level_strings[] = { 6275 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 6276 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 6277 [FW_DEVLOG_LEVEL_ERR] = "ERR", 6278 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 6279 [FW_DEVLOG_LEVEL_INFO] = "INFO", 6280 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 6281}; 6282 6283const char *devlog_facility_strings[] = { 6284 [FW_DEVLOG_FACILITY_CORE] = "CORE", 6285 [FW_DEVLOG_FACILITY_CF] = "CF", 6286 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 6287 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 6288 [FW_DEVLOG_FACILITY_RES] = "RES", 6289 [FW_DEVLOG_FACILITY_HW] = "HW", 6290 [FW_DEVLOG_FACILITY_FLR] = "FLR", 6291 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 6292 [FW_DEVLOG_FACILITY_PHY] = "PHY", 6293 [FW_DEVLOG_FACILITY_MAC] = "MAC", 6294 [FW_DEVLOG_FACILITY_PORT] = "PORT", 6295 [FW_DEVLOG_FACILITY_VI] = "VI", 6296 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 6297 [FW_DEVLOG_FACILITY_ACL] = "ACL", 6298 [FW_DEVLOG_FACILITY_TM] = "TM", 6299 [FW_DEVLOG_FACILITY_QFC] = "QFC", 6300 [FW_DEVLOG_FACILITY_DCB] = "DCB", 6301 [FW_DEVLOG_FACILITY_ETH] = "ETH", 6302 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 6303 [FW_DEVLOG_FACILITY_RI] = "RI", 6304 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 6305 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 6306 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 6307 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 6308}; 6309 6310static int 6311sysctl_devlog(SYSCTL_HANDLER_ARGS) 6312{ 6313 struct adapter *sc = arg1; 6314 struct devlog_params *dparams = &sc->params.devlog; 6315 struct fw_devlog_e *buf, *e; 6316 int i, j, rc, nentries, first = 0, m; 6317 struct sbuf *sb; 6318 uint64_t ftstamp = UINT64_MAX; 6319 6320 if (dparams->start == 0) { 6321 dparams->memtype = FW_MEMTYPE_EDC0; 6322 dparams->start = 0x84000; 6323 dparams->size = 32768; 6324 } 6325 6326 nentries = dparams->size / sizeof(struct fw_devlog_e); 6327 6328 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 6329 if (buf == NULL) 6330 return (ENOMEM); 6331 6332 m = fwmtype_to_hwmtype(dparams->memtype); 6333 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 6334 if (rc != 0) 6335 goto done; 6336 6337 for (i = 0; i < nentries; i++) { 6338 e = &buf[i]; 6339 6340 if (e->timestamp == 0) 6341 break; /* end */ 6342 6343 e->timestamp = be64toh(e->timestamp); 6344 e->seqno = be32toh(e->seqno); 6345 for (j = 0; j < 8; j++) 6346 e->params[j] = be32toh(e->params[j]); 6347 6348 if (e->timestamp < ftstamp) { 6349 ftstamp = e->timestamp; 6350 first = i; 6351 } 6352 } 6353 6354 if (buf[first].timestamp == 0) 6355 goto done; /* nothing in the log */ 6356 6357 rc = sysctl_wire_old_buffer(req, 0); 6358 if (rc != 0) 6359 goto done; 6360 6361 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6362 if (sb == NULL) { 6363 rc = ENOMEM; 6364 goto done; 6365 } 6366 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 6367 "Seq#", "Tstamp", "Level", "Facility", "Message"); 6368 6369 i = first; 6370 do { 6371 e = &buf[i]; 6372 if (e->timestamp == 0) 6373 break; /* end */ 6374 6375 sbuf_printf(sb, "%10d %15ju %8s %8s ", 6376 e->seqno, e->timestamp, 6377 (e->level < nitems(devlog_level_strings) ? 6378 devlog_level_strings[e->level] : "UNKNOWN"), 6379 (e->facility < nitems(devlog_facility_strings) ? 6380 devlog_facility_strings[e->facility] : "UNKNOWN")); 6381 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 6382 e->params[2], e->params[3], e->params[4], 6383 e->params[5], e->params[6], e->params[7]); 6384 6385 if (++i == nentries) 6386 i = 0; 6387 } while (i != first); 6388 6389 rc = sbuf_finish(sb); 6390 sbuf_delete(sb); 6391done: 6392 free(buf, M_CXGBE); 6393 return (rc); 6394} 6395 6396static int 6397sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 6398{ 6399 struct adapter *sc = arg1; 6400 struct sbuf *sb; 6401 int rc; 6402 struct tp_fcoe_stats stats[4]; 6403 6404 rc = sysctl_wire_old_buffer(req, 0); 6405 if (rc != 0) 6406 return (rc); 6407 6408 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6409 if (sb == NULL) 6410 return (ENOMEM); 6411 6412 t4_get_fcoe_stats(sc, 0, &stats[0]); 6413 t4_get_fcoe_stats(sc, 1, &stats[1]); 6414 t4_get_fcoe_stats(sc, 2, &stats[2]); 6415 t4_get_fcoe_stats(sc, 3, &stats[3]); 6416 6417 sbuf_printf(sb, " channel 0 channel 1 " 6418 "channel 2 channel 3\n"); 6419 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 6420 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 6421 stats[3].octetsDDP); 6422 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 6423 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 6424 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 6425 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 6426 stats[3].framesDrop); 6427 6428 rc = sbuf_finish(sb); 6429 sbuf_delete(sb); 6430 6431 return (rc); 6432} 6433 6434static int 6435sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 6436{ 6437 struct adapter *sc = arg1; 6438 struct sbuf *sb; 6439 int rc, i; 6440 unsigned int map, kbps, ipg, mode; 6441 unsigned int pace_tab[NTX_SCHED]; 6442 6443 rc = sysctl_wire_old_buffer(req, 0); 6444 if (rc != 0) 6445 return (rc); 6446 6447 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6448 if (sb == NULL) 6449 return (ENOMEM); 6450 6451 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 6452 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 6453 t4_read_pace_tbl(sc, pace_tab); 6454 6455 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 6456 "Class IPG (0.1 ns) Flow IPG (us)"); 6457 6458 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 6459 t4_get_tx_sched(sc, i, &kbps, &ipg); 6460 sbuf_printf(sb, "\n %u %-5s %u ", i, 6461 (mode & (1 << i)) ? "flow" : "class", map & 3); 6462 if (kbps) 6463 sbuf_printf(sb, "%9u ", kbps); 6464 else 6465 sbuf_printf(sb, " disabled "); 6466 6467 if (ipg) 6468 sbuf_printf(sb, "%13u ", ipg); 6469 else 6470 sbuf_printf(sb, " disabled "); 6471 6472 if (pace_tab[i]) 6473 sbuf_printf(sb, "%10u", pace_tab[i]); 6474 else 6475 sbuf_printf(sb, " disabled"); 6476 } 6477 6478 rc = sbuf_finish(sb); 6479 sbuf_delete(sb); 6480 6481 return (rc); 6482} 6483 6484static int 6485sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 6486{ 6487 struct adapter *sc = arg1; 6488 struct sbuf *sb; 6489 int rc, i, j; 6490 uint64_t *p0, *p1; 6491 struct lb_port_stats s[2]; 6492 static const char *stat_name[] = { 6493 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 6494 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 6495 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 6496 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 6497 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 6498 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 6499 "BG2FramesTrunc:", "BG3FramesTrunc:" 6500 }; 6501 6502 rc = sysctl_wire_old_buffer(req, 0); 6503 if (rc != 0) 6504 return (rc); 6505 6506 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6507 if (sb == NULL) 6508 return (ENOMEM); 6509 6510 memset(s, 0, sizeof(s)); 6511 6512 for (i = 0; i < 4; i += 2) { 6513 t4_get_lb_stats(sc, i, &s[0]); 6514 t4_get_lb_stats(sc, i + 1, &s[1]); 6515 6516 p0 = &s[0].octets; 6517 p1 = &s[1].octets; 6518 sbuf_printf(sb, "%s Loopback %u" 6519 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 6520 6521 for (j = 0; j < nitems(stat_name); j++) 6522 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 6523 *p0++, *p1++); 6524 } 6525 6526 rc = sbuf_finish(sb); 6527 sbuf_delete(sb); 6528 6529 return (rc); 6530} 6531 6532static int 6533sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6534{ 6535 int rc = 0; 6536 struct port_info *pi = arg1; 6537 struct sbuf *sb; 6538 static const char *linkdnreasons[] = { 6539 "non-specific", "remote fault", "autoneg failed", "reserved3", 6540 "PHY overheated", "unknown", "rx los", "reserved7" 6541 }; 6542 6543 rc = sysctl_wire_old_buffer(req, 0); 6544 if (rc != 0) 6545 return(rc); 6546 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6547 if (sb == NULL) 6548 return (ENOMEM); 6549 6550 if (pi->linkdnrc < 0) 6551 sbuf_printf(sb, "n/a"); 6552 else if (pi->linkdnrc < nitems(linkdnreasons)) 6553 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6554 else 6555 sbuf_printf(sb, "%d", pi->linkdnrc); 6556 6557 rc = sbuf_finish(sb); 6558 sbuf_delete(sb); 6559 6560 return (rc); 6561} 6562 6563struct mem_desc { 6564 unsigned int base; 6565 unsigned int limit; 6566 unsigned int idx; 6567}; 6568 6569static int 6570mem_desc_cmp(const void *a, const void *b) 6571{ 6572 return ((const struct mem_desc *)a)->base - 6573 ((const struct mem_desc *)b)->base; 6574} 6575 6576static void 6577mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6578 unsigned int to) 6579{ 6580 unsigned int size; 6581 6582 size = to - from + 1; 6583 if (size == 0) 6584 return; 6585 6586 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6587 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6588} 6589 6590static int 6591sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6592{ 6593 struct adapter *sc = arg1; 6594 struct sbuf *sb; 6595 int rc, i, n; 6596 uint32_t lo, hi, used, alloc; 6597 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6598 static const char *region[] = { 6599 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6600 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6601 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6602 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6603 "RQUDP region:", "PBL region:", "TXPBL region:", 6604 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6605 "On-chip queues:" 6606 }; 6607 struct mem_desc avail[4]; 6608 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6609 struct mem_desc *md = mem; 6610 6611 rc = sysctl_wire_old_buffer(req, 0); 6612 if (rc != 0) 6613 return (rc); 6614 6615 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6616 if (sb == NULL) 6617 return (ENOMEM); 6618 6619 for (i = 0; i < nitems(mem); i++) { 6620 mem[i].limit = 0; 6621 mem[i].idx = i; 6622 } 6623 6624 /* Find and sort the populated memory ranges */ 6625 i = 0; 6626 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6627 if (lo & F_EDRAM0_ENABLE) { 6628 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6629 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6630 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6631 avail[i].idx = 0; 6632 i++; 6633 } 6634 if (lo & F_EDRAM1_ENABLE) { 6635 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6636 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6637 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6638 avail[i].idx = 1; 6639 i++; 6640 } 6641 if (lo & F_EXT_MEM_ENABLE) { 6642 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6643 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6644 avail[i].limit = avail[i].base + 6645 (G_EXT_MEM_SIZE(hi) << 20); 6646 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6647 i++; 6648 } 6649 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6650 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6651 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6652 avail[i].limit = avail[i].base + 6653 (G_EXT_MEM1_SIZE(hi) << 20); 6654 avail[i].idx = 4; 6655 i++; 6656 } 6657 if (!i) /* no memory available */ 6658 return 0; 6659 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6660 6661 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6662 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6663 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6664 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6665 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6666 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6667 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6668 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6669 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6670 6671 /* the next few have explicit upper bounds */ 6672 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6673 md->limit = md->base - 1 + 6674 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6675 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6676 md++; 6677 6678 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6679 md->limit = md->base - 1 + 6680 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6681 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6682 md++; 6683 6684 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6685 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6686 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6687 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6688 } else { 6689 md->base = 0; 6690 md->idx = nitems(region); /* hide it */ 6691 } 6692 md++; 6693 6694#define ulp_region(reg) \ 6695 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6696 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6697 6698 ulp_region(RX_ISCSI); 6699 ulp_region(RX_TDDP); 6700 ulp_region(TX_TPT); 6701 ulp_region(RX_STAG); 6702 ulp_region(RX_RQ); 6703 ulp_region(RX_RQUDP); 6704 ulp_region(RX_PBL); 6705 ulp_region(TX_PBL); 6706#undef ulp_region 6707 6708 md->base = 0; 6709 md->idx = nitems(region); 6710 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6711 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6712 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6713 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6714 } 6715 md++; 6716 6717 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6718 md->limit = md->base + sc->tids.ntids - 1; 6719 md++; 6720 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6721 md->limit = md->base + sc->tids.ntids - 1; 6722 md++; 6723 6724 md->base = sc->vres.ocq.start; 6725 if (sc->vres.ocq.size) 6726 md->limit = md->base + sc->vres.ocq.size - 1; 6727 else 6728 md->idx = nitems(region); /* hide it */ 6729 md++; 6730 6731 /* add any address-space holes, there can be up to 3 */ 6732 for (n = 0; n < i - 1; n++) 6733 if (avail[n].limit < avail[n + 1].base) 6734 (md++)->base = avail[n].limit; 6735 if (avail[n].limit) 6736 (md++)->base = avail[n].limit; 6737 6738 n = md - mem; 6739 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6740 6741 for (lo = 0; lo < i; lo++) 6742 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6743 avail[lo].limit - 1); 6744 6745 sbuf_printf(sb, "\n"); 6746 for (i = 0; i < n; i++) { 6747 if (mem[i].idx >= nitems(region)) 6748 continue; /* skip holes */ 6749 if (!mem[i].limit) 6750 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6751 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6752 mem[i].limit); 6753 } 6754 6755 sbuf_printf(sb, "\n"); 6756 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6757 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6758 mem_region_show(sb, "uP RAM:", lo, hi); 6759 6760 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6761 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6762 mem_region_show(sb, "uP Extmem2:", lo, hi); 6763 6764 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6765 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6766 G_PMRXMAXPAGE(lo), 6767 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6768 (lo & F_PMRXNUMCHN) ? 2 : 1); 6769 6770 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6771 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6772 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6773 G_PMTXMAXPAGE(lo), 6774 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6775 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6776 sbuf_printf(sb, "%u p-structs\n", 6777 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6778 6779 for (i = 0; i < 4; i++) { 6780 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6781 if (is_t4(sc)) { 6782 used = G_USED(lo); 6783 alloc = G_ALLOC(lo); 6784 } else { 6785 used = G_T5_USED(lo); 6786 alloc = G_T5_ALLOC(lo); 6787 } 6788 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6789 i, used, alloc); 6790 } 6791 for (i = 0; i < 4; i++) { 6792 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6793 if (is_t4(sc)) { 6794 used = G_USED(lo); 6795 alloc = G_ALLOC(lo); 6796 } else { 6797 used = G_T5_USED(lo); 6798 alloc = G_T5_ALLOC(lo); 6799 } 6800 sbuf_printf(sb, 6801 "\nLoopback %d using %u pages out of %u allocated", 6802 i, used, alloc); 6803 } 6804 6805 rc = sbuf_finish(sb); 6806 sbuf_delete(sb); 6807 6808 return (rc); 6809} 6810 6811static inline void 6812tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6813{ 6814 *mask = x | y; 6815 y = htobe64(y); 6816 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6817} 6818 6819static int 6820sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6821{ 6822 struct adapter *sc = arg1; 6823 struct sbuf *sb; 6824 int rc, i, n; 6825 6826 rc = sysctl_wire_old_buffer(req, 0); 6827 if (rc != 0) 6828 return (rc); 6829 6830 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6831 if (sb == NULL) 6832 return (ENOMEM); 6833 6834 sbuf_printf(sb, 6835 "Idx Ethernet address Mask Vld Ports PF" 6836 " VF Replication P0 P1 P2 P3 ML"); 6837 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6838 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6839 for (i = 0; i < n; i++) { 6840 uint64_t tcamx, tcamy, mask; 6841 uint32_t cls_lo, cls_hi; 6842 uint8_t addr[ETHER_ADDR_LEN]; 6843 6844 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6845 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6846 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6847 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6848 6849 if (tcamx & tcamy) 6850 continue; 6851 6852 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6853 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6854 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6855 addr[3], addr[4], addr[5], (uintmax_t)mask, 6856 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6857 G_PORTMAP(cls_hi), G_PF(cls_lo), 6858 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6859 6860 if (cls_lo & F_REPLICATE) { 6861 struct fw_ldst_cmd ldst_cmd; 6862 6863 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6864 ldst_cmd.op_to_addrspace = 6865 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6866 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6867 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6868 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6869 ldst_cmd.u.mps.rplc.fid_idx = 6870 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6871 V_FW_LDST_CMD_IDX(i)); 6872 6873 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6874 "t4mps"); 6875 if (rc) 6876 break; 6877 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6878 sizeof(ldst_cmd), &ldst_cmd); 6879 end_synchronized_op(sc, 0); 6880 6881 if (rc != 0) { 6882 sbuf_printf(sb, 6883 " ------------ error %3u ------------", rc); 6884 rc = 0; 6885 } else { 6886 sbuf_printf(sb, " %08x %08x %08x %08x", 6887 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6888 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6889 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6890 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6891 } 6892 } else 6893 sbuf_printf(sb, "%36s", ""); 6894 6895 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6896 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6897 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6898 } 6899 6900 if (rc) 6901 (void) sbuf_finish(sb); 6902 else 6903 rc = sbuf_finish(sb); 6904 sbuf_delete(sb); 6905 6906 return (rc); 6907} 6908 6909static int 6910sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6911{ 6912 struct adapter *sc = arg1; 6913 struct sbuf *sb; 6914 int rc; 6915 uint16_t mtus[NMTUS]; 6916 6917 rc = sysctl_wire_old_buffer(req, 0); 6918 if (rc != 0) 6919 return (rc); 6920 6921 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6922 if (sb == NULL) 6923 return (ENOMEM); 6924 6925 t4_read_mtu_tbl(sc, mtus, NULL); 6926 6927 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6928 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6929 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6930 mtus[14], mtus[15]); 6931 6932 rc = sbuf_finish(sb); 6933 sbuf_delete(sb); 6934 6935 return (rc); 6936} 6937 6938static int 6939sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6940{ 6941 struct adapter *sc = arg1; 6942 struct sbuf *sb; 6943 int rc, i; 6944 uint32_t cnt[PM_NSTATS]; 6945 uint64_t cyc[PM_NSTATS]; 6946 static const char *rx_stats[] = { 6947 "Read:", "Write bypass:", "Write mem:", "Flush:" 6948 }; 6949 static const char *tx_stats[] = { 6950 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6951 }; 6952 6953 rc = sysctl_wire_old_buffer(req, 0); 6954 if (rc != 0) 6955 return (rc); 6956 6957 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6958 if (sb == NULL) 6959 return (ENOMEM); 6960 6961 t4_pmtx_get_stats(sc, cnt, cyc); 6962 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6963 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6964 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6965 cyc[i]); 6966 6967 t4_pmrx_get_stats(sc, cnt, cyc); 6968 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6969 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6970 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6971 cyc[i]); 6972 6973 rc = sbuf_finish(sb); 6974 sbuf_delete(sb); 6975 6976 return (rc); 6977} 6978 6979static int 6980sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6981{ 6982 struct adapter *sc = arg1; 6983 struct sbuf *sb; 6984 int rc; 6985 struct tp_rdma_stats stats; 6986 6987 rc = sysctl_wire_old_buffer(req, 0); 6988 if (rc != 0) 6989 return (rc); 6990 6991 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6992 if (sb == NULL) 6993 return (ENOMEM); 6994 6995 t4_tp_get_rdma_stats(sc, &stats); 6996 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6997 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6998 6999 rc = sbuf_finish(sb); 7000 sbuf_delete(sb); 7001 7002 return (rc); 7003} 7004 7005static int 7006sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 7007{ 7008 struct adapter *sc = arg1; 7009 struct sbuf *sb; 7010 int rc; 7011 struct tp_tcp_stats v4, v6; 7012 7013 rc = sysctl_wire_old_buffer(req, 0); 7014 if (rc != 0) 7015 return (rc); 7016 7017 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7018 if (sb == NULL) 7019 return (ENOMEM); 7020 7021 t4_tp_get_tcp_stats(sc, &v4, &v6); 7022 sbuf_printf(sb, 7023 " IP IPv6\n"); 7024 sbuf_printf(sb, "OutRsts: %20u %20u\n", 7025 v4.tcpOutRsts, v6.tcpOutRsts); 7026 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 7027 v4.tcpInSegs, v6.tcpInSegs); 7028 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 7029 v4.tcpOutSegs, v6.tcpOutSegs); 7030 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 7031 v4.tcpRetransSegs, v6.tcpRetransSegs); 7032 7033 rc = sbuf_finish(sb); 7034 sbuf_delete(sb); 7035 7036 return (rc); 7037} 7038 7039static int 7040sysctl_tids(SYSCTL_HANDLER_ARGS) 7041{ 7042 struct adapter *sc = arg1; 7043 struct sbuf *sb; 7044 int rc; 7045 struct tid_info *t = &sc->tids; 7046 7047 rc = sysctl_wire_old_buffer(req, 0); 7048 if (rc != 0) 7049 return (rc); 7050 7051 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7052 if (sb == NULL) 7053 return (ENOMEM); 7054 7055 if (t->natids) { 7056 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 7057 t->atids_in_use); 7058 } 7059 7060 if (t->ntids) { 7061 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 7062 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 7063 7064 if (b) { 7065 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 7066 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7067 t->ntids - 1); 7068 } else { 7069 sbuf_printf(sb, "TID range: %u-%u", 7070 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 7071 t->ntids - 1); 7072 } 7073 } else 7074 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 7075 sbuf_printf(sb, ", in use: %u\n", 7076 atomic_load_acq_int(&t->tids_in_use)); 7077 } 7078 7079 if (t->nstids) { 7080 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 7081 t->stid_base + t->nstids - 1, t->stids_in_use); 7082 } 7083 7084 if (t->nftids) { 7085 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 7086 t->ftid_base + t->nftids - 1); 7087 } 7088 7089 if (t->netids) { 7090 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 7091 t->etid_base + t->netids - 1); 7092 } 7093 7094 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 7095 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 7096 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 7097 7098 rc = sbuf_finish(sb); 7099 sbuf_delete(sb); 7100 7101 return (rc); 7102} 7103 7104static int 7105sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 7106{ 7107 struct adapter *sc = arg1; 7108 struct sbuf *sb; 7109 int rc; 7110 struct tp_err_stats stats; 7111 7112 rc = sysctl_wire_old_buffer(req, 0); 7113 if (rc != 0) 7114 return (rc); 7115 7116 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7117 if (sb == NULL) 7118 return (ENOMEM); 7119 7120 t4_tp_get_err_stats(sc, &stats); 7121 7122 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7123 "channel 3\n"); 7124 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 7125 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 7126 stats.macInErrs[3]); 7127 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 7128 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 7129 stats.hdrInErrs[3]); 7130 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 7131 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 7132 stats.tcpInErrs[3]); 7133 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 7134 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 7135 stats.tcp6InErrs[3]); 7136 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 7137 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 7138 stats.tnlCongDrops[3]); 7139 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 7140 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 7141 stats.tnlTxDrops[3]); 7142 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 7143 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 7144 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 7145 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 7146 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 7147 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 7148 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 7149 stats.ofldNoNeigh, stats.ofldCongDefer); 7150 7151 rc = sbuf_finish(sb); 7152 sbuf_delete(sb); 7153 7154 return (rc); 7155} 7156 7157struct field_desc { 7158 const char *name; 7159 u_int start; 7160 u_int width; 7161}; 7162 7163static void 7164field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 7165{ 7166 char buf[32]; 7167 int line_size = 0; 7168 7169 while (f->name) { 7170 uint64_t mask = (1ULL << f->width) - 1; 7171 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 7172 ((uintmax_t)v >> f->start) & mask); 7173 7174 if (line_size + len >= 79) { 7175 line_size = 8; 7176 sbuf_printf(sb, "\n "); 7177 } 7178 sbuf_printf(sb, "%s ", buf); 7179 line_size += len + 1; 7180 f++; 7181 } 7182 sbuf_printf(sb, "\n"); 7183} 7184 7185static struct field_desc tp_la0[] = { 7186 { "RcfOpCodeOut", 60, 4 }, 7187 { "State", 56, 4 }, 7188 { "WcfState", 52, 4 }, 7189 { "RcfOpcSrcOut", 50, 2 }, 7190 { "CRxError", 49, 1 }, 7191 { "ERxError", 48, 1 }, 7192 { "SanityFailed", 47, 1 }, 7193 { "SpuriousMsg", 46, 1 }, 7194 { "FlushInputMsg", 45, 1 }, 7195 { "FlushInputCpl", 44, 1 }, 7196 { "RssUpBit", 43, 1 }, 7197 { "RssFilterHit", 42, 1 }, 7198 { "Tid", 32, 10 }, 7199 { "InitTcb", 31, 1 }, 7200 { "LineNumber", 24, 7 }, 7201 { "Emsg", 23, 1 }, 7202 { "EdataOut", 22, 1 }, 7203 { "Cmsg", 21, 1 }, 7204 { "CdataOut", 20, 1 }, 7205 { "EreadPdu", 19, 1 }, 7206 { "CreadPdu", 18, 1 }, 7207 { "TunnelPkt", 17, 1 }, 7208 { "RcfPeerFin", 16, 1 }, 7209 { "RcfReasonOut", 12, 4 }, 7210 { "TxCchannel", 10, 2 }, 7211 { "RcfTxChannel", 8, 2 }, 7212 { "RxEchannel", 6, 2 }, 7213 { "RcfRxChannel", 5, 1 }, 7214 { "RcfDataOutSrdy", 4, 1 }, 7215 { "RxDvld", 3, 1 }, 7216 { "RxOoDvld", 2, 1 }, 7217 { "RxCongestion", 1, 1 }, 7218 { "TxCongestion", 0, 1 }, 7219 { NULL } 7220}; 7221 7222static struct field_desc tp_la1[] = { 7223 { "CplCmdIn", 56, 8 }, 7224 { "CplCmdOut", 48, 8 }, 7225 { "ESynOut", 47, 1 }, 7226 { "EAckOut", 46, 1 }, 7227 { "EFinOut", 45, 1 }, 7228 { "ERstOut", 44, 1 }, 7229 { "SynIn", 43, 1 }, 7230 { "AckIn", 42, 1 }, 7231 { "FinIn", 41, 1 }, 7232 { "RstIn", 40, 1 }, 7233 { "DataIn", 39, 1 }, 7234 { "DataInVld", 38, 1 }, 7235 { "PadIn", 37, 1 }, 7236 { "RxBufEmpty", 36, 1 }, 7237 { "RxDdp", 35, 1 }, 7238 { "RxFbCongestion", 34, 1 }, 7239 { "TxFbCongestion", 33, 1 }, 7240 { "TxPktSumSrdy", 32, 1 }, 7241 { "RcfUlpType", 28, 4 }, 7242 { "Eread", 27, 1 }, 7243 { "Ebypass", 26, 1 }, 7244 { "Esave", 25, 1 }, 7245 { "Static0", 24, 1 }, 7246 { "Cread", 23, 1 }, 7247 { "Cbypass", 22, 1 }, 7248 { "Csave", 21, 1 }, 7249 { "CPktOut", 20, 1 }, 7250 { "RxPagePoolFull", 18, 2 }, 7251 { "RxLpbkPkt", 17, 1 }, 7252 { "TxLpbkPkt", 16, 1 }, 7253 { "RxVfValid", 15, 1 }, 7254 { "SynLearned", 14, 1 }, 7255 { "SetDelEntry", 13, 1 }, 7256 { "SetInvEntry", 12, 1 }, 7257 { "CpcmdDvld", 11, 1 }, 7258 { "CpcmdSave", 10, 1 }, 7259 { "RxPstructsFull", 8, 2 }, 7260 { "EpcmdDvld", 7, 1 }, 7261 { "EpcmdFlush", 6, 1 }, 7262 { "EpcmdTrimPrefix", 5, 1 }, 7263 { "EpcmdTrimPostfix", 4, 1 }, 7264 { "ERssIp4Pkt", 3, 1 }, 7265 { "ERssIp6Pkt", 2, 1 }, 7266 { "ERssTcpUdpPkt", 1, 1 }, 7267 { "ERssFceFipPkt", 0, 1 }, 7268 { NULL } 7269}; 7270 7271static struct field_desc tp_la2[] = { 7272 { "CplCmdIn", 56, 8 }, 7273 { "MpsVfVld", 55, 1 }, 7274 { "MpsPf", 52, 3 }, 7275 { "MpsVf", 44, 8 }, 7276 { "SynIn", 43, 1 }, 7277 { "AckIn", 42, 1 }, 7278 { "FinIn", 41, 1 }, 7279 { "RstIn", 40, 1 }, 7280 { "DataIn", 39, 1 }, 7281 { "DataInVld", 38, 1 }, 7282 { "PadIn", 37, 1 }, 7283 { "RxBufEmpty", 36, 1 }, 7284 { "RxDdp", 35, 1 }, 7285 { "RxFbCongestion", 34, 1 }, 7286 { "TxFbCongestion", 33, 1 }, 7287 { "TxPktSumSrdy", 32, 1 }, 7288 { "RcfUlpType", 28, 4 }, 7289 { "Eread", 27, 1 }, 7290 { "Ebypass", 26, 1 }, 7291 { "Esave", 25, 1 }, 7292 { "Static0", 24, 1 }, 7293 { "Cread", 23, 1 }, 7294 { "Cbypass", 22, 1 }, 7295 { "Csave", 21, 1 }, 7296 { "CPktOut", 20, 1 }, 7297 { "RxPagePoolFull", 18, 2 }, 7298 { "RxLpbkPkt", 17, 1 }, 7299 { "TxLpbkPkt", 16, 1 }, 7300 { "RxVfValid", 15, 1 }, 7301 { "SynLearned", 14, 1 }, 7302 { "SetDelEntry", 13, 1 }, 7303 { "SetInvEntry", 12, 1 }, 7304 { "CpcmdDvld", 11, 1 }, 7305 { "CpcmdSave", 10, 1 }, 7306 { "RxPstructsFull", 8, 2 }, 7307 { "EpcmdDvld", 7, 1 }, 7308 { "EpcmdFlush", 6, 1 }, 7309 { "EpcmdTrimPrefix", 5, 1 }, 7310 { "EpcmdTrimPostfix", 4, 1 }, 7311 { "ERssIp4Pkt", 3, 1 }, 7312 { "ERssIp6Pkt", 2, 1 }, 7313 { "ERssTcpUdpPkt", 1, 1 }, 7314 { "ERssFceFipPkt", 0, 1 }, 7315 { NULL } 7316}; 7317 7318static void 7319tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 7320{ 7321 7322 field_desc_show(sb, *p, tp_la0); 7323} 7324 7325static void 7326tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 7327{ 7328 7329 if (idx) 7330 sbuf_printf(sb, "\n"); 7331 field_desc_show(sb, p[0], tp_la0); 7332 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7333 field_desc_show(sb, p[1], tp_la0); 7334} 7335 7336static void 7337tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 7338{ 7339 7340 if (idx) 7341 sbuf_printf(sb, "\n"); 7342 field_desc_show(sb, p[0], tp_la0); 7343 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7344 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 7345} 7346 7347static int 7348sysctl_tp_la(SYSCTL_HANDLER_ARGS) 7349{ 7350 struct adapter *sc = arg1; 7351 struct sbuf *sb; 7352 uint64_t *buf, *p; 7353 int rc; 7354 u_int i, inc; 7355 void (*show_func)(struct sbuf *, uint64_t *, int); 7356 7357 rc = sysctl_wire_old_buffer(req, 0); 7358 if (rc != 0) 7359 return (rc); 7360 7361 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7362 if (sb == NULL) 7363 return (ENOMEM); 7364 7365 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 7366 7367 t4_tp_read_la(sc, buf, NULL); 7368 p = buf; 7369 7370 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 7371 case 2: 7372 inc = 2; 7373 show_func = tp_la_show2; 7374 break; 7375 case 3: 7376 inc = 2; 7377 show_func = tp_la_show3; 7378 break; 7379 default: 7380 inc = 1; 7381 show_func = tp_la_show; 7382 } 7383 7384 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 7385 (*show_func)(sb, p, i); 7386 7387 rc = sbuf_finish(sb); 7388 sbuf_delete(sb); 7389 free(buf, M_CXGBE); 7390 return (rc); 7391} 7392 7393static int 7394sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 7395{ 7396 struct adapter *sc = arg1; 7397 struct sbuf *sb; 7398 int rc; 7399 u64 nrate[NCHAN], orate[NCHAN]; 7400 7401 rc = sysctl_wire_old_buffer(req, 0); 7402 if (rc != 0) 7403 return (rc); 7404 7405 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7406 if (sb == NULL) 7407 return (ENOMEM); 7408 7409 t4_get_chan_txrate(sc, nrate, orate); 7410 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 7411 "channel 3\n"); 7412 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 7413 nrate[0], nrate[1], nrate[2], nrate[3]); 7414 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 7415 orate[0], orate[1], orate[2], orate[3]); 7416 7417 rc = sbuf_finish(sb); 7418 sbuf_delete(sb); 7419 7420 return (rc); 7421} 7422 7423static int 7424sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 7425{ 7426 struct adapter *sc = arg1; 7427 struct sbuf *sb; 7428 uint32_t *buf, *p; 7429 int rc, i; 7430 7431 rc = sysctl_wire_old_buffer(req, 0); 7432 if (rc != 0) 7433 return (rc); 7434 7435 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7436 if (sb == NULL) 7437 return (ENOMEM); 7438 7439 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 7440 M_ZERO | M_WAITOK); 7441 7442 t4_ulprx_read_la(sc, buf); 7443 p = buf; 7444 7445 sbuf_printf(sb, " Pcmd Type Message" 7446 " Data"); 7447 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 7448 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 7449 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 7450 } 7451 7452 rc = sbuf_finish(sb); 7453 sbuf_delete(sb); 7454 free(buf, M_CXGBE); 7455 return (rc); 7456} 7457 7458static int 7459sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 7460{ 7461 struct adapter *sc = arg1; 7462 struct sbuf *sb; 7463 int rc, v; 7464 7465 rc = sysctl_wire_old_buffer(req, 0); 7466 if (rc != 0) 7467 return (rc); 7468 7469 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7470 if (sb == NULL) 7471 return (ENOMEM); 7472 7473 v = t4_read_reg(sc, A_SGE_STAT_CFG); 7474 if (G_STATSOURCE_T5(v) == 7) { 7475 if (G_STATMODE(v) == 0) { 7476 sbuf_printf(sb, "total %d, incomplete %d", 7477 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7478 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7479 } else if (G_STATMODE(v) == 1) { 7480 sbuf_printf(sb, "total %d, data overflow %d", 7481 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7482 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7483 } 7484 } 7485 rc = sbuf_finish(sb); 7486 sbuf_delete(sb); 7487 7488 return (rc); 7489} 7490#endif 7491 7492static uint32_t 7493fconf_to_mode(uint32_t fconf) 7494{ 7495 uint32_t mode; 7496 7497 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7498 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7499 7500 if (fconf & F_FRAGMENTATION) 7501 mode |= T4_FILTER_IP_FRAGMENT; 7502 7503 if (fconf & F_MPSHITTYPE) 7504 mode |= T4_FILTER_MPS_HIT_TYPE; 7505 7506 if (fconf & F_MACMATCH) 7507 mode |= T4_FILTER_MAC_IDX; 7508 7509 if (fconf & F_ETHERTYPE) 7510 mode |= T4_FILTER_ETH_TYPE; 7511 7512 if (fconf & F_PROTOCOL) 7513 mode |= T4_FILTER_IP_PROTO; 7514 7515 if (fconf & F_TOS) 7516 mode |= T4_FILTER_IP_TOS; 7517 7518 if (fconf & F_VLAN) 7519 mode |= T4_FILTER_VLAN; 7520 7521 if (fconf & F_VNIC_ID) 7522 mode |= T4_FILTER_VNIC; 7523 7524 if (fconf & F_PORT) 7525 mode |= T4_FILTER_PORT; 7526 7527 if (fconf & F_FCOE) 7528 mode |= T4_FILTER_FCoE; 7529 7530 return (mode); 7531} 7532 7533static uint32_t 7534mode_to_fconf(uint32_t mode) 7535{ 7536 uint32_t fconf = 0; 7537 7538 if (mode & T4_FILTER_IP_FRAGMENT) 7539 fconf |= F_FRAGMENTATION; 7540 7541 if (mode & T4_FILTER_MPS_HIT_TYPE) 7542 fconf |= F_MPSHITTYPE; 7543 7544 if (mode & T4_FILTER_MAC_IDX) 7545 fconf |= F_MACMATCH; 7546 7547 if (mode & T4_FILTER_ETH_TYPE) 7548 fconf |= F_ETHERTYPE; 7549 7550 if (mode & T4_FILTER_IP_PROTO) 7551 fconf |= F_PROTOCOL; 7552 7553 if (mode & T4_FILTER_IP_TOS) 7554 fconf |= F_TOS; 7555 7556 if (mode & T4_FILTER_VLAN) 7557 fconf |= F_VLAN; 7558 7559 if (mode & T4_FILTER_VNIC) 7560 fconf |= F_VNIC_ID; 7561 7562 if (mode & T4_FILTER_PORT) 7563 fconf |= F_PORT; 7564 7565 if (mode & T4_FILTER_FCoE) 7566 fconf |= F_FCOE; 7567 7568 return (fconf); 7569} 7570 7571static uint32_t 7572fspec_to_fconf(struct t4_filter_specification *fs) 7573{ 7574 uint32_t fconf = 0; 7575 7576 if (fs->val.frag || fs->mask.frag) 7577 fconf |= F_FRAGMENTATION; 7578 7579 if (fs->val.matchtype || fs->mask.matchtype) 7580 fconf |= F_MPSHITTYPE; 7581 7582 if (fs->val.macidx || fs->mask.macidx) 7583 fconf |= F_MACMATCH; 7584 7585 if (fs->val.ethtype || fs->mask.ethtype) 7586 fconf |= F_ETHERTYPE; 7587 7588 if (fs->val.proto || fs->mask.proto) 7589 fconf |= F_PROTOCOL; 7590 7591 if (fs->val.tos || fs->mask.tos) 7592 fconf |= F_TOS; 7593 7594 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7595 fconf |= F_VLAN; 7596 7597 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7598 fconf |= F_VNIC_ID; 7599 7600 if (fs->val.iport || fs->mask.iport) 7601 fconf |= F_PORT; 7602 7603 if (fs->val.fcoe || fs->mask.fcoe) 7604 fconf |= F_FCOE; 7605 7606 return (fconf); 7607} 7608 7609static int 7610get_filter_mode(struct adapter *sc, uint32_t *mode) 7611{ 7612 int rc; 7613 uint32_t fconf; 7614 7615 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7616 "t4getfm"); 7617 if (rc) 7618 return (rc); 7619 7620 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7621 A_TP_VLAN_PRI_MAP); 7622 7623 if (sc->params.tp.vlan_pri_map != fconf) { 7624 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7625 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7626 fconf); 7627 } 7628 7629 *mode = fconf_to_mode(fconf); 7630 7631 end_synchronized_op(sc, LOCK_HELD); 7632 return (0); 7633} 7634 7635static int 7636set_filter_mode(struct adapter *sc, uint32_t mode) 7637{ 7638 uint32_t fconf; 7639 int rc; 7640 7641 fconf = mode_to_fconf(mode); 7642 7643 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7644 "t4setfm"); 7645 if (rc) 7646 return (rc); 7647 7648 if (sc->tids.ftids_in_use > 0) { 7649 rc = EBUSY; 7650 goto done; 7651 } 7652 7653#ifdef TCP_OFFLOAD 7654 if (uld_active(sc, ULD_TOM)) { 7655 rc = EBUSY; 7656 goto done; 7657 } 7658#endif 7659 7660 rc = -t4_set_filter_mode(sc, fconf); 7661done: 7662 end_synchronized_op(sc, LOCK_HELD); 7663 return (rc); 7664} 7665 7666static inline uint64_t 7667get_filter_hits(struct adapter *sc, uint32_t fid) 7668{ 7669 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7670 uint64_t hits; 7671 7672 memwin_info(sc, 0, &mw_base, NULL); 7673 off = position_memwin(sc, 0, 7674 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7675 if (is_t4(sc)) { 7676 hits = t4_read_reg64(sc, mw_base + off + 16); 7677 hits = be64toh(hits); 7678 } else { 7679 hits = t4_read_reg(sc, mw_base + off + 24); 7680 hits = be32toh(hits); 7681 } 7682 7683 return (hits); 7684} 7685 7686static int 7687get_filter(struct adapter *sc, struct t4_filter *t) 7688{ 7689 int i, rc, nfilters = sc->tids.nftids; 7690 struct filter_entry *f; 7691 7692 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7693 "t4getf"); 7694 if (rc) 7695 return (rc); 7696 7697 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7698 t->idx >= nfilters) { 7699 t->idx = 0xffffffff; 7700 goto done; 7701 } 7702 7703 f = &sc->tids.ftid_tab[t->idx]; 7704 for (i = t->idx; i < nfilters; i++, f++) { 7705 if (f->valid) { 7706 t->idx = i; 7707 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7708 t->smtidx = f->smtidx; 7709 if (f->fs.hitcnts) 7710 t->hits = get_filter_hits(sc, t->idx); 7711 else 7712 t->hits = UINT64_MAX; 7713 t->fs = f->fs; 7714 7715 goto done; 7716 } 7717 } 7718 7719 t->idx = 0xffffffff; 7720done: 7721 end_synchronized_op(sc, LOCK_HELD); 7722 return (0); 7723} 7724 7725static int 7726set_filter(struct adapter *sc, struct t4_filter *t) 7727{ 7728 unsigned int nfilters, nports; 7729 struct filter_entry *f; 7730 int i, rc; 7731 7732 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7733 if (rc) 7734 return (rc); 7735 7736 nfilters = sc->tids.nftids; 7737 nports = sc->params.nports; 7738 7739 if (nfilters == 0) { 7740 rc = ENOTSUP; 7741 goto done; 7742 } 7743 7744 if (!(sc->flags & FULL_INIT_DONE)) { 7745 rc = EAGAIN; 7746 goto done; 7747 } 7748 7749 if (t->idx >= nfilters) { 7750 rc = EINVAL; 7751 goto done; 7752 } 7753 7754 /* Validate against the global filter mode */ 7755 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7756 sc->params.tp.vlan_pri_map) { 7757 rc = E2BIG; 7758 goto done; 7759 } 7760 7761 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7762 rc = EINVAL; 7763 goto done; 7764 } 7765 7766 if (t->fs.val.iport >= nports) { 7767 rc = EINVAL; 7768 goto done; 7769 } 7770 7771 /* Can't specify an iq if not steering to it */ 7772 if (!t->fs.dirsteer && t->fs.iq) { 7773 rc = EINVAL; 7774 goto done; 7775 } 7776 7777 /* IPv6 filter idx must be 4 aligned */ 7778 if (t->fs.type == 1 && 7779 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7780 rc = EINVAL; 7781 goto done; 7782 } 7783 7784 if (sc->tids.ftid_tab == NULL) { 7785 KASSERT(sc->tids.ftids_in_use == 0, 7786 ("%s: no memory allocated but filters_in_use > 0", 7787 __func__)); 7788 7789 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7790 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7791 if (sc->tids.ftid_tab == NULL) { 7792 rc = ENOMEM; 7793 goto done; 7794 } 7795 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7796 } 7797 7798 for (i = 0; i < 4; i++) { 7799 f = &sc->tids.ftid_tab[t->idx + i]; 7800 7801 if (f->pending || f->valid) { 7802 rc = EBUSY; 7803 goto done; 7804 } 7805 if (f->locked) { 7806 rc = EPERM; 7807 goto done; 7808 } 7809 7810 if (t->fs.type == 0) 7811 break; 7812 } 7813 7814 f = &sc->tids.ftid_tab[t->idx]; 7815 f->fs = t->fs; 7816 7817 rc = set_filter_wr(sc, t->idx); 7818done: 7819 end_synchronized_op(sc, 0); 7820 7821 if (rc == 0) { 7822 mtx_lock(&sc->tids.ftid_lock); 7823 for (;;) { 7824 if (f->pending == 0) { 7825 rc = f->valid ? 0 : EIO; 7826 break; 7827 } 7828 7829 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7830 PCATCH, "t4setfw", 0)) { 7831 rc = EINPROGRESS; 7832 break; 7833 } 7834 } 7835 mtx_unlock(&sc->tids.ftid_lock); 7836 } 7837 return (rc); 7838} 7839 7840static int 7841del_filter(struct adapter *sc, struct t4_filter *t) 7842{ 7843 unsigned int nfilters; 7844 struct filter_entry *f; 7845 int rc; 7846 7847 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7848 if (rc) 7849 return (rc); 7850 7851 nfilters = sc->tids.nftids; 7852 7853 if (nfilters == 0) { 7854 rc = ENOTSUP; 7855 goto done; 7856 } 7857 7858 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7859 t->idx >= nfilters) { 7860 rc = EINVAL; 7861 goto done; 7862 } 7863 7864 if (!(sc->flags & FULL_INIT_DONE)) { 7865 rc = EAGAIN; 7866 goto done; 7867 } 7868 7869 f = &sc->tids.ftid_tab[t->idx]; 7870 7871 if (f->pending) { 7872 rc = EBUSY; 7873 goto done; 7874 } 7875 if (f->locked) { 7876 rc = EPERM; 7877 goto done; 7878 } 7879 7880 if (f->valid) { 7881 t->fs = f->fs; /* extra info for the caller */ 7882 rc = del_filter_wr(sc, t->idx); 7883 } 7884 7885done: 7886 end_synchronized_op(sc, 0); 7887 7888 if (rc == 0) { 7889 mtx_lock(&sc->tids.ftid_lock); 7890 for (;;) { 7891 if (f->pending == 0) { 7892 rc = f->valid ? EIO : 0; 7893 break; 7894 } 7895 7896 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7897 PCATCH, "t4delfw", 0)) { 7898 rc = EINPROGRESS; 7899 break; 7900 } 7901 } 7902 mtx_unlock(&sc->tids.ftid_lock); 7903 } 7904 7905 return (rc); 7906} 7907 7908static void 7909clear_filter(struct filter_entry *f) 7910{ 7911 if (f->l2t) 7912 t4_l2t_release(f->l2t); 7913 7914 bzero(f, sizeof (*f)); 7915} 7916 7917static int 7918set_filter_wr(struct adapter *sc, int fidx) 7919{ 7920 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7921 struct fw_filter_wr *fwr; 7922 unsigned int ftid; 7923 struct wrq_cookie cookie; 7924 7925 ASSERT_SYNCHRONIZED_OP(sc); 7926 7927 if (f->fs.newdmac || f->fs.newvlan) { 7928 /* This filter needs an L2T entry; allocate one. */ 7929 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7930 if (f->l2t == NULL) 7931 return (EAGAIN); 7932 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7933 f->fs.dmac)) { 7934 t4_l2t_release(f->l2t); 7935 f->l2t = NULL; 7936 return (ENOMEM); 7937 } 7938 } 7939 7940 ftid = sc->tids.ftid_base + fidx; 7941 7942 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7943 if (fwr == NULL) 7944 return (ENOMEM); 7945 bzero(fwr, sizeof(*fwr)); 7946 7947 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7948 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7949 fwr->tid_to_iq = 7950 htobe32(V_FW_FILTER_WR_TID(ftid) | 7951 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7952 V_FW_FILTER_WR_NOREPLY(0) | 7953 V_FW_FILTER_WR_IQ(f->fs.iq)); 7954 fwr->del_filter_to_l2tix = 7955 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7956 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7957 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7958 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7959 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7960 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7961 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7962 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7963 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7964 f->fs.newvlan == VLAN_REWRITE) | 7965 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7966 f->fs.newvlan == VLAN_REWRITE) | 7967 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7968 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7969 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7970 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7971 fwr->ethtype = htobe16(f->fs.val.ethtype); 7972 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7973 fwr->frag_to_ovlan_vldm = 7974 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7975 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7976 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7977 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7978 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7979 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7980 fwr->smac_sel = 0; 7981 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7982 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7983 fwr->maci_to_matchtypem = 7984 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7985 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7986 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7987 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7988 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7989 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7990 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7991 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7992 fwr->ptcl = f->fs.val.proto; 7993 fwr->ptclm = f->fs.mask.proto; 7994 fwr->ttyp = f->fs.val.tos; 7995 fwr->ttypm = f->fs.mask.tos; 7996 fwr->ivlan = htobe16(f->fs.val.vlan); 7997 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7998 fwr->ovlan = htobe16(f->fs.val.vnic); 7999 fwr->ovlanm = htobe16(f->fs.mask.vnic); 8000 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 8001 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 8002 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 8003 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 8004 fwr->lp = htobe16(f->fs.val.dport); 8005 fwr->lpm = htobe16(f->fs.mask.dport); 8006 fwr->fp = htobe16(f->fs.val.sport); 8007 fwr->fpm = htobe16(f->fs.mask.sport); 8008 if (f->fs.newsmac) 8009 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 8010 8011 f->pending = 1; 8012 sc->tids.ftids_in_use++; 8013 8014 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8015 return (0); 8016} 8017 8018static int 8019del_filter_wr(struct adapter *sc, int fidx) 8020{ 8021 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 8022 struct fw_filter_wr *fwr; 8023 unsigned int ftid; 8024 struct wrq_cookie cookie; 8025 8026 ftid = sc->tids.ftid_base + fidx; 8027 8028 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8029 if (fwr == NULL) 8030 return (ENOMEM); 8031 bzero(fwr, sizeof (*fwr)); 8032 8033 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 8034 8035 f->pending = 1; 8036 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8037 return (0); 8038} 8039 8040int 8041t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 8042{ 8043 struct adapter *sc = iq->adapter; 8044 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 8045 unsigned int idx = GET_TID(rpl); 8046 unsigned int rc; 8047 struct filter_entry *f; 8048 8049 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 8050 rss->opcode)); 8051 8052 if (is_ftid(sc, idx)) { 8053 8054 idx -= sc->tids.ftid_base; 8055 f = &sc->tids.ftid_tab[idx]; 8056 rc = G_COOKIE(rpl->cookie); 8057 8058 mtx_lock(&sc->tids.ftid_lock); 8059 if (rc == FW_FILTER_WR_FLT_ADDED) { 8060 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 8061 __func__, idx)); 8062 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 8063 f->pending = 0; /* asynchronous setup completed */ 8064 f->valid = 1; 8065 } else { 8066 if (rc != FW_FILTER_WR_FLT_DELETED) { 8067 /* Add or delete failed, display an error */ 8068 log(LOG_ERR, 8069 "filter %u setup failed with error %u\n", 8070 idx, rc); 8071 } 8072 8073 clear_filter(f); 8074 sc->tids.ftids_in_use--; 8075 } 8076 wakeup(&sc->tids.ftid_tab); 8077 mtx_unlock(&sc->tids.ftid_lock); 8078 } 8079 8080 return (0); 8081} 8082 8083static int 8084get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 8085{ 8086 int rc; 8087 8088 if (cntxt->cid > M_CTXTQID) 8089 return (EINVAL); 8090 8091 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 8092 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 8093 return (EINVAL); 8094 8095 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 8096 if (rc) 8097 return (rc); 8098 8099 if (sc->flags & FW_OK) { 8100 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 8101 &cntxt->data[0]); 8102 if (rc == 0) 8103 goto done; 8104 } 8105 8106 /* 8107 * Read via firmware failed or wasn't even attempted. Read directly via 8108 * the backdoor. 8109 */ 8110 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 8111done: 8112 end_synchronized_op(sc, 0); 8113 return (rc); 8114} 8115 8116static int 8117load_fw(struct adapter *sc, struct t4_data *fw) 8118{ 8119 int rc; 8120 uint8_t *fw_data; 8121 8122 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 8123 if (rc) 8124 return (rc); 8125 8126 if (sc->flags & FULL_INIT_DONE) { 8127 rc = EBUSY; 8128 goto done; 8129 } 8130 8131 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 8132 if (fw_data == NULL) { 8133 rc = ENOMEM; 8134 goto done; 8135 } 8136 8137 rc = copyin(fw->data, fw_data, fw->len); 8138 if (rc == 0) 8139 rc = -t4_load_fw(sc, fw_data, fw->len); 8140 8141 free(fw_data, M_CXGBE); 8142done: 8143 end_synchronized_op(sc, 0); 8144 return (rc); 8145} 8146 8147static int 8148read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 8149{ 8150 uint32_t addr, off, remaining, i, n; 8151 uint32_t *buf, *b; 8152 uint32_t mw_base, mw_aperture; 8153 int rc; 8154 uint8_t *dst; 8155 8156 rc = validate_mem_range(sc, mr->addr, mr->len); 8157 if (rc != 0) 8158 return (rc); 8159 8160 memwin_info(sc, win, &mw_base, &mw_aperture); 8161 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 8162 addr = mr->addr; 8163 remaining = mr->len; 8164 dst = (void *)mr->data; 8165 8166 while (remaining) { 8167 off = position_memwin(sc, win, addr); 8168 8169 /* number of bytes that we'll copy in the inner loop */ 8170 n = min(remaining, mw_aperture - off); 8171 for (i = 0; i < n; i += 4) 8172 *b++ = t4_read_reg(sc, mw_base + off + i); 8173 8174 rc = copyout(buf, dst, n); 8175 if (rc != 0) 8176 break; 8177 8178 b = buf; 8179 dst += n; 8180 remaining -= n; 8181 addr += n; 8182 } 8183 8184 free(buf, M_CXGBE); 8185 return (rc); 8186} 8187 8188static int 8189read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 8190{ 8191 int rc; 8192 8193 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 8194 return (EINVAL); 8195 8196 if (i2cd->len > sizeof(i2cd->data)) 8197 return (EFBIG); 8198 8199 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 8200 if (rc) 8201 return (rc); 8202 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 8203 i2cd->offset, i2cd->len, &i2cd->data[0]); 8204 end_synchronized_op(sc, 0); 8205 8206 return (rc); 8207} 8208 8209static int 8210in_range(int val, int lo, int hi) 8211{ 8212 8213 return (val < 0 || (val <= hi && val >= lo)); 8214} 8215 8216static int 8217set_sched_class(struct adapter *sc, struct t4_sched_params *p) 8218{ 8219 int fw_subcmd, fw_type, rc; 8220 8221 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 8222 if (rc) 8223 return (rc); 8224 8225 if (!(sc->flags & FULL_INIT_DONE)) { 8226 rc = EAGAIN; 8227 goto done; 8228 } 8229 8230 /* 8231 * Translate the cxgbetool parameters into T4 firmware parameters. (The 8232 * sub-command and type are in common locations.) 8233 */ 8234 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 8235 fw_subcmd = FW_SCHED_SC_CONFIG; 8236 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 8237 fw_subcmd = FW_SCHED_SC_PARAMS; 8238 else { 8239 rc = EINVAL; 8240 goto done; 8241 } 8242 if (p->type == SCHED_CLASS_TYPE_PACKET) 8243 fw_type = FW_SCHED_TYPE_PKTSCHED; 8244 else { 8245 rc = EINVAL; 8246 goto done; 8247 } 8248 8249 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 8250 /* Vet our parameters ..*/ 8251 if (p->u.config.minmax < 0) { 8252 rc = EINVAL; 8253 goto done; 8254 } 8255 8256 /* And pass the request to the firmware ...*/ 8257 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 8258 goto done; 8259 } 8260 8261 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 8262 int fw_level; 8263 int fw_mode; 8264 int fw_rateunit; 8265 int fw_ratemode; 8266 8267 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 8268 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 8269 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 8270 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 8271 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 8272 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 8273 else { 8274 rc = EINVAL; 8275 goto done; 8276 } 8277 8278 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 8279 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 8280 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 8281 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 8282 else { 8283 rc = EINVAL; 8284 goto done; 8285 } 8286 8287 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 8288 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 8289 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 8290 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 8291 else { 8292 rc = EINVAL; 8293 goto done; 8294 } 8295 8296 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 8297 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 8298 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 8299 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 8300 else { 8301 rc = EINVAL; 8302 goto done; 8303 } 8304 8305 /* Vet our parameters ... */ 8306 if (!in_range(p->u.params.channel, 0, 3) || 8307 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 8308 !in_range(p->u.params.minrate, 0, 10000000) || 8309 !in_range(p->u.params.maxrate, 0, 10000000) || 8310 !in_range(p->u.params.weight, 0, 100)) { 8311 rc = ERANGE; 8312 goto done; 8313 } 8314 8315 /* 8316 * Translate any unset parameters into the firmware's 8317 * nomenclature and/or fail the call if the parameters 8318 * are required ... 8319 */ 8320 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 8321 p->u.params.channel < 0 || p->u.params.cl < 0) { 8322 rc = EINVAL; 8323 goto done; 8324 } 8325 if (p->u.params.minrate < 0) 8326 p->u.params.minrate = 0; 8327 if (p->u.params.maxrate < 0) { 8328 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8329 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8330 rc = EINVAL; 8331 goto done; 8332 } else 8333 p->u.params.maxrate = 0; 8334 } 8335 if (p->u.params.weight < 0) { 8336 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 8337 rc = EINVAL; 8338 goto done; 8339 } else 8340 p->u.params.weight = 0; 8341 } 8342 if (p->u.params.pktsize < 0) { 8343 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 8344 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 8345 rc = EINVAL; 8346 goto done; 8347 } else 8348 p->u.params.pktsize = 0; 8349 } 8350 8351 /* See what the firmware thinks of the request ... */ 8352 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 8353 fw_rateunit, fw_ratemode, p->u.params.channel, 8354 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 8355 p->u.params.weight, p->u.params.pktsize, 1); 8356 goto done; 8357 } 8358 8359 rc = EINVAL; 8360done: 8361 end_synchronized_op(sc, 0); 8362 return (rc); 8363} 8364 8365static int 8366set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 8367{ 8368 struct port_info *pi = NULL; 8369 struct vi_info *vi; 8370 struct sge_txq *txq; 8371 uint32_t fw_mnem, fw_queue, fw_class; 8372 int i, rc; 8373 8374 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 8375 if (rc) 8376 return (rc); 8377 8378 if (!(sc->flags & FULL_INIT_DONE)) { 8379 rc = EAGAIN; 8380 goto done; 8381 } 8382 8383 if (p->port >= sc->params.nports) { 8384 rc = EINVAL; 8385 goto done; 8386 } 8387 8388 /* XXX: Only supported for the main VI. */ 8389 pi = sc->port[p->port]; 8390 vi = &pi->vi[0]; 8391 if (!in_range(p->queue, 0, vi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 8392 rc = EINVAL; 8393 goto done; 8394 } 8395 8396 /* 8397 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 8398 * Scheduling Class in this case). 8399 */ 8400 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 8401 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 8402 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 8403 8404 /* 8405 * If op.queue is non-negative, then we're only changing the scheduling 8406 * on a single specified TX queue. 8407 */ 8408 if (p->queue >= 0) { 8409 txq = &sc->sge.txq[vi->first_txq + p->queue]; 8410 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8411 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8412 &fw_class); 8413 goto done; 8414 } 8415 8416 /* 8417 * Change the scheduling on all the TX queues for the 8418 * interface. 8419 */ 8420 for_each_txq(vi, i, txq) { 8421 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8422 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8423 &fw_class); 8424 if (rc) 8425 goto done; 8426 } 8427 8428 rc = 0; 8429done: 8430 end_synchronized_op(sc, 0); 8431 return (rc); 8432} 8433 8434int 8435t4_os_find_pci_capability(struct adapter *sc, int cap) 8436{ 8437 int i; 8438 8439 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 8440} 8441 8442int 8443t4_os_pci_save_state(struct adapter *sc) 8444{ 8445 device_t dev; 8446 struct pci_devinfo *dinfo; 8447 8448 dev = sc->dev; 8449 dinfo = device_get_ivars(dev); 8450 8451 pci_cfg_save(dev, dinfo, 0); 8452 return (0); 8453} 8454 8455int 8456t4_os_pci_restore_state(struct adapter *sc) 8457{ 8458 device_t dev; 8459 struct pci_devinfo *dinfo; 8460 8461 dev = sc->dev; 8462 dinfo = device_get_ivars(dev); 8463 8464 pci_cfg_restore(dev, dinfo); 8465 return (0); 8466} 8467 8468void 8469t4_os_portmod_changed(const struct adapter *sc, int idx) 8470{ 8471 struct port_info *pi = sc->port[idx]; 8472 struct vi_info *vi; 8473 struct ifnet *ifp; 8474 int v; 8475 static const char *mod_str[] = { 8476 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8477 }; 8478 8479 for_each_vi(pi, v, vi) { 8480 build_medialist(pi, &vi->media); 8481 } 8482 8483 ifp = pi->vi[0].ifp; 8484 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8485 if_printf(ifp, "transceiver unplugged.\n"); 8486 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8487 if_printf(ifp, "unknown transceiver inserted.\n"); 8488 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8489 if_printf(ifp, "unsupported transceiver inserted.\n"); 8490 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8491 if_printf(ifp, "%s transceiver inserted.\n", 8492 mod_str[pi->mod_type]); 8493 } else { 8494 if_printf(ifp, "transceiver (type %d) inserted.\n", 8495 pi->mod_type); 8496 } 8497} 8498 8499void 8500t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8501{ 8502 struct port_info *pi = sc->port[idx]; 8503 struct vi_info *vi; 8504 struct ifnet *ifp; 8505 int v; 8506 8507 if (link_stat) 8508 pi->linkdnrc = -1; 8509 else { 8510 if (reason >= 0) 8511 pi->linkdnrc = reason; 8512 } 8513 for_each_vi(pi, v, vi) { 8514 ifp = vi->ifp; 8515 if (ifp == NULL) 8516 continue; 8517 8518 if (link_stat) { 8519 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8520 if_link_state_change(ifp, LINK_STATE_UP); 8521 } else { 8522 if_link_state_change(ifp, LINK_STATE_DOWN); 8523 } 8524 } 8525} 8526 8527void 8528t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8529{ 8530 struct adapter *sc; 8531 8532 sx_slock(&t4_list_lock); 8533 SLIST_FOREACH(sc, &t4_list, link) { 8534 /* 8535 * func should not make any assumptions about what state sc is 8536 * in - the only guarantee is that sc->sc_lock is a valid lock. 8537 */ 8538 func(sc, arg); 8539 } 8540 sx_sunlock(&t4_list_lock); 8541} 8542 8543static int 8544t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8545{ 8546 return (0); 8547} 8548 8549static int 8550t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8551{ 8552 return (0); 8553} 8554 8555static int 8556t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8557 struct thread *td) 8558{ 8559 int rc; 8560 struct adapter *sc = dev->si_drv1; 8561 8562 rc = priv_check(td, PRIV_DRIVER); 8563 if (rc != 0) 8564 return (rc); 8565 8566 switch (cmd) { 8567 case CHELSIO_T4_GETREG: { 8568 struct t4_reg *edata = (struct t4_reg *)data; 8569 8570 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8571 return (EFAULT); 8572 8573 if (edata->size == 4) 8574 edata->val = t4_read_reg(sc, edata->addr); 8575 else if (edata->size == 8) 8576 edata->val = t4_read_reg64(sc, edata->addr); 8577 else 8578 return (EINVAL); 8579 8580 break; 8581 } 8582 case CHELSIO_T4_SETREG: { 8583 struct t4_reg *edata = (struct t4_reg *)data; 8584 8585 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8586 return (EFAULT); 8587 8588 if (edata->size == 4) { 8589 if (edata->val & 0xffffffff00000000) 8590 return (EINVAL); 8591 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8592 } else if (edata->size == 8) 8593 t4_write_reg64(sc, edata->addr, edata->val); 8594 else 8595 return (EINVAL); 8596 break; 8597 } 8598 case CHELSIO_T4_REGDUMP: { 8599 struct t4_regdump *regs = (struct t4_regdump *)data; 8600 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8601 uint8_t *buf; 8602 8603 if (regs->len < reglen) { 8604 regs->len = reglen; /* hint to the caller */ 8605 return (ENOBUFS); 8606 } 8607 8608 regs->len = reglen; 8609 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8610 t4_get_regs(sc, regs, buf); 8611 rc = copyout(buf, regs->data, reglen); 8612 free(buf, M_CXGBE); 8613 break; 8614 } 8615 case CHELSIO_T4_GET_FILTER_MODE: 8616 rc = get_filter_mode(sc, (uint32_t *)data); 8617 break; 8618 case CHELSIO_T4_SET_FILTER_MODE: 8619 rc = set_filter_mode(sc, *(uint32_t *)data); 8620 break; 8621 case CHELSIO_T4_GET_FILTER: 8622 rc = get_filter(sc, (struct t4_filter *)data); 8623 break; 8624 case CHELSIO_T4_SET_FILTER: 8625 rc = set_filter(sc, (struct t4_filter *)data); 8626 break; 8627 case CHELSIO_T4_DEL_FILTER: 8628 rc = del_filter(sc, (struct t4_filter *)data); 8629 break; 8630 case CHELSIO_T4_GET_SGE_CONTEXT: 8631 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8632 break; 8633 case CHELSIO_T4_LOAD_FW: 8634 rc = load_fw(sc, (struct t4_data *)data); 8635 break; 8636 case CHELSIO_T4_GET_MEM: 8637 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8638 break; 8639 case CHELSIO_T4_GET_I2C: 8640 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8641 break; 8642 case CHELSIO_T4_CLEAR_STATS: { 8643 int i, v; 8644 u_int port_id = *(uint32_t *)data; 8645 struct port_info *pi; 8646 struct vi_info *vi; 8647 8648 if (port_id >= sc->params.nports) 8649 return (EINVAL); 8650 pi = sc->port[port_id]; 8651 8652 /* MAC stats */ 8653 t4_clr_port_stats(sc, pi->tx_chan); 8654 pi->tx_parse_error = 0; 8655 mtx_lock(&sc->regwin_lock); 8656 for_each_vi(pi, v, vi) { 8657 if (vi->flags & VI_INIT_DONE) 8658 t4_clr_vi_stats(sc, vi->viid); 8659 } 8660 mtx_unlock(&sc->regwin_lock); 8661 8662 /* 8663 * Since this command accepts a port, clear stats for 8664 * all VIs on this port. 8665 */ 8666 for_each_vi(pi, v, vi) { 8667 if (vi->flags & VI_INIT_DONE) { 8668 struct sge_rxq *rxq; 8669 struct sge_txq *txq; 8670 struct sge_wrq *wrq; 8671 8672 for_each_rxq(vi, i, rxq) { 8673#if defined(INET) || defined(INET6) 8674 rxq->lro.lro_queued = 0; 8675 rxq->lro.lro_flushed = 0; 8676#endif 8677 rxq->rxcsum = 0; 8678 rxq->vlan_extraction = 0; 8679 } 8680 8681 for_each_txq(vi, i, txq) { 8682 txq->txcsum = 0; 8683 txq->tso_wrs = 0; 8684 txq->vlan_insertion = 0; 8685 txq->imm_wrs = 0; 8686 txq->sgl_wrs = 0; 8687 txq->txpkt_wrs = 0; 8688 txq->txpkts0_wrs = 0; 8689 txq->txpkts1_wrs = 0; 8690 txq->txpkts0_pkts = 0; 8691 txq->txpkts1_pkts = 0; 8692 mp_ring_reset_stats(txq->r); 8693 } 8694 8695#ifdef TCP_OFFLOAD 8696 /* nothing to clear for each ofld_rxq */ 8697 8698 for_each_ofld_txq(vi, i, wrq) { 8699 wrq->tx_wrs_direct = 0; 8700 wrq->tx_wrs_copied = 0; 8701 } 8702#endif 8703 8704 if (IS_MAIN_VI(vi)) { 8705 wrq = &sc->sge.ctrlq[pi->port_id]; 8706 wrq->tx_wrs_direct = 0; 8707 wrq->tx_wrs_copied = 0; 8708 } 8709 } 8710 } 8711 break; 8712 } 8713 case CHELSIO_T4_SCHED_CLASS: 8714 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8715 break; 8716 case CHELSIO_T4_SCHED_QUEUE: 8717 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8718 break; 8719 case CHELSIO_T4_GET_TRACER: 8720 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8721 break; 8722 case CHELSIO_T4_SET_TRACER: 8723 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8724 break; 8725 default: 8726 rc = EINVAL; 8727 } 8728 8729 return (rc); 8730} 8731 8732#ifdef TCP_OFFLOAD 8733void 8734t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8735 const unsigned int *pgsz_order) 8736{ 8737 struct vi_info *vi = ifp->if_softc; 8738 struct adapter *sc = vi->pi->adapter; 8739 8740 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8741 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8742 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8743 V_HPZ3(pgsz_order[3])); 8744} 8745 8746static int 8747toe_capability(struct vi_info *vi, int enable) 8748{ 8749 int rc; 8750 struct port_info *pi = vi->pi; 8751 struct adapter *sc = pi->adapter; 8752 8753 ASSERT_SYNCHRONIZED_OP(sc); 8754 8755 if (!is_offload(sc)) 8756 return (ENODEV); 8757 8758 if (enable) { 8759 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) { 8760 /* TOE is already enabled. */ 8761 return (0); 8762 } 8763 8764 /* 8765 * We need the port's queues around so that we're able to send 8766 * and receive CPLs to/from the TOE even if the ifnet for this 8767 * port has never been UP'd administratively. 8768 */ 8769 if (!(vi->flags & VI_INIT_DONE)) { 8770 rc = vi_full_init(vi); 8771 if (rc) 8772 return (rc); 8773 } 8774 if (!(pi->vi[0].flags & VI_INIT_DONE)) { 8775 rc = vi_full_init(&pi->vi[0]); 8776 if (rc) 8777 return (rc); 8778 } 8779 8780 if (isset(&sc->offload_map, pi->port_id)) { 8781 /* TOE is enabled on another VI of this port. */ 8782 pi->uld_vis++; 8783 return (0); 8784 } 8785 8786 if (!uld_active(sc, ULD_TOM)) { 8787 rc = t4_activate_uld(sc, ULD_TOM); 8788 if (rc == EAGAIN) { 8789 log(LOG_WARNING, 8790 "You must kldload t4_tom.ko before trying " 8791 "to enable TOE on a cxgbe interface.\n"); 8792 } 8793 if (rc != 0) 8794 return (rc); 8795 KASSERT(sc->tom_softc != NULL, 8796 ("%s: TOM activated but softc NULL", __func__)); 8797 KASSERT(uld_active(sc, ULD_TOM), 8798 ("%s: TOM activated but flag not set", __func__)); 8799 } 8800 8801 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8802 if (!uld_active(sc, ULD_IWARP)) 8803 (void) t4_activate_uld(sc, ULD_IWARP); 8804 if (!uld_active(sc, ULD_ISCSI)) 8805 (void) t4_activate_uld(sc, ULD_ISCSI); 8806 8807 pi->uld_vis++; 8808 setbit(&sc->offload_map, pi->port_id); 8809 } else { 8810 pi->uld_vis--; 8811 8812 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0) 8813 return (0); 8814 8815 KASSERT(uld_active(sc, ULD_TOM), 8816 ("%s: TOM never initialized?", __func__)); 8817 clrbit(&sc->offload_map, pi->port_id); 8818 } 8819 8820 return (0); 8821} 8822 8823/* 8824 * Add an upper layer driver to the global list. 8825 */ 8826int 8827t4_register_uld(struct uld_info *ui) 8828{ 8829 int rc = 0; 8830 struct uld_info *u; 8831 8832 sx_xlock(&t4_uld_list_lock); 8833 SLIST_FOREACH(u, &t4_uld_list, link) { 8834 if (u->uld_id == ui->uld_id) { 8835 rc = EEXIST; 8836 goto done; 8837 } 8838 } 8839 8840 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8841 ui->refcount = 0; 8842done: 8843 sx_xunlock(&t4_uld_list_lock); 8844 return (rc); 8845} 8846 8847int 8848t4_unregister_uld(struct uld_info *ui) 8849{ 8850 int rc = EINVAL; 8851 struct uld_info *u; 8852 8853 sx_xlock(&t4_uld_list_lock); 8854 8855 SLIST_FOREACH(u, &t4_uld_list, link) { 8856 if (u == ui) { 8857 if (ui->refcount > 0) { 8858 rc = EBUSY; 8859 goto done; 8860 } 8861 8862 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8863 rc = 0; 8864 goto done; 8865 } 8866 } 8867done: 8868 sx_xunlock(&t4_uld_list_lock); 8869 return (rc); 8870} 8871 8872int 8873t4_activate_uld(struct adapter *sc, int id) 8874{ 8875 int rc; 8876 struct uld_info *ui; 8877 8878 ASSERT_SYNCHRONIZED_OP(sc); 8879 8880 if (id < 0 || id > ULD_MAX) 8881 return (EINVAL); 8882 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8883 8884 sx_slock(&t4_uld_list_lock); 8885 8886 SLIST_FOREACH(ui, &t4_uld_list, link) { 8887 if (ui->uld_id == id) { 8888 if (!(sc->flags & FULL_INIT_DONE)) { 8889 rc = adapter_full_init(sc); 8890 if (rc != 0) 8891 break; 8892 } 8893 8894 rc = ui->activate(sc); 8895 if (rc == 0) { 8896 setbit(&sc->active_ulds, id); 8897 ui->refcount++; 8898 } 8899 break; 8900 } 8901 } 8902 8903 sx_sunlock(&t4_uld_list_lock); 8904 8905 return (rc); 8906} 8907 8908int 8909t4_deactivate_uld(struct adapter *sc, int id) 8910{ 8911 int rc; 8912 struct uld_info *ui; 8913 8914 ASSERT_SYNCHRONIZED_OP(sc); 8915 8916 if (id < 0 || id > ULD_MAX) 8917 return (EINVAL); 8918 rc = ENXIO; 8919 8920 sx_slock(&t4_uld_list_lock); 8921 8922 SLIST_FOREACH(ui, &t4_uld_list, link) { 8923 if (ui->uld_id == id) { 8924 rc = ui->deactivate(sc); 8925 if (rc == 0) { 8926 clrbit(&sc->active_ulds, id); 8927 ui->refcount--; 8928 } 8929 break; 8930 } 8931 } 8932 8933 sx_sunlock(&t4_uld_list_lock); 8934 8935 return (rc); 8936} 8937 8938int 8939uld_active(struct adapter *sc, int uld_id) 8940{ 8941 8942 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8943 8944 return (isset(&sc->active_ulds, uld_id)); 8945} 8946#endif 8947 8948/* 8949 * Come up with reasonable defaults for some of the tunables, provided they're 8950 * not set by the user (in which case we'll use the values as is). 8951 */ 8952static void 8953tweak_tunables(void) 8954{ 8955 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8956 8957 if (t4_ntxq10g < 1) { 8958#ifdef RSS 8959 t4_ntxq10g = rss_getnumbuckets(); 8960#else 8961 t4_ntxq10g = min(nc, NTXQ_10G); 8962#endif 8963 } 8964 8965 if (t4_ntxq1g < 1) { 8966#ifdef RSS 8967 /* XXX: way too many for 1GbE? */ 8968 t4_ntxq1g = rss_getnumbuckets(); 8969#else 8970 t4_ntxq1g = min(nc, NTXQ_1G); 8971#endif 8972 } 8973 8974 if (t4_ntxq_vi < 1) 8975 t4_ntxq_vi = min(nc, NTXQ_VI); 8976 8977 if (t4_nrxq10g < 1) { 8978#ifdef RSS 8979 t4_nrxq10g = rss_getnumbuckets(); 8980#else 8981 t4_nrxq10g = min(nc, NRXQ_10G); 8982#endif 8983 } 8984 8985 if (t4_nrxq1g < 1) { 8986#ifdef RSS 8987 /* XXX: way too many for 1GbE? */ 8988 t4_nrxq1g = rss_getnumbuckets(); 8989#else 8990 t4_nrxq1g = min(nc, NRXQ_1G); 8991#endif 8992 } 8993 8994 if (t4_nrxq_vi < 1) 8995 t4_nrxq_vi = min(nc, NRXQ_VI); 8996 8997#ifdef TCP_OFFLOAD 8998 if (t4_nofldtxq10g < 1) 8999 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 9000 9001 if (t4_nofldtxq1g < 1) 9002 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 9003 9004 if (t4_nofldtxq_vi < 1) 9005 t4_nofldtxq_vi = min(nc, NOFLDTXQ_VI); 9006 9007 if (t4_nofldrxq10g < 1) 9008 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 9009 9010 if (t4_nofldrxq1g < 1) 9011 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 9012 9013 if (t4_nofldrxq_vi < 1) 9014 t4_nofldrxq_vi = min(nc, NOFLDRXQ_VI); 9015 9016 if (t4_toecaps_allowed == -1) 9017 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 9018#else 9019 if (t4_toecaps_allowed == -1) 9020 t4_toecaps_allowed = 0; 9021#endif 9022 9023#ifdef DEV_NETMAP 9024 if (t4_nnmtxq_vi < 1) 9025 t4_nnmtxq_vi = min(nc, NNMTXQ_VI); 9026 9027 if (t4_nnmrxq_vi < 1) 9028 t4_nnmrxq_vi = min(nc, NNMRXQ_VI); 9029#endif 9030 9031 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 9032 t4_tmr_idx_10g = TMR_IDX_10G; 9033 9034 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 9035 t4_pktc_idx_10g = PKTC_IDX_10G; 9036 9037 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 9038 t4_tmr_idx_1g = TMR_IDX_1G; 9039 9040 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 9041 t4_pktc_idx_1g = PKTC_IDX_1G; 9042 9043 if (t4_qsize_txq < 128) 9044 t4_qsize_txq = 128; 9045 9046 if (t4_qsize_rxq < 128) 9047 t4_qsize_rxq = 128; 9048 while (t4_qsize_rxq & 7) 9049 t4_qsize_rxq++; 9050 9051 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 9052} 9053 9054static struct sx mlu; /* mod load unload */ 9055SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 9056 9057static int 9058mod_event(module_t mod, int cmd, void *arg) 9059{ 9060 int rc = 0; 9061 static int loaded = 0; 9062 9063 switch (cmd) { 9064 case MOD_LOAD: 9065 sx_xlock(&mlu); 9066 if (loaded++ == 0) { 9067 t4_sge_modload(); 9068 sx_init(&t4_list_lock, "T4/T5 adapters"); 9069 SLIST_INIT(&t4_list); 9070#ifdef TCP_OFFLOAD 9071 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 9072 SLIST_INIT(&t4_uld_list); 9073#endif 9074 t4_tracer_modload(); 9075 tweak_tunables(); 9076 } 9077 sx_xunlock(&mlu); 9078 break; 9079 9080 case MOD_UNLOAD: 9081 sx_xlock(&mlu); 9082 if (--loaded == 0) { 9083 int tries; 9084 9085 sx_slock(&t4_list_lock); 9086 if (!SLIST_EMPTY(&t4_list)) { 9087 rc = EBUSY; 9088 sx_sunlock(&t4_list_lock); 9089 goto done_unload; 9090 } 9091#ifdef TCP_OFFLOAD 9092 sx_slock(&t4_uld_list_lock); 9093 if (!SLIST_EMPTY(&t4_uld_list)) { 9094 rc = EBUSY; 9095 sx_sunlock(&t4_uld_list_lock); 9096 sx_sunlock(&t4_list_lock); 9097 goto done_unload; 9098 } 9099#endif 9100 tries = 0; 9101 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 9102 uprintf("%ju clusters with custom free routine " 9103 "still is use.\n", t4_sge_extfree_refs()); 9104 pause("t4unload", 2 * hz); 9105 } 9106#ifdef TCP_OFFLOAD 9107 sx_sunlock(&t4_uld_list_lock); 9108#endif 9109 sx_sunlock(&t4_list_lock); 9110 9111 if (t4_sge_extfree_refs() == 0) { 9112 t4_tracer_modunload(); 9113#ifdef TCP_OFFLOAD 9114 sx_destroy(&t4_uld_list_lock); 9115#endif 9116 sx_destroy(&t4_list_lock); 9117 t4_sge_modunload(); 9118 loaded = 0; 9119 } else { 9120 rc = EBUSY; 9121 loaded++; /* undo earlier decrement */ 9122 } 9123 } 9124done_unload: 9125 sx_xunlock(&mlu); 9126 break; 9127 } 9128 9129 return (rc); 9130} 9131 9132static devclass_t t4_devclass, t5_devclass; 9133static devclass_t cxgbe_devclass, cxl_devclass; 9134static devclass_t vcxgbe_devclass, vcxl_devclass; 9135 9136DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 9137MODULE_VERSION(t4nex, 1); 9138MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 9139 9140DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 9141MODULE_VERSION(t5nex, 1); 9142MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 9143 9144DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 9145MODULE_VERSION(cxgbe, 1); 9146 9147DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 9148MODULE_VERSION(cxl, 1); 9149 9150DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0); 9151MODULE_VERSION(vcxgbe, 1); 9152 9153DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0); 9154MODULE_VERSION(vcxl, 1); 9155