t4_main.c revision 286271
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 286271 2015-08-04 03:33:59Z np $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/systm.h> 40#include <sys/counter.h> 41#include <sys/module.h> 42#include <sys/malloc.h> 43#include <sys/queue.h> 44#include <sys/taskqueue.h> 45#include <sys/pciio.h> 46#include <dev/pci/pcireg.h> 47#include <dev/pci/pcivar.h> 48#include <dev/pci/pci_private.h> 49#include <sys/firmware.h> 50#include <sys/sbuf.h> 51#include <sys/smp.h> 52#include <sys/socket.h> 53#include <sys/sockio.h> 54#include <sys/sysctl.h> 55#include <net/ethernet.h> 56#include <net/if.h> 57#include <net/if_types.h> 58#include <net/if_dl.h> 59#include <net/if_vlan_var.h> 60#if defined(__i386__) || defined(__amd64__) 61#include <vm/vm.h> 62#include <vm/pmap.h> 63#endif 64 65#include "common/common.h" 66#include "common/t4_msg.h" 67#include "common/t4_regs.h" 68#include "common/t4_regs_values.h" 69#include "t4_ioctl.h" 70#include "t4_l2t.h" 71#include "t4_mp_ring.h" 72 73/* T4 bus driver interface */ 74static int t4_probe(device_t); 75static int t4_attach(device_t); 76static int t4_detach(device_t); 77static device_method_t t4_methods[] = { 78 DEVMETHOD(device_probe, t4_probe), 79 DEVMETHOD(device_attach, t4_attach), 80 DEVMETHOD(device_detach, t4_detach), 81 82 DEVMETHOD_END 83}; 84static driver_t t4_driver = { 85 "t4nex", 86 t4_methods, 87 sizeof(struct adapter) 88}; 89 90 91/* T4 port (cxgbe) interface */ 92static int cxgbe_probe(device_t); 93static int cxgbe_attach(device_t); 94static int cxgbe_detach(device_t); 95static device_method_t cxgbe_methods[] = { 96 DEVMETHOD(device_probe, cxgbe_probe), 97 DEVMETHOD(device_attach, cxgbe_attach), 98 DEVMETHOD(device_detach, cxgbe_detach), 99 { 0, 0 } 100}; 101static driver_t cxgbe_driver = { 102 "cxgbe", 103 cxgbe_methods, 104 sizeof(struct port_info) 105}; 106 107static d_ioctl_t t4_ioctl; 108static d_open_t t4_open; 109static d_close_t t4_close; 110 111static struct cdevsw t4_cdevsw = { 112 .d_version = D_VERSION, 113 .d_flags = 0, 114 .d_open = t4_open, 115 .d_close = t4_close, 116 .d_ioctl = t4_ioctl, 117 .d_name = "t4nex", 118}; 119 120/* T5 bus driver interface */ 121static int t5_probe(device_t); 122static device_method_t t5_methods[] = { 123 DEVMETHOD(device_probe, t5_probe), 124 DEVMETHOD(device_attach, t4_attach), 125 DEVMETHOD(device_detach, t4_detach), 126 127 DEVMETHOD_END 128}; 129static driver_t t5_driver = { 130 "t5nex", 131 t5_methods, 132 sizeof(struct adapter) 133}; 134 135 136/* T5 port (cxl) interface */ 137static driver_t cxl_driver = { 138 "cxl", 139 cxgbe_methods, 140 sizeof(struct port_info) 141}; 142 143static struct cdevsw t5_cdevsw = { 144 .d_version = D_VERSION, 145 .d_flags = 0, 146 .d_open = t4_open, 147 .d_close = t4_close, 148 .d_ioctl = t4_ioctl, 149 .d_name = "t5nex", 150}; 151 152/* ifnet + media interface */ 153static void cxgbe_init(void *); 154static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 155static int cxgbe_transmit(struct ifnet *, struct mbuf *); 156static void cxgbe_qflush(struct ifnet *); 157static int cxgbe_media_change(struct ifnet *); 158static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 159 160MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 161 162/* 163 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 164 * then ADAPTER_LOCK, then t4_uld_list_lock. 165 */ 166static struct sx t4_list_lock; 167SLIST_HEAD(, adapter) t4_list; 168#ifdef TCP_OFFLOAD 169static struct sx t4_uld_list_lock; 170SLIST_HEAD(, uld_info) t4_uld_list; 171#endif 172 173/* 174 * Tunables. See tweak_tunables() too. 175 * 176 * Each tunable is set to a default value here if it's known at compile-time. 177 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 178 * provide a reasonable default when the driver is loaded. 179 * 180 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 181 * T5 are under hw.cxl. 182 */ 183 184/* 185 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 186 */ 187#define NTXQ_10G 16 188static int t4_ntxq10g = -1; 189TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 190 191#define NRXQ_10G 8 192static int t4_nrxq10g = -1; 193TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 194 195#define NTXQ_1G 4 196static int t4_ntxq1g = -1; 197TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 198 199#define NRXQ_1G 2 200static int t4_nrxq1g = -1; 201TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 202 203static int t4_rsrv_noflowq = 0; 204TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 205 206#ifdef TCP_OFFLOAD 207#define NOFLDTXQ_10G 8 208static int t4_nofldtxq10g = -1; 209TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 210 211#define NOFLDRXQ_10G 2 212static int t4_nofldrxq10g = -1; 213TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 214 215#define NOFLDTXQ_1G 2 216static int t4_nofldtxq1g = -1; 217TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 218 219#define NOFLDRXQ_1G 1 220static int t4_nofldrxq1g = -1; 221TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 222#endif 223 224#ifdef DEV_NETMAP 225#define NNMTXQ_10G 2 226static int t4_nnmtxq10g = -1; 227TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 228 229#define NNMRXQ_10G 2 230static int t4_nnmrxq10g = -1; 231TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 232 233#define NNMTXQ_1G 1 234static int t4_nnmtxq1g = -1; 235TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 236 237#define NNMRXQ_1G 1 238static int t4_nnmrxq1g = -1; 239TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 240#endif 241 242/* 243 * Holdoff parameters for 10G and 1G ports. 244 */ 245#define TMR_IDX_10G 1 246static int t4_tmr_idx_10g = TMR_IDX_10G; 247TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 248 249#define PKTC_IDX_10G (-1) 250static int t4_pktc_idx_10g = PKTC_IDX_10G; 251TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 252 253#define TMR_IDX_1G 1 254static int t4_tmr_idx_1g = TMR_IDX_1G; 255TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 256 257#define PKTC_IDX_1G (-1) 258static int t4_pktc_idx_1g = PKTC_IDX_1G; 259TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 260 261/* 262 * Size (# of entries) of each tx and rx queue. 263 */ 264static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 265TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 266 267static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 268TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 269 270/* 271 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 272 */ 273static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 274TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 275 276/* 277 * Configuration file. 278 */ 279#define DEFAULT_CF "default" 280#define FLASH_CF "flash" 281#define UWIRE_CF "uwire" 282#define FPGA_CF "fpga" 283static char t4_cfg_file[32] = DEFAULT_CF; 284TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 285 286/* 287 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 288 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 289 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 290 * mark or when signalled to do so, 0 to never emit PAUSE. 291 */ 292static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 293TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 294 295/* 296 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 297 * encouraged respectively). 298 */ 299static unsigned int t4_fw_install = 1; 300TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 301 302/* 303 * ASIC features that will be used. Disable the ones you don't want so that the 304 * chip resources aren't wasted on features that will not be used. 305 */ 306static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 307TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 308 309static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 310TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 311 312static int t4_toecaps_allowed = -1; 313TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 314 315static int t4_rdmacaps_allowed = 0; 316TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 317 318static int t4_iscsicaps_allowed = 0; 319TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 320 321static int t4_fcoecaps_allowed = 0; 322TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 323 324static int t5_write_combine = 0; 325TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 326 327struct intrs_and_queues { 328 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 329 uint16_t nirq; /* Total # of vectors */ 330 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 331 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 332 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 333 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 334 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 335 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 336 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 337#ifdef TCP_OFFLOAD 338 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 339 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 340 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 341 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 342#endif 343#ifdef DEV_NETMAP 344 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 345 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 346 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 347 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 348#endif 349}; 350 351struct filter_entry { 352 uint32_t valid:1; /* filter allocated and valid */ 353 uint32_t locked:1; /* filter is administratively locked */ 354 uint32_t pending:1; /* filter action is pending firmware reply */ 355 uint32_t smtidx:8; /* Source MAC Table index for smac */ 356 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 357 358 struct t4_filter_specification fs; 359}; 360 361static int map_bars_0_and_4(struct adapter *); 362static int map_bar_2(struct adapter *); 363static void setup_memwin(struct adapter *); 364static int validate_mem_range(struct adapter *, uint32_t, int); 365static int fwmtype_to_hwmtype(int); 366static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 367 uint32_t *); 368static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 369static uint32_t position_memwin(struct adapter *, int, uint32_t); 370static int cfg_itype_and_nqueues(struct adapter *, int, int, 371 struct intrs_and_queues *); 372static int prep_firmware(struct adapter *); 373static int partition_resources(struct adapter *, const struct firmware *, 374 const char *); 375static int get_params__pre_init(struct adapter *); 376static int get_params__post_init(struct adapter *); 377static int set_params__post_init(struct adapter *); 378static void t4_set_desc(struct adapter *); 379static void build_medialist(struct port_info *, struct ifmedia *); 380static int cxgbe_init_synchronized(struct port_info *); 381static int cxgbe_uninit_synchronized(struct port_info *); 382static int setup_intr_handlers(struct adapter *); 383static void quiesce_txq(struct adapter *, struct sge_txq *); 384static void quiesce_wrq(struct adapter *, struct sge_wrq *); 385static void quiesce_iq(struct adapter *, struct sge_iq *); 386static void quiesce_fl(struct adapter *, struct sge_fl *); 387static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 388 driver_intr_t *, void *, char *); 389static int t4_free_irq(struct adapter *, struct irq *); 390static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 391 unsigned int); 392static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 393static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 394static void cxgbe_tick(void *); 395static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 396static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 397 struct mbuf *); 398static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 399static int fw_msg_not_handled(struct adapter *, const __be64 *); 400static int t4_sysctls(struct adapter *); 401static int cxgbe_sysctls(struct port_info *); 402static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 403static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 404static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 405static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 406static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 407static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 408static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 409static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 410static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 411static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 412static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 413#ifdef SBUF_DRAIN 414static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 415static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 416static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 417static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 418static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 419static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 420static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 421static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 422static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 423static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 424static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 425static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 426static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 427static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 428static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 429static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 430static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 431static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 432static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 433static int sysctl_tids(SYSCTL_HANDLER_ARGS); 434static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 435static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 436static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 437static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 438static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 439#endif 440static uint32_t fconf_to_mode(uint32_t); 441static uint32_t mode_to_fconf(uint32_t); 442static uint32_t fspec_to_fconf(struct t4_filter_specification *); 443static int get_filter_mode(struct adapter *, uint32_t *); 444static int set_filter_mode(struct adapter *, uint32_t); 445static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 446static int get_filter(struct adapter *, struct t4_filter *); 447static int set_filter(struct adapter *, struct t4_filter *); 448static int del_filter(struct adapter *, struct t4_filter *); 449static void clear_filter(struct filter_entry *); 450static int set_filter_wr(struct adapter *, int); 451static int del_filter_wr(struct adapter *, int); 452static int get_sge_context(struct adapter *, struct t4_sge_context *); 453static int load_fw(struct adapter *, struct t4_data *); 454static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 455static int read_i2c(struct adapter *, struct t4_i2c_data *); 456static int set_sched_class(struct adapter *, struct t4_sched_params *); 457static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 458#ifdef TCP_OFFLOAD 459static int toe_capability(struct port_info *, int); 460#endif 461static int mod_event(module_t, int, void *); 462 463struct { 464 uint16_t device; 465 char *desc; 466} t4_pciids[] = { 467 {0xa000, "Chelsio Terminator 4 FPGA"}, 468 {0x4400, "Chelsio T440-dbg"}, 469 {0x4401, "Chelsio T420-CR"}, 470 {0x4402, "Chelsio T422-CR"}, 471 {0x4403, "Chelsio T440-CR"}, 472 {0x4404, "Chelsio T420-BCH"}, 473 {0x4405, "Chelsio T440-BCH"}, 474 {0x4406, "Chelsio T440-CH"}, 475 {0x4407, "Chelsio T420-SO"}, 476 {0x4408, "Chelsio T420-CX"}, 477 {0x4409, "Chelsio T420-BT"}, 478 {0x440a, "Chelsio T404-BT"}, 479 {0x440e, "Chelsio T440-LP-CR"}, 480}, t5_pciids[] = { 481 {0xb000, "Chelsio Terminator 5 FPGA"}, 482 {0x5400, "Chelsio T580-dbg"}, 483 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 484 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 485 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 486 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 487 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 488 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 489 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 490 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 491 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 492 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 493 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 494 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 495 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 496#ifdef notyet 497 {0x5404, "Chelsio T520-BCH"}, 498 {0x5405, "Chelsio T540-BCH"}, 499 {0x5406, "Chelsio T540-CH"}, 500 {0x5408, "Chelsio T520-CX"}, 501 {0x540b, "Chelsio B520-SR"}, 502 {0x540c, "Chelsio B504-BT"}, 503 {0x540f, "Chelsio Amsterdam"}, 504 {0x5413, "Chelsio T580-CHR"}, 505#endif 506}; 507 508#ifdef TCP_OFFLOAD 509/* 510 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 511 * exactly the same for both rxq and ofld_rxq. 512 */ 513CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 514CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 515#endif 516 517/* No easy way to include t4_msg.h before adapter.h so we check this way */ 518CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 519CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 520 521CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 522 523static int 524t4_probe(device_t dev) 525{ 526 int i; 527 uint16_t v = pci_get_vendor(dev); 528 uint16_t d = pci_get_device(dev); 529 uint8_t f = pci_get_function(dev); 530 531 if (v != PCI_VENDOR_ID_CHELSIO) 532 return (ENXIO); 533 534 /* Attach only to PF0 of the FPGA */ 535 if (d == 0xa000 && f != 0) 536 return (ENXIO); 537 538 for (i = 0; i < nitems(t4_pciids); i++) { 539 if (d == t4_pciids[i].device) { 540 device_set_desc(dev, t4_pciids[i].desc); 541 return (BUS_PROBE_DEFAULT); 542 } 543 } 544 545 return (ENXIO); 546} 547 548static int 549t5_probe(device_t dev) 550{ 551 int i; 552 uint16_t v = pci_get_vendor(dev); 553 uint16_t d = pci_get_device(dev); 554 uint8_t f = pci_get_function(dev); 555 556 if (v != PCI_VENDOR_ID_CHELSIO) 557 return (ENXIO); 558 559 /* Attach only to PF0 of the FPGA */ 560 if (d == 0xb000 && f != 0) 561 return (ENXIO); 562 563 for (i = 0; i < nitems(t5_pciids); i++) { 564 if (d == t5_pciids[i].device) { 565 device_set_desc(dev, t5_pciids[i].desc); 566 return (BUS_PROBE_DEFAULT); 567 } 568 } 569 570 return (ENXIO); 571} 572 573static int 574t4_attach(device_t dev) 575{ 576 struct adapter *sc; 577 int rc = 0, i, n10g, n1g, rqidx, tqidx; 578 struct intrs_and_queues iaq; 579 struct sge *s; 580#ifdef TCP_OFFLOAD 581 int ofld_rqidx, ofld_tqidx; 582#endif 583#ifdef DEV_NETMAP 584 int nm_rqidx, nm_tqidx; 585#endif 586 587 sc = device_get_softc(dev); 588 sc->dev = dev; 589 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 590 591 pci_enable_busmaster(dev); 592 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 593 uint32_t v; 594 595 pci_set_max_read_req(dev, 4096); 596 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 597 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 598 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 599 600 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 601 } 602 603 sc->traceq = -1; 604 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 605 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 606 device_get_nameunit(dev)); 607 608 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 609 device_get_nameunit(dev)); 610 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 611 sx_xlock(&t4_list_lock); 612 SLIST_INSERT_HEAD(&t4_list, sc, link); 613 sx_xunlock(&t4_list_lock); 614 615 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 616 TAILQ_INIT(&sc->sfl); 617 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 618 619 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 620 621 rc = map_bars_0_and_4(sc); 622 if (rc != 0) 623 goto done; /* error message displayed already */ 624 625 /* 626 * This is the real PF# to which we're attaching. Works from within PCI 627 * passthrough environments too, where pci_get_function() could return a 628 * different PF# depending on the passthrough configuration. We need to 629 * use the real PF# in all our communication with the firmware. 630 */ 631 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 632 sc->mbox = sc->pf; 633 634 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 635 sc->an_handler = an_not_handled; 636 for (i = 0; i < nitems(sc->cpl_handler); i++) 637 sc->cpl_handler[i] = cpl_not_handled; 638 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 639 sc->fw_msg_handler[i] = fw_msg_not_handled; 640 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 641 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 642 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 643 t4_init_sge_cpl_handlers(sc); 644 645 /* Prepare the adapter for operation */ 646 rc = -t4_prep_adapter(sc); 647 if (rc != 0) { 648 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 649 goto done; 650 } 651 652 /* 653 * Do this really early, with the memory windows set up even before the 654 * character device. The userland tool's register i/o and mem read 655 * will work even in "recovery mode". 656 */ 657 setup_memwin(sc); 658 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 659 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 660 device_get_nameunit(dev)); 661 if (sc->cdev == NULL) 662 device_printf(dev, "failed to create nexus char device.\n"); 663 else 664 sc->cdev->si_drv1 = sc; 665 666 /* Go no further if recovery mode has been requested. */ 667 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 668 device_printf(dev, "recovery mode.\n"); 669 goto done; 670 } 671 672#if defined(__i386__) 673 if ((cpu_feature & CPUID_CX8) == 0) { 674 device_printf(dev, "64 bit atomics not available.\n"); 675 rc = ENOTSUP; 676 goto done; 677 } 678#endif 679 680 /* Prepare the firmware for operation */ 681 rc = prep_firmware(sc); 682 if (rc != 0) 683 goto done; /* error message displayed already */ 684 685 rc = get_params__post_init(sc); 686 if (rc != 0) 687 goto done; /* error message displayed already */ 688 689 rc = set_params__post_init(sc); 690 if (rc != 0) 691 goto done; /* error message displayed already */ 692 693 rc = map_bar_2(sc); 694 if (rc != 0) 695 goto done; /* error message displayed already */ 696 697 rc = t4_create_dma_tag(sc); 698 if (rc != 0) 699 goto done; /* error message displayed already */ 700 701 /* 702 * First pass over all the ports - allocate VIs and initialize some 703 * basic parameters like mac address, port type, etc. We also figure 704 * out whether a port is 10G or 1G and use that information when 705 * calculating how many interrupts to attempt to allocate. 706 */ 707 n10g = n1g = 0; 708 for_each_port(sc, i) { 709 struct port_info *pi; 710 711 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 712 sc->port[i] = pi; 713 714 /* These must be set before t4_port_init */ 715 pi->adapter = sc; 716 pi->port_id = i; 717 718 /* Allocate the vi and initialize parameters like mac addr */ 719 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 720 if (rc != 0) { 721 device_printf(dev, "unable to initialize port %d: %d\n", 722 i, rc); 723 free(pi, M_CXGBE); 724 sc->port[i] = NULL; 725 goto done; 726 } 727 728 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 729 pi->link_cfg.requested_fc |= t4_pause_settings; 730 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 731 pi->link_cfg.fc |= t4_pause_settings; 732 733 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 734 if (rc != 0) { 735 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 736 free(pi, M_CXGBE); 737 sc->port[i] = NULL; 738 goto done; 739 } 740 741 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 742 device_get_nameunit(dev), i); 743 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 744 sc->chan_map[pi->tx_chan] = i; 745 746 if (is_10G_port(pi) || is_40G_port(pi)) { 747 n10g++; 748 pi->tmr_idx = t4_tmr_idx_10g; 749 pi->pktc_idx = t4_pktc_idx_10g; 750 } else { 751 n1g++; 752 pi->tmr_idx = t4_tmr_idx_1g; 753 pi->pktc_idx = t4_pktc_idx_1g; 754 } 755 756 pi->xact_addr_filt = -1; 757 pi->linkdnrc = -1; 758 759 pi->qsize_rxq = t4_qsize_rxq; 760 pi->qsize_txq = t4_qsize_txq; 761 762 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 763 if (pi->dev == NULL) { 764 device_printf(dev, 765 "failed to add device for port %d.\n", i); 766 rc = ENXIO; 767 goto done; 768 } 769 device_set_softc(pi->dev, pi); 770 } 771 772 /* 773 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 774 */ 775 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 776 if (rc != 0) 777 goto done; /* error message displayed already */ 778 779 sc->intr_type = iaq.intr_type; 780 sc->intr_count = iaq.nirq; 781 782 s = &sc->sge; 783 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 784 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 785 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 786 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 787 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 788#ifdef TCP_OFFLOAD 789 if (is_offload(sc)) { 790 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 791 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 792 s->neq += s->nofldtxq + s->nofldrxq; 793 s->niq += s->nofldrxq; 794 795 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 796 M_CXGBE, M_ZERO | M_WAITOK); 797 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 798 M_CXGBE, M_ZERO | M_WAITOK); 799 } 800#endif 801#ifdef DEV_NETMAP 802 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 803 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 804 s->neq += s->nnmtxq + s->nnmrxq; 805 s->niq += s->nnmrxq; 806 807 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 808 M_CXGBE, M_ZERO | M_WAITOK); 809 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 810 M_CXGBE, M_ZERO | M_WAITOK); 811#endif 812 813 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 814 M_ZERO | M_WAITOK); 815 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 816 M_ZERO | M_WAITOK); 817 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 818 M_ZERO | M_WAITOK); 819 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 820 M_ZERO | M_WAITOK); 821 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 822 M_ZERO | M_WAITOK); 823 824 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 825 M_ZERO | M_WAITOK); 826 827 t4_init_l2t(sc, M_WAITOK); 828 829 /* 830 * Second pass over the ports. This time we know the number of rx and 831 * tx queues that each port should get. 832 */ 833 rqidx = tqidx = 0; 834#ifdef TCP_OFFLOAD 835 ofld_rqidx = ofld_tqidx = 0; 836#endif 837#ifdef DEV_NETMAP 838 nm_rqidx = nm_tqidx = 0; 839#endif 840 for_each_port(sc, i) { 841 struct port_info *pi = sc->port[i]; 842 843 if (pi == NULL) 844 continue; 845 846 pi->first_rxq = rqidx; 847 pi->first_txq = tqidx; 848 if (is_10G_port(pi) || is_40G_port(pi)) { 849 pi->flags |= iaq.intr_flags_10g; 850 pi->nrxq = iaq.nrxq10g; 851 pi->ntxq = iaq.ntxq10g; 852 } else { 853 pi->flags |= iaq.intr_flags_1g; 854 pi->nrxq = iaq.nrxq1g; 855 pi->ntxq = iaq.ntxq1g; 856 } 857 858 if (pi->ntxq > 1) 859 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 860 else 861 pi->rsrv_noflowq = 0; 862 863 rqidx += pi->nrxq; 864 tqidx += pi->ntxq; 865#ifdef TCP_OFFLOAD 866 if (is_offload(sc)) { 867 pi->first_ofld_rxq = ofld_rqidx; 868 pi->first_ofld_txq = ofld_tqidx; 869 if (is_10G_port(pi) || is_40G_port(pi)) { 870 pi->nofldrxq = iaq.nofldrxq10g; 871 pi->nofldtxq = iaq.nofldtxq10g; 872 } else { 873 pi->nofldrxq = iaq.nofldrxq1g; 874 pi->nofldtxq = iaq.nofldtxq1g; 875 } 876 ofld_rqidx += pi->nofldrxq; 877 ofld_tqidx += pi->nofldtxq; 878 } 879#endif 880#ifdef DEV_NETMAP 881 pi->first_nm_rxq = nm_rqidx; 882 pi->first_nm_txq = nm_tqidx; 883 if (is_10G_port(pi) || is_40G_port(pi)) { 884 pi->nnmrxq = iaq.nnmrxq10g; 885 pi->nnmtxq = iaq.nnmtxq10g; 886 } else { 887 pi->nnmrxq = iaq.nnmrxq1g; 888 pi->nnmtxq = iaq.nnmtxq1g; 889 } 890 nm_rqidx += pi->nnmrxq; 891 nm_tqidx += pi->nnmtxq; 892#endif 893 } 894 895 rc = setup_intr_handlers(sc); 896 if (rc != 0) { 897 device_printf(dev, 898 "failed to setup interrupt handlers: %d\n", rc); 899 goto done; 900 } 901 902 rc = bus_generic_attach(dev); 903 if (rc != 0) { 904 device_printf(dev, 905 "failed to attach all child ports: %d\n", rc); 906 goto done; 907 } 908 909 device_printf(dev, 910 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 911 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 912 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 913 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 914 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 915 916 t4_set_desc(sc); 917 918done: 919 if (rc != 0 && sc->cdev) { 920 /* cdev was created and so cxgbetool works; recover that way. */ 921 device_printf(dev, 922 "error during attach, adapter is now in recovery mode.\n"); 923 rc = 0; 924 } 925 926 if (rc != 0) 927 t4_detach(dev); 928 else 929 t4_sysctls(sc); 930 931 return (rc); 932} 933 934/* 935 * Idempotent 936 */ 937static int 938t4_detach(device_t dev) 939{ 940 struct adapter *sc; 941 struct port_info *pi; 942 int i, rc; 943 944 sc = device_get_softc(dev); 945 946 if (sc->flags & FULL_INIT_DONE) 947 t4_intr_disable(sc); 948 949 if (sc->cdev) { 950 destroy_dev(sc->cdev); 951 sc->cdev = NULL; 952 } 953 954 rc = bus_generic_detach(dev); 955 if (rc) { 956 device_printf(dev, 957 "failed to detach child devices: %d\n", rc); 958 return (rc); 959 } 960 961 for (i = 0; i < sc->intr_count; i++) 962 t4_free_irq(sc, &sc->irq[i]); 963 964 for (i = 0; i < MAX_NPORTS; i++) { 965 pi = sc->port[i]; 966 if (pi) { 967 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 968 if (pi->dev) 969 device_delete_child(dev, pi->dev); 970 971 mtx_destroy(&pi->pi_lock); 972 free(pi, M_CXGBE); 973 } 974 } 975 976 if (sc->flags & FULL_INIT_DONE) 977 adapter_full_uninit(sc); 978 979 if (sc->flags & FW_OK) 980 t4_fw_bye(sc, sc->mbox); 981 982 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 983 pci_release_msi(dev); 984 985 if (sc->regs_res) 986 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 987 sc->regs_res); 988 989 if (sc->udbs_res) 990 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 991 sc->udbs_res); 992 993 if (sc->msix_res) 994 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 995 sc->msix_res); 996 997 if (sc->l2t) 998 t4_free_l2t(sc->l2t); 999 1000#ifdef TCP_OFFLOAD 1001 free(sc->sge.ofld_rxq, M_CXGBE); 1002 free(sc->sge.ofld_txq, M_CXGBE); 1003#endif 1004#ifdef DEV_NETMAP 1005 free(sc->sge.nm_rxq, M_CXGBE); 1006 free(sc->sge.nm_txq, M_CXGBE); 1007#endif 1008 free(sc->irq, M_CXGBE); 1009 free(sc->sge.rxq, M_CXGBE); 1010 free(sc->sge.txq, M_CXGBE); 1011 free(sc->sge.ctrlq, M_CXGBE); 1012 free(sc->sge.iqmap, M_CXGBE); 1013 free(sc->sge.eqmap, M_CXGBE); 1014 free(sc->tids.ftid_tab, M_CXGBE); 1015 t4_destroy_dma_tag(sc); 1016 if (mtx_initialized(&sc->sc_lock)) { 1017 sx_xlock(&t4_list_lock); 1018 SLIST_REMOVE(&t4_list, sc, adapter, link); 1019 sx_xunlock(&t4_list_lock); 1020 mtx_destroy(&sc->sc_lock); 1021 } 1022 1023 if (mtx_initialized(&sc->tids.ftid_lock)) 1024 mtx_destroy(&sc->tids.ftid_lock); 1025 if (mtx_initialized(&sc->sfl_lock)) 1026 mtx_destroy(&sc->sfl_lock); 1027 if (mtx_initialized(&sc->ifp_lock)) 1028 mtx_destroy(&sc->ifp_lock); 1029 if (mtx_initialized(&sc->regwin_lock)) 1030 mtx_destroy(&sc->regwin_lock); 1031 1032 bzero(sc, sizeof(*sc)); 1033 1034 return (0); 1035} 1036 1037static int 1038cxgbe_probe(device_t dev) 1039{ 1040 char buf[128]; 1041 struct port_info *pi = device_get_softc(dev); 1042 1043 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1044 device_set_desc_copy(dev, buf); 1045 1046 return (BUS_PROBE_DEFAULT); 1047} 1048 1049#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1050 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1051 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1052#define T4_CAP_ENABLE (T4_CAP) 1053 1054static int 1055cxgbe_attach(device_t dev) 1056{ 1057 struct port_info *pi = device_get_softc(dev); 1058 struct ifnet *ifp; 1059 char *s; 1060 int n, o; 1061 1062 /* Allocate an ifnet and set it up */ 1063 ifp = if_alloc(IFT_ETHER); 1064 if (ifp == NULL) { 1065 device_printf(dev, "Cannot allocate ifnet\n"); 1066 return (ENOMEM); 1067 } 1068 pi->ifp = ifp; 1069 ifp->if_softc = pi; 1070 1071 callout_init(&pi->tick, CALLOUT_MPSAFE); 1072 1073 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1074 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1075 1076 ifp->if_init = cxgbe_init; 1077 ifp->if_ioctl = cxgbe_ioctl; 1078 ifp->if_transmit = cxgbe_transmit; 1079 ifp->if_qflush = cxgbe_qflush; 1080 1081 ifp->if_capabilities = T4_CAP; 1082#ifdef TCP_OFFLOAD 1083 if (is_offload(pi->adapter)) 1084 ifp->if_capabilities |= IFCAP_TOE; 1085#endif 1086 ifp->if_capenable = T4_CAP_ENABLE; 1087 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1088 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1089 1090 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1091 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1092 ifp->if_hw_tsomaxsegsize = 65536; 1093 1094 /* Initialize ifmedia for this port */ 1095 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1096 cxgbe_media_status); 1097 build_medialist(pi, &pi->media); 1098 1099 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1100 EVENTHANDLER_PRI_ANY); 1101 1102 ether_ifattach(ifp, pi->hw_addr); 1103 1104 n = 128; 1105 s = malloc(n, M_CXGBE, M_WAITOK); 1106 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1107 MPASS(n > o); 1108#ifdef TCP_OFFLOAD 1109 if (is_offload(pi->adapter)) { 1110 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1111 pi->nofldtxq, pi->nofldrxq); 1112 MPASS(n > o); 1113 } 1114#endif 1115#ifdef DEV_NETMAP 1116 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1117 pi->nnmrxq); 1118 MPASS(n > o); 1119#endif 1120 device_printf(dev, "%s\n", s); 1121 free(s, M_CXGBE); 1122 1123#ifdef DEV_NETMAP 1124 /* nm_media handled here to keep implementation private to this file */ 1125 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1126 cxgbe_media_status); 1127 build_medialist(pi, &pi->nm_media); 1128 create_netmap_ifnet(pi); /* logs errors it something fails */ 1129#endif 1130 cxgbe_sysctls(pi); 1131 1132 return (0); 1133} 1134 1135static int 1136cxgbe_detach(device_t dev) 1137{ 1138 struct port_info *pi = device_get_softc(dev); 1139 struct adapter *sc = pi->adapter; 1140 struct ifnet *ifp = pi->ifp; 1141 1142 /* Tell if_ioctl and if_init that the port is going away */ 1143 ADAPTER_LOCK(sc); 1144 SET_DOOMED(pi); 1145 wakeup(&sc->flags); 1146 while (IS_BUSY(sc)) 1147 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1148 SET_BUSY(sc); 1149#ifdef INVARIANTS 1150 sc->last_op = "t4detach"; 1151 sc->last_op_thr = curthread; 1152#endif 1153 ADAPTER_UNLOCK(sc); 1154 1155 if (pi->flags & HAS_TRACEQ) { 1156 sc->traceq = -1; /* cloner should not create ifnet */ 1157 t4_tracer_port_detach(sc); 1158 } 1159 1160 if (pi->vlan_c) 1161 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1162 1163 PORT_LOCK(pi); 1164 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1165 callout_stop(&pi->tick); 1166 PORT_UNLOCK(pi); 1167 callout_drain(&pi->tick); 1168 1169 /* Let detach proceed even if these fail. */ 1170 cxgbe_uninit_synchronized(pi); 1171 port_full_uninit(pi); 1172 1173 ifmedia_removeall(&pi->media); 1174 ether_ifdetach(pi->ifp); 1175 if_free(pi->ifp); 1176 1177#ifdef DEV_NETMAP 1178 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1179 destroy_netmap_ifnet(pi); 1180#endif 1181 1182 ADAPTER_LOCK(sc); 1183 CLR_BUSY(sc); 1184 wakeup(&sc->flags); 1185 ADAPTER_UNLOCK(sc); 1186 1187 return (0); 1188} 1189 1190static void 1191cxgbe_init(void *arg) 1192{ 1193 struct port_info *pi = arg; 1194 struct adapter *sc = pi->adapter; 1195 1196 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1197 return; 1198 cxgbe_init_synchronized(pi); 1199 end_synchronized_op(sc, 0); 1200} 1201 1202static int 1203cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1204{ 1205 int rc = 0, mtu, flags, can_sleep; 1206 struct port_info *pi = ifp->if_softc; 1207 struct adapter *sc = pi->adapter; 1208 struct ifreq *ifr = (struct ifreq *)data; 1209 uint32_t mask; 1210 1211 switch (cmd) { 1212 case SIOCSIFMTU: 1213 mtu = ifr->ifr_mtu; 1214 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1215 return (EINVAL); 1216 1217 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1218 if (rc) 1219 return (rc); 1220 ifp->if_mtu = mtu; 1221 if (pi->flags & PORT_INIT_DONE) { 1222 t4_update_fl_bufsize(ifp); 1223 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1224 rc = update_mac_settings(ifp, XGMAC_MTU); 1225 } 1226 end_synchronized_op(sc, 0); 1227 break; 1228 1229 case SIOCSIFFLAGS: 1230 can_sleep = 0; 1231redo_sifflags: 1232 rc = begin_synchronized_op(sc, pi, 1233 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1234 if (rc) 1235 return (rc); 1236 1237 if (ifp->if_flags & IFF_UP) { 1238 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1239 flags = pi->if_flags; 1240 if ((ifp->if_flags ^ flags) & 1241 (IFF_PROMISC | IFF_ALLMULTI)) { 1242 if (can_sleep == 1) { 1243 end_synchronized_op(sc, 0); 1244 can_sleep = 0; 1245 goto redo_sifflags; 1246 } 1247 rc = update_mac_settings(ifp, 1248 XGMAC_PROMISC | XGMAC_ALLMULTI); 1249 } 1250 } else { 1251 if (can_sleep == 0) { 1252 end_synchronized_op(sc, LOCK_HELD); 1253 can_sleep = 1; 1254 goto redo_sifflags; 1255 } 1256 rc = cxgbe_init_synchronized(pi); 1257 } 1258 pi->if_flags = ifp->if_flags; 1259 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1260 if (can_sleep == 0) { 1261 end_synchronized_op(sc, LOCK_HELD); 1262 can_sleep = 1; 1263 goto redo_sifflags; 1264 } 1265 rc = cxgbe_uninit_synchronized(pi); 1266 } 1267 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1268 break; 1269 1270 case SIOCADDMULTI: 1271 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1272 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1273 if (rc) 1274 return (rc); 1275 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1276 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1277 end_synchronized_op(sc, LOCK_HELD); 1278 break; 1279 1280 case SIOCSIFCAP: 1281 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1282 if (rc) 1283 return (rc); 1284 1285 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1286 if (mask & IFCAP_TXCSUM) { 1287 ifp->if_capenable ^= IFCAP_TXCSUM; 1288 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1289 1290 if (IFCAP_TSO4 & ifp->if_capenable && 1291 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1292 ifp->if_capenable &= ~IFCAP_TSO4; 1293 if_printf(ifp, 1294 "tso4 disabled due to -txcsum.\n"); 1295 } 1296 } 1297 if (mask & IFCAP_TXCSUM_IPV6) { 1298 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1299 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1300 1301 if (IFCAP_TSO6 & ifp->if_capenable && 1302 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1303 ifp->if_capenable &= ~IFCAP_TSO6; 1304 if_printf(ifp, 1305 "tso6 disabled due to -txcsum6.\n"); 1306 } 1307 } 1308 if (mask & IFCAP_RXCSUM) 1309 ifp->if_capenable ^= IFCAP_RXCSUM; 1310 if (mask & IFCAP_RXCSUM_IPV6) 1311 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1312 1313 /* 1314 * Note that we leave CSUM_TSO alone (it is always set). The 1315 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1316 * sending a TSO request our way, so it's sufficient to toggle 1317 * IFCAP_TSOx only. 1318 */ 1319 if (mask & IFCAP_TSO4) { 1320 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1321 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1322 if_printf(ifp, "enable txcsum first.\n"); 1323 rc = EAGAIN; 1324 goto fail; 1325 } 1326 ifp->if_capenable ^= IFCAP_TSO4; 1327 } 1328 if (mask & IFCAP_TSO6) { 1329 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1330 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1331 if_printf(ifp, "enable txcsum6 first.\n"); 1332 rc = EAGAIN; 1333 goto fail; 1334 } 1335 ifp->if_capenable ^= IFCAP_TSO6; 1336 } 1337 if (mask & IFCAP_LRO) { 1338#if defined(INET) || defined(INET6) 1339 int i; 1340 struct sge_rxq *rxq; 1341 1342 ifp->if_capenable ^= IFCAP_LRO; 1343 for_each_rxq(pi, i, rxq) { 1344 if (ifp->if_capenable & IFCAP_LRO) 1345 rxq->iq.flags |= IQ_LRO_ENABLED; 1346 else 1347 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1348 } 1349#endif 1350 } 1351#ifdef TCP_OFFLOAD 1352 if (mask & IFCAP_TOE) { 1353 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1354 1355 rc = toe_capability(pi, enable); 1356 if (rc != 0) 1357 goto fail; 1358 1359 ifp->if_capenable ^= mask; 1360 } 1361#endif 1362 if (mask & IFCAP_VLAN_HWTAGGING) { 1363 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1364 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1365 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1366 } 1367 if (mask & IFCAP_VLAN_MTU) { 1368 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1369 1370 /* Need to find out how to disable auto-mtu-inflation */ 1371 } 1372 if (mask & IFCAP_VLAN_HWTSO) 1373 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1374 if (mask & IFCAP_VLAN_HWCSUM) 1375 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1376 1377#ifdef VLAN_CAPABILITIES 1378 VLAN_CAPABILITIES(ifp); 1379#endif 1380fail: 1381 end_synchronized_op(sc, 0); 1382 break; 1383 1384 case SIOCSIFMEDIA: 1385 case SIOCGIFMEDIA: 1386 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1387 break; 1388 1389 default: 1390 rc = ether_ioctl(ifp, cmd, data); 1391 } 1392 1393 return (rc); 1394} 1395 1396static int 1397cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1398{ 1399 struct port_info *pi = ifp->if_softc; 1400 struct adapter *sc = pi->adapter; 1401 struct sge_txq *txq; 1402 void *items[1]; 1403 int rc; 1404 1405 M_ASSERTPKTHDR(m); 1406 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1407 1408 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1409 m_freem(m); 1410 return (ENETDOWN); 1411 } 1412 1413 rc = parse_pkt(&m); 1414 if (__predict_false(rc != 0)) { 1415 MPASS(m == NULL); /* was freed already */ 1416 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1417 return (rc); 1418 } 1419 1420 /* Select a txq. */ 1421 txq = &sc->sge.txq[pi->first_txq]; 1422 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1423 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1424 pi->rsrv_noflowq); 1425 1426 items[0] = m; 1427 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1428 if (__predict_false(rc != 0)) 1429 m_freem(m); 1430 1431 return (rc); 1432} 1433 1434static void 1435cxgbe_qflush(struct ifnet *ifp) 1436{ 1437 struct port_info *pi = ifp->if_softc; 1438 struct sge_txq *txq; 1439 int i; 1440 1441 /* queues do not exist if !PORT_INIT_DONE. */ 1442 if (pi->flags & PORT_INIT_DONE) { 1443 for_each_txq(pi, i, txq) { 1444 TXQ_LOCK(txq); 1445 txq->eq.flags &= ~EQ_ENABLED; 1446 TXQ_UNLOCK(txq); 1447 while (!mp_ring_is_idle(txq->r)) { 1448 mp_ring_check_drainage(txq->r, 0); 1449 pause("qflush", 1); 1450 } 1451 } 1452 } 1453 if_qflush(ifp); 1454} 1455 1456static int 1457cxgbe_media_change(struct ifnet *ifp) 1458{ 1459 struct port_info *pi = ifp->if_softc; 1460 1461 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1462 1463 return (EOPNOTSUPP); 1464} 1465 1466static void 1467cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1468{ 1469 struct port_info *pi = ifp->if_softc; 1470 struct ifmedia *media = NULL; 1471 struct ifmedia_entry *cur; 1472 int speed = pi->link_cfg.speed; 1473#ifdef INVARIANTS 1474 int data = (pi->port_type << 8) | pi->mod_type; 1475#endif 1476 1477 if (ifp == pi->ifp) 1478 media = &pi->media; 1479#ifdef DEV_NETMAP 1480 else if (ifp == pi->nm_ifp) 1481 media = &pi->nm_media; 1482#endif 1483 MPASS(media != NULL); 1484 1485 cur = media->ifm_cur; 1486 MPASS(cur->ifm_data == data); 1487 1488 ifmr->ifm_status = IFM_AVALID; 1489 if (!pi->link_cfg.link_ok) 1490 return; 1491 1492 ifmr->ifm_status |= IFM_ACTIVE; 1493 1494 /* active and current will differ iff current media is autoselect. */ 1495 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1496 return; 1497 1498 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1499 if (speed == SPEED_10000) 1500 ifmr->ifm_active |= IFM_10G_T; 1501 else if (speed == SPEED_1000) 1502 ifmr->ifm_active |= IFM_1000_T; 1503 else if (speed == SPEED_100) 1504 ifmr->ifm_active |= IFM_100_TX; 1505 else if (speed == SPEED_10) 1506 ifmr->ifm_active |= IFM_10_T; 1507 else 1508 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1509 speed)); 1510} 1511 1512void 1513t4_fatal_err(struct adapter *sc) 1514{ 1515 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1516 t4_intr_disable(sc); 1517 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1518 device_get_nameunit(sc->dev)); 1519} 1520 1521static int 1522map_bars_0_and_4(struct adapter *sc) 1523{ 1524 sc->regs_rid = PCIR_BAR(0); 1525 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1526 &sc->regs_rid, RF_ACTIVE); 1527 if (sc->regs_res == NULL) { 1528 device_printf(sc->dev, "cannot map registers.\n"); 1529 return (ENXIO); 1530 } 1531 sc->bt = rman_get_bustag(sc->regs_res); 1532 sc->bh = rman_get_bushandle(sc->regs_res); 1533 sc->mmio_len = rman_get_size(sc->regs_res); 1534 setbit(&sc->doorbells, DOORBELL_KDB); 1535 1536 sc->msix_rid = PCIR_BAR(4); 1537 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1538 &sc->msix_rid, RF_ACTIVE); 1539 if (sc->msix_res == NULL) { 1540 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1541 return (ENXIO); 1542 } 1543 1544 return (0); 1545} 1546 1547static int 1548map_bar_2(struct adapter *sc) 1549{ 1550 1551 /* 1552 * T4: only iWARP driver uses the userspace doorbells. There is no need 1553 * to map it if RDMA is disabled. 1554 */ 1555 if (is_t4(sc) && sc->rdmacaps == 0) 1556 return (0); 1557 1558 sc->udbs_rid = PCIR_BAR(2); 1559 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1560 &sc->udbs_rid, RF_ACTIVE); 1561 if (sc->udbs_res == NULL) { 1562 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1563 return (ENXIO); 1564 } 1565 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1566 1567 if (is_t5(sc)) { 1568 setbit(&sc->doorbells, DOORBELL_UDB); 1569#if defined(__i386__) || defined(__amd64__) 1570 if (t5_write_combine) { 1571 int rc; 1572 1573 /* 1574 * Enable write combining on BAR2. This is the 1575 * userspace doorbell BAR and is split into 128B 1576 * (UDBS_SEG_SIZE) doorbell regions, each associated 1577 * with an egress queue. The first 64B has the doorbell 1578 * and the second 64B can be used to submit a tx work 1579 * request with an implicit doorbell. 1580 */ 1581 1582 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1583 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1584 if (rc == 0) { 1585 clrbit(&sc->doorbells, DOORBELL_UDB); 1586 setbit(&sc->doorbells, DOORBELL_WCWR); 1587 setbit(&sc->doorbells, DOORBELL_UDBWC); 1588 } else { 1589 device_printf(sc->dev, 1590 "couldn't enable write combining: %d\n", 1591 rc); 1592 } 1593 1594 t4_write_reg(sc, A_SGE_STAT_CFG, 1595 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1596 } 1597#endif 1598 } 1599 1600 return (0); 1601} 1602 1603static const struct memwin t4_memwin[] = { 1604 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1605 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1606 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1607}; 1608 1609static const struct memwin t5_memwin[] = { 1610 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1611 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1612 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1613}; 1614 1615static void 1616setup_memwin(struct adapter *sc) 1617{ 1618 const struct memwin *mw; 1619 int i, n; 1620 uint32_t bar0; 1621 1622 if (is_t4(sc)) { 1623 /* 1624 * Read low 32b of bar0 indirectly via the hardware backdoor 1625 * mechanism. Works from within PCI passthrough environments 1626 * too, where rman_get_start() can return a different value. We 1627 * need to program the T4 memory window decoders with the actual 1628 * addresses that will be coming across the PCIe link. 1629 */ 1630 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1631 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1632 1633 mw = &t4_memwin[0]; 1634 n = nitems(t4_memwin); 1635 } else { 1636 /* T5 uses the relative offset inside the PCIe BAR */ 1637 bar0 = 0; 1638 1639 mw = &t5_memwin[0]; 1640 n = nitems(t5_memwin); 1641 } 1642 1643 for (i = 0; i < n; i++, mw++) { 1644 t4_write_reg(sc, 1645 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1646 (mw->base + bar0) | V_BIR(0) | 1647 V_WINDOW(ilog2(mw->aperture) - 10)); 1648 } 1649 1650 /* flush */ 1651 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1652} 1653 1654/* 1655 * Verify that the memory range specified by the addr/len pair is valid and lies 1656 * entirely within a single region (EDCx or MCx). 1657 */ 1658static int 1659validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1660{ 1661 uint32_t em, addr_len, maddr, mlen; 1662 1663 /* Memory can only be accessed in naturally aligned 4 byte units */ 1664 if (addr & 3 || len & 3 || len == 0) 1665 return (EINVAL); 1666 1667 /* Enabled memories */ 1668 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1669 if (em & F_EDRAM0_ENABLE) { 1670 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1671 maddr = G_EDRAM0_BASE(addr_len) << 20; 1672 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1673 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1674 addr + len <= maddr + mlen) 1675 return (0); 1676 } 1677 if (em & F_EDRAM1_ENABLE) { 1678 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1679 maddr = G_EDRAM1_BASE(addr_len) << 20; 1680 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1681 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1682 addr + len <= maddr + mlen) 1683 return (0); 1684 } 1685 if (em & F_EXT_MEM_ENABLE) { 1686 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1687 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1688 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1689 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1690 addr + len <= maddr + mlen) 1691 return (0); 1692 } 1693 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1694 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1695 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1696 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1697 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1698 addr + len <= maddr + mlen) 1699 return (0); 1700 } 1701 1702 return (EFAULT); 1703} 1704 1705static int 1706fwmtype_to_hwmtype(int mtype) 1707{ 1708 1709 switch (mtype) { 1710 case FW_MEMTYPE_EDC0: 1711 return (MEM_EDC0); 1712 case FW_MEMTYPE_EDC1: 1713 return (MEM_EDC1); 1714 case FW_MEMTYPE_EXTMEM: 1715 return (MEM_MC0); 1716 case FW_MEMTYPE_EXTMEM1: 1717 return (MEM_MC1); 1718 default: 1719 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1720 } 1721} 1722 1723/* 1724 * Verify that the memory range specified by the memtype/offset/len pair is 1725 * valid and lies entirely within the memtype specified. The global address of 1726 * the start of the range is returned in addr. 1727 */ 1728static int 1729validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1730 uint32_t *addr) 1731{ 1732 uint32_t em, addr_len, maddr, mlen; 1733 1734 /* Memory can only be accessed in naturally aligned 4 byte units */ 1735 if (off & 3 || len & 3 || len == 0) 1736 return (EINVAL); 1737 1738 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1739 switch (fwmtype_to_hwmtype(mtype)) { 1740 case MEM_EDC0: 1741 if (!(em & F_EDRAM0_ENABLE)) 1742 return (EINVAL); 1743 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1744 maddr = G_EDRAM0_BASE(addr_len) << 20; 1745 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1746 break; 1747 case MEM_EDC1: 1748 if (!(em & F_EDRAM1_ENABLE)) 1749 return (EINVAL); 1750 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1751 maddr = G_EDRAM1_BASE(addr_len) << 20; 1752 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1753 break; 1754 case MEM_MC: 1755 if (!(em & F_EXT_MEM_ENABLE)) 1756 return (EINVAL); 1757 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1758 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1759 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1760 break; 1761 case MEM_MC1: 1762 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1763 return (EINVAL); 1764 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1765 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1766 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1767 break; 1768 default: 1769 return (EINVAL); 1770 } 1771 1772 if (mlen > 0 && off < mlen && off + len <= mlen) { 1773 *addr = maddr + off; /* global address */ 1774 return (0); 1775 } 1776 1777 return (EFAULT); 1778} 1779 1780static void 1781memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1782{ 1783 const struct memwin *mw; 1784 1785 if (is_t4(sc)) { 1786 KASSERT(win >= 0 && win < nitems(t4_memwin), 1787 ("%s: incorrect memwin# (%d)", __func__, win)); 1788 mw = &t4_memwin[win]; 1789 } else { 1790 KASSERT(win >= 0 && win < nitems(t5_memwin), 1791 ("%s: incorrect memwin# (%d)", __func__, win)); 1792 mw = &t5_memwin[win]; 1793 } 1794 1795 if (base != NULL) 1796 *base = mw->base; 1797 if (aperture != NULL) 1798 *aperture = mw->aperture; 1799} 1800 1801/* 1802 * Positions the memory window such that it can be used to access the specified 1803 * address in the chip's address space. The return value is the offset of addr 1804 * from the start of the window. 1805 */ 1806static uint32_t 1807position_memwin(struct adapter *sc, int n, uint32_t addr) 1808{ 1809 uint32_t start, pf; 1810 uint32_t reg; 1811 1812 KASSERT(n >= 0 && n <= 3, 1813 ("%s: invalid window %d.", __func__, n)); 1814 KASSERT((addr & 3) == 0, 1815 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1816 1817 if (is_t4(sc)) { 1818 pf = 0; 1819 start = addr & ~0xf; /* start must be 16B aligned */ 1820 } else { 1821 pf = V_PFNUM(sc->pf); 1822 start = addr & ~0x7f; /* start must be 128B aligned */ 1823 } 1824 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1825 1826 t4_write_reg(sc, reg, start | pf); 1827 t4_read_reg(sc, reg); 1828 1829 return (addr - start); 1830} 1831 1832static int 1833cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1834 struct intrs_and_queues *iaq) 1835{ 1836 int rc, itype, navail, nrxq10g, nrxq1g, n; 1837 int nofldrxq10g = 0, nofldrxq1g = 0; 1838 int nnmrxq10g = 0, nnmrxq1g = 0; 1839 1840 bzero(iaq, sizeof(*iaq)); 1841 1842 iaq->ntxq10g = t4_ntxq10g; 1843 iaq->ntxq1g = t4_ntxq1g; 1844 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1845 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1846 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1847#ifdef TCP_OFFLOAD 1848 if (is_offload(sc)) { 1849 iaq->nofldtxq10g = t4_nofldtxq10g; 1850 iaq->nofldtxq1g = t4_nofldtxq1g; 1851 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1852 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1853 } 1854#endif 1855#ifdef DEV_NETMAP 1856 iaq->nnmtxq10g = t4_nnmtxq10g; 1857 iaq->nnmtxq1g = t4_nnmtxq1g; 1858 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1859 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1860#endif 1861 1862 for (itype = INTR_MSIX; itype; itype >>= 1) { 1863 1864 if ((itype & t4_intr_types) == 0) 1865 continue; /* not allowed */ 1866 1867 if (itype == INTR_MSIX) 1868 navail = pci_msix_count(sc->dev); 1869 else if (itype == INTR_MSI) 1870 navail = pci_msi_count(sc->dev); 1871 else 1872 navail = 1; 1873restart: 1874 if (navail == 0) 1875 continue; 1876 1877 iaq->intr_type = itype; 1878 iaq->intr_flags_10g = 0; 1879 iaq->intr_flags_1g = 0; 1880 1881 /* 1882 * Best option: an interrupt vector for errors, one for the 1883 * firmware event queue, and one for every rxq (NIC, TOE, and 1884 * netmap). 1885 */ 1886 iaq->nirq = T4_EXTRA_INTR; 1887 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1888 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1889 if (iaq->nirq <= navail && 1890 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1891 iaq->intr_flags_10g = INTR_ALL; 1892 iaq->intr_flags_1g = INTR_ALL; 1893 goto allocate; 1894 } 1895 1896 /* 1897 * Second best option: a vector for errors, one for the firmware 1898 * event queue, and vectors for either all the NIC rx queues or 1899 * all the TOE rx queues. The queues that don't get vectors 1900 * will forward their interrupts to those that do. 1901 * 1902 * Note: netmap rx queues cannot be created early and so they 1903 * can't be setup to receive forwarded interrupts for others. 1904 */ 1905 iaq->nirq = T4_EXTRA_INTR; 1906 if (nrxq10g >= nofldrxq10g) { 1907 iaq->intr_flags_10g = INTR_RXQ; 1908 iaq->nirq += n10g * nrxq10g; 1909#ifdef DEV_NETMAP 1910 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1911#endif 1912 } else { 1913 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1914 iaq->nirq += n10g * nofldrxq10g; 1915#ifdef DEV_NETMAP 1916 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1917#endif 1918 } 1919 if (nrxq1g >= nofldrxq1g) { 1920 iaq->intr_flags_1g = INTR_RXQ; 1921 iaq->nirq += n1g * nrxq1g; 1922#ifdef DEV_NETMAP 1923 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1924#endif 1925 } else { 1926 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1927 iaq->nirq += n1g * nofldrxq1g; 1928#ifdef DEV_NETMAP 1929 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1930#endif 1931 } 1932 if (iaq->nirq <= navail && 1933 (itype != INTR_MSI || powerof2(iaq->nirq))) 1934 goto allocate; 1935 1936 /* 1937 * Next best option: an interrupt vector for errors, one for the 1938 * firmware event queue, and at least one per port. At this 1939 * point we know we'll have to downsize nrxq and/or nofldrxq 1940 * and/or nnmrxq to fit what's available to us. 1941 */ 1942 iaq->nirq = T4_EXTRA_INTR; 1943 iaq->nirq += n10g + n1g; 1944 if (iaq->nirq <= navail) { 1945 int leftover = navail - iaq->nirq; 1946 1947 if (n10g > 0) { 1948 int target = max(nrxq10g, nofldrxq10g); 1949 1950 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 1951 INTR_RXQ : INTR_OFLD_RXQ; 1952 1953 n = 1; 1954 while (n < target && leftover >= n10g) { 1955 leftover -= n10g; 1956 iaq->nirq += n10g; 1957 n++; 1958 } 1959 iaq->nrxq10g = min(n, nrxq10g); 1960#ifdef TCP_OFFLOAD 1961 iaq->nofldrxq10g = min(n, nofldrxq10g); 1962#endif 1963#ifdef DEV_NETMAP 1964 iaq->nnmrxq10g = min(n, nnmrxq10g); 1965#endif 1966 } 1967 1968 if (n1g > 0) { 1969 int target = max(nrxq1g, nofldrxq1g); 1970 1971 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 1972 INTR_RXQ : INTR_OFLD_RXQ; 1973 1974 n = 1; 1975 while (n < target && leftover >= n1g) { 1976 leftover -= n1g; 1977 iaq->nirq += n1g; 1978 n++; 1979 } 1980 iaq->nrxq1g = min(n, nrxq1g); 1981#ifdef TCP_OFFLOAD 1982 iaq->nofldrxq1g = min(n, nofldrxq1g); 1983#endif 1984#ifdef DEV_NETMAP 1985 iaq->nnmrxq1g = min(n, nnmrxq1g); 1986#endif 1987 } 1988 1989 if (itype != INTR_MSI || powerof2(iaq->nirq)) 1990 goto allocate; 1991 } 1992 1993 /* 1994 * Least desirable option: one interrupt vector for everything. 1995 */ 1996 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1997 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 1998#ifdef TCP_OFFLOAD 1999 if (is_offload(sc)) 2000 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2001#endif 2002#ifdef DEV_NETMAP 2003 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2004#endif 2005 2006allocate: 2007 navail = iaq->nirq; 2008 rc = 0; 2009 if (itype == INTR_MSIX) 2010 rc = pci_alloc_msix(sc->dev, &navail); 2011 else if (itype == INTR_MSI) 2012 rc = pci_alloc_msi(sc->dev, &navail); 2013 2014 if (rc == 0) { 2015 if (navail == iaq->nirq) 2016 return (0); 2017 2018 /* 2019 * Didn't get the number requested. Use whatever number 2020 * the kernel is willing to allocate (it's in navail). 2021 */ 2022 device_printf(sc->dev, "fewer vectors than requested, " 2023 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2024 itype, iaq->nirq, navail); 2025 pci_release_msi(sc->dev); 2026 goto restart; 2027 } 2028 2029 device_printf(sc->dev, 2030 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2031 itype, rc, iaq->nirq, navail); 2032 } 2033 2034 device_printf(sc->dev, 2035 "failed to find a usable interrupt type. " 2036 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2037 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2038 2039 return (ENXIO); 2040} 2041 2042#define FW_VERSION(chip) ( \ 2043 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2044 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2045 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2046 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2047#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2048 2049struct fw_info { 2050 uint8_t chip; 2051 char *kld_name; 2052 char *fw_mod_name; 2053 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2054} fw_info[] = { 2055 { 2056 .chip = CHELSIO_T4, 2057 .kld_name = "t4fw_cfg", 2058 .fw_mod_name = "t4fw", 2059 .fw_hdr = { 2060 .chip = FW_HDR_CHIP_T4, 2061 .fw_ver = htobe32_const(FW_VERSION(T4)), 2062 .intfver_nic = FW_INTFVER(T4, NIC), 2063 .intfver_vnic = FW_INTFVER(T4, VNIC), 2064 .intfver_ofld = FW_INTFVER(T4, OFLD), 2065 .intfver_ri = FW_INTFVER(T4, RI), 2066 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2067 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2068 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2069 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2070 }, 2071 }, { 2072 .chip = CHELSIO_T5, 2073 .kld_name = "t5fw_cfg", 2074 .fw_mod_name = "t5fw", 2075 .fw_hdr = { 2076 .chip = FW_HDR_CHIP_T5, 2077 .fw_ver = htobe32_const(FW_VERSION(T5)), 2078 .intfver_nic = FW_INTFVER(T5, NIC), 2079 .intfver_vnic = FW_INTFVER(T5, VNIC), 2080 .intfver_ofld = FW_INTFVER(T5, OFLD), 2081 .intfver_ri = FW_INTFVER(T5, RI), 2082 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2083 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2084 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2085 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2086 }, 2087 } 2088}; 2089 2090static struct fw_info * 2091find_fw_info(int chip) 2092{ 2093 int i; 2094 2095 for (i = 0; i < nitems(fw_info); i++) { 2096 if (fw_info[i].chip == chip) 2097 return (&fw_info[i]); 2098 } 2099 return (NULL); 2100} 2101 2102/* 2103 * Is the given firmware API compatible with the one the driver was compiled 2104 * with? 2105 */ 2106static int 2107fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2108{ 2109 2110 /* short circuit if it's the exact same firmware version */ 2111 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2112 return (1); 2113 2114 /* 2115 * XXX: Is this too conservative? Perhaps I should limit this to the 2116 * features that are supported in the driver. 2117 */ 2118#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2119 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2120 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2121 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2122 return (1); 2123#undef SAME_INTF 2124 2125 return (0); 2126} 2127 2128/* 2129 * The firmware in the KLD is usable, but should it be installed? This routine 2130 * explains itself in detail if it indicates the KLD firmware should be 2131 * installed. 2132 */ 2133static int 2134should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2135{ 2136 const char *reason; 2137 2138 if (!card_fw_usable) { 2139 reason = "incompatible or unusable"; 2140 goto install; 2141 } 2142 2143 if (k > c) { 2144 reason = "older than the version bundled with this driver"; 2145 goto install; 2146 } 2147 2148 if (t4_fw_install == 2 && k != c) { 2149 reason = "different than the version bundled with this driver"; 2150 goto install; 2151 } 2152 2153 return (0); 2154 2155install: 2156 if (t4_fw_install == 0) { 2157 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2158 "but the driver is prohibited from installing a different " 2159 "firmware on the card.\n", 2160 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2161 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2162 2163 return (0); 2164 } 2165 2166 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2167 "installing firmware %u.%u.%u.%u on card.\n", 2168 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2169 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2170 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2171 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2172 2173 return (1); 2174} 2175/* 2176 * Establish contact with the firmware and determine if we are the master driver 2177 * or not, and whether we are responsible for chip initialization. 2178 */ 2179static int 2180prep_firmware(struct adapter *sc) 2181{ 2182 const struct firmware *fw = NULL, *default_cfg; 2183 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2184 enum dev_state state; 2185 struct fw_info *fw_info; 2186 struct fw_hdr *card_fw; /* fw on the card */ 2187 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2188 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2189 against */ 2190 2191 /* Contact firmware. */ 2192 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2193 if (rc < 0 || state == DEV_STATE_ERR) { 2194 rc = -rc; 2195 device_printf(sc->dev, 2196 "failed to connect to the firmware: %d, %d.\n", rc, state); 2197 return (rc); 2198 } 2199 pf = rc; 2200 if (pf == sc->mbox) 2201 sc->flags |= MASTER_PF; 2202 else if (state == DEV_STATE_UNINIT) { 2203 /* 2204 * We didn't get to be the master so we definitely won't be 2205 * configuring the chip. It's a bug if someone else hasn't 2206 * configured it already. 2207 */ 2208 device_printf(sc->dev, "couldn't be master(%d), " 2209 "device not already initialized either(%d).\n", rc, state); 2210 return (EDOOFUS); 2211 } 2212 2213 /* This is the firmware whose headers the driver was compiled against */ 2214 fw_info = find_fw_info(chip_id(sc)); 2215 if (fw_info == NULL) { 2216 device_printf(sc->dev, 2217 "unable to look up firmware information for chip %d.\n", 2218 chip_id(sc)); 2219 return (EINVAL); 2220 } 2221 drv_fw = &fw_info->fw_hdr; 2222 2223 /* 2224 * The firmware KLD contains many modules. The KLD name is also the 2225 * name of the module that contains the default config file. 2226 */ 2227 default_cfg = firmware_get(fw_info->kld_name); 2228 2229 /* Read the header of the firmware on the card */ 2230 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2231 rc = -t4_read_flash(sc, FLASH_FW_START, 2232 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2233 if (rc == 0) 2234 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2235 else { 2236 device_printf(sc->dev, 2237 "Unable to read card's firmware header: %d\n", rc); 2238 card_fw_usable = 0; 2239 } 2240 2241 /* This is the firmware in the KLD */ 2242 fw = firmware_get(fw_info->fw_mod_name); 2243 if (fw != NULL) { 2244 kld_fw = (const void *)fw->data; 2245 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2246 } else { 2247 kld_fw = NULL; 2248 kld_fw_usable = 0; 2249 } 2250 2251 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2252 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2253 /* 2254 * Common case: the firmware on the card is an exact match and 2255 * the KLD is an exact match too, or the KLD is 2256 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2257 * here -- use cxgbetool loadfw if you want to reinstall the 2258 * same firmware as the one on the card. 2259 */ 2260 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2261 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2262 be32toh(card_fw->fw_ver))) { 2263 2264 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2265 if (rc != 0) { 2266 device_printf(sc->dev, 2267 "failed to install firmware: %d\n", rc); 2268 goto done; 2269 } 2270 2271 /* Installed successfully, update the cached header too. */ 2272 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2273 card_fw_usable = 1; 2274 need_fw_reset = 0; /* already reset as part of load_fw */ 2275 } 2276 2277 if (!card_fw_usable) { 2278 uint32_t d, c, k; 2279 2280 d = ntohl(drv_fw->fw_ver); 2281 c = ntohl(card_fw->fw_ver); 2282 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2283 2284 device_printf(sc->dev, "Cannot find a usable firmware: " 2285 "fw_install %d, chip state %d, " 2286 "driver compiled with %d.%d.%d.%d, " 2287 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2288 t4_fw_install, state, 2289 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2290 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2291 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2292 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2293 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2294 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2295 rc = EINVAL; 2296 goto done; 2297 } 2298 2299 /* We're using whatever's on the card and it's known to be good. */ 2300 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2301 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2302 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2303 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2304 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2305 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2306 t4_get_tp_version(sc, &sc->params.tp_vers); 2307 2308 /* Reset device */ 2309 if (need_fw_reset && 2310 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2311 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2312 if (rc != ETIMEDOUT && rc != EIO) 2313 t4_fw_bye(sc, sc->mbox); 2314 goto done; 2315 } 2316 sc->flags |= FW_OK; 2317 2318 rc = get_params__pre_init(sc); 2319 if (rc != 0) 2320 goto done; /* error message displayed already */ 2321 2322 /* Partition adapter resources as specified in the config file. */ 2323 if (state == DEV_STATE_UNINIT) { 2324 2325 KASSERT(sc->flags & MASTER_PF, 2326 ("%s: trying to change chip settings when not master.", 2327 __func__)); 2328 2329 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2330 if (rc != 0) 2331 goto done; /* error message displayed already */ 2332 2333 t4_tweak_chip_settings(sc); 2334 2335 /* get basic stuff going */ 2336 rc = -t4_fw_initialize(sc, sc->mbox); 2337 if (rc != 0) { 2338 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2339 goto done; 2340 } 2341 } else { 2342 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2343 sc->cfcsum = 0; 2344 } 2345 2346done: 2347 free(card_fw, M_CXGBE); 2348 if (fw != NULL) 2349 firmware_put(fw, FIRMWARE_UNLOAD); 2350 if (default_cfg != NULL) 2351 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2352 2353 return (rc); 2354} 2355 2356#define FW_PARAM_DEV(param) \ 2357 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2358 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2359#define FW_PARAM_PFVF(param) \ 2360 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2361 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2362 2363/* 2364 * Partition chip resources for use between various PFs, VFs, etc. 2365 */ 2366static int 2367partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2368 const char *name_prefix) 2369{ 2370 const struct firmware *cfg = NULL; 2371 int rc = 0; 2372 struct fw_caps_config_cmd caps; 2373 uint32_t mtype, moff, finicsum, cfcsum; 2374 2375 /* 2376 * Figure out what configuration file to use. Pick the default config 2377 * file for the card if the user hasn't specified one explicitly. 2378 */ 2379 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2380 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2381 /* Card specific overrides go here. */ 2382 if (pci_get_device(sc->dev) == 0x440a) 2383 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2384 if (is_fpga(sc)) 2385 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2386 } 2387 2388 /* 2389 * We need to load another module if the profile is anything except 2390 * "default" or "flash". 2391 */ 2392 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2393 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2394 char s[32]; 2395 2396 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2397 cfg = firmware_get(s); 2398 if (cfg == NULL) { 2399 if (default_cfg != NULL) { 2400 device_printf(sc->dev, 2401 "unable to load module \"%s\" for " 2402 "configuration profile \"%s\", will use " 2403 "the default config file instead.\n", 2404 s, sc->cfg_file); 2405 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2406 "%s", DEFAULT_CF); 2407 } else { 2408 device_printf(sc->dev, 2409 "unable to load module \"%s\" for " 2410 "configuration profile \"%s\", will use " 2411 "the config file on the card's flash " 2412 "instead.\n", s, sc->cfg_file); 2413 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2414 "%s", FLASH_CF); 2415 } 2416 } 2417 } 2418 2419 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2420 default_cfg == NULL) { 2421 device_printf(sc->dev, 2422 "default config file not available, will use the config " 2423 "file on the card's flash instead.\n"); 2424 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2425 } 2426 2427 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2428 u_int cflen, i, n; 2429 const uint32_t *cfdata; 2430 uint32_t param, val, addr, off, mw_base, mw_aperture; 2431 2432 KASSERT(cfg != NULL || default_cfg != NULL, 2433 ("%s: no config to upload", __func__)); 2434 2435 /* 2436 * Ask the firmware where it wants us to upload the config file. 2437 */ 2438 param = FW_PARAM_DEV(CF); 2439 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2440 if (rc != 0) { 2441 /* No support for config file? Shouldn't happen. */ 2442 device_printf(sc->dev, 2443 "failed to query config file location: %d.\n", rc); 2444 goto done; 2445 } 2446 mtype = G_FW_PARAMS_PARAM_Y(val); 2447 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2448 2449 /* 2450 * XXX: sheer laziness. We deliberately added 4 bytes of 2451 * useless stuffing/comments at the end of the config file so 2452 * it's ok to simply throw away the last remaining bytes when 2453 * the config file is not an exact multiple of 4. This also 2454 * helps with the validate_mt_off_len check. 2455 */ 2456 if (cfg != NULL) { 2457 cflen = cfg->datasize & ~3; 2458 cfdata = cfg->data; 2459 } else { 2460 cflen = default_cfg->datasize & ~3; 2461 cfdata = default_cfg->data; 2462 } 2463 2464 if (cflen > FLASH_CFG_MAX_SIZE) { 2465 device_printf(sc->dev, 2466 "config file too long (%d, max allowed is %d). " 2467 "Will try to use the config on the card, if any.\n", 2468 cflen, FLASH_CFG_MAX_SIZE); 2469 goto use_config_on_flash; 2470 } 2471 2472 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2473 if (rc != 0) { 2474 device_printf(sc->dev, 2475 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2476 "Will try to use the config on the card, if any.\n", 2477 __func__, mtype, moff, cflen, rc); 2478 goto use_config_on_flash; 2479 } 2480 2481 memwin_info(sc, 2, &mw_base, &mw_aperture); 2482 while (cflen) { 2483 off = position_memwin(sc, 2, addr); 2484 n = min(cflen, mw_aperture - off); 2485 for (i = 0; i < n; i += 4) 2486 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2487 cflen -= n; 2488 addr += n; 2489 } 2490 } else { 2491use_config_on_flash: 2492 mtype = FW_MEMTYPE_FLASH; 2493 moff = t4_flash_cfg_addr(sc); 2494 } 2495 2496 bzero(&caps, sizeof(caps)); 2497 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2498 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2499 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2500 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2501 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2502 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2503 if (rc != 0) { 2504 device_printf(sc->dev, 2505 "failed to pre-process config file: %d " 2506 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2507 goto done; 2508 } 2509 2510 finicsum = be32toh(caps.finicsum); 2511 cfcsum = be32toh(caps.cfcsum); 2512 if (finicsum != cfcsum) { 2513 device_printf(sc->dev, 2514 "WARNING: config file checksum mismatch: %08x %08x\n", 2515 finicsum, cfcsum); 2516 } 2517 sc->cfcsum = cfcsum; 2518 2519#define LIMIT_CAPS(x) do { \ 2520 caps.x &= htobe16(t4_##x##_allowed); \ 2521} while (0) 2522 2523 /* 2524 * Let the firmware know what features will (not) be used so it can tune 2525 * things accordingly. 2526 */ 2527 LIMIT_CAPS(linkcaps); 2528 LIMIT_CAPS(niccaps); 2529 LIMIT_CAPS(toecaps); 2530 LIMIT_CAPS(rdmacaps); 2531 LIMIT_CAPS(iscsicaps); 2532 LIMIT_CAPS(fcoecaps); 2533#undef LIMIT_CAPS 2534 2535 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2536 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2537 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2538 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2539 if (rc != 0) { 2540 device_printf(sc->dev, 2541 "failed to process config file: %d.\n", rc); 2542 } 2543done: 2544 if (cfg != NULL) 2545 firmware_put(cfg, FIRMWARE_UNLOAD); 2546 return (rc); 2547} 2548 2549/* 2550 * Retrieve parameters that are needed (or nice to have) very early. 2551 */ 2552static int 2553get_params__pre_init(struct adapter *sc) 2554{ 2555 int rc; 2556 uint32_t param[2], val[2]; 2557 struct fw_devlog_cmd cmd; 2558 struct devlog_params *dlog = &sc->params.devlog; 2559 2560 param[0] = FW_PARAM_DEV(PORTVEC); 2561 param[1] = FW_PARAM_DEV(CCLK); 2562 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2563 if (rc != 0) { 2564 device_printf(sc->dev, 2565 "failed to query parameters (pre_init): %d.\n", rc); 2566 return (rc); 2567 } 2568 2569 sc->params.portvec = val[0]; 2570 sc->params.nports = bitcount32(val[0]); 2571 sc->params.vpd.cclk = val[1]; 2572 2573 /* Read device log parameters. */ 2574 bzero(&cmd, sizeof(cmd)); 2575 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2576 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2577 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2578 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2579 if (rc != 0) { 2580 device_printf(sc->dev, 2581 "failed to get devlog parameters: %d.\n", rc); 2582 bzero(dlog, sizeof (*dlog)); 2583 rc = 0; /* devlog isn't critical for device operation */ 2584 } else { 2585 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2586 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2587 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2588 dlog->size = be32toh(cmd.memsize_devlog); 2589 } 2590 2591 return (rc); 2592} 2593 2594/* 2595 * Retrieve various parameters that are of interest to the driver. The device 2596 * has been initialized by the firmware at this point. 2597 */ 2598static int 2599get_params__post_init(struct adapter *sc) 2600{ 2601 int rc; 2602 uint32_t param[7], val[7]; 2603 struct fw_caps_config_cmd caps; 2604 2605 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2606 param[1] = FW_PARAM_PFVF(EQ_START); 2607 param[2] = FW_PARAM_PFVF(FILTER_START); 2608 param[3] = FW_PARAM_PFVF(FILTER_END); 2609 param[4] = FW_PARAM_PFVF(L2T_START); 2610 param[5] = FW_PARAM_PFVF(L2T_END); 2611 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2612 if (rc != 0) { 2613 device_printf(sc->dev, 2614 "failed to query parameters (post_init): %d.\n", rc); 2615 return (rc); 2616 } 2617 2618 sc->sge.iq_start = val[0]; 2619 sc->sge.eq_start = val[1]; 2620 sc->tids.ftid_base = val[2]; 2621 sc->tids.nftids = val[3] - val[2] + 1; 2622 sc->params.ftid_min = val[2]; 2623 sc->params.ftid_max = val[3]; 2624 sc->vres.l2t.start = val[4]; 2625 sc->vres.l2t.size = val[5] - val[4] + 1; 2626 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2627 ("%s: L2 table size (%u) larger than expected (%u)", 2628 __func__, sc->vres.l2t.size, L2T_SIZE)); 2629 2630 /* get capabilites */ 2631 bzero(&caps, sizeof(caps)); 2632 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2633 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2634 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2635 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2636 if (rc != 0) { 2637 device_printf(sc->dev, 2638 "failed to get card capabilities: %d.\n", rc); 2639 return (rc); 2640 } 2641 2642#define READ_CAPS(x) do { \ 2643 sc->x = htobe16(caps.x); \ 2644} while (0) 2645 READ_CAPS(linkcaps); 2646 READ_CAPS(niccaps); 2647 READ_CAPS(toecaps); 2648 READ_CAPS(rdmacaps); 2649 READ_CAPS(iscsicaps); 2650 READ_CAPS(fcoecaps); 2651 2652 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2653 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2654 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2655 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2656 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2657 if (rc != 0) { 2658 device_printf(sc->dev, 2659 "failed to query NIC parameters: %d.\n", rc); 2660 return (rc); 2661 } 2662 sc->tids.etid_base = val[0]; 2663 sc->params.etid_min = val[0]; 2664 sc->tids.netids = val[1] - val[0] + 1; 2665 sc->params.netids = sc->tids.netids; 2666 sc->params.eo_wr_cred = val[2]; 2667 sc->params.ethoffload = 1; 2668 } 2669 2670 if (sc->toecaps) { 2671 /* query offload-related parameters */ 2672 param[0] = FW_PARAM_DEV(NTID); 2673 param[1] = FW_PARAM_PFVF(SERVER_START); 2674 param[2] = FW_PARAM_PFVF(SERVER_END); 2675 param[3] = FW_PARAM_PFVF(TDDP_START); 2676 param[4] = FW_PARAM_PFVF(TDDP_END); 2677 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2678 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2679 if (rc != 0) { 2680 device_printf(sc->dev, 2681 "failed to query TOE parameters: %d.\n", rc); 2682 return (rc); 2683 } 2684 sc->tids.ntids = val[0]; 2685 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2686 sc->tids.stid_base = val[1]; 2687 sc->tids.nstids = val[2] - val[1] + 1; 2688 sc->vres.ddp.start = val[3]; 2689 sc->vres.ddp.size = val[4] - val[3] + 1; 2690 sc->params.ofldq_wr_cred = val[5]; 2691 sc->params.offload = 1; 2692 } 2693 if (sc->rdmacaps) { 2694 param[0] = FW_PARAM_PFVF(STAG_START); 2695 param[1] = FW_PARAM_PFVF(STAG_END); 2696 param[2] = FW_PARAM_PFVF(RQ_START); 2697 param[3] = FW_PARAM_PFVF(RQ_END); 2698 param[4] = FW_PARAM_PFVF(PBL_START); 2699 param[5] = FW_PARAM_PFVF(PBL_END); 2700 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2701 if (rc != 0) { 2702 device_printf(sc->dev, 2703 "failed to query RDMA parameters(1): %d.\n", rc); 2704 return (rc); 2705 } 2706 sc->vres.stag.start = val[0]; 2707 sc->vres.stag.size = val[1] - val[0] + 1; 2708 sc->vres.rq.start = val[2]; 2709 sc->vres.rq.size = val[3] - val[2] + 1; 2710 sc->vres.pbl.start = val[4]; 2711 sc->vres.pbl.size = val[5] - val[4] + 1; 2712 2713 param[0] = FW_PARAM_PFVF(SQRQ_START); 2714 param[1] = FW_PARAM_PFVF(SQRQ_END); 2715 param[2] = FW_PARAM_PFVF(CQ_START); 2716 param[3] = FW_PARAM_PFVF(CQ_END); 2717 param[4] = FW_PARAM_PFVF(OCQ_START); 2718 param[5] = FW_PARAM_PFVF(OCQ_END); 2719 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2720 if (rc != 0) { 2721 device_printf(sc->dev, 2722 "failed to query RDMA parameters(2): %d.\n", rc); 2723 return (rc); 2724 } 2725 sc->vres.qp.start = val[0]; 2726 sc->vres.qp.size = val[1] - val[0] + 1; 2727 sc->vres.cq.start = val[2]; 2728 sc->vres.cq.size = val[3] - val[2] + 1; 2729 sc->vres.ocq.start = val[4]; 2730 sc->vres.ocq.size = val[5] - val[4] + 1; 2731 } 2732 if (sc->iscsicaps) { 2733 param[0] = FW_PARAM_PFVF(ISCSI_START); 2734 param[1] = FW_PARAM_PFVF(ISCSI_END); 2735 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2736 if (rc != 0) { 2737 device_printf(sc->dev, 2738 "failed to query iSCSI parameters: %d.\n", rc); 2739 return (rc); 2740 } 2741 sc->vres.iscsi.start = val[0]; 2742 sc->vres.iscsi.size = val[1] - val[0] + 1; 2743 } 2744 2745 /* 2746 * We've got the params we wanted to query via the firmware. Now grab 2747 * some others directly from the chip. 2748 */ 2749 rc = t4_read_chip_settings(sc); 2750 2751 return (rc); 2752} 2753 2754static int 2755set_params__post_init(struct adapter *sc) 2756{ 2757 uint32_t param, val; 2758 2759 /* ask for encapsulated CPLs */ 2760 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2761 val = 1; 2762 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2763 2764 return (0); 2765} 2766 2767#undef FW_PARAM_PFVF 2768#undef FW_PARAM_DEV 2769 2770static void 2771t4_set_desc(struct adapter *sc) 2772{ 2773 char buf[128]; 2774 struct adapter_params *p = &sc->params; 2775 2776 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2777 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2778 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2779 2780 device_set_desc_copy(sc->dev, buf); 2781} 2782 2783static void 2784build_medialist(struct port_info *pi, struct ifmedia *media) 2785{ 2786 int data, m; 2787 2788 PORT_LOCK(pi); 2789 2790 ifmedia_removeall(media); 2791 2792 m = IFM_ETHER | IFM_FDX; 2793 data = (pi->port_type << 8) | pi->mod_type; 2794 2795 switch(pi->port_type) { 2796 case FW_PORT_TYPE_BT_XFI: 2797 case FW_PORT_TYPE_BT_XAUI: 2798 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2799 /* fall through */ 2800 2801 case FW_PORT_TYPE_BT_SGMII: 2802 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2803 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2804 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2805 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2806 break; 2807 2808 case FW_PORT_TYPE_CX4: 2809 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2810 ifmedia_set(media, m | IFM_10G_CX4); 2811 break; 2812 2813 case FW_PORT_TYPE_QSFP_10G: 2814 case FW_PORT_TYPE_SFP: 2815 case FW_PORT_TYPE_FIBER_XFI: 2816 case FW_PORT_TYPE_FIBER_XAUI: 2817 switch (pi->mod_type) { 2818 2819 case FW_PORT_MOD_TYPE_LR: 2820 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2821 ifmedia_set(media, m | IFM_10G_LR); 2822 break; 2823 2824 case FW_PORT_MOD_TYPE_SR: 2825 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2826 ifmedia_set(media, m | IFM_10G_SR); 2827 break; 2828 2829 case FW_PORT_MOD_TYPE_LRM: 2830 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2831 ifmedia_set(media, m | IFM_10G_LRM); 2832 break; 2833 2834 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2835 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2836 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2837 ifmedia_set(media, m | IFM_10G_TWINAX); 2838 break; 2839 2840 case FW_PORT_MOD_TYPE_NONE: 2841 m &= ~IFM_FDX; 2842 ifmedia_add(media, m | IFM_NONE, data, NULL); 2843 ifmedia_set(media, m | IFM_NONE); 2844 break; 2845 2846 case FW_PORT_MOD_TYPE_NA: 2847 case FW_PORT_MOD_TYPE_ER: 2848 default: 2849 device_printf(pi->dev, 2850 "unknown port_type (%d), mod_type (%d)\n", 2851 pi->port_type, pi->mod_type); 2852 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2853 ifmedia_set(media, m | IFM_UNKNOWN); 2854 break; 2855 } 2856 break; 2857 2858 case FW_PORT_TYPE_QSFP: 2859 switch (pi->mod_type) { 2860 2861 case FW_PORT_MOD_TYPE_LR: 2862 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2863 ifmedia_set(media, m | IFM_40G_LR4); 2864 break; 2865 2866 case FW_PORT_MOD_TYPE_SR: 2867 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2868 ifmedia_set(media, m | IFM_40G_SR4); 2869 break; 2870 2871 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2872 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2873 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2874 ifmedia_set(media, m | IFM_40G_CR4); 2875 break; 2876 2877 case FW_PORT_MOD_TYPE_NONE: 2878 m &= ~IFM_FDX; 2879 ifmedia_add(media, m | IFM_NONE, data, NULL); 2880 ifmedia_set(media, m | IFM_NONE); 2881 break; 2882 2883 default: 2884 device_printf(pi->dev, 2885 "unknown port_type (%d), mod_type (%d)\n", 2886 pi->port_type, pi->mod_type); 2887 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2888 ifmedia_set(media, m | IFM_UNKNOWN); 2889 break; 2890 } 2891 break; 2892 2893 default: 2894 device_printf(pi->dev, 2895 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2896 pi->mod_type); 2897 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2898 ifmedia_set(media, m | IFM_UNKNOWN); 2899 break; 2900 } 2901 2902 PORT_UNLOCK(pi); 2903} 2904 2905#define FW_MAC_EXACT_CHUNK 7 2906 2907/* 2908 * Program the port's XGMAC based on parameters in ifnet. The caller also 2909 * indicates which parameters should be programmed (the rest are left alone). 2910 */ 2911int 2912update_mac_settings(struct ifnet *ifp, int flags) 2913{ 2914 int rc = 0; 2915 struct port_info *pi = ifp->if_softc; 2916 struct adapter *sc = pi->adapter; 2917 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2918 uint16_t viid = 0xffff; 2919 int16_t *xact_addr_filt = NULL; 2920 2921 ASSERT_SYNCHRONIZED_OP(sc); 2922 KASSERT(flags, ("%s: not told what to update.", __func__)); 2923 2924 if (ifp == pi->ifp) { 2925 viid = pi->viid; 2926 xact_addr_filt = &pi->xact_addr_filt; 2927 } 2928#ifdef DEV_NETMAP 2929 else if (ifp == pi->nm_ifp) { 2930 viid = pi->nm_viid; 2931 xact_addr_filt = &pi->nm_xact_addr_filt; 2932 } 2933#endif 2934 if (flags & XGMAC_MTU) 2935 mtu = ifp->if_mtu; 2936 2937 if (flags & XGMAC_PROMISC) 2938 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2939 2940 if (flags & XGMAC_ALLMULTI) 2941 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2942 2943 if (flags & XGMAC_VLANEX) 2944 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2945 2946 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 2947 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 2948 1, vlanex, false); 2949 if (rc) { 2950 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 2951 rc); 2952 return (rc); 2953 } 2954 } 2955 2956 if (flags & XGMAC_UCADDR) { 2957 uint8_t ucaddr[ETHER_ADDR_LEN]; 2958 2959 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2960 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 2961 true, true); 2962 if (rc < 0) { 2963 rc = -rc; 2964 if_printf(ifp, "change_mac failed: %d\n", rc); 2965 return (rc); 2966 } else { 2967 *xact_addr_filt = rc; 2968 rc = 0; 2969 } 2970 } 2971 2972 if (flags & XGMAC_MCADDRS) { 2973 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2974 int del = 1; 2975 uint64_t hash = 0; 2976 struct ifmultiaddr *ifma; 2977 int i = 0, j; 2978 2979 if_maddr_rlock(ifp); 2980 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2981 if (ifma->ifma_addr->sa_family != AF_LINK) 2982 continue; 2983 mcaddr[i] = 2984 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2985 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 2986 i++; 2987 2988 if (i == FW_MAC_EXACT_CHUNK) { 2989 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 2990 i, mcaddr, NULL, &hash, 0); 2991 if (rc < 0) { 2992 rc = -rc; 2993 for (j = 0; j < i; j++) { 2994 if_printf(ifp, 2995 "failed to add mc address" 2996 " %02x:%02x:%02x:" 2997 "%02x:%02x:%02x rc=%d\n", 2998 mcaddr[j][0], mcaddr[j][1], 2999 mcaddr[j][2], mcaddr[j][3], 3000 mcaddr[j][4], mcaddr[j][5], 3001 rc); 3002 } 3003 goto mcfail; 3004 } 3005 del = 0; 3006 i = 0; 3007 } 3008 } 3009 if (i > 0) { 3010 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3011 mcaddr, NULL, &hash, 0); 3012 if (rc < 0) { 3013 rc = -rc; 3014 for (j = 0; j < i; j++) { 3015 if_printf(ifp, 3016 "failed to add mc address" 3017 " %02x:%02x:%02x:" 3018 "%02x:%02x:%02x rc=%d\n", 3019 mcaddr[j][0], mcaddr[j][1], 3020 mcaddr[j][2], mcaddr[j][3], 3021 mcaddr[j][4], mcaddr[j][5], 3022 rc); 3023 } 3024 goto mcfail; 3025 } 3026 } 3027 3028 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3029 if (rc != 0) 3030 if_printf(ifp, "failed to set mc address hash: %d", rc); 3031mcfail: 3032 if_maddr_runlock(ifp); 3033 } 3034 3035 return (rc); 3036} 3037 3038/* 3039 * {begin|end}_synchronized_op must be called from the same thread. 3040 */ 3041int 3042begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3043 char *wmesg) 3044{ 3045 int rc, pri; 3046 3047#ifdef WITNESS 3048 /* the caller thinks it's ok to sleep, but is it really? */ 3049 if (flags & SLEEP_OK) 3050 pause("t4slptst", 1); 3051#endif 3052 3053 if (INTR_OK) 3054 pri = PCATCH; 3055 else 3056 pri = 0; 3057 3058 ADAPTER_LOCK(sc); 3059 for (;;) { 3060 3061 if (pi && IS_DOOMED(pi)) { 3062 rc = ENXIO; 3063 goto done; 3064 } 3065 3066 if (!IS_BUSY(sc)) { 3067 rc = 0; 3068 break; 3069 } 3070 3071 if (!(flags & SLEEP_OK)) { 3072 rc = EBUSY; 3073 goto done; 3074 } 3075 3076 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3077 rc = EINTR; 3078 goto done; 3079 } 3080 } 3081 3082 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3083 SET_BUSY(sc); 3084#ifdef INVARIANTS 3085 sc->last_op = wmesg; 3086 sc->last_op_thr = curthread; 3087#endif 3088 3089done: 3090 if (!(flags & HOLD_LOCK) || rc) 3091 ADAPTER_UNLOCK(sc); 3092 3093 return (rc); 3094} 3095 3096/* 3097 * {begin|end}_synchronized_op must be called from the same thread. 3098 */ 3099void 3100end_synchronized_op(struct adapter *sc, int flags) 3101{ 3102 3103 if (flags & LOCK_HELD) 3104 ADAPTER_LOCK_ASSERT_OWNED(sc); 3105 else 3106 ADAPTER_LOCK(sc); 3107 3108 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3109 CLR_BUSY(sc); 3110 wakeup(&sc->flags); 3111 ADAPTER_UNLOCK(sc); 3112} 3113 3114static int 3115cxgbe_init_synchronized(struct port_info *pi) 3116{ 3117 struct adapter *sc = pi->adapter; 3118 struct ifnet *ifp = pi->ifp; 3119 int rc = 0, i; 3120 struct sge_txq *txq; 3121 3122 ASSERT_SYNCHRONIZED_OP(sc); 3123 3124 if (isset(&sc->open_device_map, pi->port_id)) { 3125 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3126 ("mismatch between open_device_map and if_drv_flags")); 3127 return (0); /* already running */ 3128 } 3129 3130 if (!(sc->flags & FULL_INIT_DONE) && 3131 ((rc = adapter_full_init(sc)) != 0)) 3132 return (rc); /* error message displayed already */ 3133 3134 if (!(pi->flags & PORT_INIT_DONE) && 3135 ((rc = port_full_init(pi)) != 0)) 3136 return (rc); /* error message displayed already */ 3137 3138 rc = update_mac_settings(ifp, XGMAC_ALL); 3139 if (rc) 3140 goto done; /* error message displayed already */ 3141 3142 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3143 if (rc != 0) { 3144 if_printf(ifp, "enable_vi failed: %d\n", rc); 3145 goto done; 3146 } 3147 3148 /* 3149 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3150 * if this changes. 3151 */ 3152 3153 for_each_txq(pi, i, txq) { 3154 TXQ_LOCK(txq); 3155 txq->eq.flags |= EQ_ENABLED; 3156 TXQ_UNLOCK(txq); 3157 } 3158 3159 /* 3160 * The first iq of the first port to come up is used for tracing. 3161 */ 3162 if (sc->traceq < 0) { 3163 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3164 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3165 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3166 V_QUEUENUMBER(sc->traceq)); 3167 pi->flags |= HAS_TRACEQ; 3168 } 3169 3170 /* all ok */ 3171 setbit(&sc->open_device_map, pi->port_id); 3172 PORT_LOCK(pi); 3173 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3174 PORT_UNLOCK(pi); 3175 3176 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3177done: 3178 if (rc != 0) 3179 cxgbe_uninit_synchronized(pi); 3180 3181 return (rc); 3182} 3183 3184/* 3185 * Idempotent. 3186 */ 3187static int 3188cxgbe_uninit_synchronized(struct port_info *pi) 3189{ 3190 struct adapter *sc = pi->adapter; 3191 struct ifnet *ifp = pi->ifp; 3192 int rc, i; 3193 struct sge_txq *txq; 3194 3195 ASSERT_SYNCHRONIZED_OP(sc); 3196 3197 /* 3198 * Disable the VI so that all its data in either direction is discarded 3199 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3200 * tick) intact as the TP can deliver negative advice or data that it's 3201 * holding in its RAM (for an offloaded connection) even after the VI is 3202 * disabled. 3203 */ 3204 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3205 if (rc) { 3206 if_printf(ifp, "disable_vi failed: %d\n", rc); 3207 return (rc); 3208 } 3209 3210 for_each_txq(pi, i, txq) { 3211 TXQ_LOCK(txq); 3212 txq->eq.flags &= ~EQ_ENABLED; 3213 TXQ_UNLOCK(txq); 3214 } 3215 3216 clrbit(&sc->open_device_map, pi->port_id); 3217 PORT_LOCK(pi); 3218 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3219 PORT_UNLOCK(pi); 3220 3221 pi->link_cfg.link_ok = 0; 3222 pi->link_cfg.speed = 0; 3223 pi->linkdnrc = -1; 3224 t4_os_link_changed(sc, pi->port_id, 0, -1); 3225 3226 return (0); 3227} 3228 3229/* 3230 * It is ok for this function to fail midway and return right away. t4_detach 3231 * will walk the entire sc->irq list and clean up whatever is valid. 3232 */ 3233static int 3234setup_intr_handlers(struct adapter *sc) 3235{ 3236 int rc, rid, p, q; 3237 char s[8]; 3238 struct irq *irq; 3239 struct port_info *pi; 3240 struct sge_rxq *rxq; 3241#ifdef TCP_OFFLOAD 3242 struct sge_ofld_rxq *ofld_rxq; 3243#endif 3244#ifdef DEV_NETMAP 3245 struct sge_nm_rxq *nm_rxq; 3246#endif 3247 3248 /* 3249 * Setup interrupts. 3250 */ 3251 irq = &sc->irq[0]; 3252 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3253 if (sc->intr_count == 1) 3254 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3255 3256 /* Multiple interrupts. */ 3257 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3258 ("%s: too few intr.", __func__)); 3259 3260 /* The first one is always error intr */ 3261 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3262 if (rc != 0) 3263 return (rc); 3264 irq++; 3265 rid++; 3266 3267 /* The second one is always the firmware event queue */ 3268 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3269 if (rc != 0) 3270 return (rc); 3271 irq++; 3272 rid++; 3273 3274 for_each_port(sc, p) { 3275 pi = sc->port[p]; 3276 3277 if (pi->flags & INTR_RXQ) { 3278 for_each_rxq(pi, q, rxq) { 3279 snprintf(s, sizeof(s), "%d.%d", p, q); 3280 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3281 s); 3282 if (rc != 0) 3283 return (rc); 3284 irq++; 3285 rid++; 3286 } 3287 } 3288#ifdef TCP_OFFLOAD 3289 if (pi->flags & INTR_OFLD_RXQ) { 3290 for_each_ofld_rxq(pi, q, ofld_rxq) { 3291 snprintf(s, sizeof(s), "%d,%d", p, q); 3292 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3293 ofld_rxq, s); 3294 if (rc != 0) 3295 return (rc); 3296 irq++; 3297 rid++; 3298 } 3299 } 3300#endif 3301#ifdef DEV_NETMAP 3302 if (pi->flags & INTR_NM_RXQ) { 3303 for_each_nm_rxq(pi, q, nm_rxq) { 3304 snprintf(s, sizeof(s), "%d-%d", p, q); 3305 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3306 nm_rxq, s); 3307 if (rc != 0) 3308 return (rc); 3309 irq++; 3310 rid++; 3311 } 3312 } 3313#endif 3314 } 3315 MPASS(irq == &sc->irq[sc->intr_count]); 3316 3317 return (0); 3318} 3319 3320int 3321adapter_full_init(struct adapter *sc) 3322{ 3323 int rc, i; 3324 3325 ASSERT_SYNCHRONIZED_OP(sc); 3326 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3327 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3328 ("%s: FULL_INIT_DONE already", __func__)); 3329 3330 /* 3331 * queues that belong to the adapter (not any particular port). 3332 */ 3333 rc = t4_setup_adapter_queues(sc); 3334 if (rc != 0) 3335 goto done; 3336 3337 for (i = 0; i < nitems(sc->tq); i++) { 3338 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3339 taskqueue_thread_enqueue, &sc->tq[i]); 3340 if (sc->tq[i] == NULL) { 3341 device_printf(sc->dev, 3342 "failed to allocate task queue %d\n", i); 3343 rc = ENOMEM; 3344 goto done; 3345 } 3346 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3347 device_get_nameunit(sc->dev), i); 3348 } 3349 3350 t4_intr_enable(sc); 3351 sc->flags |= FULL_INIT_DONE; 3352done: 3353 if (rc != 0) 3354 adapter_full_uninit(sc); 3355 3356 return (rc); 3357} 3358 3359int 3360adapter_full_uninit(struct adapter *sc) 3361{ 3362 int i; 3363 3364 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3365 3366 t4_teardown_adapter_queues(sc); 3367 3368 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3369 taskqueue_free(sc->tq[i]); 3370 sc->tq[i] = NULL; 3371 } 3372 3373 sc->flags &= ~FULL_INIT_DONE; 3374 3375 return (0); 3376} 3377 3378int 3379port_full_init(struct port_info *pi) 3380{ 3381 struct adapter *sc = pi->adapter; 3382 struct ifnet *ifp = pi->ifp; 3383 uint16_t *rss; 3384 struct sge_rxq *rxq; 3385 int rc, i, j; 3386 3387 ASSERT_SYNCHRONIZED_OP(sc); 3388 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3389 ("%s: PORT_INIT_DONE already", __func__)); 3390 3391 sysctl_ctx_init(&pi->ctx); 3392 pi->flags |= PORT_SYSCTL_CTX; 3393 3394 /* 3395 * Allocate tx/rx/fl queues for this port. 3396 */ 3397 rc = t4_setup_port_queues(pi); 3398 if (rc != 0) 3399 goto done; /* error message displayed already */ 3400 3401 /* 3402 * Setup RSS for this port. Save a copy of the RSS table for later use. 3403 */ 3404 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3405 for (i = 0; i < pi->rss_size;) { 3406 for_each_rxq(pi, j, rxq) { 3407 rss[i++] = rxq->iq.abs_id; 3408 if (i == pi->rss_size) 3409 break; 3410 } 3411 } 3412 3413 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3414 pi->rss_size); 3415 if (rc != 0) { 3416 if_printf(ifp, "rss_config failed: %d\n", rc); 3417 goto done; 3418 } 3419 3420 pi->rss = rss; 3421 pi->flags |= PORT_INIT_DONE; 3422done: 3423 if (rc != 0) 3424 port_full_uninit(pi); 3425 3426 return (rc); 3427} 3428 3429/* 3430 * Idempotent. 3431 */ 3432int 3433port_full_uninit(struct port_info *pi) 3434{ 3435 struct adapter *sc = pi->adapter; 3436 int i; 3437 struct sge_rxq *rxq; 3438 struct sge_txq *txq; 3439#ifdef TCP_OFFLOAD 3440 struct sge_ofld_rxq *ofld_rxq; 3441 struct sge_wrq *ofld_txq; 3442#endif 3443 3444 if (pi->flags & PORT_INIT_DONE) { 3445 3446 /* Need to quiesce queues. */ 3447 3448 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3449 3450 for_each_txq(pi, i, txq) { 3451 quiesce_txq(sc, txq); 3452 } 3453 3454#ifdef TCP_OFFLOAD 3455 for_each_ofld_txq(pi, i, ofld_txq) { 3456 quiesce_wrq(sc, ofld_txq); 3457 } 3458#endif 3459 3460 for_each_rxq(pi, i, rxq) { 3461 quiesce_iq(sc, &rxq->iq); 3462 quiesce_fl(sc, &rxq->fl); 3463 } 3464 3465#ifdef TCP_OFFLOAD 3466 for_each_ofld_rxq(pi, i, ofld_rxq) { 3467 quiesce_iq(sc, &ofld_rxq->iq); 3468 quiesce_fl(sc, &ofld_rxq->fl); 3469 } 3470#endif 3471 free(pi->rss, M_CXGBE); 3472 } 3473 3474 t4_teardown_port_queues(pi); 3475 pi->flags &= ~PORT_INIT_DONE; 3476 3477 return (0); 3478} 3479 3480static void 3481quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3482{ 3483 struct sge_eq *eq = &txq->eq; 3484 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3485 3486 (void) sc; /* unused */ 3487 3488#ifdef INVARIANTS 3489 TXQ_LOCK(txq); 3490 MPASS((eq->flags & EQ_ENABLED) == 0); 3491 TXQ_UNLOCK(txq); 3492#endif 3493 3494 /* Wait for the mp_ring to empty. */ 3495 while (!mp_ring_is_idle(txq->r)) { 3496 mp_ring_check_drainage(txq->r, 0); 3497 pause("rquiesce", 1); 3498 } 3499 3500 /* Then wait for the hardware to finish. */ 3501 while (spg->cidx != htobe16(eq->pidx)) 3502 pause("equiesce", 1); 3503 3504 /* Finally, wait for the driver to reclaim all descriptors. */ 3505 while (eq->cidx != eq->pidx) 3506 pause("dquiesce", 1); 3507} 3508 3509static void 3510quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3511{ 3512 3513 /* XXXTX */ 3514} 3515 3516static void 3517quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3518{ 3519 (void) sc; /* unused */ 3520 3521 /* Synchronize with the interrupt handler */ 3522 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3523 pause("iqfree", 1); 3524} 3525 3526static void 3527quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3528{ 3529 mtx_lock(&sc->sfl_lock); 3530 FL_LOCK(fl); 3531 fl->flags |= FL_DOOMED; 3532 FL_UNLOCK(fl); 3533 mtx_unlock(&sc->sfl_lock); 3534 3535 callout_drain(&sc->sfl_callout); 3536 KASSERT((fl->flags & FL_STARVING) == 0, 3537 ("%s: still starving", __func__)); 3538} 3539 3540static int 3541t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3542 driver_intr_t *handler, void *arg, char *name) 3543{ 3544 int rc; 3545 3546 irq->rid = rid; 3547 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3548 RF_SHAREABLE | RF_ACTIVE); 3549 if (irq->res == NULL) { 3550 device_printf(sc->dev, 3551 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3552 return (ENOMEM); 3553 } 3554 3555 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3556 NULL, handler, arg, &irq->tag); 3557 if (rc != 0) { 3558 device_printf(sc->dev, 3559 "failed to setup interrupt for rid %d, name %s: %d\n", 3560 rid, name, rc); 3561 } else if (name) 3562 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3563 3564 return (rc); 3565} 3566 3567static int 3568t4_free_irq(struct adapter *sc, struct irq *irq) 3569{ 3570 if (irq->tag) 3571 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3572 if (irq->res) 3573 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3574 3575 bzero(irq, sizeof(*irq)); 3576 3577 return (0); 3578} 3579 3580static void 3581reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3582 unsigned int end) 3583{ 3584 uint32_t *p = (uint32_t *)(buf + start); 3585 3586 for ( ; start <= end; start += sizeof(uint32_t)) 3587 *p++ = t4_read_reg(sc, start); 3588} 3589 3590static void 3591t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3592{ 3593 int i, n; 3594 const unsigned int *reg_ranges; 3595 static const unsigned int t4_reg_ranges[] = { 3596 0x1008, 0x1108, 3597 0x1180, 0x11b4, 3598 0x11fc, 0x123c, 3599 0x1300, 0x173c, 3600 0x1800, 0x18fc, 3601 0x3000, 0x30d8, 3602 0x30e0, 0x5924, 3603 0x5960, 0x59d4, 3604 0x5a00, 0x5af8, 3605 0x6000, 0x6098, 3606 0x6100, 0x6150, 3607 0x6200, 0x6208, 3608 0x6240, 0x6248, 3609 0x6280, 0x6338, 3610 0x6370, 0x638c, 3611 0x6400, 0x643c, 3612 0x6500, 0x6524, 3613 0x6a00, 0x6a38, 3614 0x6a60, 0x6a78, 3615 0x6b00, 0x6b84, 3616 0x6bf0, 0x6c84, 3617 0x6cf0, 0x6d84, 3618 0x6df0, 0x6e84, 3619 0x6ef0, 0x6f84, 3620 0x6ff0, 0x7084, 3621 0x70f0, 0x7184, 3622 0x71f0, 0x7284, 3623 0x72f0, 0x7384, 3624 0x73f0, 0x7450, 3625 0x7500, 0x7530, 3626 0x7600, 0x761c, 3627 0x7680, 0x76cc, 3628 0x7700, 0x7798, 3629 0x77c0, 0x77fc, 3630 0x7900, 0x79fc, 3631 0x7b00, 0x7c38, 3632 0x7d00, 0x7efc, 3633 0x8dc0, 0x8e1c, 3634 0x8e30, 0x8e78, 3635 0x8ea0, 0x8f6c, 3636 0x8fc0, 0x9074, 3637 0x90fc, 0x90fc, 3638 0x9400, 0x9458, 3639 0x9600, 0x96bc, 3640 0x9800, 0x9808, 3641 0x9820, 0x983c, 3642 0x9850, 0x9864, 3643 0x9c00, 0x9c6c, 3644 0x9c80, 0x9cec, 3645 0x9d00, 0x9d6c, 3646 0x9d80, 0x9dec, 3647 0x9e00, 0x9e6c, 3648 0x9e80, 0x9eec, 3649 0x9f00, 0x9f6c, 3650 0x9f80, 0x9fec, 3651 0xd004, 0xd03c, 3652 0xdfc0, 0xdfe0, 3653 0xe000, 0xea7c, 3654 0xf000, 0x11110, 3655 0x11118, 0x11190, 3656 0x19040, 0x1906c, 3657 0x19078, 0x19080, 3658 0x1908c, 0x19124, 3659 0x19150, 0x191b0, 3660 0x191d0, 0x191e8, 3661 0x19238, 0x1924c, 3662 0x193f8, 0x19474, 3663 0x19490, 0x194f8, 3664 0x19800, 0x19f30, 3665 0x1a000, 0x1a06c, 3666 0x1a0b0, 0x1a120, 3667 0x1a128, 0x1a138, 3668 0x1a190, 0x1a1c4, 3669 0x1a1fc, 0x1a1fc, 3670 0x1e040, 0x1e04c, 3671 0x1e284, 0x1e28c, 3672 0x1e2c0, 0x1e2c0, 3673 0x1e2e0, 0x1e2e0, 3674 0x1e300, 0x1e384, 3675 0x1e3c0, 0x1e3c8, 3676 0x1e440, 0x1e44c, 3677 0x1e684, 0x1e68c, 3678 0x1e6c0, 0x1e6c0, 3679 0x1e6e0, 0x1e6e0, 3680 0x1e700, 0x1e784, 3681 0x1e7c0, 0x1e7c8, 3682 0x1e840, 0x1e84c, 3683 0x1ea84, 0x1ea8c, 3684 0x1eac0, 0x1eac0, 3685 0x1eae0, 0x1eae0, 3686 0x1eb00, 0x1eb84, 3687 0x1ebc0, 0x1ebc8, 3688 0x1ec40, 0x1ec4c, 3689 0x1ee84, 0x1ee8c, 3690 0x1eec0, 0x1eec0, 3691 0x1eee0, 0x1eee0, 3692 0x1ef00, 0x1ef84, 3693 0x1efc0, 0x1efc8, 3694 0x1f040, 0x1f04c, 3695 0x1f284, 0x1f28c, 3696 0x1f2c0, 0x1f2c0, 3697 0x1f2e0, 0x1f2e0, 3698 0x1f300, 0x1f384, 3699 0x1f3c0, 0x1f3c8, 3700 0x1f440, 0x1f44c, 3701 0x1f684, 0x1f68c, 3702 0x1f6c0, 0x1f6c0, 3703 0x1f6e0, 0x1f6e0, 3704 0x1f700, 0x1f784, 3705 0x1f7c0, 0x1f7c8, 3706 0x1f840, 0x1f84c, 3707 0x1fa84, 0x1fa8c, 3708 0x1fac0, 0x1fac0, 3709 0x1fae0, 0x1fae0, 3710 0x1fb00, 0x1fb84, 3711 0x1fbc0, 0x1fbc8, 3712 0x1fc40, 0x1fc4c, 3713 0x1fe84, 0x1fe8c, 3714 0x1fec0, 0x1fec0, 3715 0x1fee0, 0x1fee0, 3716 0x1ff00, 0x1ff84, 3717 0x1ffc0, 0x1ffc8, 3718 0x20000, 0x2002c, 3719 0x20100, 0x2013c, 3720 0x20190, 0x201c8, 3721 0x20200, 0x20318, 3722 0x20400, 0x20528, 3723 0x20540, 0x20614, 3724 0x21000, 0x21040, 3725 0x2104c, 0x21060, 3726 0x210c0, 0x210ec, 3727 0x21200, 0x21268, 3728 0x21270, 0x21284, 3729 0x212fc, 0x21388, 3730 0x21400, 0x21404, 3731 0x21500, 0x21518, 3732 0x2152c, 0x2153c, 3733 0x21550, 0x21554, 3734 0x21600, 0x21600, 3735 0x21608, 0x21628, 3736 0x21630, 0x2163c, 3737 0x21700, 0x2171c, 3738 0x21780, 0x2178c, 3739 0x21800, 0x21c38, 3740 0x21c80, 0x21d7c, 3741 0x21e00, 0x21e04, 3742 0x22000, 0x2202c, 3743 0x22100, 0x2213c, 3744 0x22190, 0x221c8, 3745 0x22200, 0x22318, 3746 0x22400, 0x22528, 3747 0x22540, 0x22614, 3748 0x23000, 0x23040, 3749 0x2304c, 0x23060, 3750 0x230c0, 0x230ec, 3751 0x23200, 0x23268, 3752 0x23270, 0x23284, 3753 0x232fc, 0x23388, 3754 0x23400, 0x23404, 3755 0x23500, 0x23518, 3756 0x2352c, 0x2353c, 3757 0x23550, 0x23554, 3758 0x23600, 0x23600, 3759 0x23608, 0x23628, 3760 0x23630, 0x2363c, 3761 0x23700, 0x2371c, 3762 0x23780, 0x2378c, 3763 0x23800, 0x23c38, 3764 0x23c80, 0x23d7c, 3765 0x23e00, 0x23e04, 3766 0x24000, 0x2402c, 3767 0x24100, 0x2413c, 3768 0x24190, 0x241c8, 3769 0x24200, 0x24318, 3770 0x24400, 0x24528, 3771 0x24540, 0x24614, 3772 0x25000, 0x25040, 3773 0x2504c, 0x25060, 3774 0x250c0, 0x250ec, 3775 0x25200, 0x25268, 3776 0x25270, 0x25284, 3777 0x252fc, 0x25388, 3778 0x25400, 0x25404, 3779 0x25500, 0x25518, 3780 0x2552c, 0x2553c, 3781 0x25550, 0x25554, 3782 0x25600, 0x25600, 3783 0x25608, 0x25628, 3784 0x25630, 0x2563c, 3785 0x25700, 0x2571c, 3786 0x25780, 0x2578c, 3787 0x25800, 0x25c38, 3788 0x25c80, 0x25d7c, 3789 0x25e00, 0x25e04, 3790 0x26000, 0x2602c, 3791 0x26100, 0x2613c, 3792 0x26190, 0x261c8, 3793 0x26200, 0x26318, 3794 0x26400, 0x26528, 3795 0x26540, 0x26614, 3796 0x27000, 0x27040, 3797 0x2704c, 0x27060, 3798 0x270c0, 0x270ec, 3799 0x27200, 0x27268, 3800 0x27270, 0x27284, 3801 0x272fc, 0x27388, 3802 0x27400, 0x27404, 3803 0x27500, 0x27518, 3804 0x2752c, 0x2753c, 3805 0x27550, 0x27554, 3806 0x27600, 0x27600, 3807 0x27608, 0x27628, 3808 0x27630, 0x2763c, 3809 0x27700, 0x2771c, 3810 0x27780, 0x2778c, 3811 0x27800, 0x27c38, 3812 0x27c80, 0x27d7c, 3813 0x27e00, 0x27e04 3814 }; 3815 static const unsigned int t5_reg_ranges[] = { 3816 0x1008, 0x1148, 3817 0x1180, 0x11b4, 3818 0x11fc, 0x123c, 3819 0x1280, 0x173c, 3820 0x1800, 0x18fc, 3821 0x3000, 0x3028, 3822 0x3060, 0x30d8, 3823 0x30e0, 0x30fc, 3824 0x3140, 0x357c, 3825 0x35a8, 0x35cc, 3826 0x35ec, 0x35ec, 3827 0x3600, 0x5624, 3828 0x56cc, 0x575c, 3829 0x580c, 0x5814, 3830 0x5890, 0x58bc, 3831 0x5940, 0x59dc, 3832 0x59fc, 0x5a18, 3833 0x5a60, 0x5a9c, 3834 0x5b94, 0x5bfc, 3835 0x6000, 0x6040, 3836 0x6058, 0x614c, 3837 0x7700, 0x7798, 3838 0x77c0, 0x78fc, 3839 0x7b00, 0x7c54, 3840 0x7d00, 0x7efc, 3841 0x8dc0, 0x8de0, 3842 0x8df8, 0x8e84, 3843 0x8ea0, 0x8f84, 3844 0x8fc0, 0x90f8, 3845 0x9400, 0x9470, 3846 0x9600, 0x96f4, 3847 0x9800, 0x9808, 3848 0x9820, 0x983c, 3849 0x9850, 0x9864, 3850 0x9c00, 0x9c6c, 3851 0x9c80, 0x9cec, 3852 0x9d00, 0x9d6c, 3853 0x9d80, 0x9dec, 3854 0x9e00, 0x9e6c, 3855 0x9e80, 0x9eec, 3856 0x9f00, 0x9f6c, 3857 0x9f80, 0xa020, 3858 0xd004, 0xd03c, 3859 0xdfc0, 0xdfe0, 3860 0xe000, 0x11088, 3861 0x1109c, 0x11110, 3862 0x11118, 0x1117c, 3863 0x11190, 0x11204, 3864 0x19040, 0x1906c, 3865 0x19078, 0x19080, 3866 0x1908c, 0x19124, 3867 0x19150, 0x191b0, 3868 0x191d0, 0x191e8, 3869 0x19238, 0x19290, 3870 0x193f8, 0x19474, 3871 0x19490, 0x194cc, 3872 0x194f0, 0x194f8, 3873 0x19c00, 0x19c60, 3874 0x19c94, 0x19e10, 3875 0x19e50, 0x19f34, 3876 0x19f40, 0x19f50, 3877 0x19f90, 0x19fe4, 3878 0x1a000, 0x1a06c, 3879 0x1a0b0, 0x1a120, 3880 0x1a128, 0x1a138, 3881 0x1a190, 0x1a1c4, 3882 0x1a1fc, 0x1a1fc, 3883 0x1e008, 0x1e00c, 3884 0x1e040, 0x1e04c, 3885 0x1e284, 0x1e290, 3886 0x1e2c0, 0x1e2c0, 3887 0x1e2e0, 0x1e2e0, 3888 0x1e300, 0x1e384, 3889 0x1e3c0, 0x1e3c8, 3890 0x1e408, 0x1e40c, 3891 0x1e440, 0x1e44c, 3892 0x1e684, 0x1e690, 3893 0x1e6c0, 0x1e6c0, 3894 0x1e6e0, 0x1e6e0, 3895 0x1e700, 0x1e784, 3896 0x1e7c0, 0x1e7c8, 3897 0x1e808, 0x1e80c, 3898 0x1e840, 0x1e84c, 3899 0x1ea84, 0x1ea90, 3900 0x1eac0, 0x1eac0, 3901 0x1eae0, 0x1eae0, 3902 0x1eb00, 0x1eb84, 3903 0x1ebc0, 0x1ebc8, 3904 0x1ec08, 0x1ec0c, 3905 0x1ec40, 0x1ec4c, 3906 0x1ee84, 0x1ee90, 3907 0x1eec0, 0x1eec0, 3908 0x1eee0, 0x1eee0, 3909 0x1ef00, 0x1ef84, 3910 0x1efc0, 0x1efc8, 3911 0x1f008, 0x1f00c, 3912 0x1f040, 0x1f04c, 3913 0x1f284, 0x1f290, 3914 0x1f2c0, 0x1f2c0, 3915 0x1f2e0, 0x1f2e0, 3916 0x1f300, 0x1f384, 3917 0x1f3c0, 0x1f3c8, 3918 0x1f408, 0x1f40c, 3919 0x1f440, 0x1f44c, 3920 0x1f684, 0x1f690, 3921 0x1f6c0, 0x1f6c0, 3922 0x1f6e0, 0x1f6e0, 3923 0x1f700, 0x1f784, 3924 0x1f7c0, 0x1f7c8, 3925 0x1f808, 0x1f80c, 3926 0x1f840, 0x1f84c, 3927 0x1fa84, 0x1fa90, 3928 0x1fac0, 0x1fac0, 3929 0x1fae0, 0x1fae0, 3930 0x1fb00, 0x1fb84, 3931 0x1fbc0, 0x1fbc8, 3932 0x1fc08, 0x1fc0c, 3933 0x1fc40, 0x1fc4c, 3934 0x1fe84, 0x1fe90, 3935 0x1fec0, 0x1fec0, 3936 0x1fee0, 0x1fee0, 3937 0x1ff00, 0x1ff84, 3938 0x1ffc0, 0x1ffc8, 3939 0x30000, 0x30030, 3940 0x30100, 0x30144, 3941 0x30190, 0x301d0, 3942 0x30200, 0x30318, 3943 0x30400, 0x3052c, 3944 0x30540, 0x3061c, 3945 0x30800, 0x30834, 3946 0x308c0, 0x30908, 3947 0x30910, 0x309ac, 3948 0x30a00, 0x30a2c, 3949 0x30a44, 0x30a50, 3950 0x30a74, 0x30c24, 3951 0x30d00, 0x30d00, 3952 0x30d08, 0x30d14, 3953 0x30d1c, 0x30d20, 3954 0x30d3c, 0x30d50, 3955 0x31200, 0x3120c, 3956 0x31220, 0x31220, 3957 0x31240, 0x31240, 3958 0x31600, 0x3160c, 3959 0x31a00, 0x31a1c, 3960 0x31e00, 0x31e20, 3961 0x31e38, 0x31e3c, 3962 0x31e80, 0x31e80, 3963 0x31e88, 0x31ea8, 3964 0x31eb0, 0x31eb4, 3965 0x31ec8, 0x31ed4, 3966 0x31fb8, 0x32004, 3967 0x32200, 0x32200, 3968 0x32208, 0x32240, 3969 0x32248, 0x32280, 3970 0x32288, 0x322c0, 3971 0x322c8, 0x322fc, 3972 0x32600, 0x32630, 3973 0x32a00, 0x32abc, 3974 0x32b00, 0x32b70, 3975 0x33000, 0x33048, 3976 0x33060, 0x3309c, 3977 0x330f0, 0x33148, 3978 0x33160, 0x3319c, 3979 0x331f0, 0x332e4, 3980 0x332f8, 0x333e4, 3981 0x333f8, 0x33448, 3982 0x33460, 0x3349c, 3983 0x334f0, 0x33548, 3984 0x33560, 0x3359c, 3985 0x335f0, 0x336e4, 3986 0x336f8, 0x337e4, 3987 0x337f8, 0x337fc, 3988 0x33814, 0x33814, 3989 0x3382c, 0x3382c, 3990 0x33880, 0x3388c, 3991 0x338e8, 0x338ec, 3992 0x33900, 0x33948, 3993 0x33960, 0x3399c, 3994 0x339f0, 0x33ae4, 3995 0x33af8, 0x33b10, 3996 0x33b28, 0x33b28, 3997 0x33b3c, 0x33b50, 3998 0x33bf0, 0x33c10, 3999 0x33c28, 0x33c28, 4000 0x33c3c, 0x33c50, 4001 0x33cf0, 0x33cfc, 4002 0x34000, 0x34030, 4003 0x34100, 0x34144, 4004 0x34190, 0x341d0, 4005 0x34200, 0x34318, 4006 0x34400, 0x3452c, 4007 0x34540, 0x3461c, 4008 0x34800, 0x34834, 4009 0x348c0, 0x34908, 4010 0x34910, 0x349ac, 4011 0x34a00, 0x34a2c, 4012 0x34a44, 0x34a50, 4013 0x34a74, 0x34c24, 4014 0x34d00, 0x34d00, 4015 0x34d08, 0x34d14, 4016 0x34d1c, 0x34d20, 4017 0x34d3c, 0x34d50, 4018 0x35200, 0x3520c, 4019 0x35220, 0x35220, 4020 0x35240, 0x35240, 4021 0x35600, 0x3560c, 4022 0x35a00, 0x35a1c, 4023 0x35e00, 0x35e20, 4024 0x35e38, 0x35e3c, 4025 0x35e80, 0x35e80, 4026 0x35e88, 0x35ea8, 4027 0x35eb0, 0x35eb4, 4028 0x35ec8, 0x35ed4, 4029 0x35fb8, 0x36004, 4030 0x36200, 0x36200, 4031 0x36208, 0x36240, 4032 0x36248, 0x36280, 4033 0x36288, 0x362c0, 4034 0x362c8, 0x362fc, 4035 0x36600, 0x36630, 4036 0x36a00, 0x36abc, 4037 0x36b00, 0x36b70, 4038 0x37000, 0x37048, 4039 0x37060, 0x3709c, 4040 0x370f0, 0x37148, 4041 0x37160, 0x3719c, 4042 0x371f0, 0x372e4, 4043 0x372f8, 0x373e4, 4044 0x373f8, 0x37448, 4045 0x37460, 0x3749c, 4046 0x374f0, 0x37548, 4047 0x37560, 0x3759c, 4048 0x375f0, 0x376e4, 4049 0x376f8, 0x377e4, 4050 0x377f8, 0x377fc, 4051 0x37814, 0x37814, 4052 0x3782c, 0x3782c, 4053 0x37880, 0x3788c, 4054 0x378e8, 0x378ec, 4055 0x37900, 0x37948, 4056 0x37960, 0x3799c, 4057 0x379f0, 0x37ae4, 4058 0x37af8, 0x37b10, 4059 0x37b28, 0x37b28, 4060 0x37b3c, 0x37b50, 4061 0x37bf0, 0x37c10, 4062 0x37c28, 0x37c28, 4063 0x37c3c, 0x37c50, 4064 0x37cf0, 0x37cfc, 4065 0x38000, 0x38030, 4066 0x38100, 0x38144, 4067 0x38190, 0x381d0, 4068 0x38200, 0x38318, 4069 0x38400, 0x3852c, 4070 0x38540, 0x3861c, 4071 0x38800, 0x38834, 4072 0x388c0, 0x38908, 4073 0x38910, 0x389ac, 4074 0x38a00, 0x38a2c, 4075 0x38a44, 0x38a50, 4076 0x38a74, 0x38c24, 4077 0x38d00, 0x38d00, 4078 0x38d08, 0x38d14, 4079 0x38d1c, 0x38d20, 4080 0x38d3c, 0x38d50, 4081 0x39200, 0x3920c, 4082 0x39220, 0x39220, 4083 0x39240, 0x39240, 4084 0x39600, 0x3960c, 4085 0x39a00, 0x39a1c, 4086 0x39e00, 0x39e20, 4087 0x39e38, 0x39e3c, 4088 0x39e80, 0x39e80, 4089 0x39e88, 0x39ea8, 4090 0x39eb0, 0x39eb4, 4091 0x39ec8, 0x39ed4, 4092 0x39fb8, 0x3a004, 4093 0x3a200, 0x3a200, 4094 0x3a208, 0x3a240, 4095 0x3a248, 0x3a280, 4096 0x3a288, 0x3a2c0, 4097 0x3a2c8, 0x3a2fc, 4098 0x3a600, 0x3a630, 4099 0x3aa00, 0x3aabc, 4100 0x3ab00, 0x3ab70, 4101 0x3b000, 0x3b048, 4102 0x3b060, 0x3b09c, 4103 0x3b0f0, 0x3b148, 4104 0x3b160, 0x3b19c, 4105 0x3b1f0, 0x3b2e4, 4106 0x3b2f8, 0x3b3e4, 4107 0x3b3f8, 0x3b448, 4108 0x3b460, 0x3b49c, 4109 0x3b4f0, 0x3b548, 4110 0x3b560, 0x3b59c, 4111 0x3b5f0, 0x3b6e4, 4112 0x3b6f8, 0x3b7e4, 4113 0x3b7f8, 0x3b7fc, 4114 0x3b814, 0x3b814, 4115 0x3b82c, 0x3b82c, 4116 0x3b880, 0x3b88c, 4117 0x3b8e8, 0x3b8ec, 4118 0x3b900, 0x3b948, 4119 0x3b960, 0x3b99c, 4120 0x3b9f0, 0x3bae4, 4121 0x3baf8, 0x3bb10, 4122 0x3bb28, 0x3bb28, 4123 0x3bb3c, 0x3bb50, 4124 0x3bbf0, 0x3bc10, 4125 0x3bc28, 0x3bc28, 4126 0x3bc3c, 0x3bc50, 4127 0x3bcf0, 0x3bcfc, 4128 0x3c000, 0x3c030, 4129 0x3c100, 0x3c144, 4130 0x3c190, 0x3c1d0, 4131 0x3c200, 0x3c318, 4132 0x3c400, 0x3c52c, 4133 0x3c540, 0x3c61c, 4134 0x3c800, 0x3c834, 4135 0x3c8c0, 0x3c908, 4136 0x3c910, 0x3c9ac, 4137 0x3ca00, 0x3ca2c, 4138 0x3ca44, 0x3ca50, 4139 0x3ca74, 0x3cc24, 4140 0x3cd00, 0x3cd00, 4141 0x3cd08, 0x3cd14, 4142 0x3cd1c, 0x3cd20, 4143 0x3cd3c, 0x3cd50, 4144 0x3d200, 0x3d20c, 4145 0x3d220, 0x3d220, 4146 0x3d240, 0x3d240, 4147 0x3d600, 0x3d60c, 4148 0x3da00, 0x3da1c, 4149 0x3de00, 0x3de20, 4150 0x3de38, 0x3de3c, 4151 0x3de80, 0x3de80, 4152 0x3de88, 0x3dea8, 4153 0x3deb0, 0x3deb4, 4154 0x3dec8, 0x3ded4, 4155 0x3dfb8, 0x3e004, 4156 0x3e200, 0x3e200, 4157 0x3e208, 0x3e240, 4158 0x3e248, 0x3e280, 4159 0x3e288, 0x3e2c0, 4160 0x3e2c8, 0x3e2fc, 4161 0x3e600, 0x3e630, 4162 0x3ea00, 0x3eabc, 4163 0x3eb00, 0x3eb70, 4164 0x3f000, 0x3f048, 4165 0x3f060, 0x3f09c, 4166 0x3f0f0, 0x3f148, 4167 0x3f160, 0x3f19c, 4168 0x3f1f0, 0x3f2e4, 4169 0x3f2f8, 0x3f3e4, 4170 0x3f3f8, 0x3f448, 4171 0x3f460, 0x3f49c, 4172 0x3f4f0, 0x3f548, 4173 0x3f560, 0x3f59c, 4174 0x3f5f0, 0x3f6e4, 4175 0x3f6f8, 0x3f7e4, 4176 0x3f7f8, 0x3f7fc, 4177 0x3f814, 0x3f814, 4178 0x3f82c, 0x3f82c, 4179 0x3f880, 0x3f88c, 4180 0x3f8e8, 0x3f8ec, 4181 0x3f900, 0x3f948, 4182 0x3f960, 0x3f99c, 4183 0x3f9f0, 0x3fae4, 4184 0x3faf8, 0x3fb10, 4185 0x3fb28, 0x3fb28, 4186 0x3fb3c, 0x3fb50, 4187 0x3fbf0, 0x3fc10, 4188 0x3fc28, 0x3fc28, 4189 0x3fc3c, 0x3fc50, 4190 0x3fcf0, 0x3fcfc, 4191 0x40000, 0x4000c, 4192 0x40040, 0x40068, 4193 0x4007c, 0x40144, 4194 0x40180, 0x4018c, 4195 0x40200, 0x40298, 4196 0x402ac, 0x4033c, 4197 0x403f8, 0x403fc, 4198 0x41304, 0x413c4, 4199 0x41400, 0x4141c, 4200 0x41480, 0x414d0, 4201 0x44000, 0x44078, 4202 0x440c0, 0x44278, 4203 0x442c0, 0x44478, 4204 0x444c0, 0x44678, 4205 0x446c0, 0x44878, 4206 0x448c0, 0x449fc, 4207 0x45000, 0x45068, 4208 0x45080, 0x45084, 4209 0x450a0, 0x450b0, 4210 0x45200, 0x45268, 4211 0x45280, 0x45284, 4212 0x452a0, 0x452b0, 4213 0x460c0, 0x460e4, 4214 0x47000, 0x4708c, 4215 0x47200, 0x47250, 4216 0x47400, 0x47420, 4217 0x47600, 0x47618, 4218 0x47800, 0x47814, 4219 0x48000, 0x4800c, 4220 0x48040, 0x48068, 4221 0x4807c, 0x48144, 4222 0x48180, 0x4818c, 4223 0x48200, 0x48298, 4224 0x482ac, 0x4833c, 4225 0x483f8, 0x483fc, 4226 0x49304, 0x493c4, 4227 0x49400, 0x4941c, 4228 0x49480, 0x494d0, 4229 0x4c000, 0x4c078, 4230 0x4c0c0, 0x4c278, 4231 0x4c2c0, 0x4c478, 4232 0x4c4c0, 0x4c678, 4233 0x4c6c0, 0x4c878, 4234 0x4c8c0, 0x4c9fc, 4235 0x4d000, 0x4d068, 4236 0x4d080, 0x4d084, 4237 0x4d0a0, 0x4d0b0, 4238 0x4d200, 0x4d268, 4239 0x4d280, 0x4d284, 4240 0x4d2a0, 0x4d2b0, 4241 0x4e0c0, 0x4e0e4, 4242 0x4f000, 0x4f08c, 4243 0x4f200, 0x4f250, 4244 0x4f400, 0x4f420, 4245 0x4f600, 0x4f618, 4246 0x4f800, 0x4f814, 4247 0x50000, 0x500cc, 4248 0x50400, 0x50400, 4249 0x50800, 0x508cc, 4250 0x50c00, 0x50c00, 4251 0x51000, 0x5101c, 4252 0x51300, 0x51308, 4253 }; 4254 4255 if (is_t4(sc)) { 4256 reg_ranges = &t4_reg_ranges[0]; 4257 n = nitems(t4_reg_ranges); 4258 } else { 4259 reg_ranges = &t5_reg_ranges[0]; 4260 n = nitems(t5_reg_ranges); 4261 } 4262 4263 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4264 for (i = 0; i < n; i += 2) 4265 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4266} 4267 4268static void 4269cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4270{ 4271 struct ifnet *ifp = pi->ifp; 4272 struct sge_txq *txq; 4273 int i, drops; 4274 struct port_stats *s = &pi->stats; 4275 struct timeval tv; 4276 const struct timeval interval = {0, 250000}; /* 250ms */ 4277 4278 getmicrotime(&tv); 4279 timevalsub(&tv, &interval); 4280 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4281 return; 4282 4283 t4_get_port_stats(sc, pi->tx_chan, s); 4284 4285 ifp->if_opackets = s->tx_frames - s->tx_pause; 4286 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4287 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4288 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4289 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4290 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4291 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4292 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4293 s->rx_trunc3; 4294 for (i = 0; i < NCHAN; i++) { 4295 if (pi->rx_chan_map & (1 << i)) { 4296 uint32_t v; 4297 4298 mtx_lock(&sc->regwin_lock); 4299 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4300 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4301 mtx_unlock(&sc->regwin_lock); 4302 ifp->if_iqdrops += v; 4303 } 4304 } 4305 4306 drops = s->tx_drop; 4307 for_each_txq(pi, i, txq) 4308 drops += counter_u64_fetch(txq->r->drops); 4309 ifp->if_snd.ifq_drops = drops; 4310 4311 ifp->if_oerrors = s->tx_error_frames; 4312 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4313 s->rx_fcs_err + s->rx_len_err; 4314 4315 getmicrotime(&pi->last_refreshed); 4316} 4317 4318static void 4319cxgbe_tick(void *arg) 4320{ 4321 struct port_info *pi = arg; 4322 struct adapter *sc = pi->adapter; 4323 struct ifnet *ifp = pi->ifp; 4324 4325 PORT_LOCK(pi); 4326 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4327 PORT_UNLOCK(pi); 4328 return; /* without scheduling another callout */ 4329 } 4330 4331 cxgbe_refresh_stats(sc, pi); 4332 4333 callout_schedule(&pi->tick, hz); 4334 PORT_UNLOCK(pi); 4335} 4336 4337static void 4338cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4339{ 4340 struct ifnet *vlan; 4341 4342 if (arg != ifp || ifp->if_type != IFT_ETHER) 4343 return; 4344 4345 vlan = VLAN_DEVAT(ifp, vid); 4346 VLAN_SETCOOKIE(vlan, ifp); 4347} 4348 4349static int 4350cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4351{ 4352 4353#ifdef INVARIANTS 4354 panic("%s: opcode 0x%02x on iq %p with payload %p", 4355 __func__, rss->opcode, iq, m); 4356#else 4357 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4358 __func__, rss->opcode, iq, m); 4359 m_freem(m); 4360#endif 4361 return (EDOOFUS); 4362} 4363 4364int 4365t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4366{ 4367 uintptr_t *loc, new; 4368 4369 if (opcode >= nitems(sc->cpl_handler)) 4370 return (EINVAL); 4371 4372 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4373 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4374 atomic_store_rel_ptr(loc, new); 4375 4376 return (0); 4377} 4378 4379static int 4380an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4381{ 4382 4383#ifdef INVARIANTS 4384 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4385#else 4386 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4387 __func__, iq, ctrl); 4388#endif 4389 return (EDOOFUS); 4390} 4391 4392int 4393t4_register_an_handler(struct adapter *sc, an_handler_t h) 4394{ 4395 uintptr_t *loc, new; 4396 4397 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4398 loc = (uintptr_t *) &sc->an_handler; 4399 atomic_store_rel_ptr(loc, new); 4400 4401 return (0); 4402} 4403 4404static int 4405fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4406{ 4407 const struct cpl_fw6_msg *cpl = 4408 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4409 4410#ifdef INVARIANTS 4411 panic("%s: fw_msg type %d", __func__, cpl->type); 4412#else 4413 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4414#endif 4415 return (EDOOFUS); 4416} 4417 4418int 4419t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4420{ 4421 uintptr_t *loc, new; 4422 4423 if (type >= nitems(sc->fw_msg_handler)) 4424 return (EINVAL); 4425 4426 /* 4427 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4428 * handler dispatch table. Reject any attempt to install a handler for 4429 * this subtype. 4430 */ 4431 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4432 return (EINVAL); 4433 4434 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4435 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4436 atomic_store_rel_ptr(loc, new); 4437 4438 return (0); 4439} 4440 4441static int 4442t4_sysctls(struct adapter *sc) 4443{ 4444 struct sysctl_ctx_list *ctx; 4445 struct sysctl_oid *oid; 4446 struct sysctl_oid_list *children, *c0; 4447 static char *caps[] = { 4448 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4449 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4450 "\6HASHFILTER\7ETHOFLD", 4451 "\20\1TOE", /* caps[2] toecaps */ 4452 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4453 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4454 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4455 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4456 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4457 "\4PO_INITIAOR\5PO_TARGET" 4458 }; 4459 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4460 4461 ctx = device_get_sysctl_ctx(sc->dev); 4462 4463 /* 4464 * dev.t4nex.X. 4465 */ 4466 oid = device_get_sysctl_tree(sc->dev); 4467 c0 = children = SYSCTL_CHILDREN(oid); 4468 4469 sc->sc_do_rxcopy = 1; 4470 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4471 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4472 4473 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4474 sc->params.nports, "# of ports"); 4475 4476 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4477 NULL, chip_rev(sc), "chip hardware revision"); 4478 4479 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4480 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4481 4482 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4483 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4484 4485 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4486 sc->cfcsum, "config file checksum"); 4487 4488 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4489 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4490 sysctl_bitfield, "A", "available doorbells"); 4491 4492 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4493 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4494 sysctl_bitfield, "A", "available link capabilities"); 4495 4496 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4497 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4498 sysctl_bitfield, "A", "available NIC capabilities"); 4499 4500 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4501 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4502 sysctl_bitfield, "A", "available TCP offload capabilities"); 4503 4504 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4505 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4506 sysctl_bitfield, "A", "available RDMA capabilities"); 4507 4508 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4509 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4510 sysctl_bitfield, "A", "available iSCSI capabilities"); 4511 4512 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4513 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4514 sysctl_bitfield, "A", "available FCoE capabilities"); 4515 4516 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4517 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4518 4519 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4520 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4521 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4522 "interrupt holdoff timer values (us)"); 4523 4524 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4525 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4526 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4527 "interrupt holdoff packet counter values"); 4528 4529 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4530 NULL, sc->tids.nftids, "number of filters"); 4531 4532 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4533 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4534 "chip temperature (in Celsius)"); 4535 4536 t4_sge_sysctls(sc, ctx, children); 4537 4538 sc->lro_timeout = 100; 4539 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4540 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4541 4542 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 4543 &sc->debug_flags, 0, "flags to enable runtime debugging"); 4544 4545#ifdef SBUF_DRAIN 4546 /* 4547 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4548 */ 4549 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4550 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4551 "logs and miscellaneous information"); 4552 children = SYSCTL_CHILDREN(oid); 4553 4554 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4555 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4556 sysctl_cctrl, "A", "congestion control"); 4557 4558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4559 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4560 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4561 4562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4563 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4564 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4565 4566 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4567 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4568 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4569 4570 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4571 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4572 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4573 4574 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4575 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4576 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4577 4578 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4579 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4580 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4581 4582 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4583 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4584 sysctl_cim_la, "A", "CIM logic analyzer"); 4585 4586 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4587 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4588 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4589 4590 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4591 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4592 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4593 4594 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4595 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4596 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4597 4598 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4599 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4600 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4601 4602 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4603 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4604 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4605 4606 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4607 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4608 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4609 4610 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4611 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4612 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4613 4614 if (is_t5(sc)) { 4615 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4616 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4617 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4618 4619 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4620 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4621 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4622 } 4623 4624 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4625 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4626 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4627 4628 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4629 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4630 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4631 4632 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4633 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4634 sysctl_cpl_stats, "A", "CPL statistics"); 4635 4636 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4637 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4638 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4639 4640 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4641 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4642 sysctl_devlog, "A", "firmware's device log"); 4643 4644 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4645 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4646 sysctl_fcoe_stats, "A", "FCoE statistics"); 4647 4648 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4649 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4650 sysctl_hw_sched, "A", "hardware scheduler "); 4651 4652 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4653 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4654 sysctl_l2t, "A", "hardware L2 table"); 4655 4656 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4657 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4658 sysctl_lb_stats, "A", "loopback statistics"); 4659 4660 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4661 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4662 sysctl_meminfo, "A", "memory regions"); 4663 4664 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4665 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4666 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4667 4668 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4669 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4670 sysctl_path_mtus, "A", "path MTUs"); 4671 4672 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4673 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4674 sysctl_pm_stats, "A", "PM statistics"); 4675 4676 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4677 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4678 sysctl_rdma_stats, "A", "RDMA statistics"); 4679 4680 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4681 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4682 sysctl_tcp_stats, "A", "TCP statistics"); 4683 4684 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4685 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4686 sysctl_tids, "A", "TID information"); 4687 4688 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4689 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4690 sysctl_tp_err_stats, "A", "TP error statistics"); 4691 4692 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4693 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4694 sysctl_tp_la, "A", "TP logic analyzer"); 4695 4696 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4697 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4698 sysctl_tx_rate, "A", "Tx rate"); 4699 4700 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4701 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4702 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4703 4704 if (is_t5(sc)) { 4705 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4706 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4707 sysctl_wcwr_stats, "A", "write combined work requests"); 4708 } 4709#endif 4710 4711#ifdef TCP_OFFLOAD 4712 if (is_offload(sc)) { 4713 /* 4714 * dev.t4nex.X.toe. 4715 */ 4716 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4717 NULL, "TOE parameters"); 4718 children = SYSCTL_CHILDREN(oid); 4719 4720 sc->tt.sndbuf = 256 * 1024; 4721 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4722 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4723 4724 sc->tt.ddp = 0; 4725 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4726 &sc->tt.ddp, 0, "DDP allowed"); 4727 4728 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4729 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4730 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4731 4732 sc->tt.ddp_thres = 4733 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4734 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4735 &sc->tt.ddp_thres, 0, "DDP threshold"); 4736 4737 sc->tt.rx_coalesce = 1; 4738 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4739 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4740 4741 sc->tt.tx_align = 1; 4742 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4743 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4744 } 4745#endif 4746 4747 4748 return (0); 4749} 4750 4751static int 4752cxgbe_sysctls(struct port_info *pi) 4753{ 4754 struct sysctl_ctx_list *ctx; 4755 struct sysctl_oid *oid; 4756 struct sysctl_oid_list *children; 4757 struct adapter *sc = pi->adapter; 4758 4759 ctx = device_get_sysctl_ctx(pi->dev); 4760 4761 /* 4762 * dev.cxgbe.X. 4763 */ 4764 oid = device_get_sysctl_tree(pi->dev); 4765 children = SYSCTL_CHILDREN(oid); 4766 4767 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4768 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4769 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4770 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4771 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4772 "PHY temperature (in Celsius)"); 4773 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4774 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4775 "PHY firmware version"); 4776 } 4777 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4778 &pi->nrxq, 0, "# of rx queues"); 4779 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4780 &pi->ntxq, 0, "# of tx queues"); 4781 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4782 &pi->first_rxq, 0, "index of first rx queue"); 4783 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4784 &pi->first_txq, 0, "index of first tx queue"); 4785 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4786 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4787 "Reserve queue 0 for non-flowid packets"); 4788 4789#ifdef TCP_OFFLOAD 4790 if (is_offload(sc)) { 4791 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4792 &pi->nofldrxq, 0, 4793 "# of rx queues for offloaded TCP connections"); 4794 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4795 &pi->nofldtxq, 0, 4796 "# of tx queues for offloaded TCP connections"); 4797 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4798 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4799 "index of first TOE rx queue"); 4800 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4801 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4802 "index of first TOE tx queue"); 4803 } 4804#endif 4805#ifdef DEV_NETMAP 4806 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4807 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4808 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4809 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4810 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4811 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4812 "index of first netmap rx queue"); 4813 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4814 CTLFLAG_RD, &pi->first_nm_txq, 0, 4815 "index of first netmap tx queue"); 4816#endif 4817 4818 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4819 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4820 "holdoff timer index"); 4821 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4822 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4823 "holdoff packet counter index"); 4824 4825 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4826 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4827 "rx queue size"); 4828 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4829 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4830 "tx queue size"); 4831 4832 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4833 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4834 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4835 4836 /* 4837 * dev.cxgbe.X.stats. 4838 */ 4839 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4840 NULL, "port statistics"); 4841 children = SYSCTL_CHILDREN(oid); 4842 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4843 &pi->tx_parse_error, 0, 4844 "# of tx packets with invalid length or # of segments"); 4845 4846#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4847 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4848 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4849 sysctl_handle_t4_reg64, "QU", desc) 4850 4851 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4852 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4853 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4854 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4855 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4856 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4857 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4858 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4859 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4860 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4861 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4862 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4863 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4864 "# of tx frames in this range", 4865 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4866 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4867 "# of tx frames in this range", 4868 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4869 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4870 "# of tx frames in this range", 4871 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4872 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4873 "# of tx frames in this range", 4874 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4875 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4876 "# of tx frames in this range", 4877 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4878 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4879 "# of tx frames in this range", 4880 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4881 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4882 "# of tx frames in this range", 4883 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4884 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4885 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4886 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4887 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4888 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4889 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4890 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4891 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4892 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4893 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4894 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4895 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4896 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4897 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4898 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4899 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4900 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4901 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4902 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4903 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4904 4905 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4906 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4907 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4908 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4909 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4910 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4911 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4912 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4913 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4914 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4915 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4916 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4917 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4918 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4919 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4920 "# of frames received with bad FCS", 4921 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4922 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4923 "# of frames received with length error", 4924 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4925 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4926 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4927 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4928 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4929 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4930 "# of rx frames in this range", 4931 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4932 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4933 "# of rx frames in this range", 4934 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4935 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4936 "# of rx frames in this range", 4937 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4938 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4939 "# of rx frames in this range", 4940 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4941 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4942 "# of rx frames in this range", 4943 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4944 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4945 "# of rx frames in this range", 4946 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4947 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4948 "# of rx frames in this range", 4949 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4950 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4951 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4952 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4953 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4954 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4955 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4956 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4957 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4958 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4959 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4960 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4961 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4962 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4963 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4964 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4965 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4966 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4967 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4968 4969#undef SYSCTL_ADD_T4_REG64 4970 4971#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4972 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4973 &pi->stats.name, desc) 4974 4975 /* We get these from port_stats and they may be stale by upto 1s */ 4976 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4977 "# drops due to buffer-group 0 overflows"); 4978 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4979 "# drops due to buffer-group 1 overflows"); 4980 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4981 "# drops due to buffer-group 2 overflows"); 4982 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4983 "# drops due to buffer-group 3 overflows"); 4984 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4985 "# of buffer-group 0 truncated packets"); 4986 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4987 "# of buffer-group 1 truncated packets"); 4988 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4989 "# of buffer-group 2 truncated packets"); 4990 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4991 "# of buffer-group 3 truncated packets"); 4992 4993#undef SYSCTL_ADD_T4_PORTSTAT 4994 4995 return (0); 4996} 4997 4998static int 4999sysctl_int_array(SYSCTL_HANDLER_ARGS) 5000{ 5001 int rc, *i, space = 0; 5002 struct sbuf sb; 5003 5004 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5005 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5006 if (space) 5007 sbuf_printf(&sb, " "); 5008 sbuf_printf(&sb, "%d", *i); 5009 space = 1; 5010 } 5011 sbuf_finish(&sb); 5012 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5013 sbuf_delete(&sb); 5014 return (rc); 5015} 5016 5017static int 5018sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5019{ 5020 int rc; 5021 struct sbuf *sb; 5022 5023 rc = sysctl_wire_old_buffer(req, 0); 5024 if (rc != 0) 5025 return(rc); 5026 5027 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5028 if (sb == NULL) 5029 return (ENOMEM); 5030 5031 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5032 rc = sbuf_finish(sb); 5033 sbuf_delete(sb); 5034 5035 return (rc); 5036} 5037 5038static int 5039sysctl_btphy(SYSCTL_HANDLER_ARGS) 5040{ 5041 struct port_info *pi = arg1; 5042 int op = arg2; 5043 struct adapter *sc = pi->adapter; 5044 u_int v; 5045 int rc; 5046 5047 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5048 if (rc) 5049 return (rc); 5050 /* XXX: magic numbers */ 5051 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5052 &v); 5053 end_synchronized_op(sc, 0); 5054 if (rc) 5055 return (rc); 5056 if (op == 0) 5057 v /= 256; 5058 5059 rc = sysctl_handle_int(oidp, &v, 0, req); 5060 return (rc); 5061} 5062 5063static int 5064sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5065{ 5066 struct port_info *pi = arg1; 5067 int rc, val; 5068 5069 val = pi->rsrv_noflowq; 5070 rc = sysctl_handle_int(oidp, &val, 0, req); 5071 if (rc != 0 || req->newptr == NULL) 5072 return (rc); 5073 5074 if ((val >= 1) && (pi->ntxq > 1)) 5075 pi->rsrv_noflowq = 1; 5076 else 5077 pi->rsrv_noflowq = 0; 5078 5079 return (rc); 5080} 5081 5082static int 5083sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5084{ 5085 struct port_info *pi = arg1; 5086 struct adapter *sc = pi->adapter; 5087 int idx, rc, i; 5088 struct sge_rxq *rxq; 5089#ifdef TCP_OFFLOAD 5090 struct sge_ofld_rxq *ofld_rxq; 5091#endif 5092 uint8_t v; 5093 5094 idx = pi->tmr_idx; 5095 5096 rc = sysctl_handle_int(oidp, &idx, 0, req); 5097 if (rc != 0 || req->newptr == NULL) 5098 return (rc); 5099 5100 if (idx < 0 || idx >= SGE_NTIMERS) 5101 return (EINVAL); 5102 5103 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5104 "t4tmr"); 5105 if (rc) 5106 return (rc); 5107 5108 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5109 for_each_rxq(pi, i, rxq) { 5110#ifdef atomic_store_rel_8 5111 atomic_store_rel_8(&rxq->iq.intr_params, v); 5112#else 5113 rxq->iq.intr_params = v; 5114#endif 5115 } 5116#ifdef TCP_OFFLOAD 5117 for_each_ofld_rxq(pi, i, ofld_rxq) { 5118#ifdef atomic_store_rel_8 5119 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5120#else 5121 ofld_rxq->iq.intr_params = v; 5122#endif 5123 } 5124#endif 5125 pi->tmr_idx = idx; 5126 5127 end_synchronized_op(sc, LOCK_HELD); 5128 return (0); 5129} 5130 5131static int 5132sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5133{ 5134 struct port_info *pi = arg1; 5135 struct adapter *sc = pi->adapter; 5136 int idx, rc; 5137 5138 idx = pi->pktc_idx; 5139 5140 rc = sysctl_handle_int(oidp, &idx, 0, req); 5141 if (rc != 0 || req->newptr == NULL) 5142 return (rc); 5143 5144 if (idx < -1 || idx >= SGE_NCOUNTERS) 5145 return (EINVAL); 5146 5147 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5148 "t4pktc"); 5149 if (rc) 5150 return (rc); 5151 5152 if (pi->flags & PORT_INIT_DONE) 5153 rc = EBUSY; /* cannot be changed once the queues are created */ 5154 else 5155 pi->pktc_idx = idx; 5156 5157 end_synchronized_op(sc, LOCK_HELD); 5158 return (rc); 5159} 5160 5161static int 5162sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5163{ 5164 struct port_info *pi = arg1; 5165 struct adapter *sc = pi->adapter; 5166 int qsize, rc; 5167 5168 qsize = pi->qsize_rxq; 5169 5170 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5171 if (rc != 0 || req->newptr == NULL) 5172 return (rc); 5173 5174 if (qsize < 128 || (qsize & 7)) 5175 return (EINVAL); 5176 5177 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5178 "t4rxqs"); 5179 if (rc) 5180 return (rc); 5181 5182 if (pi->flags & PORT_INIT_DONE) 5183 rc = EBUSY; /* cannot be changed once the queues are created */ 5184 else 5185 pi->qsize_rxq = qsize; 5186 5187 end_synchronized_op(sc, LOCK_HELD); 5188 return (rc); 5189} 5190 5191static int 5192sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5193{ 5194 struct port_info *pi = arg1; 5195 struct adapter *sc = pi->adapter; 5196 int qsize, rc; 5197 5198 qsize = pi->qsize_txq; 5199 5200 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5201 if (rc != 0 || req->newptr == NULL) 5202 return (rc); 5203 5204 if (qsize < 128 || qsize > 65536) 5205 return (EINVAL); 5206 5207 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5208 "t4txqs"); 5209 if (rc) 5210 return (rc); 5211 5212 if (pi->flags & PORT_INIT_DONE) 5213 rc = EBUSY; /* cannot be changed once the queues are created */ 5214 else 5215 pi->qsize_txq = qsize; 5216 5217 end_synchronized_op(sc, LOCK_HELD); 5218 return (rc); 5219} 5220 5221static int 5222sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5223{ 5224 struct port_info *pi = arg1; 5225 struct adapter *sc = pi->adapter; 5226 struct link_config *lc = &pi->link_cfg; 5227 int rc; 5228 5229 if (req->newptr == NULL) { 5230 struct sbuf *sb; 5231 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5232 5233 rc = sysctl_wire_old_buffer(req, 0); 5234 if (rc != 0) 5235 return(rc); 5236 5237 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5238 if (sb == NULL) 5239 return (ENOMEM); 5240 5241 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5242 rc = sbuf_finish(sb); 5243 sbuf_delete(sb); 5244 } else { 5245 char s[2]; 5246 int n; 5247 5248 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5249 s[1] = 0; 5250 5251 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5252 if (rc != 0) 5253 return(rc); 5254 5255 if (s[1] != 0) 5256 return (EINVAL); 5257 if (s[0] < '0' || s[0] > '9') 5258 return (EINVAL); /* not a number */ 5259 n = s[0] - '0'; 5260 if (n & ~(PAUSE_TX | PAUSE_RX)) 5261 return (EINVAL); /* some other bit is set too */ 5262 5263 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5264 if (rc) 5265 return (rc); 5266 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5267 int link_ok = lc->link_ok; 5268 5269 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5270 lc->requested_fc |= n; 5271 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5272 lc->link_ok = link_ok; /* restore */ 5273 } 5274 end_synchronized_op(sc, 0); 5275 } 5276 5277 return (rc); 5278} 5279 5280static int 5281sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5282{ 5283 struct adapter *sc = arg1; 5284 int reg = arg2; 5285 uint64_t val; 5286 5287 val = t4_read_reg64(sc, reg); 5288 5289 return (sysctl_handle_64(oidp, &val, 0, req)); 5290} 5291 5292static int 5293sysctl_temperature(SYSCTL_HANDLER_ARGS) 5294{ 5295 struct adapter *sc = arg1; 5296 int rc, t; 5297 uint32_t param, val; 5298 5299 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5300 if (rc) 5301 return (rc); 5302 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5303 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5304 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5305 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5306 end_synchronized_op(sc, 0); 5307 if (rc) 5308 return (rc); 5309 5310 /* unknown is returned as 0 but we display -1 in that case */ 5311 t = val == 0 ? -1 : val; 5312 5313 rc = sysctl_handle_int(oidp, &t, 0, req); 5314 return (rc); 5315} 5316 5317#ifdef SBUF_DRAIN 5318static int 5319sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5320{ 5321 struct adapter *sc = arg1; 5322 struct sbuf *sb; 5323 int rc, i; 5324 uint16_t incr[NMTUS][NCCTRL_WIN]; 5325 static const char *dec_fac[] = { 5326 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5327 "0.9375" 5328 }; 5329 5330 rc = sysctl_wire_old_buffer(req, 0); 5331 if (rc != 0) 5332 return (rc); 5333 5334 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5335 if (sb == NULL) 5336 return (ENOMEM); 5337 5338 t4_read_cong_tbl(sc, incr); 5339 5340 for (i = 0; i < NCCTRL_WIN; ++i) { 5341 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5342 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5343 incr[5][i], incr[6][i], incr[7][i]); 5344 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5345 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5346 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5347 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5348 } 5349 5350 rc = sbuf_finish(sb); 5351 sbuf_delete(sb); 5352 5353 return (rc); 5354} 5355 5356static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5357 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5358 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5359 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5360}; 5361 5362static int 5363sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5364{ 5365 struct adapter *sc = arg1; 5366 struct sbuf *sb; 5367 int rc, i, n, qid = arg2; 5368 uint32_t *buf, *p; 5369 char *qtype; 5370 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5371 5372 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5373 ("%s: bad qid %d\n", __func__, qid)); 5374 5375 if (qid < CIM_NUM_IBQ) { 5376 /* inbound queue */ 5377 qtype = "IBQ"; 5378 n = 4 * CIM_IBQ_SIZE; 5379 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5380 rc = t4_read_cim_ibq(sc, qid, buf, n); 5381 } else { 5382 /* outbound queue */ 5383 qtype = "OBQ"; 5384 qid -= CIM_NUM_IBQ; 5385 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5386 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5387 rc = t4_read_cim_obq(sc, qid, buf, n); 5388 } 5389 5390 if (rc < 0) { 5391 rc = -rc; 5392 goto done; 5393 } 5394 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5395 5396 rc = sysctl_wire_old_buffer(req, 0); 5397 if (rc != 0) 5398 goto done; 5399 5400 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5401 if (sb == NULL) { 5402 rc = ENOMEM; 5403 goto done; 5404 } 5405 5406 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5407 for (i = 0, p = buf; i < n; i += 16, p += 4) 5408 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5409 p[2], p[3]); 5410 5411 rc = sbuf_finish(sb); 5412 sbuf_delete(sb); 5413done: 5414 free(buf, M_CXGBE); 5415 return (rc); 5416} 5417 5418static int 5419sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5420{ 5421 struct adapter *sc = arg1; 5422 u_int cfg; 5423 struct sbuf *sb; 5424 uint32_t *buf, *p; 5425 int rc; 5426 5427 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5428 if (rc != 0) 5429 return (rc); 5430 5431 rc = sysctl_wire_old_buffer(req, 0); 5432 if (rc != 0) 5433 return (rc); 5434 5435 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5436 if (sb == NULL) 5437 return (ENOMEM); 5438 5439 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5440 M_ZERO | M_WAITOK); 5441 5442 rc = -t4_cim_read_la(sc, buf, NULL); 5443 if (rc != 0) 5444 goto done; 5445 5446 sbuf_printf(sb, "Status Data PC%s", 5447 cfg & F_UPDBGLACAPTPCONLY ? "" : 5448 " LS0Stat LS0Addr LS0Data"); 5449 5450 KASSERT((sc->params.cim_la_size & 7) == 0, 5451 ("%s: p will walk off the end of buf", __func__)); 5452 5453 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5454 if (cfg & F_UPDBGLACAPTPCONLY) { 5455 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5456 p[6], p[7]); 5457 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5458 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5459 p[4] & 0xff, p[5] >> 8); 5460 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5461 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5462 p[1] & 0xf, p[2] >> 4); 5463 } else { 5464 sbuf_printf(sb, 5465 "\n %02x %x%07x %x%07x %08x %08x " 5466 "%08x%08x%08x%08x", 5467 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5468 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5469 p[6], p[7]); 5470 } 5471 } 5472 5473 rc = sbuf_finish(sb); 5474 sbuf_delete(sb); 5475done: 5476 free(buf, M_CXGBE); 5477 return (rc); 5478} 5479 5480static int 5481sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5482{ 5483 struct adapter *sc = arg1; 5484 u_int i; 5485 struct sbuf *sb; 5486 uint32_t *buf, *p; 5487 int rc; 5488 5489 rc = sysctl_wire_old_buffer(req, 0); 5490 if (rc != 0) 5491 return (rc); 5492 5493 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5494 if (sb == NULL) 5495 return (ENOMEM); 5496 5497 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5498 M_ZERO | M_WAITOK); 5499 5500 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5501 p = buf; 5502 5503 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5504 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5505 p[1], p[0]); 5506 } 5507 5508 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5509 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5510 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5511 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5512 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5513 (p[1] >> 2) | ((p[2] & 3) << 30), 5514 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5515 p[0] & 1); 5516 } 5517 5518 rc = sbuf_finish(sb); 5519 sbuf_delete(sb); 5520 free(buf, M_CXGBE); 5521 return (rc); 5522} 5523 5524static int 5525sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5526{ 5527 struct adapter *sc = arg1; 5528 u_int i; 5529 struct sbuf *sb; 5530 uint32_t *buf, *p; 5531 int rc; 5532 5533 rc = sysctl_wire_old_buffer(req, 0); 5534 if (rc != 0) 5535 return (rc); 5536 5537 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5538 if (sb == NULL) 5539 return (ENOMEM); 5540 5541 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5542 M_ZERO | M_WAITOK); 5543 5544 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5545 p = buf; 5546 5547 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5548 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5549 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5550 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5551 p[4], p[3], p[2], p[1], p[0]); 5552 } 5553 5554 sbuf_printf(sb, "\n\nCntl ID Data"); 5555 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5556 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5557 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5558 } 5559 5560 rc = sbuf_finish(sb); 5561 sbuf_delete(sb); 5562 free(buf, M_CXGBE); 5563 return (rc); 5564} 5565 5566static int 5567sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5568{ 5569 struct adapter *sc = arg1; 5570 struct sbuf *sb; 5571 int rc, i; 5572 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5573 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5574 uint16_t thres[CIM_NUM_IBQ]; 5575 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5576 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5577 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5578 5579 if (is_t4(sc)) { 5580 cim_num_obq = CIM_NUM_OBQ; 5581 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5582 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5583 } else { 5584 cim_num_obq = CIM_NUM_OBQ_T5; 5585 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5586 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5587 } 5588 nq = CIM_NUM_IBQ + cim_num_obq; 5589 5590 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5591 if (rc == 0) 5592 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5593 if (rc != 0) 5594 return (rc); 5595 5596 t4_read_cimq_cfg(sc, base, size, thres); 5597 5598 rc = sysctl_wire_old_buffer(req, 0); 5599 if (rc != 0) 5600 return (rc); 5601 5602 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5603 if (sb == NULL) 5604 return (ENOMEM); 5605 5606 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5607 5608 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5609 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5610 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5611 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5612 G_QUEREMFLITS(p[2]) * 16); 5613 for ( ; i < nq; i++, p += 4, wr += 2) 5614 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5615 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5616 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5617 G_QUEREMFLITS(p[2]) * 16); 5618 5619 rc = sbuf_finish(sb); 5620 sbuf_delete(sb); 5621 5622 return (rc); 5623} 5624 5625static int 5626sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5627{ 5628 struct adapter *sc = arg1; 5629 struct sbuf *sb; 5630 int rc; 5631 struct tp_cpl_stats stats; 5632 5633 rc = sysctl_wire_old_buffer(req, 0); 5634 if (rc != 0) 5635 return (rc); 5636 5637 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5638 if (sb == NULL) 5639 return (ENOMEM); 5640 5641 t4_tp_get_cpl_stats(sc, &stats); 5642 5643 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5644 "channel 3\n"); 5645 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5646 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5647 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5648 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5649 5650 rc = sbuf_finish(sb); 5651 sbuf_delete(sb); 5652 5653 return (rc); 5654} 5655 5656static int 5657sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5658{ 5659 struct adapter *sc = arg1; 5660 struct sbuf *sb; 5661 int rc; 5662 struct tp_usm_stats stats; 5663 5664 rc = sysctl_wire_old_buffer(req, 0); 5665 if (rc != 0) 5666 return(rc); 5667 5668 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5669 if (sb == NULL) 5670 return (ENOMEM); 5671 5672 t4_get_usm_stats(sc, &stats); 5673 5674 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5675 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5676 sbuf_printf(sb, "Drops: %u", stats.drops); 5677 5678 rc = sbuf_finish(sb); 5679 sbuf_delete(sb); 5680 5681 return (rc); 5682} 5683 5684const char *devlog_level_strings[] = { 5685 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5686 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5687 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5688 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5689 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5690 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5691}; 5692 5693const char *devlog_facility_strings[] = { 5694 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5695 [FW_DEVLOG_FACILITY_CF] = "CF", 5696 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5697 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5698 [FW_DEVLOG_FACILITY_RES] = "RES", 5699 [FW_DEVLOG_FACILITY_HW] = "HW", 5700 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5701 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5702 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5703 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5704 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5705 [FW_DEVLOG_FACILITY_VI] = "VI", 5706 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5707 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5708 [FW_DEVLOG_FACILITY_TM] = "TM", 5709 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5710 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5711 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5712 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5713 [FW_DEVLOG_FACILITY_RI] = "RI", 5714 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5715 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5716 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5717 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5718}; 5719 5720static int 5721sysctl_devlog(SYSCTL_HANDLER_ARGS) 5722{ 5723 struct adapter *sc = arg1; 5724 struct devlog_params *dparams = &sc->params.devlog; 5725 struct fw_devlog_e *buf, *e; 5726 int i, j, rc, nentries, first = 0, m; 5727 struct sbuf *sb; 5728 uint64_t ftstamp = UINT64_MAX; 5729 5730 if (dparams->start == 0) { 5731 dparams->memtype = FW_MEMTYPE_EDC0; 5732 dparams->start = 0x84000; 5733 dparams->size = 32768; 5734 } 5735 5736 nentries = dparams->size / sizeof(struct fw_devlog_e); 5737 5738 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5739 if (buf == NULL) 5740 return (ENOMEM); 5741 5742 m = fwmtype_to_hwmtype(dparams->memtype); 5743 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5744 if (rc != 0) 5745 goto done; 5746 5747 for (i = 0; i < nentries; i++) { 5748 e = &buf[i]; 5749 5750 if (e->timestamp == 0) 5751 break; /* end */ 5752 5753 e->timestamp = be64toh(e->timestamp); 5754 e->seqno = be32toh(e->seqno); 5755 for (j = 0; j < 8; j++) 5756 e->params[j] = be32toh(e->params[j]); 5757 5758 if (e->timestamp < ftstamp) { 5759 ftstamp = e->timestamp; 5760 first = i; 5761 } 5762 } 5763 5764 if (buf[first].timestamp == 0) 5765 goto done; /* nothing in the log */ 5766 5767 rc = sysctl_wire_old_buffer(req, 0); 5768 if (rc != 0) 5769 goto done; 5770 5771 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5772 if (sb == NULL) { 5773 rc = ENOMEM; 5774 goto done; 5775 } 5776 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5777 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5778 5779 i = first; 5780 do { 5781 e = &buf[i]; 5782 if (e->timestamp == 0) 5783 break; /* end */ 5784 5785 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5786 e->seqno, e->timestamp, 5787 (e->level < nitems(devlog_level_strings) ? 5788 devlog_level_strings[e->level] : "UNKNOWN"), 5789 (e->facility < nitems(devlog_facility_strings) ? 5790 devlog_facility_strings[e->facility] : "UNKNOWN")); 5791 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5792 e->params[2], e->params[3], e->params[4], 5793 e->params[5], e->params[6], e->params[7]); 5794 5795 if (++i == nentries) 5796 i = 0; 5797 } while (i != first); 5798 5799 rc = sbuf_finish(sb); 5800 sbuf_delete(sb); 5801done: 5802 free(buf, M_CXGBE); 5803 return (rc); 5804} 5805 5806static int 5807sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5808{ 5809 struct adapter *sc = arg1; 5810 struct sbuf *sb; 5811 int rc; 5812 struct tp_fcoe_stats stats[4]; 5813 5814 rc = sysctl_wire_old_buffer(req, 0); 5815 if (rc != 0) 5816 return (rc); 5817 5818 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5819 if (sb == NULL) 5820 return (ENOMEM); 5821 5822 t4_get_fcoe_stats(sc, 0, &stats[0]); 5823 t4_get_fcoe_stats(sc, 1, &stats[1]); 5824 t4_get_fcoe_stats(sc, 2, &stats[2]); 5825 t4_get_fcoe_stats(sc, 3, &stats[3]); 5826 5827 sbuf_printf(sb, " channel 0 channel 1 " 5828 "channel 2 channel 3\n"); 5829 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5830 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5831 stats[3].octetsDDP); 5832 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5833 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5834 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5835 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5836 stats[3].framesDrop); 5837 5838 rc = sbuf_finish(sb); 5839 sbuf_delete(sb); 5840 5841 return (rc); 5842} 5843 5844static int 5845sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5846{ 5847 struct adapter *sc = arg1; 5848 struct sbuf *sb; 5849 int rc, i; 5850 unsigned int map, kbps, ipg, mode; 5851 unsigned int pace_tab[NTX_SCHED]; 5852 5853 rc = sysctl_wire_old_buffer(req, 0); 5854 if (rc != 0) 5855 return (rc); 5856 5857 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5858 if (sb == NULL) 5859 return (ENOMEM); 5860 5861 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5862 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5863 t4_read_pace_tbl(sc, pace_tab); 5864 5865 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5866 "Class IPG (0.1 ns) Flow IPG (us)"); 5867 5868 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5869 t4_get_tx_sched(sc, i, &kbps, &ipg); 5870 sbuf_printf(sb, "\n %u %-5s %u ", i, 5871 (mode & (1 << i)) ? "flow" : "class", map & 3); 5872 if (kbps) 5873 sbuf_printf(sb, "%9u ", kbps); 5874 else 5875 sbuf_printf(sb, " disabled "); 5876 5877 if (ipg) 5878 sbuf_printf(sb, "%13u ", ipg); 5879 else 5880 sbuf_printf(sb, " disabled "); 5881 5882 if (pace_tab[i]) 5883 sbuf_printf(sb, "%10u", pace_tab[i]); 5884 else 5885 sbuf_printf(sb, " disabled"); 5886 } 5887 5888 rc = sbuf_finish(sb); 5889 sbuf_delete(sb); 5890 5891 return (rc); 5892} 5893 5894static int 5895sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5896{ 5897 struct adapter *sc = arg1; 5898 struct sbuf *sb; 5899 int rc, i, j; 5900 uint64_t *p0, *p1; 5901 struct lb_port_stats s[2]; 5902 static const char *stat_name[] = { 5903 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5904 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5905 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5906 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5907 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5908 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5909 "BG2FramesTrunc:", "BG3FramesTrunc:" 5910 }; 5911 5912 rc = sysctl_wire_old_buffer(req, 0); 5913 if (rc != 0) 5914 return (rc); 5915 5916 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5917 if (sb == NULL) 5918 return (ENOMEM); 5919 5920 memset(s, 0, sizeof(s)); 5921 5922 for (i = 0; i < 4; i += 2) { 5923 t4_get_lb_stats(sc, i, &s[0]); 5924 t4_get_lb_stats(sc, i + 1, &s[1]); 5925 5926 p0 = &s[0].octets; 5927 p1 = &s[1].octets; 5928 sbuf_printf(sb, "%s Loopback %u" 5929 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5930 5931 for (j = 0; j < nitems(stat_name); j++) 5932 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5933 *p0++, *p1++); 5934 } 5935 5936 rc = sbuf_finish(sb); 5937 sbuf_delete(sb); 5938 5939 return (rc); 5940} 5941 5942static int 5943sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5944{ 5945 int rc = 0; 5946 struct port_info *pi = arg1; 5947 struct sbuf *sb; 5948 static const char *linkdnreasons[] = { 5949 "non-specific", "remote fault", "autoneg failed", "reserved3", 5950 "PHY overheated", "unknown", "rx los", "reserved7" 5951 }; 5952 5953 rc = sysctl_wire_old_buffer(req, 0); 5954 if (rc != 0) 5955 return(rc); 5956 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5957 if (sb == NULL) 5958 return (ENOMEM); 5959 5960 if (pi->linkdnrc < 0) 5961 sbuf_printf(sb, "n/a"); 5962 else if (pi->linkdnrc < nitems(linkdnreasons)) 5963 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5964 else 5965 sbuf_printf(sb, "%d", pi->linkdnrc); 5966 5967 rc = sbuf_finish(sb); 5968 sbuf_delete(sb); 5969 5970 return (rc); 5971} 5972 5973struct mem_desc { 5974 unsigned int base; 5975 unsigned int limit; 5976 unsigned int idx; 5977}; 5978 5979static int 5980mem_desc_cmp(const void *a, const void *b) 5981{ 5982 return ((const struct mem_desc *)a)->base - 5983 ((const struct mem_desc *)b)->base; 5984} 5985 5986static void 5987mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5988 unsigned int to) 5989{ 5990 unsigned int size; 5991 5992 size = to - from + 1; 5993 if (size == 0) 5994 return; 5995 5996 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5997 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5998} 5999 6000static int 6001sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6002{ 6003 struct adapter *sc = arg1; 6004 struct sbuf *sb; 6005 int rc, i, n; 6006 uint32_t lo, hi, used, alloc; 6007 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6008 static const char *region[] = { 6009 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6010 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6011 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6012 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6013 "RQUDP region:", "PBL region:", "TXPBL region:", 6014 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6015 "On-chip queues:" 6016 }; 6017 struct mem_desc avail[4]; 6018 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6019 struct mem_desc *md = mem; 6020 6021 rc = sysctl_wire_old_buffer(req, 0); 6022 if (rc != 0) 6023 return (rc); 6024 6025 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6026 if (sb == NULL) 6027 return (ENOMEM); 6028 6029 for (i = 0; i < nitems(mem); i++) { 6030 mem[i].limit = 0; 6031 mem[i].idx = i; 6032 } 6033 6034 /* Find and sort the populated memory ranges */ 6035 i = 0; 6036 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6037 if (lo & F_EDRAM0_ENABLE) { 6038 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6039 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6040 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6041 avail[i].idx = 0; 6042 i++; 6043 } 6044 if (lo & F_EDRAM1_ENABLE) { 6045 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6046 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6047 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6048 avail[i].idx = 1; 6049 i++; 6050 } 6051 if (lo & F_EXT_MEM_ENABLE) { 6052 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6053 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6054 avail[i].limit = avail[i].base + 6055 (G_EXT_MEM_SIZE(hi) << 20); 6056 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6057 i++; 6058 } 6059 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6060 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6061 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6062 avail[i].limit = avail[i].base + 6063 (G_EXT_MEM1_SIZE(hi) << 20); 6064 avail[i].idx = 4; 6065 i++; 6066 } 6067 if (!i) /* no memory available */ 6068 return 0; 6069 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6070 6071 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6072 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6073 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6074 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6075 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6076 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6077 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6078 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6079 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6080 6081 /* the next few have explicit upper bounds */ 6082 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6083 md->limit = md->base - 1 + 6084 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6085 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6086 md++; 6087 6088 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6089 md->limit = md->base - 1 + 6090 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6091 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6092 md++; 6093 6094 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6095 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6096 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6097 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6098 } else { 6099 md->base = 0; 6100 md->idx = nitems(region); /* hide it */ 6101 } 6102 md++; 6103 6104#define ulp_region(reg) \ 6105 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6106 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6107 6108 ulp_region(RX_ISCSI); 6109 ulp_region(RX_TDDP); 6110 ulp_region(TX_TPT); 6111 ulp_region(RX_STAG); 6112 ulp_region(RX_RQ); 6113 ulp_region(RX_RQUDP); 6114 ulp_region(RX_PBL); 6115 ulp_region(TX_PBL); 6116#undef ulp_region 6117 6118 md->base = 0; 6119 md->idx = nitems(region); 6120 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6121 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6122 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6123 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6124 } 6125 md++; 6126 6127 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6128 md->limit = md->base + sc->tids.ntids - 1; 6129 md++; 6130 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6131 md->limit = md->base + sc->tids.ntids - 1; 6132 md++; 6133 6134 md->base = sc->vres.ocq.start; 6135 if (sc->vres.ocq.size) 6136 md->limit = md->base + sc->vres.ocq.size - 1; 6137 else 6138 md->idx = nitems(region); /* hide it */ 6139 md++; 6140 6141 /* add any address-space holes, there can be up to 3 */ 6142 for (n = 0; n < i - 1; n++) 6143 if (avail[n].limit < avail[n + 1].base) 6144 (md++)->base = avail[n].limit; 6145 if (avail[n].limit) 6146 (md++)->base = avail[n].limit; 6147 6148 n = md - mem; 6149 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6150 6151 for (lo = 0; lo < i; lo++) 6152 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6153 avail[lo].limit - 1); 6154 6155 sbuf_printf(sb, "\n"); 6156 for (i = 0; i < n; i++) { 6157 if (mem[i].idx >= nitems(region)) 6158 continue; /* skip holes */ 6159 if (!mem[i].limit) 6160 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6161 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6162 mem[i].limit); 6163 } 6164 6165 sbuf_printf(sb, "\n"); 6166 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6167 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6168 mem_region_show(sb, "uP RAM:", lo, hi); 6169 6170 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6171 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6172 mem_region_show(sb, "uP Extmem2:", lo, hi); 6173 6174 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6175 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6176 G_PMRXMAXPAGE(lo), 6177 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6178 (lo & F_PMRXNUMCHN) ? 2 : 1); 6179 6180 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6181 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6182 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6183 G_PMTXMAXPAGE(lo), 6184 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6185 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6186 sbuf_printf(sb, "%u p-structs\n", 6187 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6188 6189 for (i = 0; i < 4; i++) { 6190 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6191 if (is_t4(sc)) { 6192 used = G_USED(lo); 6193 alloc = G_ALLOC(lo); 6194 } else { 6195 used = G_T5_USED(lo); 6196 alloc = G_T5_ALLOC(lo); 6197 } 6198 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6199 i, used, alloc); 6200 } 6201 for (i = 0; i < 4; i++) { 6202 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6203 if (is_t4(sc)) { 6204 used = G_USED(lo); 6205 alloc = G_ALLOC(lo); 6206 } else { 6207 used = G_T5_USED(lo); 6208 alloc = G_T5_ALLOC(lo); 6209 } 6210 sbuf_printf(sb, 6211 "\nLoopback %d using %u pages out of %u allocated", 6212 i, used, alloc); 6213 } 6214 6215 rc = sbuf_finish(sb); 6216 sbuf_delete(sb); 6217 6218 return (rc); 6219} 6220 6221static inline void 6222tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6223{ 6224 *mask = x | y; 6225 y = htobe64(y); 6226 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6227} 6228 6229static int 6230sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6231{ 6232 struct adapter *sc = arg1; 6233 struct sbuf *sb; 6234 int rc, i, n; 6235 6236 rc = sysctl_wire_old_buffer(req, 0); 6237 if (rc != 0) 6238 return (rc); 6239 6240 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6241 if (sb == NULL) 6242 return (ENOMEM); 6243 6244 sbuf_printf(sb, 6245 "Idx Ethernet address Mask Vld Ports PF" 6246 " VF Replication P0 P1 P2 P3 ML"); 6247 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6248 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6249 for (i = 0; i < n; i++) { 6250 uint64_t tcamx, tcamy, mask; 6251 uint32_t cls_lo, cls_hi; 6252 uint8_t addr[ETHER_ADDR_LEN]; 6253 6254 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6255 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6256 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6257 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6258 6259 if (tcamx & tcamy) 6260 continue; 6261 6262 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6263 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6264 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6265 addr[3], addr[4], addr[5], (uintmax_t)mask, 6266 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6267 G_PORTMAP(cls_hi), G_PF(cls_lo), 6268 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6269 6270 if (cls_lo & F_REPLICATE) { 6271 struct fw_ldst_cmd ldst_cmd; 6272 6273 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6274 ldst_cmd.op_to_addrspace = 6275 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6276 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6277 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6278 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6279 ldst_cmd.u.mps.fid_ctl = 6280 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6281 V_FW_LDST_CMD_CTL(i)); 6282 6283 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6284 "t4mps"); 6285 if (rc) 6286 break; 6287 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6288 sizeof(ldst_cmd), &ldst_cmd); 6289 end_synchronized_op(sc, 0); 6290 6291 if (rc != 0) { 6292 sbuf_printf(sb, 6293 " ------------ error %3u ------------", rc); 6294 rc = 0; 6295 } else { 6296 sbuf_printf(sb, " %08x %08x %08x %08x", 6297 be32toh(ldst_cmd.u.mps.rplc127_96), 6298 be32toh(ldst_cmd.u.mps.rplc95_64), 6299 be32toh(ldst_cmd.u.mps.rplc63_32), 6300 be32toh(ldst_cmd.u.mps.rplc31_0)); 6301 } 6302 } else 6303 sbuf_printf(sb, "%36s", ""); 6304 6305 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6306 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6307 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6308 } 6309 6310 if (rc) 6311 (void) sbuf_finish(sb); 6312 else 6313 rc = sbuf_finish(sb); 6314 sbuf_delete(sb); 6315 6316 return (rc); 6317} 6318 6319static int 6320sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6321{ 6322 struct adapter *sc = arg1; 6323 struct sbuf *sb; 6324 int rc; 6325 uint16_t mtus[NMTUS]; 6326 6327 rc = sysctl_wire_old_buffer(req, 0); 6328 if (rc != 0) 6329 return (rc); 6330 6331 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6332 if (sb == NULL) 6333 return (ENOMEM); 6334 6335 t4_read_mtu_tbl(sc, mtus, NULL); 6336 6337 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6338 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6339 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6340 mtus[14], mtus[15]); 6341 6342 rc = sbuf_finish(sb); 6343 sbuf_delete(sb); 6344 6345 return (rc); 6346} 6347 6348static int 6349sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6350{ 6351 struct adapter *sc = arg1; 6352 struct sbuf *sb; 6353 int rc, i; 6354 uint32_t cnt[PM_NSTATS]; 6355 uint64_t cyc[PM_NSTATS]; 6356 static const char *rx_stats[] = { 6357 "Read:", "Write bypass:", "Write mem:", "Flush:" 6358 }; 6359 static const char *tx_stats[] = { 6360 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6361 }; 6362 6363 rc = sysctl_wire_old_buffer(req, 0); 6364 if (rc != 0) 6365 return (rc); 6366 6367 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6368 if (sb == NULL) 6369 return (ENOMEM); 6370 6371 t4_pmtx_get_stats(sc, cnt, cyc); 6372 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6373 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6374 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6375 cyc[i]); 6376 6377 t4_pmrx_get_stats(sc, cnt, cyc); 6378 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6379 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6380 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6381 cyc[i]); 6382 6383 rc = sbuf_finish(sb); 6384 sbuf_delete(sb); 6385 6386 return (rc); 6387} 6388 6389static int 6390sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6391{ 6392 struct adapter *sc = arg1; 6393 struct sbuf *sb; 6394 int rc; 6395 struct tp_rdma_stats stats; 6396 6397 rc = sysctl_wire_old_buffer(req, 0); 6398 if (rc != 0) 6399 return (rc); 6400 6401 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6402 if (sb == NULL) 6403 return (ENOMEM); 6404 6405 t4_tp_get_rdma_stats(sc, &stats); 6406 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6407 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6408 6409 rc = sbuf_finish(sb); 6410 sbuf_delete(sb); 6411 6412 return (rc); 6413} 6414 6415static int 6416sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6417{ 6418 struct adapter *sc = arg1; 6419 struct sbuf *sb; 6420 int rc; 6421 struct tp_tcp_stats v4, v6; 6422 6423 rc = sysctl_wire_old_buffer(req, 0); 6424 if (rc != 0) 6425 return (rc); 6426 6427 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6428 if (sb == NULL) 6429 return (ENOMEM); 6430 6431 t4_tp_get_tcp_stats(sc, &v4, &v6); 6432 sbuf_printf(sb, 6433 " IP IPv6\n"); 6434 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6435 v4.tcpOutRsts, v6.tcpOutRsts); 6436 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6437 v4.tcpInSegs, v6.tcpInSegs); 6438 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6439 v4.tcpOutSegs, v6.tcpOutSegs); 6440 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6441 v4.tcpRetransSegs, v6.tcpRetransSegs); 6442 6443 rc = sbuf_finish(sb); 6444 sbuf_delete(sb); 6445 6446 return (rc); 6447} 6448 6449static int 6450sysctl_tids(SYSCTL_HANDLER_ARGS) 6451{ 6452 struct adapter *sc = arg1; 6453 struct sbuf *sb; 6454 int rc; 6455 struct tid_info *t = &sc->tids; 6456 6457 rc = sysctl_wire_old_buffer(req, 0); 6458 if (rc != 0) 6459 return (rc); 6460 6461 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6462 if (sb == NULL) 6463 return (ENOMEM); 6464 6465 if (t->natids) { 6466 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6467 t->atids_in_use); 6468 } 6469 6470 if (t->ntids) { 6471 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6472 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6473 6474 if (b) { 6475 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6476 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6477 t->ntids - 1); 6478 } else { 6479 sbuf_printf(sb, "TID range: %u-%u", 6480 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6481 t->ntids - 1); 6482 } 6483 } else 6484 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6485 sbuf_printf(sb, ", in use: %u\n", 6486 atomic_load_acq_int(&t->tids_in_use)); 6487 } 6488 6489 if (t->nstids) { 6490 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6491 t->stid_base + t->nstids - 1, t->stids_in_use); 6492 } 6493 6494 if (t->nftids) { 6495 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6496 t->ftid_base + t->nftids - 1); 6497 } 6498 6499 if (t->netids) { 6500 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6501 t->etid_base + t->netids - 1); 6502 } 6503 6504 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6505 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6506 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6507 6508 rc = sbuf_finish(sb); 6509 sbuf_delete(sb); 6510 6511 return (rc); 6512} 6513 6514static int 6515sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6516{ 6517 struct adapter *sc = arg1; 6518 struct sbuf *sb; 6519 int rc; 6520 struct tp_err_stats stats; 6521 6522 rc = sysctl_wire_old_buffer(req, 0); 6523 if (rc != 0) 6524 return (rc); 6525 6526 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6527 if (sb == NULL) 6528 return (ENOMEM); 6529 6530 t4_tp_get_err_stats(sc, &stats); 6531 6532 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6533 "channel 3\n"); 6534 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6535 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6536 stats.macInErrs[3]); 6537 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6538 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6539 stats.hdrInErrs[3]); 6540 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6541 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6542 stats.tcpInErrs[3]); 6543 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6544 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6545 stats.tcp6InErrs[3]); 6546 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6547 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6548 stats.tnlCongDrops[3]); 6549 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6550 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6551 stats.tnlTxDrops[3]); 6552 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6553 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6554 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6555 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6556 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6557 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6558 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6559 stats.ofldNoNeigh, stats.ofldCongDefer); 6560 6561 rc = sbuf_finish(sb); 6562 sbuf_delete(sb); 6563 6564 return (rc); 6565} 6566 6567struct field_desc { 6568 const char *name; 6569 u_int start; 6570 u_int width; 6571}; 6572 6573static void 6574field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6575{ 6576 char buf[32]; 6577 int line_size = 0; 6578 6579 while (f->name) { 6580 uint64_t mask = (1ULL << f->width) - 1; 6581 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6582 ((uintmax_t)v >> f->start) & mask); 6583 6584 if (line_size + len >= 79) { 6585 line_size = 8; 6586 sbuf_printf(sb, "\n "); 6587 } 6588 sbuf_printf(sb, "%s ", buf); 6589 line_size += len + 1; 6590 f++; 6591 } 6592 sbuf_printf(sb, "\n"); 6593} 6594 6595static struct field_desc tp_la0[] = { 6596 { "RcfOpCodeOut", 60, 4 }, 6597 { "State", 56, 4 }, 6598 { "WcfState", 52, 4 }, 6599 { "RcfOpcSrcOut", 50, 2 }, 6600 { "CRxError", 49, 1 }, 6601 { "ERxError", 48, 1 }, 6602 { "SanityFailed", 47, 1 }, 6603 { "SpuriousMsg", 46, 1 }, 6604 { "FlushInputMsg", 45, 1 }, 6605 { "FlushInputCpl", 44, 1 }, 6606 { "RssUpBit", 43, 1 }, 6607 { "RssFilterHit", 42, 1 }, 6608 { "Tid", 32, 10 }, 6609 { "InitTcb", 31, 1 }, 6610 { "LineNumber", 24, 7 }, 6611 { "Emsg", 23, 1 }, 6612 { "EdataOut", 22, 1 }, 6613 { "Cmsg", 21, 1 }, 6614 { "CdataOut", 20, 1 }, 6615 { "EreadPdu", 19, 1 }, 6616 { "CreadPdu", 18, 1 }, 6617 { "TunnelPkt", 17, 1 }, 6618 { "RcfPeerFin", 16, 1 }, 6619 { "RcfReasonOut", 12, 4 }, 6620 { "TxCchannel", 10, 2 }, 6621 { "RcfTxChannel", 8, 2 }, 6622 { "RxEchannel", 6, 2 }, 6623 { "RcfRxChannel", 5, 1 }, 6624 { "RcfDataOutSrdy", 4, 1 }, 6625 { "RxDvld", 3, 1 }, 6626 { "RxOoDvld", 2, 1 }, 6627 { "RxCongestion", 1, 1 }, 6628 { "TxCongestion", 0, 1 }, 6629 { NULL } 6630}; 6631 6632static struct field_desc tp_la1[] = { 6633 { "CplCmdIn", 56, 8 }, 6634 { "CplCmdOut", 48, 8 }, 6635 { "ESynOut", 47, 1 }, 6636 { "EAckOut", 46, 1 }, 6637 { "EFinOut", 45, 1 }, 6638 { "ERstOut", 44, 1 }, 6639 { "SynIn", 43, 1 }, 6640 { "AckIn", 42, 1 }, 6641 { "FinIn", 41, 1 }, 6642 { "RstIn", 40, 1 }, 6643 { "DataIn", 39, 1 }, 6644 { "DataInVld", 38, 1 }, 6645 { "PadIn", 37, 1 }, 6646 { "RxBufEmpty", 36, 1 }, 6647 { "RxDdp", 35, 1 }, 6648 { "RxFbCongestion", 34, 1 }, 6649 { "TxFbCongestion", 33, 1 }, 6650 { "TxPktSumSrdy", 32, 1 }, 6651 { "RcfUlpType", 28, 4 }, 6652 { "Eread", 27, 1 }, 6653 { "Ebypass", 26, 1 }, 6654 { "Esave", 25, 1 }, 6655 { "Static0", 24, 1 }, 6656 { "Cread", 23, 1 }, 6657 { "Cbypass", 22, 1 }, 6658 { "Csave", 21, 1 }, 6659 { "CPktOut", 20, 1 }, 6660 { "RxPagePoolFull", 18, 2 }, 6661 { "RxLpbkPkt", 17, 1 }, 6662 { "TxLpbkPkt", 16, 1 }, 6663 { "RxVfValid", 15, 1 }, 6664 { "SynLearned", 14, 1 }, 6665 { "SetDelEntry", 13, 1 }, 6666 { "SetInvEntry", 12, 1 }, 6667 { "CpcmdDvld", 11, 1 }, 6668 { "CpcmdSave", 10, 1 }, 6669 { "RxPstructsFull", 8, 2 }, 6670 { "EpcmdDvld", 7, 1 }, 6671 { "EpcmdFlush", 6, 1 }, 6672 { "EpcmdTrimPrefix", 5, 1 }, 6673 { "EpcmdTrimPostfix", 4, 1 }, 6674 { "ERssIp4Pkt", 3, 1 }, 6675 { "ERssIp6Pkt", 2, 1 }, 6676 { "ERssTcpUdpPkt", 1, 1 }, 6677 { "ERssFceFipPkt", 0, 1 }, 6678 { NULL } 6679}; 6680 6681static struct field_desc tp_la2[] = { 6682 { "CplCmdIn", 56, 8 }, 6683 { "MpsVfVld", 55, 1 }, 6684 { "MpsPf", 52, 3 }, 6685 { "MpsVf", 44, 8 }, 6686 { "SynIn", 43, 1 }, 6687 { "AckIn", 42, 1 }, 6688 { "FinIn", 41, 1 }, 6689 { "RstIn", 40, 1 }, 6690 { "DataIn", 39, 1 }, 6691 { "DataInVld", 38, 1 }, 6692 { "PadIn", 37, 1 }, 6693 { "RxBufEmpty", 36, 1 }, 6694 { "RxDdp", 35, 1 }, 6695 { "RxFbCongestion", 34, 1 }, 6696 { "TxFbCongestion", 33, 1 }, 6697 { "TxPktSumSrdy", 32, 1 }, 6698 { "RcfUlpType", 28, 4 }, 6699 { "Eread", 27, 1 }, 6700 { "Ebypass", 26, 1 }, 6701 { "Esave", 25, 1 }, 6702 { "Static0", 24, 1 }, 6703 { "Cread", 23, 1 }, 6704 { "Cbypass", 22, 1 }, 6705 { "Csave", 21, 1 }, 6706 { "CPktOut", 20, 1 }, 6707 { "RxPagePoolFull", 18, 2 }, 6708 { "RxLpbkPkt", 17, 1 }, 6709 { "TxLpbkPkt", 16, 1 }, 6710 { "RxVfValid", 15, 1 }, 6711 { "SynLearned", 14, 1 }, 6712 { "SetDelEntry", 13, 1 }, 6713 { "SetInvEntry", 12, 1 }, 6714 { "CpcmdDvld", 11, 1 }, 6715 { "CpcmdSave", 10, 1 }, 6716 { "RxPstructsFull", 8, 2 }, 6717 { "EpcmdDvld", 7, 1 }, 6718 { "EpcmdFlush", 6, 1 }, 6719 { "EpcmdTrimPrefix", 5, 1 }, 6720 { "EpcmdTrimPostfix", 4, 1 }, 6721 { "ERssIp4Pkt", 3, 1 }, 6722 { "ERssIp6Pkt", 2, 1 }, 6723 { "ERssTcpUdpPkt", 1, 1 }, 6724 { "ERssFceFipPkt", 0, 1 }, 6725 { NULL } 6726}; 6727 6728static void 6729tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6730{ 6731 6732 field_desc_show(sb, *p, tp_la0); 6733} 6734 6735static void 6736tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6737{ 6738 6739 if (idx) 6740 sbuf_printf(sb, "\n"); 6741 field_desc_show(sb, p[0], tp_la0); 6742 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6743 field_desc_show(sb, p[1], tp_la0); 6744} 6745 6746static void 6747tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6748{ 6749 6750 if (idx) 6751 sbuf_printf(sb, "\n"); 6752 field_desc_show(sb, p[0], tp_la0); 6753 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6754 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6755} 6756 6757static int 6758sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6759{ 6760 struct adapter *sc = arg1; 6761 struct sbuf *sb; 6762 uint64_t *buf, *p; 6763 int rc; 6764 u_int i, inc; 6765 void (*show_func)(struct sbuf *, uint64_t *, int); 6766 6767 rc = sysctl_wire_old_buffer(req, 0); 6768 if (rc != 0) 6769 return (rc); 6770 6771 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6772 if (sb == NULL) 6773 return (ENOMEM); 6774 6775 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6776 6777 t4_tp_read_la(sc, buf, NULL); 6778 p = buf; 6779 6780 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6781 case 2: 6782 inc = 2; 6783 show_func = tp_la_show2; 6784 break; 6785 case 3: 6786 inc = 2; 6787 show_func = tp_la_show3; 6788 break; 6789 default: 6790 inc = 1; 6791 show_func = tp_la_show; 6792 } 6793 6794 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6795 (*show_func)(sb, p, i); 6796 6797 rc = sbuf_finish(sb); 6798 sbuf_delete(sb); 6799 free(buf, M_CXGBE); 6800 return (rc); 6801} 6802 6803static int 6804sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6805{ 6806 struct adapter *sc = arg1; 6807 struct sbuf *sb; 6808 int rc; 6809 u64 nrate[NCHAN], orate[NCHAN]; 6810 6811 rc = sysctl_wire_old_buffer(req, 0); 6812 if (rc != 0) 6813 return (rc); 6814 6815 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6816 if (sb == NULL) 6817 return (ENOMEM); 6818 6819 t4_get_chan_txrate(sc, nrate, orate); 6820 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6821 "channel 3\n"); 6822 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6823 nrate[0], nrate[1], nrate[2], nrate[3]); 6824 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6825 orate[0], orate[1], orate[2], orate[3]); 6826 6827 rc = sbuf_finish(sb); 6828 sbuf_delete(sb); 6829 6830 return (rc); 6831} 6832 6833static int 6834sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6835{ 6836 struct adapter *sc = arg1; 6837 struct sbuf *sb; 6838 uint32_t *buf, *p; 6839 int rc, i; 6840 6841 rc = sysctl_wire_old_buffer(req, 0); 6842 if (rc != 0) 6843 return (rc); 6844 6845 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6846 if (sb == NULL) 6847 return (ENOMEM); 6848 6849 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6850 M_ZERO | M_WAITOK); 6851 6852 t4_ulprx_read_la(sc, buf); 6853 p = buf; 6854 6855 sbuf_printf(sb, " Pcmd Type Message" 6856 " Data"); 6857 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6858 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6859 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6860 } 6861 6862 rc = sbuf_finish(sb); 6863 sbuf_delete(sb); 6864 free(buf, M_CXGBE); 6865 return (rc); 6866} 6867 6868static int 6869sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6870{ 6871 struct adapter *sc = arg1; 6872 struct sbuf *sb; 6873 int rc, v; 6874 6875 rc = sysctl_wire_old_buffer(req, 0); 6876 if (rc != 0) 6877 return (rc); 6878 6879 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6880 if (sb == NULL) 6881 return (ENOMEM); 6882 6883 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6884 if (G_STATSOURCE_T5(v) == 7) { 6885 if (G_STATMODE(v) == 0) { 6886 sbuf_printf(sb, "total %d, incomplete %d", 6887 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6888 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6889 } else if (G_STATMODE(v) == 1) { 6890 sbuf_printf(sb, "total %d, data overflow %d", 6891 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6892 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6893 } 6894 } 6895 rc = sbuf_finish(sb); 6896 sbuf_delete(sb); 6897 6898 return (rc); 6899} 6900#endif 6901 6902static uint32_t 6903fconf_to_mode(uint32_t fconf) 6904{ 6905 uint32_t mode; 6906 6907 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6908 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6909 6910 if (fconf & F_FRAGMENTATION) 6911 mode |= T4_FILTER_IP_FRAGMENT; 6912 6913 if (fconf & F_MPSHITTYPE) 6914 mode |= T4_FILTER_MPS_HIT_TYPE; 6915 6916 if (fconf & F_MACMATCH) 6917 mode |= T4_FILTER_MAC_IDX; 6918 6919 if (fconf & F_ETHERTYPE) 6920 mode |= T4_FILTER_ETH_TYPE; 6921 6922 if (fconf & F_PROTOCOL) 6923 mode |= T4_FILTER_IP_PROTO; 6924 6925 if (fconf & F_TOS) 6926 mode |= T4_FILTER_IP_TOS; 6927 6928 if (fconf & F_VLAN) 6929 mode |= T4_FILTER_VLAN; 6930 6931 if (fconf & F_VNIC_ID) 6932 mode |= T4_FILTER_VNIC; 6933 6934 if (fconf & F_PORT) 6935 mode |= T4_FILTER_PORT; 6936 6937 if (fconf & F_FCOE) 6938 mode |= T4_FILTER_FCoE; 6939 6940 return (mode); 6941} 6942 6943static uint32_t 6944mode_to_fconf(uint32_t mode) 6945{ 6946 uint32_t fconf = 0; 6947 6948 if (mode & T4_FILTER_IP_FRAGMENT) 6949 fconf |= F_FRAGMENTATION; 6950 6951 if (mode & T4_FILTER_MPS_HIT_TYPE) 6952 fconf |= F_MPSHITTYPE; 6953 6954 if (mode & T4_FILTER_MAC_IDX) 6955 fconf |= F_MACMATCH; 6956 6957 if (mode & T4_FILTER_ETH_TYPE) 6958 fconf |= F_ETHERTYPE; 6959 6960 if (mode & T4_FILTER_IP_PROTO) 6961 fconf |= F_PROTOCOL; 6962 6963 if (mode & T4_FILTER_IP_TOS) 6964 fconf |= F_TOS; 6965 6966 if (mode & T4_FILTER_VLAN) 6967 fconf |= F_VLAN; 6968 6969 if (mode & T4_FILTER_VNIC) 6970 fconf |= F_VNIC_ID; 6971 6972 if (mode & T4_FILTER_PORT) 6973 fconf |= F_PORT; 6974 6975 if (mode & T4_FILTER_FCoE) 6976 fconf |= F_FCOE; 6977 6978 return (fconf); 6979} 6980 6981static uint32_t 6982fspec_to_fconf(struct t4_filter_specification *fs) 6983{ 6984 uint32_t fconf = 0; 6985 6986 if (fs->val.frag || fs->mask.frag) 6987 fconf |= F_FRAGMENTATION; 6988 6989 if (fs->val.matchtype || fs->mask.matchtype) 6990 fconf |= F_MPSHITTYPE; 6991 6992 if (fs->val.macidx || fs->mask.macidx) 6993 fconf |= F_MACMATCH; 6994 6995 if (fs->val.ethtype || fs->mask.ethtype) 6996 fconf |= F_ETHERTYPE; 6997 6998 if (fs->val.proto || fs->mask.proto) 6999 fconf |= F_PROTOCOL; 7000 7001 if (fs->val.tos || fs->mask.tos) 7002 fconf |= F_TOS; 7003 7004 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7005 fconf |= F_VLAN; 7006 7007 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7008 fconf |= F_VNIC_ID; 7009 7010 if (fs->val.iport || fs->mask.iport) 7011 fconf |= F_PORT; 7012 7013 if (fs->val.fcoe || fs->mask.fcoe) 7014 fconf |= F_FCOE; 7015 7016 return (fconf); 7017} 7018 7019static int 7020get_filter_mode(struct adapter *sc, uint32_t *mode) 7021{ 7022 int rc; 7023 uint32_t fconf; 7024 7025 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7026 "t4getfm"); 7027 if (rc) 7028 return (rc); 7029 7030 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7031 A_TP_VLAN_PRI_MAP); 7032 7033 if (sc->params.tp.vlan_pri_map != fconf) { 7034 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7035 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7036 fconf); 7037 } 7038 7039 *mode = fconf_to_mode(fconf); 7040 7041 end_synchronized_op(sc, LOCK_HELD); 7042 return (0); 7043} 7044 7045static int 7046set_filter_mode(struct adapter *sc, uint32_t mode) 7047{ 7048 uint32_t fconf; 7049 int rc; 7050 7051 fconf = mode_to_fconf(mode); 7052 7053 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7054 "t4setfm"); 7055 if (rc) 7056 return (rc); 7057 7058 if (sc->tids.ftids_in_use > 0) { 7059 rc = EBUSY; 7060 goto done; 7061 } 7062 7063#ifdef TCP_OFFLOAD 7064 if (uld_active(sc, ULD_TOM)) { 7065 rc = EBUSY; 7066 goto done; 7067 } 7068#endif 7069 7070 rc = -t4_set_filter_mode(sc, fconf); 7071done: 7072 end_synchronized_op(sc, LOCK_HELD); 7073 return (rc); 7074} 7075 7076static inline uint64_t 7077get_filter_hits(struct adapter *sc, uint32_t fid) 7078{ 7079 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7080 uint64_t hits; 7081 7082 memwin_info(sc, 0, &mw_base, NULL); 7083 off = position_memwin(sc, 0, 7084 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7085 if (is_t4(sc)) { 7086 hits = t4_read_reg64(sc, mw_base + off + 16); 7087 hits = be64toh(hits); 7088 } else { 7089 hits = t4_read_reg(sc, mw_base + off + 24); 7090 hits = be32toh(hits); 7091 } 7092 7093 return (hits); 7094} 7095 7096static int 7097get_filter(struct adapter *sc, struct t4_filter *t) 7098{ 7099 int i, rc, nfilters = sc->tids.nftids; 7100 struct filter_entry *f; 7101 7102 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7103 "t4getf"); 7104 if (rc) 7105 return (rc); 7106 7107 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7108 t->idx >= nfilters) { 7109 t->idx = 0xffffffff; 7110 goto done; 7111 } 7112 7113 f = &sc->tids.ftid_tab[t->idx]; 7114 for (i = t->idx; i < nfilters; i++, f++) { 7115 if (f->valid) { 7116 t->idx = i; 7117 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7118 t->smtidx = f->smtidx; 7119 if (f->fs.hitcnts) 7120 t->hits = get_filter_hits(sc, t->idx); 7121 else 7122 t->hits = UINT64_MAX; 7123 t->fs = f->fs; 7124 7125 goto done; 7126 } 7127 } 7128 7129 t->idx = 0xffffffff; 7130done: 7131 end_synchronized_op(sc, LOCK_HELD); 7132 return (0); 7133} 7134 7135static int 7136set_filter(struct adapter *sc, struct t4_filter *t) 7137{ 7138 unsigned int nfilters, nports; 7139 struct filter_entry *f; 7140 int i, rc; 7141 7142 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7143 if (rc) 7144 return (rc); 7145 7146 nfilters = sc->tids.nftids; 7147 nports = sc->params.nports; 7148 7149 if (nfilters == 0) { 7150 rc = ENOTSUP; 7151 goto done; 7152 } 7153 7154 if (!(sc->flags & FULL_INIT_DONE)) { 7155 rc = EAGAIN; 7156 goto done; 7157 } 7158 7159 if (t->idx >= nfilters) { 7160 rc = EINVAL; 7161 goto done; 7162 } 7163 7164 /* Validate against the global filter mode */ 7165 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7166 sc->params.tp.vlan_pri_map) { 7167 rc = E2BIG; 7168 goto done; 7169 } 7170 7171 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7172 rc = EINVAL; 7173 goto done; 7174 } 7175 7176 if (t->fs.val.iport >= nports) { 7177 rc = EINVAL; 7178 goto done; 7179 } 7180 7181 /* Can't specify an iq if not steering to it */ 7182 if (!t->fs.dirsteer && t->fs.iq) { 7183 rc = EINVAL; 7184 goto done; 7185 } 7186 7187 /* IPv6 filter idx must be 4 aligned */ 7188 if (t->fs.type == 1 && 7189 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7190 rc = EINVAL; 7191 goto done; 7192 } 7193 7194 if (sc->tids.ftid_tab == NULL) { 7195 KASSERT(sc->tids.ftids_in_use == 0, 7196 ("%s: no memory allocated but filters_in_use > 0", 7197 __func__)); 7198 7199 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7200 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7201 if (sc->tids.ftid_tab == NULL) { 7202 rc = ENOMEM; 7203 goto done; 7204 } 7205 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7206 } 7207 7208 for (i = 0; i < 4; i++) { 7209 f = &sc->tids.ftid_tab[t->idx + i]; 7210 7211 if (f->pending || f->valid) { 7212 rc = EBUSY; 7213 goto done; 7214 } 7215 if (f->locked) { 7216 rc = EPERM; 7217 goto done; 7218 } 7219 7220 if (t->fs.type == 0) 7221 break; 7222 } 7223 7224 f = &sc->tids.ftid_tab[t->idx]; 7225 f->fs = t->fs; 7226 7227 rc = set_filter_wr(sc, t->idx); 7228done: 7229 end_synchronized_op(sc, 0); 7230 7231 if (rc == 0) { 7232 mtx_lock(&sc->tids.ftid_lock); 7233 for (;;) { 7234 if (f->pending == 0) { 7235 rc = f->valid ? 0 : EIO; 7236 break; 7237 } 7238 7239 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7240 PCATCH, "t4setfw", 0)) { 7241 rc = EINPROGRESS; 7242 break; 7243 } 7244 } 7245 mtx_unlock(&sc->tids.ftid_lock); 7246 } 7247 return (rc); 7248} 7249 7250static int 7251del_filter(struct adapter *sc, struct t4_filter *t) 7252{ 7253 unsigned int nfilters; 7254 struct filter_entry *f; 7255 int rc; 7256 7257 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7258 if (rc) 7259 return (rc); 7260 7261 nfilters = sc->tids.nftids; 7262 7263 if (nfilters == 0) { 7264 rc = ENOTSUP; 7265 goto done; 7266 } 7267 7268 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7269 t->idx >= nfilters) { 7270 rc = EINVAL; 7271 goto done; 7272 } 7273 7274 if (!(sc->flags & FULL_INIT_DONE)) { 7275 rc = EAGAIN; 7276 goto done; 7277 } 7278 7279 f = &sc->tids.ftid_tab[t->idx]; 7280 7281 if (f->pending) { 7282 rc = EBUSY; 7283 goto done; 7284 } 7285 if (f->locked) { 7286 rc = EPERM; 7287 goto done; 7288 } 7289 7290 if (f->valid) { 7291 t->fs = f->fs; /* extra info for the caller */ 7292 rc = del_filter_wr(sc, t->idx); 7293 } 7294 7295done: 7296 end_synchronized_op(sc, 0); 7297 7298 if (rc == 0) { 7299 mtx_lock(&sc->tids.ftid_lock); 7300 for (;;) { 7301 if (f->pending == 0) { 7302 rc = f->valid ? EIO : 0; 7303 break; 7304 } 7305 7306 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7307 PCATCH, "t4delfw", 0)) { 7308 rc = EINPROGRESS; 7309 break; 7310 } 7311 } 7312 mtx_unlock(&sc->tids.ftid_lock); 7313 } 7314 7315 return (rc); 7316} 7317 7318static void 7319clear_filter(struct filter_entry *f) 7320{ 7321 if (f->l2t) 7322 t4_l2t_release(f->l2t); 7323 7324 bzero(f, sizeof (*f)); 7325} 7326 7327static int 7328set_filter_wr(struct adapter *sc, int fidx) 7329{ 7330 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7331 struct fw_filter_wr *fwr; 7332 unsigned int ftid; 7333 struct wrq_cookie cookie; 7334 7335 ASSERT_SYNCHRONIZED_OP(sc); 7336 7337 if (f->fs.newdmac || f->fs.newvlan) { 7338 /* This filter needs an L2T entry; allocate one. */ 7339 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7340 if (f->l2t == NULL) 7341 return (EAGAIN); 7342 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7343 f->fs.dmac)) { 7344 t4_l2t_release(f->l2t); 7345 f->l2t = NULL; 7346 return (ENOMEM); 7347 } 7348 } 7349 7350 ftid = sc->tids.ftid_base + fidx; 7351 7352 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7353 if (fwr == NULL) 7354 return (ENOMEM); 7355 bzero(fwr, sizeof(*fwr)); 7356 7357 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7358 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7359 fwr->tid_to_iq = 7360 htobe32(V_FW_FILTER_WR_TID(ftid) | 7361 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7362 V_FW_FILTER_WR_NOREPLY(0) | 7363 V_FW_FILTER_WR_IQ(f->fs.iq)); 7364 fwr->del_filter_to_l2tix = 7365 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7366 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7367 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7368 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7369 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7370 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7371 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7372 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7373 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7374 f->fs.newvlan == VLAN_REWRITE) | 7375 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7376 f->fs.newvlan == VLAN_REWRITE) | 7377 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7378 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7379 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7380 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7381 fwr->ethtype = htobe16(f->fs.val.ethtype); 7382 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7383 fwr->frag_to_ovlan_vldm = 7384 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7385 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7386 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7387 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7388 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7389 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7390 fwr->smac_sel = 0; 7391 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7392 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7393 fwr->maci_to_matchtypem = 7394 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7395 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7396 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7397 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7398 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7399 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7400 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7401 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7402 fwr->ptcl = f->fs.val.proto; 7403 fwr->ptclm = f->fs.mask.proto; 7404 fwr->ttyp = f->fs.val.tos; 7405 fwr->ttypm = f->fs.mask.tos; 7406 fwr->ivlan = htobe16(f->fs.val.vlan); 7407 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7408 fwr->ovlan = htobe16(f->fs.val.vnic); 7409 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7410 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7411 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7412 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7413 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7414 fwr->lp = htobe16(f->fs.val.dport); 7415 fwr->lpm = htobe16(f->fs.mask.dport); 7416 fwr->fp = htobe16(f->fs.val.sport); 7417 fwr->fpm = htobe16(f->fs.mask.sport); 7418 if (f->fs.newsmac) 7419 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7420 7421 f->pending = 1; 7422 sc->tids.ftids_in_use++; 7423 7424 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7425 return (0); 7426} 7427 7428static int 7429del_filter_wr(struct adapter *sc, int fidx) 7430{ 7431 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7432 struct fw_filter_wr *fwr; 7433 unsigned int ftid; 7434 struct wrq_cookie cookie; 7435 7436 ftid = sc->tids.ftid_base + fidx; 7437 7438 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7439 if (fwr == NULL) 7440 return (ENOMEM); 7441 bzero(fwr, sizeof (*fwr)); 7442 7443 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7444 7445 f->pending = 1; 7446 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7447 return (0); 7448} 7449 7450int 7451t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7452{ 7453 struct adapter *sc = iq->adapter; 7454 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7455 unsigned int idx = GET_TID(rpl); 7456 unsigned int rc; 7457 struct filter_entry *f; 7458 7459 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7460 rss->opcode)); 7461 7462 if (is_ftid(sc, idx)) { 7463 7464 idx -= sc->tids.ftid_base; 7465 f = &sc->tids.ftid_tab[idx]; 7466 rc = G_COOKIE(rpl->cookie); 7467 7468 mtx_lock(&sc->tids.ftid_lock); 7469 if (rc == FW_FILTER_WR_FLT_ADDED) { 7470 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7471 __func__, idx)); 7472 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7473 f->pending = 0; /* asynchronous setup completed */ 7474 f->valid = 1; 7475 } else { 7476 if (rc != FW_FILTER_WR_FLT_DELETED) { 7477 /* Add or delete failed, display an error */ 7478 log(LOG_ERR, 7479 "filter %u setup failed with error %u\n", 7480 idx, rc); 7481 } 7482 7483 clear_filter(f); 7484 sc->tids.ftids_in_use--; 7485 } 7486 wakeup(&sc->tids.ftid_tab); 7487 mtx_unlock(&sc->tids.ftid_lock); 7488 } 7489 7490 return (0); 7491} 7492 7493static int 7494get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7495{ 7496 int rc; 7497 7498 if (cntxt->cid > M_CTXTQID) 7499 return (EINVAL); 7500 7501 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7502 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7503 return (EINVAL); 7504 7505 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7506 if (rc) 7507 return (rc); 7508 7509 if (sc->flags & FW_OK) { 7510 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7511 &cntxt->data[0]); 7512 if (rc == 0) 7513 goto done; 7514 } 7515 7516 /* 7517 * Read via firmware failed or wasn't even attempted. Read directly via 7518 * the backdoor. 7519 */ 7520 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7521done: 7522 end_synchronized_op(sc, 0); 7523 return (rc); 7524} 7525 7526static int 7527load_fw(struct adapter *sc, struct t4_data *fw) 7528{ 7529 int rc; 7530 uint8_t *fw_data; 7531 7532 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7533 if (rc) 7534 return (rc); 7535 7536 if (sc->flags & FULL_INIT_DONE) { 7537 rc = EBUSY; 7538 goto done; 7539 } 7540 7541 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7542 if (fw_data == NULL) { 7543 rc = ENOMEM; 7544 goto done; 7545 } 7546 7547 rc = copyin(fw->data, fw_data, fw->len); 7548 if (rc == 0) 7549 rc = -t4_load_fw(sc, fw_data, fw->len); 7550 7551 free(fw_data, M_CXGBE); 7552done: 7553 end_synchronized_op(sc, 0); 7554 return (rc); 7555} 7556 7557static int 7558read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7559{ 7560 uint32_t addr, off, remaining, i, n; 7561 uint32_t *buf, *b; 7562 uint32_t mw_base, mw_aperture; 7563 int rc; 7564 uint8_t *dst; 7565 7566 rc = validate_mem_range(sc, mr->addr, mr->len); 7567 if (rc != 0) 7568 return (rc); 7569 7570 memwin_info(sc, win, &mw_base, &mw_aperture); 7571 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7572 addr = mr->addr; 7573 remaining = mr->len; 7574 dst = (void *)mr->data; 7575 7576 while (remaining) { 7577 off = position_memwin(sc, win, addr); 7578 7579 /* number of bytes that we'll copy in the inner loop */ 7580 n = min(remaining, mw_aperture - off); 7581 for (i = 0; i < n; i += 4) 7582 *b++ = t4_read_reg(sc, mw_base + off + i); 7583 7584 rc = copyout(buf, dst, n); 7585 if (rc != 0) 7586 break; 7587 7588 b = buf; 7589 dst += n; 7590 remaining -= n; 7591 addr += n; 7592 } 7593 7594 free(buf, M_CXGBE); 7595 return (rc); 7596} 7597 7598static int 7599read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7600{ 7601 int rc; 7602 7603 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7604 return (EINVAL); 7605 7606 if (i2cd->len > sizeof(i2cd->data)) 7607 return (EFBIG); 7608 7609 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7610 if (rc) 7611 return (rc); 7612 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7613 i2cd->offset, i2cd->len, &i2cd->data[0]); 7614 end_synchronized_op(sc, 0); 7615 7616 return (rc); 7617} 7618 7619static int 7620in_range(int val, int lo, int hi) 7621{ 7622 7623 return (val < 0 || (val <= hi && val >= lo)); 7624} 7625 7626static int 7627set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7628{ 7629 int fw_subcmd, fw_type, rc; 7630 7631 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7632 if (rc) 7633 return (rc); 7634 7635 if (!(sc->flags & FULL_INIT_DONE)) { 7636 rc = EAGAIN; 7637 goto done; 7638 } 7639 7640 /* 7641 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7642 * sub-command and type are in common locations.) 7643 */ 7644 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7645 fw_subcmd = FW_SCHED_SC_CONFIG; 7646 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7647 fw_subcmd = FW_SCHED_SC_PARAMS; 7648 else { 7649 rc = EINVAL; 7650 goto done; 7651 } 7652 if (p->type == SCHED_CLASS_TYPE_PACKET) 7653 fw_type = FW_SCHED_TYPE_PKTSCHED; 7654 else { 7655 rc = EINVAL; 7656 goto done; 7657 } 7658 7659 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7660 /* Vet our parameters ..*/ 7661 if (p->u.config.minmax < 0) { 7662 rc = EINVAL; 7663 goto done; 7664 } 7665 7666 /* And pass the request to the firmware ...*/ 7667 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7668 goto done; 7669 } 7670 7671 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7672 int fw_level; 7673 int fw_mode; 7674 int fw_rateunit; 7675 int fw_ratemode; 7676 7677 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7678 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7679 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7680 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7681 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7682 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7683 else { 7684 rc = EINVAL; 7685 goto done; 7686 } 7687 7688 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7689 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7690 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7691 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7692 else { 7693 rc = EINVAL; 7694 goto done; 7695 } 7696 7697 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7698 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7699 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7700 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7701 else { 7702 rc = EINVAL; 7703 goto done; 7704 } 7705 7706 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7707 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7708 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7709 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7710 else { 7711 rc = EINVAL; 7712 goto done; 7713 } 7714 7715 /* Vet our parameters ... */ 7716 if (!in_range(p->u.params.channel, 0, 3) || 7717 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7718 !in_range(p->u.params.minrate, 0, 10000000) || 7719 !in_range(p->u.params.maxrate, 0, 10000000) || 7720 !in_range(p->u.params.weight, 0, 100)) { 7721 rc = ERANGE; 7722 goto done; 7723 } 7724 7725 /* 7726 * Translate any unset parameters into the firmware's 7727 * nomenclature and/or fail the call if the parameters 7728 * are required ... 7729 */ 7730 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7731 p->u.params.channel < 0 || p->u.params.cl < 0) { 7732 rc = EINVAL; 7733 goto done; 7734 } 7735 if (p->u.params.minrate < 0) 7736 p->u.params.minrate = 0; 7737 if (p->u.params.maxrate < 0) { 7738 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7739 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7740 rc = EINVAL; 7741 goto done; 7742 } else 7743 p->u.params.maxrate = 0; 7744 } 7745 if (p->u.params.weight < 0) { 7746 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7747 rc = EINVAL; 7748 goto done; 7749 } else 7750 p->u.params.weight = 0; 7751 } 7752 if (p->u.params.pktsize < 0) { 7753 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7754 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7755 rc = EINVAL; 7756 goto done; 7757 } else 7758 p->u.params.pktsize = 0; 7759 } 7760 7761 /* See what the firmware thinks of the request ... */ 7762 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7763 fw_rateunit, fw_ratemode, p->u.params.channel, 7764 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7765 p->u.params.weight, p->u.params.pktsize, 1); 7766 goto done; 7767 } 7768 7769 rc = EINVAL; 7770done: 7771 end_synchronized_op(sc, 0); 7772 return (rc); 7773} 7774 7775static int 7776set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7777{ 7778 struct port_info *pi = NULL; 7779 struct sge_txq *txq; 7780 uint32_t fw_mnem, fw_queue, fw_class; 7781 int i, rc; 7782 7783 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7784 if (rc) 7785 return (rc); 7786 7787 if (!(sc->flags & FULL_INIT_DONE)) { 7788 rc = EAGAIN; 7789 goto done; 7790 } 7791 7792 if (p->port >= sc->params.nports) { 7793 rc = EINVAL; 7794 goto done; 7795 } 7796 7797 pi = sc->port[p->port]; 7798 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7799 rc = EINVAL; 7800 goto done; 7801 } 7802 7803 /* 7804 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7805 * Scheduling Class in this case). 7806 */ 7807 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7808 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7809 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7810 7811 /* 7812 * If op.queue is non-negative, then we're only changing the scheduling 7813 * on a single specified TX queue. 7814 */ 7815 if (p->queue >= 0) { 7816 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7817 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7818 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7819 &fw_class); 7820 goto done; 7821 } 7822 7823 /* 7824 * Change the scheduling on all the TX queues for the 7825 * interface. 7826 */ 7827 for_each_txq(pi, i, txq) { 7828 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7829 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7830 &fw_class); 7831 if (rc) 7832 goto done; 7833 } 7834 7835 rc = 0; 7836done: 7837 end_synchronized_op(sc, 0); 7838 return (rc); 7839} 7840 7841int 7842t4_os_find_pci_capability(struct adapter *sc, int cap) 7843{ 7844 int i; 7845 7846 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7847} 7848 7849int 7850t4_os_pci_save_state(struct adapter *sc) 7851{ 7852 device_t dev; 7853 struct pci_devinfo *dinfo; 7854 7855 dev = sc->dev; 7856 dinfo = device_get_ivars(dev); 7857 7858 pci_cfg_save(dev, dinfo, 0); 7859 return (0); 7860} 7861 7862int 7863t4_os_pci_restore_state(struct adapter *sc) 7864{ 7865 device_t dev; 7866 struct pci_devinfo *dinfo; 7867 7868 dev = sc->dev; 7869 dinfo = device_get_ivars(dev); 7870 7871 pci_cfg_restore(dev, dinfo); 7872 return (0); 7873} 7874 7875void 7876t4_os_portmod_changed(const struct adapter *sc, int idx) 7877{ 7878 struct port_info *pi = sc->port[idx]; 7879 static const char *mod_str[] = { 7880 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7881 }; 7882 7883 build_medialist(pi, &pi->media); 7884#ifdef DEV_NETMAP 7885 build_medialist(pi, &pi->nm_media); 7886#endif 7887 7888 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7889 if_printf(pi->ifp, "transceiver unplugged.\n"); 7890 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7891 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7892 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7893 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7894 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7895 if_printf(pi->ifp, "%s transceiver inserted.\n", 7896 mod_str[pi->mod_type]); 7897 } else { 7898 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7899 pi->mod_type); 7900 } 7901} 7902 7903void 7904t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7905{ 7906 struct port_info *pi = sc->port[idx]; 7907 struct ifnet *ifp = pi->ifp; 7908 7909 if (link_stat) { 7910 pi->linkdnrc = -1; 7911 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7912 if_link_state_change(ifp, LINK_STATE_UP); 7913 } else { 7914 if (reason >= 0) 7915 pi->linkdnrc = reason; 7916 if_link_state_change(ifp, LINK_STATE_DOWN); 7917 } 7918} 7919 7920void 7921t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7922{ 7923 struct adapter *sc; 7924 7925 sx_slock(&t4_list_lock); 7926 SLIST_FOREACH(sc, &t4_list, link) { 7927 /* 7928 * func should not make any assumptions about what state sc is 7929 * in - the only guarantee is that sc->sc_lock is a valid lock. 7930 */ 7931 func(sc, arg); 7932 } 7933 sx_sunlock(&t4_list_lock); 7934} 7935 7936static int 7937t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7938{ 7939 return (0); 7940} 7941 7942static int 7943t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7944{ 7945 return (0); 7946} 7947 7948static int 7949t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7950 struct thread *td) 7951{ 7952 int rc; 7953 struct adapter *sc = dev->si_drv1; 7954 7955 rc = priv_check(td, PRIV_DRIVER); 7956 if (rc != 0) 7957 return (rc); 7958 7959 switch (cmd) { 7960 case CHELSIO_T4_GETREG: { 7961 struct t4_reg *edata = (struct t4_reg *)data; 7962 7963 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7964 return (EFAULT); 7965 7966 if (edata->size == 4) 7967 edata->val = t4_read_reg(sc, edata->addr); 7968 else if (edata->size == 8) 7969 edata->val = t4_read_reg64(sc, edata->addr); 7970 else 7971 return (EINVAL); 7972 7973 break; 7974 } 7975 case CHELSIO_T4_SETREG: { 7976 struct t4_reg *edata = (struct t4_reg *)data; 7977 7978 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7979 return (EFAULT); 7980 7981 if (edata->size == 4) { 7982 if (edata->val & 0xffffffff00000000) 7983 return (EINVAL); 7984 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 7985 } else if (edata->size == 8) 7986 t4_write_reg64(sc, edata->addr, edata->val); 7987 else 7988 return (EINVAL); 7989 break; 7990 } 7991 case CHELSIO_T4_REGDUMP: { 7992 struct t4_regdump *regs = (struct t4_regdump *)data; 7993 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 7994 uint8_t *buf; 7995 7996 if (regs->len < reglen) { 7997 regs->len = reglen; /* hint to the caller */ 7998 return (ENOBUFS); 7999 } 8000 8001 regs->len = reglen; 8002 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8003 t4_get_regs(sc, regs, buf); 8004 rc = copyout(buf, regs->data, reglen); 8005 free(buf, M_CXGBE); 8006 break; 8007 } 8008 case CHELSIO_T4_GET_FILTER_MODE: 8009 rc = get_filter_mode(sc, (uint32_t *)data); 8010 break; 8011 case CHELSIO_T4_SET_FILTER_MODE: 8012 rc = set_filter_mode(sc, *(uint32_t *)data); 8013 break; 8014 case CHELSIO_T4_GET_FILTER: 8015 rc = get_filter(sc, (struct t4_filter *)data); 8016 break; 8017 case CHELSIO_T4_SET_FILTER: 8018 rc = set_filter(sc, (struct t4_filter *)data); 8019 break; 8020 case CHELSIO_T4_DEL_FILTER: 8021 rc = del_filter(sc, (struct t4_filter *)data); 8022 break; 8023 case CHELSIO_T4_GET_SGE_CONTEXT: 8024 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8025 break; 8026 case CHELSIO_T4_LOAD_FW: 8027 rc = load_fw(sc, (struct t4_data *)data); 8028 break; 8029 case CHELSIO_T4_GET_MEM: 8030 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8031 break; 8032 case CHELSIO_T4_GET_I2C: 8033 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8034 break; 8035 case CHELSIO_T4_CLEAR_STATS: { 8036 int i; 8037 u_int port_id = *(uint32_t *)data; 8038 struct port_info *pi; 8039 8040 if (port_id >= sc->params.nports) 8041 return (EINVAL); 8042 pi = sc->port[port_id]; 8043 8044 /* MAC stats */ 8045 t4_clr_port_stats(sc, pi->tx_chan); 8046 pi->tx_parse_error = 0; 8047 8048 if (pi->flags & PORT_INIT_DONE) { 8049 struct sge_rxq *rxq; 8050 struct sge_txq *txq; 8051 struct sge_wrq *wrq; 8052 8053 for_each_rxq(pi, i, rxq) { 8054#if defined(INET) || defined(INET6) 8055 rxq->lro.lro_queued = 0; 8056 rxq->lro.lro_flushed = 0; 8057#endif 8058 rxq->rxcsum = 0; 8059 rxq->vlan_extraction = 0; 8060 } 8061 8062 for_each_txq(pi, i, txq) { 8063 txq->txcsum = 0; 8064 txq->tso_wrs = 0; 8065 txq->vlan_insertion = 0; 8066 txq->imm_wrs = 0; 8067 txq->sgl_wrs = 0; 8068 txq->txpkt_wrs = 0; 8069 txq->txpkts0_wrs = 0; 8070 txq->txpkts1_wrs = 0; 8071 txq->txpkts0_pkts = 0; 8072 txq->txpkts1_pkts = 0; 8073 mp_ring_reset_stats(txq->r); 8074 } 8075 8076#ifdef TCP_OFFLOAD 8077 /* nothing to clear for each ofld_rxq */ 8078 8079 for_each_ofld_txq(pi, i, wrq) { 8080 wrq->tx_wrs_direct = 0; 8081 wrq->tx_wrs_copied = 0; 8082 } 8083#endif 8084 wrq = &sc->sge.ctrlq[pi->port_id]; 8085 wrq->tx_wrs_direct = 0; 8086 wrq->tx_wrs_copied = 0; 8087 } 8088 break; 8089 } 8090 case CHELSIO_T4_SCHED_CLASS: 8091 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8092 break; 8093 case CHELSIO_T4_SCHED_QUEUE: 8094 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8095 break; 8096 case CHELSIO_T4_GET_TRACER: 8097 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8098 break; 8099 case CHELSIO_T4_SET_TRACER: 8100 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8101 break; 8102 default: 8103 rc = EINVAL; 8104 } 8105 8106 return (rc); 8107} 8108 8109#ifdef TCP_OFFLOAD 8110void 8111t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8112 const unsigned int *pgsz_order) 8113{ 8114 struct port_info *pi = ifp->if_softc; 8115 struct adapter *sc = pi->adapter; 8116 8117 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8118 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8119 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8120 V_HPZ3(pgsz_order[3])); 8121} 8122 8123static int 8124toe_capability(struct port_info *pi, int enable) 8125{ 8126 int rc; 8127 struct adapter *sc = pi->adapter; 8128 8129 ASSERT_SYNCHRONIZED_OP(sc); 8130 8131 if (!is_offload(sc)) 8132 return (ENODEV); 8133 8134 if (enable) { 8135 /* 8136 * We need the port's queues around so that we're able to send 8137 * and receive CPLs to/from the TOE even if the ifnet for this 8138 * port has never been UP'd administratively. 8139 */ 8140 if (!(pi->flags & PORT_INIT_DONE)) { 8141 rc = cxgbe_init_synchronized(pi); 8142 if (rc) 8143 return (rc); 8144 } 8145 8146 if (isset(&sc->offload_map, pi->port_id)) 8147 return (0); 8148 8149 if (!uld_active(sc, ULD_TOM)) { 8150 rc = t4_activate_uld(sc, ULD_TOM); 8151 if (rc == EAGAIN) { 8152 log(LOG_WARNING, 8153 "You must kldload t4_tom.ko before trying " 8154 "to enable TOE on a cxgbe interface.\n"); 8155 } 8156 if (rc != 0) 8157 return (rc); 8158 KASSERT(sc->tom_softc != NULL, 8159 ("%s: TOM activated but softc NULL", __func__)); 8160 KASSERT(uld_active(sc, ULD_TOM), 8161 ("%s: TOM activated but flag not set", __func__)); 8162 } 8163 8164 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8165 if (!uld_active(sc, ULD_IWARP)) 8166 (void) t4_activate_uld(sc, ULD_IWARP); 8167 if (!uld_active(sc, ULD_ISCSI)) 8168 (void) t4_activate_uld(sc, ULD_ISCSI); 8169 8170 setbit(&sc->offload_map, pi->port_id); 8171 } else { 8172 if (!isset(&sc->offload_map, pi->port_id)) 8173 return (0); 8174 8175 KASSERT(uld_active(sc, ULD_TOM), 8176 ("%s: TOM never initialized?", __func__)); 8177 clrbit(&sc->offload_map, pi->port_id); 8178 } 8179 8180 return (0); 8181} 8182 8183/* 8184 * Add an upper layer driver to the global list. 8185 */ 8186int 8187t4_register_uld(struct uld_info *ui) 8188{ 8189 int rc = 0; 8190 struct uld_info *u; 8191 8192 sx_xlock(&t4_uld_list_lock); 8193 SLIST_FOREACH(u, &t4_uld_list, link) { 8194 if (u->uld_id == ui->uld_id) { 8195 rc = EEXIST; 8196 goto done; 8197 } 8198 } 8199 8200 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8201 ui->refcount = 0; 8202done: 8203 sx_xunlock(&t4_uld_list_lock); 8204 return (rc); 8205} 8206 8207int 8208t4_unregister_uld(struct uld_info *ui) 8209{ 8210 int rc = EINVAL; 8211 struct uld_info *u; 8212 8213 sx_xlock(&t4_uld_list_lock); 8214 8215 SLIST_FOREACH(u, &t4_uld_list, link) { 8216 if (u == ui) { 8217 if (ui->refcount > 0) { 8218 rc = EBUSY; 8219 goto done; 8220 } 8221 8222 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8223 rc = 0; 8224 goto done; 8225 } 8226 } 8227done: 8228 sx_xunlock(&t4_uld_list_lock); 8229 return (rc); 8230} 8231 8232int 8233t4_activate_uld(struct adapter *sc, int id) 8234{ 8235 int rc; 8236 struct uld_info *ui; 8237 8238 ASSERT_SYNCHRONIZED_OP(sc); 8239 8240 if (id < 0 || id > ULD_MAX) 8241 return (EINVAL); 8242 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8243 8244 sx_slock(&t4_uld_list_lock); 8245 8246 SLIST_FOREACH(ui, &t4_uld_list, link) { 8247 if (ui->uld_id == id) { 8248 if (!(sc->flags & FULL_INIT_DONE)) { 8249 rc = adapter_full_init(sc); 8250 if (rc != 0) 8251 break; 8252 } 8253 8254 rc = ui->activate(sc); 8255 if (rc == 0) { 8256 setbit(&sc->active_ulds, id); 8257 ui->refcount++; 8258 } 8259 break; 8260 } 8261 } 8262 8263 sx_sunlock(&t4_uld_list_lock); 8264 8265 return (rc); 8266} 8267 8268int 8269t4_deactivate_uld(struct adapter *sc, int id) 8270{ 8271 int rc; 8272 struct uld_info *ui; 8273 8274 ASSERT_SYNCHRONIZED_OP(sc); 8275 8276 if (id < 0 || id > ULD_MAX) 8277 return (EINVAL); 8278 rc = ENXIO; 8279 8280 sx_slock(&t4_uld_list_lock); 8281 8282 SLIST_FOREACH(ui, &t4_uld_list, link) { 8283 if (ui->uld_id == id) { 8284 rc = ui->deactivate(sc); 8285 if (rc == 0) { 8286 clrbit(&sc->active_ulds, id); 8287 ui->refcount--; 8288 } 8289 break; 8290 } 8291 } 8292 8293 sx_sunlock(&t4_uld_list_lock); 8294 8295 return (rc); 8296} 8297 8298int 8299uld_active(struct adapter *sc, int uld_id) 8300{ 8301 8302 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8303 8304 return (isset(&sc->active_ulds, uld_id)); 8305} 8306#endif 8307 8308/* 8309 * Come up with reasonable defaults for some of the tunables, provided they're 8310 * not set by the user (in which case we'll use the values as is). 8311 */ 8312static void 8313tweak_tunables(void) 8314{ 8315 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8316 8317 if (t4_ntxq10g < 1) 8318 t4_ntxq10g = min(nc, NTXQ_10G); 8319 8320 if (t4_ntxq1g < 1) 8321 t4_ntxq1g = min(nc, NTXQ_1G); 8322 8323 if (t4_nrxq10g < 1) 8324 t4_nrxq10g = min(nc, NRXQ_10G); 8325 8326 if (t4_nrxq1g < 1) 8327 t4_nrxq1g = min(nc, NRXQ_1G); 8328 8329#ifdef TCP_OFFLOAD 8330 if (t4_nofldtxq10g < 1) 8331 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8332 8333 if (t4_nofldtxq1g < 1) 8334 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8335 8336 if (t4_nofldrxq10g < 1) 8337 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8338 8339 if (t4_nofldrxq1g < 1) 8340 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8341 8342 if (t4_toecaps_allowed == -1) 8343 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8344#else 8345 if (t4_toecaps_allowed == -1) 8346 t4_toecaps_allowed = 0; 8347#endif 8348 8349#ifdef DEV_NETMAP 8350 if (t4_nnmtxq10g < 1) 8351 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8352 8353 if (t4_nnmtxq1g < 1) 8354 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8355 8356 if (t4_nnmrxq10g < 1) 8357 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8358 8359 if (t4_nnmrxq1g < 1) 8360 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8361#endif 8362 8363 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8364 t4_tmr_idx_10g = TMR_IDX_10G; 8365 8366 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8367 t4_pktc_idx_10g = PKTC_IDX_10G; 8368 8369 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8370 t4_tmr_idx_1g = TMR_IDX_1G; 8371 8372 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8373 t4_pktc_idx_1g = PKTC_IDX_1G; 8374 8375 if (t4_qsize_txq < 128) 8376 t4_qsize_txq = 128; 8377 8378 if (t4_qsize_rxq < 128) 8379 t4_qsize_rxq = 128; 8380 while (t4_qsize_rxq & 7) 8381 t4_qsize_rxq++; 8382 8383 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8384} 8385 8386static struct sx mlu; /* mod load unload */ 8387SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8388 8389static int 8390mod_event(module_t mod, int cmd, void *arg) 8391{ 8392 int rc = 0; 8393 static int loaded = 0; 8394 8395 switch (cmd) { 8396 case MOD_LOAD: 8397 sx_xlock(&mlu); 8398 if (loaded++ == 0) { 8399 t4_sge_modload(); 8400 sx_init(&t4_list_lock, "T4/T5 adapters"); 8401 SLIST_INIT(&t4_list); 8402#ifdef TCP_OFFLOAD 8403 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8404 SLIST_INIT(&t4_uld_list); 8405#endif 8406 t4_tracer_modload(); 8407 tweak_tunables(); 8408 } 8409 sx_xunlock(&mlu); 8410 break; 8411 8412 case MOD_UNLOAD: 8413 sx_xlock(&mlu); 8414 if (--loaded == 0) { 8415 int tries; 8416 8417 sx_slock(&t4_list_lock); 8418 if (!SLIST_EMPTY(&t4_list)) { 8419 rc = EBUSY; 8420 sx_sunlock(&t4_list_lock); 8421 goto done_unload; 8422 } 8423#ifdef TCP_OFFLOAD 8424 sx_slock(&t4_uld_list_lock); 8425 if (!SLIST_EMPTY(&t4_uld_list)) { 8426 rc = EBUSY; 8427 sx_sunlock(&t4_uld_list_lock); 8428 sx_sunlock(&t4_list_lock); 8429 goto done_unload; 8430 } 8431#endif 8432 tries = 0; 8433 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8434 uprintf("%ju clusters with custom free routine " 8435 "still is use.\n", t4_sge_extfree_refs()); 8436 pause("t4unload", 2 * hz); 8437 } 8438#ifdef TCP_OFFLOAD 8439 sx_sunlock(&t4_uld_list_lock); 8440#endif 8441 sx_sunlock(&t4_list_lock); 8442 8443 if (t4_sge_extfree_refs() == 0) { 8444 t4_tracer_modunload(); 8445#ifdef TCP_OFFLOAD 8446 sx_destroy(&t4_uld_list_lock); 8447#endif 8448 sx_destroy(&t4_list_lock); 8449 t4_sge_modunload(); 8450 loaded = 0; 8451 } else { 8452 rc = EBUSY; 8453 loaded++; /* undo earlier decrement */ 8454 } 8455 } 8456done_unload: 8457 sx_xunlock(&mlu); 8458 break; 8459 } 8460 8461 return (rc); 8462} 8463 8464static devclass_t t4_devclass, t5_devclass; 8465static devclass_t cxgbe_devclass, cxl_devclass; 8466 8467DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8468MODULE_VERSION(t4nex, 1); 8469MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8470 8471DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8472MODULE_VERSION(t5nex, 1); 8473MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8474 8475DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8476MODULE_VERSION(cxgbe, 1); 8477 8478DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8479MODULE_VERSION(cxl, 1); 8480