t4_main.c revision 287149
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 287149 2015-08-26 01:41:55Z np $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/systm.h> 40#include <sys/counter.h> 41#include <sys/module.h> 42#include <sys/malloc.h> 43#include <sys/queue.h> 44#include <sys/taskqueue.h> 45#include <sys/pciio.h> 46#include <dev/pci/pcireg.h> 47#include <dev/pci/pcivar.h> 48#include <dev/pci/pci_private.h> 49#include <sys/firmware.h> 50#include <sys/sbuf.h> 51#include <sys/smp.h> 52#include <sys/socket.h> 53#include <sys/sockio.h> 54#include <sys/sysctl.h> 55#include <net/ethernet.h> 56#include <net/if.h> 57#include <net/if_types.h> 58#include <net/if_dl.h> 59#include <net/if_vlan_var.h> 60#if defined(__i386__) || defined(__amd64__) 61#include <vm/vm.h> 62#include <vm/pmap.h> 63#endif 64 65#include "common/common.h" 66#include "common/t4_msg.h" 67#include "common/t4_regs.h" 68#include "common/t4_regs_values.h" 69#include "t4_ioctl.h" 70#include "t4_l2t.h" 71#include "t4_mp_ring.h" 72 73/* T4 bus driver interface */ 74static int t4_probe(device_t); 75static int t4_attach(device_t); 76static int t4_detach(device_t); 77static device_method_t t4_methods[] = { 78 DEVMETHOD(device_probe, t4_probe), 79 DEVMETHOD(device_attach, t4_attach), 80 DEVMETHOD(device_detach, t4_detach), 81 82 DEVMETHOD_END 83}; 84static driver_t t4_driver = { 85 "t4nex", 86 t4_methods, 87 sizeof(struct adapter) 88}; 89 90 91/* T4 port (cxgbe) interface */ 92static int cxgbe_probe(device_t); 93static int cxgbe_attach(device_t); 94static int cxgbe_detach(device_t); 95static device_method_t cxgbe_methods[] = { 96 DEVMETHOD(device_probe, cxgbe_probe), 97 DEVMETHOD(device_attach, cxgbe_attach), 98 DEVMETHOD(device_detach, cxgbe_detach), 99 { 0, 0 } 100}; 101static driver_t cxgbe_driver = { 102 "cxgbe", 103 cxgbe_methods, 104 sizeof(struct port_info) 105}; 106 107static d_ioctl_t t4_ioctl; 108static d_open_t t4_open; 109static d_close_t t4_close; 110 111static struct cdevsw t4_cdevsw = { 112 .d_version = D_VERSION, 113 .d_flags = 0, 114 .d_open = t4_open, 115 .d_close = t4_close, 116 .d_ioctl = t4_ioctl, 117 .d_name = "t4nex", 118}; 119 120/* T5 bus driver interface */ 121static int t5_probe(device_t); 122static device_method_t t5_methods[] = { 123 DEVMETHOD(device_probe, t5_probe), 124 DEVMETHOD(device_attach, t4_attach), 125 DEVMETHOD(device_detach, t4_detach), 126 127 DEVMETHOD_END 128}; 129static driver_t t5_driver = { 130 "t5nex", 131 t5_methods, 132 sizeof(struct adapter) 133}; 134 135 136/* T5 port (cxl) interface */ 137static driver_t cxl_driver = { 138 "cxl", 139 cxgbe_methods, 140 sizeof(struct port_info) 141}; 142 143static struct cdevsw t5_cdevsw = { 144 .d_version = D_VERSION, 145 .d_flags = 0, 146 .d_open = t4_open, 147 .d_close = t4_close, 148 .d_ioctl = t4_ioctl, 149 .d_name = "t5nex", 150}; 151 152/* ifnet + media interface */ 153static void cxgbe_init(void *); 154static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 155static int cxgbe_transmit(struct ifnet *, struct mbuf *); 156static void cxgbe_qflush(struct ifnet *); 157static int cxgbe_media_change(struct ifnet *); 158static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 159 160MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 161 162/* 163 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 164 * then ADAPTER_LOCK, then t4_uld_list_lock. 165 */ 166static struct sx t4_list_lock; 167SLIST_HEAD(, adapter) t4_list; 168#ifdef TCP_OFFLOAD 169static struct sx t4_uld_list_lock; 170SLIST_HEAD(, uld_info) t4_uld_list; 171#endif 172 173/* 174 * Tunables. See tweak_tunables() too. 175 * 176 * Each tunable is set to a default value here if it's known at compile-time. 177 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 178 * provide a reasonable default when the driver is loaded. 179 * 180 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 181 * T5 are under hw.cxl. 182 */ 183 184/* 185 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 186 */ 187#define NTXQ_10G 16 188static int t4_ntxq10g = -1; 189TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 190 191#define NRXQ_10G 8 192static int t4_nrxq10g = -1; 193TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 194 195#define NTXQ_1G 4 196static int t4_ntxq1g = -1; 197TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 198 199#define NRXQ_1G 2 200static int t4_nrxq1g = -1; 201TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 202 203static int t4_rsrv_noflowq = 0; 204TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 205 206#ifdef TCP_OFFLOAD 207#define NOFLDTXQ_10G 8 208static int t4_nofldtxq10g = -1; 209TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 210 211#define NOFLDRXQ_10G 2 212static int t4_nofldrxq10g = -1; 213TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 214 215#define NOFLDTXQ_1G 2 216static int t4_nofldtxq1g = -1; 217TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 218 219#define NOFLDRXQ_1G 1 220static int t4_nofldrxq1g = -1; 221TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 222#endif 223 224#ifdef DEV_NETMAP 225#define NNMTXQ_10G 2 226static int t4_nnmtxq10g = -1; 227TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 228 229#define NNMRXQ_10G 2 230static int t4_nnmrxq10g = -1; 231TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 232 233#define NNMTXQ_1G 1 234static int t4_nnmtxq1g = -1; 235TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 236 237#define NNMRXQ_1G 1 238static int t4_nnmrxq1g = -1; 239TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 240#endif 241 242/* 243 * Holdoff parameters for 10G and 1G ports. 244 */ 245#define TMR_IDX_10G 1 246static int t4_tmr_idx_10g = TMR_IDX_10G; 247TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 248 249#define PKTC_IDX_10G (-1) 250static int t4_pktc_idx_10g = PKTC_IDX_10G; 251TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 252 253#define TMR_IDX_1G 1 254static int t4_tmr_idx_1g = TMR_IDX_1G; 255TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 256 257#define PKTC_IDX_1G (-1) 258static int t4_pktc_idx_1g = PKTC_IDX_1G; 259TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 260 261/* 262 * Size (# of entries) of each tx and rx queue. 263 */ 264static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 265TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 266 267static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 268TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 269 270/* 271 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 272 */ 273static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 274TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 275 276/* 277 * Configuration file. 278 */ 279#define DEFAULT_CF "default" 280#define FLASH_CF "flash" 281#define UWIRE_CF "uwire" 282#define FPGA_CF "fpga" 283static char t4_cfg_file[32] = DEFAULT_CF; 284TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 285 286/* 287 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 288 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 289 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 290 * mark or when signalled to do so, 0 to never emit PAUSE. 291 */ 292static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 293TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 294 295/* 296 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 297 * encouraged respectively). 298 */ 299static unsigned int t4_fw_install = 1; 300TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 301 302/* 303 * ASIC features that will be used. Disable the ones you don't want so that the 304 * chip resources aren't wasted on features that will not be used. 305 */ 306static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 307TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 308 309static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 310TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 311 312static int t4_toecaps_allowed = -1; 313TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 314 315static int t4_rdmacaps_allowed = 0; 316TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 317 318static int t4_iscsicaps_allowed = 0; 319TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 320 321static int t4_fcoecaps_allowed = 0; 322TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 323 324static int t5_write_combine = 0; 325TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 326 327struct intrs_and_queues { 328 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 329 uint16_t nirq; /* Total # of vectors */ 330 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 331 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 332 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 333 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 334 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 335 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 336 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 337#ifdef TCP_OFFLOAD 338 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 339 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 340 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 341 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 342#endif 343#ifdef DEV_NETMAP 344 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 345 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 346 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 347 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 348#endif 349}; 350 351struct filter_entry { 352 uint32_t valid:1; /* filter allocated and valid */ 353 uint32_t locked:1; /* filter is administratively locked */ 354 uint32_t pending:1; /* filter action is pending firmware reply */ 355 uint32_t smtidx:8; /* Source MAC Table index for smac */ 356 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 357 358 struct t4_filter_specification fs; 359}; 360 361static int map_bars_0_and_4(struct adapter *); 362static int map_bar_2(struct adapter *); 363static void setup_memwin(struct adapter *); 364static int validate_mem_range(struct adapter *, uint32_t, int); 365static int fwmtype_to_hwmtype(int); 366static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 367 uint32_t *); 368static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 369static uint32_t position_memwin(struct adapter *, int, uint32_t); 370static int cfg_itype_and_nqueues(struct adapter *, int, int, 371 struct intrs_and_queues *); 372static int prep_firmware(struct adapter *); 373static int partition_resources(struct adapter *, const struct firmware *, 374 const char *); 375static int get_params__pre_init(struct adapter *); 376static int get_params__post_init(struct adapter *); 377static int set_params__post_init(struct adapter *); 378static void t4_set_desc(struct adapter *); 379static void build_medialist(struct port_info *, struct ifmedia *); 380static int cxgbe_init_synchronized(struct port_info *); 381static int cxgbe_uninit_synchronized(struct port_info *); 382static int setup_intr_handlers(struct adapter *); 383static void quiesce_txq(struct adapter *, struct sge_txq *); 384static void quiesce_wrq(struct adapter *, struct sge_wrq *); 385static void quiesce_iq(struct adapter *, struct sge_iq *); 386static void quiesce_fl(struct adapter *, struct sge_fl *); 387static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 388 driver_intr_t *, void *, char *); 389static int t4_free_irq(struct adapter *, struct irq *); 390static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 391 unsigned int); 392static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 393static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 394static void cxgbe_tick(void *); 395static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 396static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 397 struct mbuf *); 398static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 399static int fw_msg_not_handled(struct adapter *, const __be64 *); 400static int t4_sysctls(struct adapter *); 401static int cxgbe_sysctls(struct port_info *); 402static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 403static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 404static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 405static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 406static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 407static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 408static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 409static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 410static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 411static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 412static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 413#ifdef SBUF_DRAIN 414static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 415static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 416static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 417static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 418static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 419static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 420static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 421static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 422static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 423static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 424static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 425static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 426static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 427static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 428static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 429static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 430static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 431static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 432static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 433static int sysctl_tids(SYSCTL_HANDLER_ARGS); 434static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 435static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 436static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 437static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 438static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 439#endif 440static uint32_t fconf_to_mode(uint32_t); 441static uint32_t mode_to_fconf(uint32_t); 442static uint32_t fspec_to_fconf(struct t4_filter_specification *); 443static int get_filter_mode(struct adapter *, uint32_t *); 444static int set_filter_mode(struct adapter *, uint32_t); 445static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 446static int get_filter(struct adapter *, struct t4_filter *); 447static int set_filter(struct adapter *, struct t4_filter *); 448static int del_filter(struct adapter *, struct t4_filter *); 449static void clear_filter(struct filter_entry *); 450static int set_filter_wr(struct adapter *, int); 451static int del_filter_wr(struct adapter *, int); 452static int get_sge_context(struct adapter *, struct t4_sge_context *); 453static int load_fw(struct adapter *, struct t4_data *); 454static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 455static int read_i2c(struct adapter *, struct t4_i2c_data *); 456static int set_sched_class(struct adapter *, struct t4_sched_params *); 457static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 458#ifdef TCP_OFFLOAD 459static int toe_capability(struct port_info *, int); 460#endif 461static int mod_event(module_t, int, void *); 462 463struct { 464 uint16_t device; 465 char *desc; 466} t4_pciids[] = { 467 {0xa000, "Chelsio Terminator 4 FPGA"}, 468 {0x4400, "Chelsio T440-dbg"}, 469 {0x4401, "Chelsio T420-CR"}, 470 {0x4402, "Chelsio T422-CR"}, 471 {0x4403, "Chelsio T440-CR"}, 472 {0x4404, "Chelsio T420-BCH"}, 473 {0x4405, "Chelsio T440-BCH"}, 474 {0x4406, "Chelsio T440-CH"}, 475 {0x4407, "Chelsio T420-SO"}, 476 {0x4408, "Chelsio T420-CX"}, 477 {0x4409, "Chelsio T420-BT"}, 478 {0x440a, "Chelsio T404-BT"}, 479 {0x440e, "Chelsio T440-LP-CR"}, 480}, t5_pciids[] = { 481 {0xb000, "Chelsio Terminator 5 FPGA"}, 482 {0x5400, "Chelsio T580-dbg"}, 483 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 484 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 485 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 486 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 487 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 488 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 489 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 490 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 491 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 492 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 493 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 494 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 495 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 496#ifdef notyet 497 {0x5404, "Chelsio T520-BCH"}, 498 {0x5405, "Chelsio T540-BCH"}, 499 {0x5406, "Chelsio T540-CH"}, 500 {0x5408, "Chelsio T520-CX"}, 501 {0x540b, "Chelsio B520-SR"}, 502 {0x540c, "Chelsio B504-BT"}, 503 {0x540f, "Chelsio Amsterdam"}, 504 {0x5413, "Chelsio T580-CHR"}, 505#endif 506}; 507 508#ifdef TCP_OFFLOAD 509/* 510 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 511 * exactly the same for both rxq and ofld_rxq. 512 */ 513CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 514CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 515#endif 516 517/* No easy way to include t4_msg.h before adapter.h so we check this way */ 518CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 519CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 520 521CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 522 523static int 524t4_probe(device_t dev) 525{ 526 int i; 527 uint16_t v = pci_get_vendor(dev); 528 uint16_t d = pci_get_device(dev); 529 uint8_t f = pci_get_function(dev); 530 531 if (v != PCI_VENDOR_ID_CHELSIO) 532 return (ENXIO); 533 534 /* Attach only to PF0 of the FPGA */ 535 if (d == 0xa000 && f != 0) 536 return (ENXIO); 537 538 for (i = 0; i < nitems(t4_pciids); i++) { 539 if (d == t4_pciids[i].device) { 540 device_set_desc(dev, t4_pciids[i].desc); 541 return (BUS_PROBE_DEFAULT); 542 } 543 } 544 545 return (ENXIO); 546} 547 548static int 549t5_probe(device_t dev) 550{ 551 int i; 552 uint16_t v = pci_get_vendor(dev); 553 uint16_t d = pci_get_device(dev); 554 uint8_t f = pci_get_function(dev); 555 556 if (v != PCI_VENDOR_ID_CHELSIO) 557 return (ENXIO); 558 559 /* Attach only to PF0 of the FPGA */ 560 if (d == 0xb000 && f != 0) 561 return (ENXIO); 562 563 for (i = 0; i < nitems(t5_pciids); i++) { 564 if (d == t5_pciids[i].device) { 565 device_set_desc(dev, t5_pciids[i].desc); 566 return (BUS_PROBE_DEFAULT); 567 } 568 } 569 570 return (ENXIO); 571} 572 573static int 574t4_attach(device_t dev) 575{ 576 struct adapter *sc; 577 int rc = 0, i, n10g, n1g, rqidx, tqidx; 578 struct intrs_and_queues iaq; 579 struct sge *s; 580#ifdef TCP_OFFLOAD 581 int ofld_rqidx, ofld_tqidx; 582#endif 583#ifdef DEV_NETMAP 584 int nm_rqidx, nm_tqidx; 585#endif 586 587 sc = device_get_softc(dev); 588 sc->dev = dev; 589 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 590 591 pci_enable_busmaster(dev); 592 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 593 uint32_t v; 594 595 pci_set_max_read_req(dev, 4096); 596 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 597 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 598 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 599 600 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 601 } 602 603 sc->traceq = -1; 604 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 605 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 606 device_get_nameunit(dev)); 607 608 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 609 device_get_nameunit(dev)); 610 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 611 sx_xlock(&t4_list_lock); 612 SLIST_INSERT_HEAD(&t4_list, sc, link); 613 sx_xunlock(&t4_list_lock); 614 615 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 616 TAILQ_INIT(&sc->sfl); 617 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 618 619 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 620 621 rc = map_bars_0_and_4(sc); 622 if (rc != 0) 623 goto done; /* error message displayed already */ 624 625 /* 626 * This is the real PF# to which we're attaching. Works from within PCI 627 * passthrough environments too, where pci_get_function() could return a 628 * different PF# depending on the passthrough configuration. We need to 629 * use the real PF# in all our communication with the firmware. 630 */ 631 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 632 sc->mbox = sc->pf; 633 634 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 635 sc->an_handler = an_not_handled; 636 for (i = 0; i < nitems(sc->cpl_handler); i++) 637 sc->cpl_handler[i] = cpl_not_handled; 638 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 639 sc->fw_msg_handler[i] = fw_msg_not_handled; 640 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 641 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 642 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 643 t4_init_sge_cpl_handlers(sc); 644 645 /* Prepare the adapter for operation */ 646 rc = -t4_prep_adapter(sc); 647 if (rc != 0) { 648 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 649 goto done; 650 } 651 652 /* 653 * Do this really early, with the memory windows set up even before the 654 * character device. The userland tool's register i/o and mem read 655 * will work even in "recovery mode". 656 */ 657 setup_memwin(sc); 658 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 659 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 660 device_get_nameunit(dev)); 661 if (sc->cdev == NULL) 662 device_printf(dev, "failed to create nexus char device.\n"); 663 else 664 sc->cdev->si_drv1 = sc; 665 666 /* Go no further if recovery mode has been requested. */ 667 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 668 device_printf(dev, "recovery mode.\n"); 669 goto done; 670 } 671 672#if defined(__i386__) 673 if ((cpu_feature & CPUID_CX8) == 0) { 674 device_printf(dev, "64 bit atomics not available.\n"); 675 rc = ENOTSUP; 676 goto done; 677 } 678#endif 679 680 /* Prepare the firmware for operation */ 681 rc = prep_firmware(sc); 682 if (rc != 0) 683 goto done; /* error message displayed already */ 684 685 rc = get_params__post_init(sc); 686 if (rc != 0) 687 goto done; /* error message displayed already */ 688 689 rc = set_params__post_init(sc); 690 if (rc != 0) 691 goto done; /* error message displayed already */ 692 693 rc = map_bar_2(sc); 694 if (rc != 0) 695 goto done; /* error message displayed already */ 696 697 rc = t4_create_dma_tag(sc); 698 if (rc != 0) 699 goto done; /* error message displayed already */ 700 701 /* 702 * First pass over all the ports - allocate VIs and initialize some 703 * basic parameters like mac address, port type, etc. We also figure 704 * out whether a port is 10G or 1G and use that information when 705 * calculating how many interrupts to attempt to allocate. 706 */ 707 n10g = n1g = 0; 708 for_each_port(sc, i) { 709 struct port_info *pi; 710 711 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 712 sc->port[i] = pi; 713 714 /* These must be set before t4_port_init */ 715 pi->adapter = sc; 716 pi->port_id = i; 717 718 /* Allocate the vi and initialize parameters like mac addr */ 719 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 720 if (rc != 0) { 721 device_printf(dev, "unable to initialize port %d: %d\n", 722 i, rc); 723 free(pi, M_CXGBE); 724 sc->port[i] = NULL; 725 goto done; 726 } 727 728 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 729 pi->link_cfg.requested_fc |= t4_pause_settings; 730 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 731 pi->link_cfg.fc |= t4_pause_settings; 732 733 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 734 if (rc != 0) { 735 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 736 free(pi, M_CXGBE); 737 sc->port[i] = NULL; 738 goto done; 739 } 740 741 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 742 device_get_nameunit(dev), i); 743 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 744 sc->chan_map[pi->tx_chan] = i; 745 746 if (is_10G_port(pi) || is_40G_port(pi)) { 747 n10g++; 748 pi->tmr_idx = t4_tmr_idx_10g; 749 pi->pktc_idx = t4_pktc_idx_10g; 750 } else { 751 n1g++; 752 pi->tmr_idx = t4_tmr_idx_1g; 753 pi->pktc_idx = t4_pktc_idx_1g; 754 } 755 756 pi->xact_addr_filt = -1; 757 pi->linkdnrc = -1; 758 759 pi->qsize_rxq = t4_qsize_rxq; 760 pi->qsize_txq = t4_qsize_txq; 761 762 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 763 if (pi->dev == NULL) { 764 device_printf(dev, 765 "failed to add device for port %d.\n", i); 766 rc = ENXIO; 767 goto done; 768 } 769 device_set_softc(pi->dev, pi); 770 } 771 772 /* 773 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 774 */ 775 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 776 if (rc != 0) 777 goto done; /* error message displayed already */ 778 779 sc->intr_type = iaq.intr_type; 780 sc->intr_count = iaq.nirq; 781 782 s = &sc->sge; 783 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 784 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 785 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 786 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 787 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 788#ifdef TCP_OFFLOAD 789 if (is_offload(sc)) { 790 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 791 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 792 s->neq += s->nofldtxq + s->nofldrxq; 793 s->niq += s->nofldrxq; 794 795 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 796 M_CXGBE, M_ZERO | M_WAITOK); 797 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 798 M_CXGBE, M_ZERO | M_WAITOK); 799 } 800#endif 801#ifdef DEV_NETMAP 802 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 803 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 804 s->neq += s->nnmtxq + s->nnmrxq; 805 s->niq += s->nnmrxq; 806 807 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 808 M_CXGBE, M_ZERO | M_WAITOK); 809 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 810 M_CXGBE, M_ZERO | M_WAITOK); 811#endif 812 813 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 814 M_ZERO | M_WAITOK); 815 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 816 M_ZERO | M_WAITOK); 817 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 818 M_ZERO | M_WAITOK); 819 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 820 M_ZERO | M_WAITOK); 821 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 822 M_ZERO | M_WAITOK); 823 824 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 825 M_ZERO | M_WAITOK); 826 827 t4_init_l2t(sc, M_WAITOK); 828 829 /* 830 * Second pass over the ports. This time we know the number of rx and 831 * tx queues that each port should get. 832 */ 833 rqidx = tqidx = 0; 834#ifdef TCP_OFFLOAD 835 ofld_rqidx = ofld_tqidx = 0; 836#endif 837#ifdef DEV_NETMAP 838 nm_rqidx = nm_tqidx = 0; 839#endif 840 for_each_port(sc, i) { 841 struct port_info *pi = sc->port[i]; 842 843 if (pi == NULL) 844 continue; 845 846 pi->first_rxq = rqidx; 847 pi->first_txq = tqidx; 848 if (is_10G_port(pi) || is_40G_port(pi)) { 849 pi->flags |= iaq.intr_flags_10g; 850 pi->nrxq = iaq.nrxq10g; 851 pi->ntxq = iaq.ntxq10g; 852 } else { 853 pi->flags |= iaq.intr_flags_1g; 854 pi->nrxq = iaq.nrxq1g; 855 pi->ntxq = iaq.ntxq1g; 856 } 857 858 if (pi->ntxq > 1) 859 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 860 else 861 pi->rsrv_noflowq = 0; 862 863 rqidx += pi->nrxq; 864 tqidx += pi->ntxq; 865#ifdef TCP_OFFLOAD 866 if (is_offload(sc)) { 867 pi->first_ofld_rxq = ofld_rqidx; 868 pi->first_ofld_txq = ofld_tqidx; 869 if (is_10G_port(pi) || is_40G_port(pi)) { 870 pi->nofldrxq = iaq.nofldrxq10g; 871 pi->nofldtxq = iaq.nofldtxq10g; 872 } else { 873 pi->nofldrxq = iaq.nofldrxq1g; 874 pi->nofldtxq = iaq.nofldtxq1g; 875 } 876 ofld_rqidx += pi->nofldrxq; 877 ofld_tqidx += pi->nofldtxq; 878 } 879#endif 880#ifdef DEV_NETMAP 881 pi->first_nm_rxq = nm_rqidx; 882 pi->first_nm_txq = nm_tqidx; 883 if (is_10G_port(pi) || is_40G_port(pi)) { 884 pi->nnmrxq = iaq.nnmrxq10g; 885 pi->nnmtxq = iaq.nnmtxq10g; 886 } else { 887 pi->nnmrxq = iaq.nnmrxq1g; 888 pi->nnmtxq = iaq.nnmtxq1g; 889 } 890 nm_rqidx += pi->nnmrxq; 891 nm_tqidx += pi->nnmtxq; 892#endif 893 } 894 895 rc = setup_intr_handlers(sc); 896 if (rc != 0) { 897 device_printf(dev, 898 "failed to setup interrupt handlers: %d\n", rc); 899 goto done; 900 } 901 902 rc = bus_generic_attach(dev); 903 if (rc != 0) { 904 device_printf(dev, 905 "failed to attach all child ports: %d\n", rc); 906 goto done; 907 } 908 909 device_printf(dev, 910 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 911 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 912 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 913 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 914 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 915 916 t4_set_desc(sc); 917 918done: 919 if (rc != 0 && sc->cdev) { 920 /* cdev was created and so cxgbetool works; recover that way. */ 921 device_printf(dev, 922 "error during attach, adapter is now in recovery mode.\n"); 923 rc = 0; 924 } 925 926 if (rc != 0) 927 t4_detach(dev); 928 else 929 t4_sysctls(sc); 930 931 return (rc); 932} 933 934/* 935 * Idempotent 936 */ 937static int 938t4_detach(device_t dev) 939{ 940 struct adapter *sc; 941 struct port_info *pi; 942 int i, rc; 943 944 sc = device_get_softc(dev); 945 946 if (sc->flags & FULL_INIT_DONE) 947 t4_intr_disable(sc); 948 949 if (sc->cdev) { 950 destroy_dev(sc->cdev); 951 sc->cdev = NULL; 952 } 953 954 rc = bus_generic_detach(dev); 955 if (rc) { 956 device_printf(dev, 957 "failed to detach child devices: %d\n", rc); 958 return (rc); 959 } 960 961 for (i = 0; i < sc->intr_count; i++) 962 t4_free_irq(sc, &sc->irq[i]); 963 964 for (i = 0; i < MAX_NPORTS; i++) { 965 pi = sc->port[i]; 966 if (pi) { 967 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 968 if (pi->dev) 969 device_delete_child(dev, pi->dev); 970 971 mtx_destroy(&pi->pi_lock); 972 free(pi, M_CXGBE); 973 } 974 } 975 976 if (sc->flags & FULL_INIT_DONE) 977 adapter_full_uninit(sc); 978 979 if (sc->flags & FW_OK) 980 t4_fw_bye(sc, sc->mbox); 981 982 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 983 pci_release_msi(dev); 984 985 if (sc->regs_res) 986 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 987 sc->regs_res); 988 989 if (sc->udbs_res) 990 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 991 sc->udbs_res); 992 993 if (sc->msix_res) 994 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 995 sc->msix_res); 996 997 if (sc->l2t) 998 t4_free_l2t(sc->l2t); 999 1000#ifdef TCP_OFFLOAD 1001 free(sc->sge.ofld_rxq, M_CXGBE); 1002 free(sc->sge.ofld_txq, M_CXGBE); 1003#endif 1004#ifdef DEV_NETMAP 1005 free(sc->sge.nm_rxq, M_CXGBE); 1006 free(sc->sge.nm_txq, M_CXGBE); 1007#endif 1008 free(sc->irq, M_CXGBE); 1009 free(sc->sge.rxq, M_CXGBE); 1010 free(sc->sge.txq, M_CXGBE); 1011 free(sc->sge.ctrlq, M_CXGBE); 1012 free(sc->sge.iqmap, M_CXGBE); 1013 free(sc->sge.eqmap, M_CXGBE); 1014 free(sc->tids.ftid_tab, M_CXGBE); 1015 t4_destroy_dma_tag(sc); 1016 if (mtx_initialized(&sc->sc_lock)) { 1017 sx_xlock(&t4_list_lock); 1018 SLIST_REMOVE(&t4_list, sc, adapter, link); 1019 sx_xunlock(&t4_list_lock); 1020 mtx_destroy(&sc->sc_lock); 1021 } 1022 1023 if (mtx_initialized(&sc->tids.ftid_lock)) 1024 mtx_destroy(&sc->tids.ftid_lock); 1025 if (mtx_initialized(&sc->sfl_lock)) 1026 mtx_destroy(&sc->sfl_lock); 1027 if (mtx_initialized(&sc->ifp_lock)) 1028 mtx_destroy(&sc->ifp_lock); 1029 if (mtx_initialized(&sc->regwin_lock)) 1030 mtx_destroy(&sc->regwin_lock); 1031 1032 bzero(sc, sizeof(*sc)); 1033 1034 return (0); 1035} 1036 1037static int 1038cxgbe_probe(device_t dev) 1039{ 1040 char buf[128]; 1041 struct port_info *pi = device_get_softc(dev); 1042 1043 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1044 device_set_desc_copy(dev, buf); 1045 1046 return (BUS_PROBE_DEFAULT); 1047} 1048 1049#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1050 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1051 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1052#define T4_CAP_ENABLE (T4_CAP) 1053 1054static int 1055cxgbe_attach(device_t dev) 1056{ 1057 struct port_info *pi = device_get_softc(dev); 1058 struct ifnet *ifp; 1059 char *s; 1060 int n, o; 1061 1062 /* Allocate an ifnet and set it up */ 1063 ifp = if_alloc(IFT_ETHER); 1064 if (ifp == NULL) { 1065 device_printf(dev, "Cannot allocate ifnet\n"); 1066 return (ENOMEM); 1067 } 1068 pi->ifp = ifp; 1069 ifp->if_softc = pi; 1070 1071 callout_init(&pi->tick, CALLOUT_MPSAFE); 1072 1073 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1074 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1075 1076 ifp->if_init = cxgbe_init; 1077 ifp->if_ioctl = cxgbe_ioctl; 1078 ifp->if_transmit = cxgbe_transmit; 1079 ifp->if_qflush = cxgbe_qflush; 1080 1081 ifp->if_capabilities = T4_CAP; 1082#ifdef TCP_OFFLOAD 1083 if (is_offload(pi->adapter)) 1084 ifp->if_capabilities |= IFCAP_TOE; 1085#endif 1086 ifp->if_capenable = T4_CAP_ENABLE; 1087 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1088 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1089 1090 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1091 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1092 ifp->if_hw_tsomaxsegsize = 65536; 1093 1094 /* Initialize ifmedia for this port */ 1095 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1096 cxgbe_media_status); 1097 build_medialist(pi, &pi->media); 1098 1099 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1100 EVENTHANDLER_PRI_ANY); 1101 1102 ether_ifattach(ifp, pi->hw_addr); 1103 1104 n = 128; 1105 s = malloc(n, M_CXGBE, M_WAITOK); 1106 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1107 MPASS(n > o); 1108#ifdef TCP_OFFLOAD 1109 if (is_offload(pi->adapter)) { 1110 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1111 pi->nofldtxq, pi->nofldrxq); 1112 MPASS(n > o); 1113 } 1114#endif 1115#ifdef DEV_NETMAP 1116 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1117 pi->nnmrxq); 1118 MPASS(n > o); 1119#endif 1120 device_printf(dev, "%s\n", s); 1121 free(s, M_CXGBE); 1122 1123#ifdef DEV_NETMAP 1124 /* nm_media handled here to keep implementation private to this file */ 1125 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1126 cxgbe_media_status); 1127 build_medialist(pi, &pi->nm_media); 1128 create_netmap_ifnet(pi); /* logs errors it something fails */ 1129#endif 1130 cxgbe_sysctls(pi); 1131 1132 return (0); 1133} 1134 1135static int 1136cxgbe_detach(device_t dev) 1137{ 1138 struct port_info *pi = device_get_softc(dev); 1139 struct adapter *sc = pi->adapter; 1140 struct ifnet *ifp = pi->ifp; 1141 1142 /* Tell if_ioctl and if_init that the port is going away */ 1143 ADAPTER_LOCK(sc); 1144 SET_DOOMED(pi); 1145 wakeup(&sc->flags); 1146 while (IS_BUSY(sc)) 1147 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1148 SET_BUSY(sc); 1149#ifdef INVARIANTS 1150 sc->last_op = "t4detach"; 1151 sc->last_op_thr = curthread; 1152 sc->last_op_flags = 0; 1153#endif 1154 ADAPTER_UNLOCK(sc); 1155 1156 if (pi->flags & HAS_TRACEQ) { 1157 sc->traceq = -1; /* cloner should not create ifnet */ 1158 t4_tracer_port_detach(sc); 1159 } 1160 1161 if (pi->vlan_c) 1162 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1163 1164 PORT_LOCK(pi); 1165 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1166 callout_stop(&pi->tick); 1167 PORT_UNLOCK(pi); 1168 callout_drain(&pi->tick); 1169 1170 /* Let detach proceed even if these fail. */ 1171 cxgbe_uninit_synchronized(pi); 1172 port_full_uninit(pi); 1173 1174 ifmedia_removeall(&pi->media); 1175 ether_ifdetach(pi->ifp); 1176 if_free(pi->ifp); 1177 1178#ifdef DEV_NETMAP 1179 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1180 destroy_netmap_ifnet(pi); 1181#endif 1182 1183 ADAPTER_LOCK(sc); 1184 CLR_BUSY(sc); 1185 wakeup(&sc->flags); 1186 ADAPTER_UNLOCK(sc); 1187 1188 return (0); 1189} 1190 1191static void 1192cxgbe_init(void *arg) 1193{ 1194 struct port_info *pi = arg; 1195 struct adapter *sc = pi->adapter; 1196 1197 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1198 return; 1199 cxgbe_init_synchronized(pi); 1200 end_synchronized_op(sc, 0); 1201} 1202 1203static int 1204cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1205{ 1206 int rc = 0, mtu, flags, can_sleep; 1207 struct port_info *pi = ifp->if_softc; 1208 struct adapter *sc = pi->adapter; 1209 struct ifreq *ifr = (struct ifreq *)data; 1210 uint32_t mask; 1211 1212 switch (cmd) { 1213 case SIOCSIFMTU: 1214 mtu = ifr->ifr_mtu; 1215 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1216 return (EINVAL); 1217 1218 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1219 if (rc) 1220 return (rc); 1221 ifp->if_mtu = mtu; 1222 if (pi->flags & PORT_INIT_DONE) { 1223 t4_update_fl_bufsize(ifp); 1224 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1225 rc = update_mac_settings(ifp, XGMAC_MTU); 1226 } 1227 end_synchronized_op(sc, 0); 1228 break; 1229 1230 case SIOCSIFFLAGS: 1231 can_sleep = 0; 1232redo_sifflags: 1233 rc = begin_synchronized_op(sc, pi, 1234 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1235 if (rc) 1236 return (rc); 1237 1238 if (ifp->if_flags & IFF_UP) { 1239 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1240 flags = pi->if_flags; 1241 if ((ifp->if_flags ^ flags) & 1242 (IFF_PROMISC | IFF_ALLMULTI)) { 1243 if (can_sleep == 1) { 1244 end_synchronized_op(sc, 0); 1245 can_sleep = 0; 1246 goto redo_sifflags; 1247 } 1248 rc = update_mac_settings(ifp, 1249 XGMAC_PROMISC | XGMAC_ALLMULTI); 1250 } 1251 } else { 1252 if (can_sleep == 0) { 1253 end_synchronized_op(sc, LOCK_HELD); 1254 can_sleep = 1; 1255 goto redo_sifflags; 1256 } 1257 rc = cxgbe_init_synchronized(pi); 1258 } 1259 pi->if_flags = ifp->if_flags; 1260 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1261 if (can_sleep == 0) { 1262 end_synchronized_op(sc, LOCK_HELD); 1263 can_sleep = 1; 1264 goto redo_sifflags; 1265 } 1266 rc = cxgbe_uninit_synchronized(pi); 1267 } 1268 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1269 break; 1270 1271 case SIOCADDMULTI: 1272 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1273 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1274 if (rc) 1275 return (rc); 1276 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1277 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1278 end_synchronized_op(sc, LOCK_HELD); 1279 break; 1280 1281 case SIOCSIFCAP: 1282 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1283 if (rc) 1284 return (rc); 1285 1286 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1287 if (mask & IFCAP_TXCSUM) { 1288 ifp->if_capenable ^= IFCAP_TXCSUM; 1289 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1290 1291 if (IFCAP_TSO4 & ifp->if_capenable && 1292 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1293 ifp->if_capenable &= ~IFCAP_TSO4; 1294 if_printf(ifp, 1295 "tso4 disabled due to -txcsum.\n"); 1296 } 1297 } 1298 if (mask & IFCAP_TXCSUM_IPV6) { 1299 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1300 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1301 1302 if (IFCAP_TSO6 & ifp->if_capenable && 1303 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1304 ifp->if_capenable &= ~IFCAP_TSO6; 1305 if_printf(ifp, 1306 "tso6 disabled due to -txcsum6.\n"); 1307 } 1308 } 1309 if (mask & IFCAP_RXCSUM) 1310 ifp->if_capenable ^= IFCAP_RXCSUM; 1311 if (mask & IFCAP_RXCSUM_IPV6) 1312 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1313 1314 /* 1315 * Note that we leave CSUM_TSO alone (it is always set). The 1316 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1317 * sending a TSO request our way, so it's sufficient to toggle 1318 * IFCAP_TSOx only. 1319 */ 1320 if (mask & IFCAP_TSO4) { 1321 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1322 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1323 if_printf(ifp, "enable txcsum first.\n"); 1324 rc = EAGAIN; 1325 goto fail; 1326 } 1327 ifp->if_capenable ^= IFCAP_TSO4; 1328 } 1329 if (mask & IFCAP_TSO6) { 1330 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1331 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1332 if_printf(ifp, "enable txcsum6 first.\n"); 1333 rc = EAGAIN; 1334 goto fail; 1335 } 1336 ifp->if_capenable ^= IFCAP_TSO6; 1337 } 1338 if (mask & IFCAP_LRO) { 1339#if defined(INET) || defined(INET6) 1340 int i; 1341 struct sge_rxq *rxq; 1342 1343 ifp->if_capenable ^= IFCAP_LRO; 1344 for_each_rxq(pi, i, rxq) { 1345 if (ifp->if_capenable & IFCAP_LRO) 1346 rxq->iq.flags |= IQ_LRO_ENABLED; 1347 else 1348 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1349 } 1350#endif 1351 } 1352#ifdef TCP_OFFLOAD 1353 if (mask & IFCAP_TOE) { 1354 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1355 1356 rc = toe_capability(pi, enable); 1357 if (rc != 0) 1358 goto fail; 1359 1360 ifp->if_capenable ^= mask; 1361 } 1362#endif 1363 if (mask & IFCAP_VLAN_HWTAGGING) { 1364 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1365 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1366 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1367 } 1368 if (mask & IFCAP_VLAN_MTU) { 1369 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1370 1371 /* Need to find out how to disable auto-mtu-inflation */ 1372 } 1373 if (mask & IFCAP_VLAN_HWTSO) 1374 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1375 if (mask & IFCAP_VLAN_HWCSUM) 1376 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1377 1378#ifdef VLAN_CAPABILITIES 1379 VLAN_CAPABILITIES(ifp); 1380#endif 1381fail: 1382 end_synchronized_op(sc, 0); 1383 break; 1384 1385 case SIOCSIFMEDIA: 1386 case SIOCGIFMEDIA: 1387 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1388 break; 1389 1390 case SIOCGI2C: { 1391 struct ifi2creq i2c; 1392 1393 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1394 if (rc != 0) 1395 break; 1396 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1397 rc = EPERM; 1398 break; 1399 } 1400 if (i2c.len > sizeof(i2c.data)) { 1401 rc = EINVAL; 1402 break; 1403 } 1404 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1405 if (rc) 1406 return (rc); 1407 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1408 i2c.offset, i2c.len, &i2c.data[0]); 1409 end_synchronized_op(sc, 0); 1410 if (rc == 0) 1411 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1412 break; 1413 } 1414 1415 default: 1416 rc = ether_ioctl(ifp, cmd, data); 1417 } 1418 1419 return (rc); 1420} 1421 1422static int 1423cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1424{ 1425 struct port_info *pi = ifp->if_softc; 1426 struct adapter *sc = pi->adapter; 1427 struct sge_txq *txq; 1428 void *items[1]; 1429 int rc; 1430 1431 M_ASSERTPKTHDR(m); 1432 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1433 1434 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1435 m_freem(m); 1436 return (ENETDOWN); 1437 } 1438 1439 rc = parse_pkt(&m); 1440 if (__predict_false(rc != 0)) { 1441 MPASS(m == NULL); /* was freed already */ 1442 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1443 return (rc); 1444 } 1445 1446 /* Select a txq. */ 1447 txq = &sc->sge.txq[pi->first_txq]; 1448 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1449 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1450 pi->rsrv_noflowq); 1451 1452 items[0] = m; 1453 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1454 if (__predict_false(rc != 0)) 1455 m_freem(m); 1456 1457 return (rc); 1458} 1459 1460static void 1461cxgbe_qflush(struct ifnet *ifp) 1462{ 1463 struct port_info *pi = ifp->if_softc; 1464 struct sge_txq *txq; 1465 int i; 1466 1467 /* queues do not exist if !PORT_INIT_DONE. */ 1468 if (pi->flags & PORT_INIT_DONE) { 1469 for_each_txq(pi, i, txq) { 1470 TXQ_LOCK(txq); 1471 txq->eq.flags &= ~EQ_ENABLED; 1472 TXQ_UNLOCK(txq); 1473 while (!mp_ring_is_idle(txq->r)) { 1474 mp_ring_check_drainage(txq->r, 0); 1475 pause("qflush", 1); 1476 } 1477 } 1478 } 1479 if_qflush(ifp); 1480} 1481 1482static int 1483cxgbe_media_change(struct ifnet *ifp) 1484{ 1485 struct port_info *pi = ifp->if_softc; 1486 1487 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1488 1489 return (EOPNOTSUPP); 1490} 1491 1492static void 1493cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1494{ 1495 struct port_info *pi = ifp->if_softc; 1496 struct ifmedia *media = NULL; 1497 struct ifmedia_entry *cur; 1498 int speed = pi->link_cfg.speed; 1499#ifdef INVARIANTS 1500 int data = (pi->port_type << 8) | pi->mod_type; 1501#endif 1502 1503 if (ifp == pi->ifp) 1504 media = &pi->media; 1505#ifdef DEV_NETMAP 1506 else if (ifp == pi->nm_ifp) 1507 media = &pi->nm_media; 1508#endif 1509 MPASS(media != NULL); 1510 1511 cur = media->ifm_cur; 1512 MPASS(cur->ifm_data == data); 1513 1514 ifmr->ifm_status = IFM_AVALID; 1515 if (!pi->link_cfg.link_ok) 1516 return; 1517 1518 ifmr->ifm_status |= IFM_ACTIVE; 1519 1520 /* active and current will differ iff current media is autoselect. */ 1521 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1522 return; 1523 1524 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1525 if (speed == SPEED_10000) 1526 ifmr->ifm_active |= IFM_10G_T; 1527 else if (speed == SPEED_1000) 1528 ifmr->ifm_active |= IFM_1000_T; 1529 else if (speed == SPEED_100) 1530 ifmr->ifm_active |= IFM_100_TX; 1531 else if (speed == SPEED_10) 1532 ifmr->ifm_active |= IFM_10_T; 1533 else 1534 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1535 speed)); 1536} 1537 1538void 1539t4_fatal_err(struct adapter *sc) 1540{ 1541 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1542 t4_intr_disable(sc); 1543 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1544 device_get_nameunit(sc->dev)); 1545} 1546 1547static int 1548map_bars_0_and_4(struct adapter *sc) 1549{ 1550 sc->regs_rid = PCIR_BAR(0); 1551 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1552 &sc->regs_rid, RF_ACTIVE); 1553 if (sc->regs_res == NULL) { 1554 device_printf(sc->dev, "cannot map registers.\n"); 1555 return (ENXIO); 1556 } 1557 sc->bt = rman_get_bustag(sc->regs_res); 1558 sc->bh = rman_get_bushandle(sc->regs_res); 1559 sc->mmio_len = rman_get_size(sc->regs_res); 1560 setbit(&sc->doorbells, DOORBELL_KDB); 1561 1562 sc->msix_rid = PCIR_BAR(4); 1563 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1564 &sc->msix_rid, RF_ACTIVE); 1565 if (sc->msix_res == NULL) { 1566 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1567 return (ENXIO); 1568 } 1569 1570 return (0); 1571} 1572 1573static int 1574map_bar_2(struct adapter *sc) 1575{ 1576 1577 /* 1578 * T4: only iWARP driver uses the userspace doorbells. There is no need 1579 * to map it if RDMA is disabled. 1580 */ 1581 if (is_t4(sc) && sc->rdmacaps == 0) 1582 return (0); 1583 1584 sc->udbs_rid = PCIR_BAR(2); 1585 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1586 &sc->udbs_rid, RF_ACTIVE); 1587 if (sc->udbs_res == NULL) { 1588 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1589 return (ENXIO); 1590 } 1591 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1592 1593 if (is_t5(sc)) { 1594 setbit(&sc->doorbells, DOORBELL_UDB); 1595#if defined(__i386__) || defined(__amd64__) 1596 if (t5_write_combine) { 1597 int rc; 1598 1599 /* 1600 * Enable write combining on BAR2. This is the 1601 * userspace doorbell BAR and is split into 128B 1602 * (UDBS_SEG_SIZE) doorbell regions, each associated 1603 * with an egress queue. The first 64B has the doorbell 1604 * and the second 64B can be used to submit a tx work 1605 * request with an implicit doorbell. 1606 */ 1607 1608 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1609 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1610 if (rc == 0) { 1611 clrbit(&sc->doorbells, DOORBELL_UDB); 1612 setbit(&sc->doorbells, DOORBELL_WCWR); 1613 setbit(&sc->doorbells, DOORBELL_UDBWC); 1614 } else { 1615 device_printf(sc->dev, 1616 "couldn't enable write combining: %d\n", 1617 rc); 1618 } 1619 1620 t4_write_reg(sc, A_SGE_STAT_CFG, 1621 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1622 } 1623#endif 1624 } 1625 1626 return (0); 1627} 1628 1629static const struct memwin t4_memwin[] = { 1630 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1631 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1632 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1633}; 1634 1635static const struct memwin t5_memwin[] = { 1636 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1637 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1638 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1639}; 1640 1641static void 1642setup_memwin(struct adapter *sc) 1643{ 1644 const struct memwin *mw; 1645 int i, n; 1646 uint32_t bar0; 1647 1648 if (is_t4(sc)) { 1649 /* 1650 * Read low 32b of bar0 indirectly via the hardware backdoor 1651 * mechanism. Works from within PCI passthrough environments 1652 * too, where rman_get_start() can return a different value. We 1653 * need to program the T4 memory window decoders with the actual 1654 * addresses that will be coming across the PCIe link. 1655 */ 1656 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1657 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1658 1659 mw = &t4_memwin[0]; 1660 n = nitems(t4_memwin); 1661 } else { 1662 /* T5 uses the relative offset inside the PCIe BAR */ 1663 bar0 = 0; 1664 1665 mw = &t5_memwin[0]; 1666 n = nitems(t5_memwin); 1667 } 1668 1669 for (i = 0; i < n; i++, mw++) { 1670 t4_write_reg(sc, 1671 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1672 (mw->base + bar0) | V_BIR(0) | 1673 V_WINDOW(ilog2(mw->aperture) - 10)); 1674 } 1675 1676 /* flush */ 1677 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1678} 1679 1680/* 1681 * Verify that the memory range specified by the addr/len pair is valid and lies 1682 * entirely within a single region (EDCx or MCx). 1683 */ 1684static int 1685validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1686{ 1687 uint32_t em, addr_len, maddr, mlen; 1688 1689 /* Memory can only be accessed in naturally aligned 4 byte units */ 1690 if (addr & 3 || len & 3 || len == 0) 1691 return (EINVAL); 1692 1693 /* Enabled memories */ 1694 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1695 if (em & F_EDRAM0_ENABLE) { 1696 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1697 maddr = G_EDRAM0_BASE(addr_len) << 20; 1698 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1699 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1700 addr + len <= maddr + mlen) 1701 return (0); 1702 } 1703 if (em & F_EDRAM1_ENABLE) { 1704 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1705 maddr = G_EDRAM1_BASE(addr_len) << 20; 1706 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1707 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1708 addr + len <= maddr + mlen) 1709 return (0); 1710 } 1711 if (em & F_EXT_MEM_ENABLE) { 1712 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1713 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1714 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1715 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1716 addr + len <= maddr + mlen) 1717 return (0); 1718 } 1719 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1720 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1721 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1722 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1723 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1724 addr + len <= maddr + mlen) 1725 return (0); 1726 } 1727 1728 return (EFAULT); 1729} 1730 1731static int 1732fwmtype_to_hwmtype(int mtype) 1733{ 1734 1735 switch (mtype) { 1736 case FW_MEMTYPE_EDC0: 1737 return (MEM_EDC0); 1738 case FW_MEMTYPE_EDC1: 1739 return (MEM_EDC1); 1740 case FW_MEMTYPE_EXTMEM: 1741 return (MEM_MC0); 1742 case FW_MEMTYPE_EXTMEM1: 1743 return (MEM_MC1); 1744 default: 1745 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1746 } 1747} 1748 1749/* 1750 * Verify that the memory range specified by the memtype/offset/len pair is 1751 * valid and lies entirely within the memtype specified. The global address of 1752 * the start of the range is returned in addr. 1753 */ 1754static int 1755validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1756 uint32_t *addr) 1757{ 1758 uint32_t em, addr_len, maddr, mlen; 1759 1760 /* Memory can only be accessed in naturally aligned 4 byte units */ 1761 if (off & 3 || len & 3 || len == 0) 1762 return (EINVAL); 1763 1764 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1765 switch (fwmtype_to_hwmtype(mtype)) { 1766 case MEM_EDC0: 1767 if (!(em & F_EDRAM0_ENABLE)) 1768 return (EINVAL); 1769 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1770 maddr = G_EDRAM0_BASE(addr_len) << 20; 1771 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1772 break; 1773 case MEM_EDC1: 1774 if (!(em & F_EDRAM1_ENABLE)) 1775 return (EINVAL); 1776 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1777 maddr = G_EDRAM1_BASE(addr_len) << 20; 1778 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1779 break; 1780 case MEM_MC: 1781 if (!(em & F_EXT_MEM_ENABLE)) 1782 return (EINVAL); 1783 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1784 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1785 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1786 break; 1787 case MEM_MC1: 1788 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1789 return (EINVAL); 1790 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1791 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1792 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1793 break; 1794 default: 1795 return (EINVAL); 1796 } 1797 1798 if (mlen > 0 && off < mlen && off + len <= mlen) { 1799 *addr = maddr + off; /* global address */ 1800 return (0); 1801 } 1802 1803 return (EFAULT); 1804} 1805 1806static void 1807memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1808{ 1809 const struct memwin *mw; 1810 1811 if (is_t4(sc)) { 1812 KASSERT(win >= 0 && win < nitems(t4_memwin), 1813 ("%s: incorrect memwin# (%d)", __func__, win)); 1814 mw = &t4_memwin[win]; 1815 } else { 1816 KASSERT(win >= 0 && win < nitems(t5_memwin), 1817 ("%s: incorrect memwin# (%d)", __func__, win)); 1818 mw = &t5_memwin[win]; 1819 } 1820 1821 if (base != NULL) 1822 *base = mw->base; 1823 if (aperture != NULL) 1824 *aperture = mw->aperture; 1825} 1826 1827/* 1828 * Positions the memory window such that it can be used to access the specified 1829 * address in the chip's address space. The return value is the offset of addr 1830 * from the start of the window. 1831 */ 1832static uint32_t 1833position_memwin(struct adapter *sc, int n, uint32_t addr) 1834{ 1835 uint32_t start, pf; 1836 uint32_t reg; 1837 1838 KASSERT(n >= 0 && n <= 3, 1839 ("%s: invalid window %d.", __func__, n)); 1840 KASSERT((addr & 3) == 0, 1841 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1842 1843 if (is_t4(sc)) { 1844 pf = 0; 1845 start = addr & ~0xf; /* start must be 16B aligned */ 1846 } else { 1847 pf = V_PFNUM(sc->pf); 1848 start = addr & ~0x7f; /* start must be 128B aligned */ 1849 } 1850 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1851 1852 t4_write_reg(sc, reg, start | pf); 1853 t4_read_reg(sc, reg); 1854 1855 return (addr - start); 1856} 1857 1858static int 1859cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1860 struct intrs_and_queues *iaq) 1861{ 1862 int rc, itype, navail, nrxq10g, nrxq1g, n; 1863 int nofldrxq10g = 0, nofldrxq1g = 0; 1864 int nnmrxq10g = 0, nnmrxq1g = 0; 1865 1866 bzero(iaq, sizeof(*iaq)); 1867 1868 iaq->ntxq10g = t4_ntxq10g; 1869 iaq->ntxq1g = t4_ntxq1g; 1870 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1871 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1872 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1873#ifdef TCP_OFFLOAD 1874 if (is_offload(sc)) { 1875 iaq->nofldtxq10g = t4_nofldtxq10g; 1876 iaq->nofldtxq1g = t4_nofldtxq1g; 1877 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1878 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1879 } 1880#endif 1881#ifdef DEV_NETMAP 1882 iaq->nnmtxq10g = t4_nnmtxq10g; 1883 iaq->nnmtxq1g = t4_nnmtxq1g; 1884 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1885 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1886#endif 1887 1888 for (itype = INTR_MSIX; itype; itype >>= 1) { 1889 1890 if ((itype & t4_intr_types) == 0) 1891 continue; /* not allowed */ 1892 1893 if (itype == INTR_MSIX) 1894 navail = pci_msix_count(sc->dev); 1895 else if (itype == INTR_MSI) 1896 navail = pci_msi_count(sc->dev); 1897 else 1898 navail = 1; 1899restart: 1900 if (navail == 0) 1901 continue; 1902 1903 iaq->intr_type = itype; 1904 iaq->intr_flags_10g = 0; 1905 iaq->intr_flags_1g = 0; 1906 1907 /* 1908 * Best option: an interrupt vector for errors, one for the 1909 * firmware event queue, and one for every rxq (NIC, TOE, and 1910 * netmap). 1911 */ 1912 iaq->nirq = T4_EXTRA_INTR; 1913 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1914 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1915 if (iaq->nirq <= navail && 1916 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1917 iaq->intr_flags_10g = INTR_ALL; 1918 iaq->intr_flags_1g = INTR_ALL; 1919 goto allocate; 1920 } 1921 1922 /* 1923 * Second best option: a vector for errors, one for the firmware 1924 * event queue, and vectors for either all the NIC rx queues or 1925 * all the TOE rx queues. The queues that don't get vectors 1926 * will forward their interrupts to those that do. 1927 * 1928 * Note: netmap rx queues cannot be created early and so they 1929 * can't be setup to receive forwarded interrupts for others. 1930 */ 1931 iaq->nirq = T4_EXTRA_INTR; 1932 if (nrxq10g >= nofldrxq10g) { 1933 iaq->intr_flags_10g = INTR_RXQ; 1934 iaq->nirq += n10g * nrxq10g; 1935#ifdef DEV_NETMAP 1936 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1937#endif 1938 } else { 1939 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1940 iaq->nirq += n10g * nofldrxq10g; 1941#ifdef DEV_NETMAP 1942 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1943#endif 1944 } 1945 if (nrxq1g >= nofldrxq1g) { 1946 iaq->intr_flags_1g = INTR_RXQ; 1947 iaq->nirq += n1g * nrxq1g; 1948#ifdef DEV_NETMAP 1949 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1950#endif 1951 } else { 1952 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1953 iaq->nirq += n1g * nofldrxq1g; 1954#ifdef DEV_NETMAP 1955 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1956#endif 1957 } 1958 if (iaq->nirq <= navail && 1959 (itype != INTR_MSI || powerof2(iaq->nirq))) 1960 goto allocate; 1961 1962 /* 1963 * Next best option: an interrupt vector for errors, one for the 1964 * firmware event queue, and at least one per port. At this 1965 * point we know we'll have to downsize nrxq and/or nofldrxq 1966 * and/or nnmrxq to fit what's available to us. 1967 */ 1968 iaq->nirq = T4_EXTRA_INTR; 1969 iaq->nirq += n10g + n1g; 1970 if (iaq->nirq <= navail) { 1971 int leftover = navail - iaq->nirq; 1972 1973 if (n10g > 0) { 1974 int target = max(nrxq10g, nofldrxq10g); 1975 1976 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 1977 INTR_RXQ : INTR_OFLD_RXQ; 1978 1979 n = 1; 1980 while (n < target && leftover >= n10g) { 1981 leftover -= n10g; 1982 iaq->nirq += n10g; 1983 n++; 1984 } 1985 iaq->nrxq10g = min(n, nrxq10g); 1986#ifdef TCP_OFFLOAD 1987 iaq->nofldrxq10g = min(n, nofldrxq10g); 1988#endif 1989#ifdef DEV_NETMAP 1990 iaq->nnmrxq10g = min(n, nnmrxq10g); 1991#endif 1992 } 1993 1994 if (n1g > 0) { 1995 int target = max(nrxq1g, nofldrxq1g); 1996 1997 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 1998 INTR_RXQ : INTR_OFLD_RXQ; 1999 2000 n = 1; 2001 while (n < target && leftover >= n1g) { 2002 leftover -= n1g; 2003 iaq->nirq += n1g; 2004 n++; 2005 } 2006 iaq->nrxq1g = min(n, nrxq1g); 2007#ifdef TCP_OFFLOAD 2008 iaq->nofldrxq1g = min(n, nofldrxq1g); 2009#endif 2010#ifdef DEV_NETMAP 2011 iaq->nnmrxq1g = min(n, nnmrxq1g); 2012#endif 2013 } 2014 2015 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2016 goto allocate; 2017 } 2018 2019 /* 2020 * Least desirable option: one interrupt vector for everything. 2021 */ 2022 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2023 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2024#ifdef TCP_OFFLOAD 2025 if (is_offload(sc)) 2026 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2027#endif 2028#ifdef DEV_NETMAP 2029 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2030#endif 2031 2032allocate: 2033 navail = iaq->nirq; 2034 rc = 0; 2035 if (itype == INTR_MSIX) 2036 rc = pci_alloc_msix(sc->dev, &navail); 2037 else if (itype == INTR_MSI) 2038 rc = pci_alloc_msi(sc->dev, &navail); 2039 2040 if (rc == 0) { 2041 if (navail == iaq->nirq) 2042 return (0); 2043 2044 /* 2045 * Didn't get the number requested. Use whatever number 2046 * the kernel is willing to allocate (it's in navail). 2047 */ 2048 device_printf(sc->dev, "fewer vectors than requested, " 2049 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2050 itype, iaq->nirq, navail); 2051 pci_release_msi(sc->dev); 2052 goto restart; 2053 } 2054 2055 device_printf(sc->dev, 2056 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2057 itype, rc, iaq->nirq, navail); 2058 } 2059 2060 device_printf(sc->dev, 2061 "failed to find a usable interrupt type. " 2062 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2063 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2064 2065 return (ENXIO); 2066} 2067 2068#define FW_VERSION(chip) ( \ 2069 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2070 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2071 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2072 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2073#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2074 2075struct fw_info { 2076 uint8_t chip; 2077 char *kld_name; 2078 char *fw_mod_name; 2079 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2080} fw_info[] = { 2081 { 2082 .chip = CHELSIO_T4, 2083 .kld_name = "t4fw_cfg", 2084 .fw_mod_name = "t4fw", 2085 .fw_hdr = { 2086 .chip = FW_HDR_CHIP_T4, 2087 .fw_ver = htobe32_const(FW_VERSION(T4)), 2088 .intfver_nic = FW_INTFVER(T4, NIC), 2089 .intfver_vnic = FW_INTFVER(T4, VNIC), 2090 .intfver_ofld = FW_INTFVER(T4, OFLD), 2091 .intfver_ri = FW_INTFVER(T4, RI), 2092 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2093 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2094 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2095 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2096 }, 2097 }, { 2098 .chip = CHELSIO_T5, 2099 .kld_name = "t5fw_cfg", 2100 .fw_mod_name = "t5fw", 2101 .fw_hdr = { 2102 .chip = FW_HDR_CHIP_T5, 2103 .fw_ver = htobe32_const(FW_VERSION(T5)), 2104 .intfver_nic = FW_INTFVER(T5, NIC), 2105 .intfver_vnic = FW_INTFVER(T5, VNIC), 2106 .intfver_ofld = FW_INTFVER(T5, OFLD), 2107 .intfver_ri = FW_INTFVER(T5, RI), 2108 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2109 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2110 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2111 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2112 }, 2113 } 2114}; 2115 2116static struct fw_info * 2117find_fw_info(int chip) 2118{ 2119 int i; 2120 2121 for (i = 0; i < nitems(fw_info); i++) { 2122 if (fw_info[i].chip == chip) 2123 return (&fw_info[i]); 2124 } 2125 return (NULL); 2126} 2127 2128/* 2129 * Is the given firmware API compatible with the one the driver was compiled 2130 * with? 2131 */ 2132static int 2133fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2134{ 2135 2136 /* short circuit if it's the exact same firmware version */ 2137 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2138 return (1); 2139 2140 /* 2141 * XXX: Is this too conservative? Perhaps I should limit this to the 2142 * features that are supported in the driver. 2143 */ 2144#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2145 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2146 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2147 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2148 return (1); 2149#undef SAME_INTF 2150 2151 return (0); 2152} 2153 2154/* 2155 * The firmware in the KLD is usable, but should it be installed? This routine 2156 * explains itself in detail if it indicates the KLD firmware should be 2157 * installed. 2158 */ 2159static int 2160should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2161{ 2162 const char *reason; 2163 2164 if (!card_fw_usable) { 2165 reason = "incompatible or unusable"; 2166 goto install; 2167 } 2168 2169 if (k > c) { 2170 reason = "older than the version bundled with this driver"; 2171 goto install; 2172 } 2173 2174 if (t4_fw_install == 2 && k != c) { 2175 reason = "different than the version bundled with this driver"; 2176 goto install; 2177 } 2178 2179 return (0); 2180 2181install: 2182 if (t4_fw_install == 0) { 2183 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2184 "but the driver is prohibited from installing a different " 2185 "firmware on the card.\n", 2186 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2187 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2188 2189 return (0); 2190 } 2191 2192 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2193 "installing firmware %u.%u.%u.%u on card.\n", 2194 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2195 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2196 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2197 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2198 2199 return (1); 2200} 2201/* 2202 * Establish contact with the firmware and determine if we are the master driver 2203 * or not, and whether we are responsible for chip initialization. 2204 */ 2205static int 2206prep_firmware(struct adapter *sc) 2207{ 2208 const struct firmware *fw = NULL, *default_cfg; 2209 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2210 enum dev_state state; 2211 struct fw_info *fw_info; 2212 struct fw_hdr *card_fw; /* fw on the card */ 2213 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2214 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2215 against */ 2216 2217 /* Contact firmware. */ 2218 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2219 if (rc < 0 || state == DEV_STATE_ERR) { 2220 rc = -rc; 2221 device_printf(sc->dev, 2222 "failed to connect to the firmware: %d, %d.\n", rc, state); 2223 return (rc); 2224 } 2225 pf = rc; 2226 if (pf == sc->mbox) 2227 sc->flags |= MASTER_PF; 2228 else if (state == DEV_STATE_UNINIT) { 2229 /* 2230 * We didn't get to be the master so we definitely won't be 2231 * configuring the chip. It's a bug if someone else hasn't 2232 * configured it already. 2233 */ 2234 device_printf(sc->dev, "couldn't be master(%d), " 2235 "device not already initialized either(%d).\n", rc, state); 2236 return (EDOOFUS); 2237 } 2238 2239 /* This is the firmware whose headers the driver was compiled against */ 2240 fw_info = find_fw_info(chip_id(sc)); 2241 if (fw_info == NULL) { 2242 device_printf(sc->dev, 2243 "unable to look up firmware information for chip %d.\n", 2244 chip_id(sc)); 2245 return (EINVAL); 2246 } 2247 drv_fw = &fw_info->fw_hdr; 2248 2249 /* 2250 * The firmware KLD contains many modules. The KLD name is also the 2251 * name of the module that contains the default config file. 2252 */ 2253 default_cfg = firmware_get(fw_info->kld_name); 2254 2255 /* Read the header of the firmware on the card */ 2256 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2257 rc = -t4_read_flash(sc, FLASH_FW_START, 2258 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2259 if (rc == 0) 2260 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2261 else { 2262 device_printf(sc->dev, 2263 "Unable to read card's firmware header: %d\n", rc); 2264 card_fw_usable = 0; 2265 } 2266 2267 /* This is the firmware in the KLD */ 2268 fw = firmware_get(fw_info->fw_mod_name); 2269 if (fw != NULL) { 2270 kld_fw = (const void *)fw->data; 2271 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2272 } else { 2273 kld_fw = NULL; 2274 kld_fw_usable = 0; 2275 } 2276 2277 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2278 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2279 /* 2280 * Common case: the firmware on the card is an exact match and 2281 * the KLD is an exact match too, or the KLD is 2282 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2283 * here -- use cxgbetool loadfw if you want to reinstall the 2284 * same firmware as the one on the card. 2285 */ 2286 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2287 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2288 be32toh(card_fw->fw_ver))) { 2289 2290 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2291 if (rc != 0) { 2292 device_printf(sc->dev, 2293 "failed to install firmware: %d\n", rc); 2294 goto done; 2295 } 2296 2297 /* Installed successfully, update the cached header too. */ 2298 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2299 card_fw_usable = 1; 2300 need_fw_reset = 0; /* already reset as part of load_fw */ 2301 } 2302 2303 if (!card_fw_usable) { 2304 uint32_t d, c, k; 2305 2306 d = ntohl(drv_fw->fw_ver); 2307 c = ntohl(card_fw->fw_ver); 2308 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2309 2310 device_printf(sc->dev, "Cannot find a usable firmware: " 2311 "fw_install %d, chip state %d, " 2312 "driver compiled with %d.%d.%d.%d, " 2313 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2314 t4_fw_install, state, 2315 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2316 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2317 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2318 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2319 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2320 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2321 rc = EINVAL; 2322 goto done; 2323 } 2324 2325 /* We're using whatever's on the card and it's known to be good. */ 2326 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2327 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2328 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2329 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2330 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2331 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2332 t4_get_tp_version(sc, &sc->params.tp_vers); 2333 2334 /* Reset device */ 2335 if (need_fw_reset && 2336 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2337 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2338 if (rc != ETIMEDOUT && rc != EIO) 2339 t4_fw_bye(sc, sc->mbox); 2340 goto done; 2341 } 2342 sc->flags |= FW_OK; 2343 2344 rc = get_params__pre_init(sc); 2345 if (rc != 0) 2346 goto done; /* error message displayed already */ 2347 2348 /* Partition adapter resources as specified in the config file. */ 2349 if (state == DEV_STATE_UNINIT) { 2350 2351 KASSERT(sc->flags & MASTER_PF, 2352 ("%s: trying to change chip settings when not master.", 2353 __func__)); 2354 2355 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2356 if (rc != 0) 2357 goto done; /* error message displayed already */ 2358 2359 t4_tweak_chip_settings(sc); 2360 2361 /* get basic stuff going */ 2362 rc = -t4_fw_initialize(sc, sc->mbox); 2363 if (rc != 0) { 2364 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2365 goto done; 2366 } 2367 } else { 2368 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2369 sc->cfcsum = 0; 2370 } 2371 2372done: 2373 free(card_fw, M_CXGBE); 2374 if (fw != NULL) 2375 firmware_put(fw, FIRMWARE_UNLOAD); 2376 if (default_cfg != NULL) 2377 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2378 2379 return (rc); 2380} 2381 2382#define FW_PARAM_DEV(param) \ 2383 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2384 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2385#define FW_PARAM_PFVF(param) \ 2386 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2387 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2388 2389/* 2390 * Partition chip resources for use between various PFs, VFs, etc. 2391 */ 2392static int 2393partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2394 const char *name_prefix) 2395{ 2396 const struct firmware *cfg = NULL; 2397 int rc = 0; 2398 struct fw_caps_config_cmd caps; 2399 uint32_t mtype, moff, finicsum, cfcsum; 2400 2401 /* 2402 * Figure out what configuration file to use. Pick the default config 2403 * file for the card if the user hasn't specified one explicitly. 2404 */ 2405 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2406 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2407 /* Card specific overrides go here. */ 2408 if (pci_get_device(sc->dev) == 0x440a) 2409 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2410 if (is_fpga(sc)) 2411 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2412 } 2413 2414 /* 2415 * We need to load another module if the profile is anything except 2416 * "default" or "flash". 2417 */ 2418 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2419 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2420 char s[32]; 2421 2422 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2423 cfg = firmware_get(s); 2424 if (cfg == NULL) { 2425 if (default_cfg != NULL) { 2426 device_printf(sc->dev, 2427 "unable to load module \"%s\" for " 2428 "configuration profile \"%s\", will use " 2429 "the default config file instead.\n", 2430 s, sc->cfg_file); 2431 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2432 "%s", DEFAULT_CF); 2433 } else { 2434 device_printf(sc->dev, 2435 "unable to load module \"%s\" for " 2436 "configuration profile \"%s\", will use " 2437 "the config file on the card's flash " 2438 "instead.\n", s, sc->cfg_file); 2439 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2440 "%s", FLASH_CF); 2441 } 2442 } 2443 } 2444 2445 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2446 default_cfg == NULL) { 2447 device_printf(sc->dev, 2448 "default config file not available, will use the config " 2449 "file on the card's flash instead.\n"); 2450 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2451 } 2452 2453 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2454 u_int cflen, i, n; 2455 const uint32_t *cfdata; 2456 uint32_t param, val, addr, off, mw_base, mw_aperture; 2457 2458 KASSERT(cfg != NULL || default_cfg != NULL, 2459 ("%s: no config to upload", __func__)); 2460 2461 /* 2462 * Ask the firmware where it wants us to upload the config file. 2463 */ 2464 param = FW_PARAM_DEV(CF); 2465 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2466 if (rc != 0) { 2467 /* No support for config file? Shouldn't happen. */ 2468 device_printf(sc->dev, 2469 "failed to query config file location: %d.\n", rc); 2470 goto done; 2471 } 2472 mtype = G_FW_PARAMS_PARAM_Y(val); 2473 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2474 2475 /* 2476 * XXX: sheer laziness. We deliberately added 4 bytes of 2477 * useless stuffing/comments at the end of the config file so 2478 * it's ok to simply throw away the last remaining bytes when 2479 * the config file is not an exact multiple of 4. This also 2480 * helps with the validate_mt_off_len check. 2481 */ 2482 if (cfg != NULL) { 2483 cflen = cfg->datasize & ~3; 2484 cfdata = cfg->data; 2485 } else { 2486 cflen = default_cfg->datasize & ~3; 2487 cfdata = default_cfg->data; 2488 } 2489 2490 if (cflen > FLASH_CFG_MAX_SIZE) { 2491 device_printf(sc->dev, 2492 "config file too long (%d, max allowed is %d). " 2493 "Will try to use the config on the card, if any.\n", 2494 cflen, FLASH_CFG_MAX_SIZE); 2495 goto use_config_on_flash; 2496 } 2497 2498 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2499 if (rc != 0) { 2500 device_printf(sc->dev, 2501 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2502 "Will try to use the config on the card, if any.\n", 2503 __func__, mtype, moff, cflen, rc); 2504 goto use_config_on_flash; 2505 } 2506 2507 memwin_info(sc, 2, &mw_base, &mw_aperture); 2508 while (cflen) { 2509 off = position_memwin(sc, 2, addr); 2510 n = min(cflen, mw_aperture - off); 2511 for (i = 0; i < n; i += 4) 2512 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2513 cflen -= n; 2514 addr += n; 2515 } 2516 } else { 2517use_config_on_flash: 2518 mtype = FW_MEMTYPE_FLASH; 2519 moff = t4_flash_cfg_addr(sc); 2520 } 2521 2522 bzero(&caps, sizeof(caps)); 2523 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2524 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2525 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2526 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2527 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2528 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2529 if (rc != 0) { 2530 device_printf(sc->dev, 2531 "failed to pre-process config file: %d " 2532 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2533 goto done; 2534 } 2535 2536 finicsum = be32toh(caps.finicsum); 2537 cfcsum = be32toh(caps.cfcsum); 2538 if (finicsum != cfcsum) { 2539 device_printf(sc->dev, 2540 "WARNING: config file checksum mismatch: %08x %08x\n", 2541 finicsum, cfcsum); 2542 } 2543 sc->cfcsum = cfcsum; 2544 2545#define LIMIT_CAPS(x) do { \ 2546 caps.x &= htobe16(t4_##x##_allowed); \ 2547} while (0) 2548 2549 /* 2550 * Let the firmware know what features will (not) be used so it can tune 2551 * things accordingly. 2552 */ 2553 LIMIT_CAPS(linkcaps); 2554 LIMIT_CAPS(niccaps); 2555 LIMIT_CAPS(toecaps); 2556 LIMIT_CAPS(rdmacaps); 2557 LIMIT_CAPS(iscsicaps); 2558 LIMIT_CAPS(fcoecaps); 2559#undef LIMIT_CAPS 2560 2561 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2562 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2563 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2564 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2565 if (rc != 0) { 2566 device_printf(sc->dev, 2567 "failed to process config file: %d.\n", rc); 2568 } 2569done: 2570 if (cfg != NULL) 2571 firmware_put(cfg, FIRMWARE_UNLOAD); 2572 return (rc); 2573} 2574 2575/* 2576 * Retrieve parameters that are needed (or nice to have) very early. 2577 */ 2578static int 2579get_params__pre_init(struct adapter *sc) 2580{ 2581 int rc; 2582 uint32_t param[2], val[2]; 2583 struct fw_devlog_cmd cmd; 2584 struct devlog_params *dlog = &sc->params.devlog; 2585 2586 param[0] = FW_PARAM_DEV(PORTVEC); 2587 param[1] = FW_PARAM_DEV(CCLK); 2588 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2589 if (rc != 0) { 2590 device_printf(sc->dev, 2591 "failed to query parameters (pre_init): %d.\n", rc); 2592 return (rc); 2593 } 2594 2595 sc->params.portvec = val[0]; 2596 sc->params.nports = bitcount32(val[0]); 2597 sc->params.vpd.cclk = val[1]; 2598 2599 /* Read device log parameters. */ 2600 bzero(&cmd, sizeof(cmd)); 2601 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2602 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2603 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2604 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2605 if (rc != 0) { 2606 device_printf(sc->dev, 2607 "failed to get devlog parameters: %d.\n", rc); 2608 bzero(dlog, sizeof (*dlog)); 2609 rc = 0; /* devlog isn't critical for device operation */ 2610 } else { 2611 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2612 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2613 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2614 dlog->size = be32toh(cmd.memsize_devlog); 2615 } 2616 2617 return (rc); 2618} 2619 2620/* 2621 * Retrieve various parameters that are of interest to the driver. The device 2622 * has been initialized by the firmware at this point. 2623 */ 2624static int 2625get_params__post_init(struct adapter *sc) 2626{ 2627 int rc; 2628 uint32_t param[7], val[7]; 2629 struct fw_caps_config_cmd caps; 2630 2631 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2632 param[1] = FW_PARAM_PFVF(EQ_START); 2633 param[2] = FW_PARAM_PFVF(FILTER_START); 2634 param[3] = FW_PARAM_PFVF(FILTER_END); 2635 param[4] = FW_PARAM_PFVF(L2T_START); 2636 param[5] = FW_PARAM_PFVF(L2T_END); 2637 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2638 if (rc != 0) { 2639 device_printf(sc->dev, 2640 "failed to query parameters (post_init): %d.\n", rc); 2641 return (rc); 2642 } 2643 2644 sc->sge.iq_start = val[0]; 2645 sc->sge.eq_start = val[1]; 2646 sc->tids.ftid_base = val[2]; 2647 sc->tids.nftids = val[3] - val[2] + 1; 2648 sc->params.ftid_min = val[2]; 2649 sc->params.ftid_max = val[3]; 2650 sc->vres.l2t.start = val[4]; 2651 sc->vres.l2t.size = val[5] - val[4] + 1; 2652 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2653 ("%s: L2 table size (%u) larger than expected (%u)", 2654 __func__, sc->vres.l2t.size, L2T_SIZE)); 2655 2656 /* get capabilites */ 2657 bzero(&caps, sizeof(caps)); 2658 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2659 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2660 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2661 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2662 if (rc != 0) { 2663 device_printf(sc->dev, 2664 "failed to get card capabilities: %d.\n", rc); 2665 return (rc); 2666 } 2667 2668#define READ_CAPS(x) do { \ 2669 sc->x = htobe16(caps.x); \ 2670} while (0) 2671 READ_CAPS(linkcaps); 2672 READ_CAPS(niccaps); 2673 READ_CAPS(toecaps); 2674 READ_CAPS(rdmacaps); 2675 READ_CAPS(iscsicaps); 2676 READ_CAPS(fcoecaps); 2677 2678 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2679 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2680 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2681 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2682 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2683 if (rc != 0) { 2684 device_printf(sc->dev, 2685 "failed to query NIC parameters: %d.\n", rc); 2686 return (rc); 2687 } 2688 sc->tids.etid_base = val[0]; 2689 sc->params.etid_min = val[0]; 2690 sc->tids.netids = val[1] - val[0] + 1; 2691 sc->params.netids = sc->tids.netids; 2692 sc->params.eo_wr_cred = val[2]; 2693 sc->params.ethoffload = 1; 2694 } 2695 2696 if (sc->toecaps) { 2697 /* query offload-related parameters */ 2698 param[0] = FW_PARAM_DEV(NTID); 2699 param[1] = FW_PARAM_PFVF(SERVER_START); 2700 param[2] = FW_PARAM_PFVF(SERVER_END); 2701 param[3] = FW_PARAM_PFVF(TDDP_START); 2702 param[4] = FW_PARAM_PFVF(TDDP_END); 2703 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2704 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2705 if (rc != 0) { 2706 device_printf(sc->dev, 2707 "failed to query TOE parameters: %d.\n", rc); 2708 return (rc); 2709 } 2710 sc->tids.ntids = val[0]; 2711 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2712 sc->tids.stid_base = val[1]; 2713 sc->tids.nstids = val[2] - val[1] + 1; 2714 sc->vres.ddp.start = val[3]; 2715 sc->vres.ddp.size = val[4] - val[3] + 1; 2716 sc->params.ofldq_wr_cred = val[5]; 2717 sc->params.offload = 1; 2718 } 2719 if (sc->rdmacaps) { 2720 param[0] = FW_PARAM_PFVF(STAG_START); 2721 param[1] = FW_PARAM_PFVF(STAG_END); 2722 param[2] = FW_PARAM_PFVF(RQ_START); 2723 param[3] = FW_PARAM_PFVF(RQ_END); 2724 param[4] = FW_PARAM_PFVF(PBL_START); 2725 param[5] = FW_PARAM_PFVF(PBL_END); 2726 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2727 if (rc != 0) { 2728 device_printf(sc->dev, 2729 "failed to query RDMA parameters(1): %d.\n", rc); 2730 return (rc); 2731 } 2732 sc->vres.stag.start = val[0]; 2733 sc->vres.stag.size = val[1] - val[0] + 1; 2734 sc->vres.rq.start = val[2]; 2735 sc->vres.rq.size = val[3] - val[2] + 1; 2736 sc->vres.pbl.start = val[4]; 2737 sc->vres.pbl.size = val[5] - val[4] + 1; 2738 2739 param[0] = FW_PARAM_PFVF(SQRQ_START); 2740 param[1] = FW_PARAM_PFVF(SQRQ_END); 2741 param[2] = FW_PARAM_PFVF(CQ_START); 2742 param[3] = FW_PARAM_PFVF(CQ_END); 2743 param[4] = FW_PARAM_PFVF(OCQ_START); 2744 param[5] = FW_PARAM_PFVF(OCQ_END); 2745 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2746 if (rc != 0) { 2747 device_printf(sc->dev, 2748 "failed to query RDMA parameters(2): %d.\n", rc); 2749 return (rc); 2750 } 2751 sc->vres.qp.start = val[0]; 2752 sc->vres.qp.size = val[1] - val[0] + 1; 2753 sc->vres.cq.start = val[2]; 2754 sc->vres.cq.size = val[3] - val[2] + 1; 2755 sc->vres.ocq.start = val[4]; 2756 sc->vres.ocq.size = val[5] - val[4] + 1; 2757 } 2758 if (sc->iscsicaps) { 2759 param[0] = FW_PARAM_PFVF(ISCSI_START); 2760 param[1] = FW_PARAM_PFVF(ISCSI_END); 2761 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2762 if (rc != 0) { 2763 device_printf(sc->dev, 2764 "failed to query iSCSI parameters: %d.\n", rc); 2765 return (rc); 2766 } 2767 sc->vres.iscsi.start = val[0]; 2768 sc->vres.iscsi.size = val[1] - val[0] + 1; 2769 } 2770 2771 /* 2772 * We've got the params we wanted to query via the firmware. Now grab 2773 * some others directly from the chip. 2774 */ 2775 rc = t4_read_chip_settings(sc); 2776 2777 return (rc); 2778} 2779 2780static int 2781set_params__post_init(struct adapter *sc) 2782{ 2783 uint32_t param, val; 2784 2785 /* ask for encapsulated CPLs */ 2786 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2787 val = 1; 2788 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2789 2790 return (0); 2791} 2792 2793#undef FW_PARAM_PFVF 2794#undef FW_PARAM_DEV 2795 2796static void 2797t4_set_desc(struct adapter *sc) 2798{ 2799 char buf[128]; 2800 struct adapter_params *p = &sc->params; 2801 2802 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2803 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2804 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2805 2806 device_set_desc_copy(sc->dev, buf); 2807} 2808 2809static void 2810build_medialist(struct port_info *pi, struct ifmedia *media) 2811{ 2812 int data, m; 2813 2814 PORT_LOCK(pi); 2815 2816 ifmedia_removeall(media); 2817 2818 m = IFM_ETHER | IFM_FDX; 2819 data = (pi->port_type << 8) | pi->mod_type; 2820 2821 switch(pi->port_type) { 2822 case FW_PORT_TYPE_BT_XFI: 2823 case FW_PORT_TYPE_BT_XAUI: 2824 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2825 /* fall through */ 2826 2827 case FW_PORT_TYPE_BT_SGMII: 2828 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2829 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2830 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2831 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2832 break; 2833 2834 case FW_PORT_TYPE_CX4: 2835 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2836 ifmedia_set(media, m | IFM_10G_CX4); 2837 break; 2838 2839 case FW_PORT_TYPE_QSFP_10G: 2840 case FW_PORT_TYPE_SFP: 2841 case FW_PORT_TYPE_FIBER_XFI: 2842 case FW_PORT_TYPE_FIBER_XAUI: 2843 switch (pi->mod_type) { 2844 2845 case FW_PORT_MOD_TYPE_LR: 2846 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2847 ifmedia_set(media, m | IFM_10G_LR); 2848 break; 2849 2850 case FW_PORT_MOD_TYPE_SR: 2851 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2852 ifmedia_set(media, m | IFM_10G_SR); 2853 break; 2854 2855 case FW_PORT_MOD_TYPE_LRM: 2856 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2857 ifmedia_set(media, m | IFM_10G_LRM); 2858 break; 2859 2860 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2861 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2862 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2863 ifmedia_set(media, m | IFM_10G_TWINAX); 2864 break; 2865 2866 case FW_PORT_MOD_TYPE_NONE: 2867 m &= ~IFM_FDX; 2868 ifmedia_add(media, m | IFM_NONE, data, NULL); 2869 ifmedia_set(media, m | IFM_NONE); 2870 break; 2871 2872 case FW_PORT_MOD_TYPE_NA: 2873 case FW_PORT_MOD_TYPE_ER: 2874 default: 2875 device_printf(pi->dev, 2876 "unknown port_type (%d), mod_type (%d)\n", 2877 pi->port_type, pi->mod_type); 2878 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2879 ifmedia_set(media, m | IFM_UNKNOWN); 2880 break; 2881 } 2882 break; 2883 2884 case FW_PORT_TYPE_QSFP: 2885 switch (pi->mod_type) { 2886 2887 case FW_PORT_MOD_TYPE_LR: 2888 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2889 ifmedia_set(media, m | IFM_40G_LR4); 2890 break; 2891 2892 case FW_PORT_MOD_TYPE_SR: 2893 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2894 ifmedia_set(media, m | IFM_40G_SR4); 2895 break; 2896 2897 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2898 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2899 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2900 ifmedia_set(media, m | IFM_40G_CR4); 2901 break; 2902 2903 case FW_PORT_MOD_TYPE_NONE: 2904 m &= ~IFM_FDX; 2905 ifmedia_add(media, m | IFM_NONE, data, NULL); 2906 ifmedia_set(media, m | IFM_NONE); 2907 break; 2908 2909 default: 2910 device_printf(pi->dev, 2911 "unknown port_type (%d), mod_type (%d)\n", 2912 pi->port_type, pi->mod_type); 2913 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2914 ifmedia_set(media, m | IFM_UNKNOWN); 2915 break; 2916 } 2917 break; 2918 2919 default: 2920 device_printf(pi->dev, 2921 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2922 pi->mod_type); 2923 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2924 ifmedia_set(media, m | IFM_UNKNOWN); 2925 break; 2926 } 2927 2928 PORT_UNLOCK(pi); 2929} 2930 2931#define FW_MAC_EXACT_CHUNK 7 2932 2933/* 2934 * Program the port's XGMAC based on parameters in ifnet. The caller also 2935 * indicates which parameters should be programmed (the rest are left alone). 2936 */ 2937int 2938update_mac_settings(struct ifnet *ifp, int flags) 2939{ 2940 int rc = 0; 2941 struct port_info *pi = ifp->if_softc; 2942 struct adapter *sc = pi->adapter; 2943 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2944 uint16_t viid = 0xffff; 2945 int16_t *xact_addr_filt = NULL; 2946 2947 ASSERT_SYNCHRONIZED_OP(sc); 2948 KASSERT(flags, ("%s: not told what to update.", __func__)); 2949 2950 if (ifp == pi->ifp) { 2951 viid = pi->viid; 2952 xact_addr_filt = &pi->xact_addr_filt; 2953 } 2954#ifdef DEV_NETMAP 2955 else if (ifp == pi->nm_ifp) { 2956 viid = pi->nm_viid; 2957 xact_addr_filt = &pi->nm_xact_addr_filt; 2958 } 2959#endif 2960 if (flags & XGMAC_MTU) 2961 mtu = ifp->if_mtu; 2962 2963 if (flags & XGMAC_PROMISC) 2964 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2965 2966 if (flags & XGMAC_ALLMULTI) 2967 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2968 2969 if (flags & XGMAC_VLANEX) 2970 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2971 2972 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 2973 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 2974 1, vlanex, false); 2975 if (rc) { 2976 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 2977 rc); 2978 return (rc); 2979 } 2980 } 2981 2982 if (flags & XGMAC_UCADDR) { 2983 uint8_t ucaddr[ETHER_ADDR_LEN]; 2984 2985 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2986 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 2987 true, true); 2988 if (rc < 0) { 2989 rc = -rc; 2990 if_printf(ifp, "change_mac failed: %d\n", rc); 2991 return (rc); 2992 } else { 2993 *xact_addr_filt = rc; 2994 rc = 0; 2995 } 2996 } 2997 2998 if (flags & XGMAC_MCADDRS) { 2999 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3000 int del = 1; 3001 uint64_t hash = 0; 3002 struct ifmultiaddr *ifma; 3003 int i = 0, j; 3004 3005 if_maddr_rlock(ifp); 3006 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3007 if (ifma->ifma_addr->sa_family != AF_LINK) 3008 continue; 3009 mcaddr[i] = 3010 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3011 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3012 i++; 3013 3014 if (i == FW_MAC_EXACT_CHUNK) { 3015 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3016 i, mcaddr, NULL, &hash, 0); 3017 if (rc < 0) { 3018 rc = -rc; 3019 for (j = 0; j < i; j++) { 3020 if_printf(ifp, 3021 "failed to add mc address" 3022 " %02x:%02x:%02x:" 3023 "%02x:%02x:%02x rc=%d\n", 3024 mcaddr[j][0], mcaddr[j][1], 3025 mcaddr[j][2], mcaddr[j][3], 3026 mcaddr[j][4], mcaddr[j][5], 3027 rc); 3028 } 3029 goto mcfail; 3030 } 3031 del = 0; 3032 i = 0; 3033 } 3034 } 3035 if (i > 0) { 3036 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3037 mcaddr, NULL, &hash, 0); 3038 if (rc < 0) { 3039 rc = -rc; 3040 for (j = 0; j < i; j++) { 3041 if_printf(ifp, 3042 "failed to add mc address" 3043 " %02x:%02x:%02x:" 3044 "%02x:%02x:%02x rc=%d\n", 3045 mcaddr[j][0], mcaddr[j][1], 3046 mcaddr[j][2], mcaddr[j][3], 3047 mcaddr[j][4], mcaddr[j][5], 3048 rc); 3049 } 3050 goto mcfail; 3051 } 3052 } 3053 3054 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3055 if (rc != 0) 3056 if_printf(ifp, "failed to set mc address hash: %d", rc); 3057mcfail: 3058 if_maddr_runlock(ifp); 3059 } 3060 3061 return (rc); 3062} 3063 3064/* 3065 * {begin|end}_synchronized_op must be called from the same thread. 3066 */ 3067int 3068begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3069 char *wmesg) 3070{ 3071 int rc, pri; 3072 3073#ifdef WITNESS 3074 /* the caller thinks it's ok to sleep, but is it really? */ 3075 if (flags & SLEEP_OK) 3076 pause("t4slptst", 1); 3077#endif 3078 3079 if (INTR_OK) 3080 pri = PCATCH; 3081 else 3082 pri = 0; 3083 3084 ADAPTER_LOCK(sc); 3085 for (;;) { 3086 3087 if (pi && IS_DOOMED(pi)) { 3088 rc = ENXIO; 3089 goto done; 3090 } 3091 3092 if (!IS_BUSY(sc)) { 3093 rc = 0; 3094 break; 3095 } 3096 3097 if (!(flags & SLEEP_OK)) { 3098 rc = EBUSY; 3099 goto done; 3100 } 3101 3102 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3103 rc = EINTR; 3104 goto done; 3105 } 3106 } 3107 3108 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3109 SET_BUSY(sc); 3110#ifdef INVARIANTS 3111 sc->last_op = wmesg; 3112 sc->last_op_thr = curthread; 3113 sc->last_op_flags = flags; 3114#endif 3115 3116done: 3117 if (!(flags & HOLD_LOCK) || rc) 3118 ADAPTER_UNLOCK(sc); 3119 3120 return (rc); 3121} 3122 3123/* 3124 * {begin|end}_synchronized_op must be called from the same thread. 3125 */ 3126void 3127end_synchronized_op(struct adapter *sc, int flags) 3128{ 3129 3130 if (flags & LOCK_HELD) 3131 ADAPTER_LOCK_ASSERT_OWNED(sc); 3132 else 3133 ADAPTER_LOCK(sc); 3134 3135 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3136 CLR_BUSY(sc); 3137 wakeup(&sc->flags); 3138 ADAPTER_UNLOCK(sc); 3139} 3140 3141static int 3142cxgbe_init_synchronized(struct port_info *pi) 3143{ 3144 struct adapter *sc = pi->adapter; 3145 struct ifnet *ifp = pi->ifp; 3146 int rc = 0, i; 3147 struct sge_txq *txq; 3148 3149 ASSERT_SYNCHRONIZED_OP(sc); 3150 3151 if (isset(&sc->open_device_map, pi->port_id)) { 3152 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3153 ("mismatch between open_device_map and if_drv_flags")); 3154 return (0); /* already running */ 3155 } 3156 3157 if (!(sc->flags & FULL_INIT_DONE) && 3158 ((rc = adapter_full_init(sc)) != 0)) 3159 return (rc); /* error message displayed already */ 3160 3161 if (!(pi->flags & PORT_INIT_DONE) && 3162 ((rc = port_full_init(pi)) != 0)) 3163 return (rc); /* error message displayed already */ 3164 3165 rc = update_mac_settings(ifp, XGMAC_ALL); 3166 if (rc) 3167 goto done; /* error message displayed already */ 3168 3169 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3170 if (rc != 0) { 3171 if_printf(ifp, "enable_vi failed: %d\n", rc); 3172 goto done; 3173 } 3174 3175 /* 3176 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3177 * if this changes. 3178 */ 3179 3180 for_each_txq(pi, i, txq) { 3181 TXQ_LOCK(txq); 3182 txq->eq.flags |= EQ_ENABLED; 3183 TXQ_UNLOCK(txq); 3184 } 3185 3186 /* 3187 * The first iq of the first port to come up is used for tracing. 3188 */ 3189 if (sc->traceq < 0) { 3190 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3191 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3192 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3193 V_QUEUENUMBER(sc->traceq)); 3194 pi->flags |= HAS_TRACEQ; 3195 } 3196 3197 /* all ok */ 3198 setbit(&sc->open_device_map, pi->port_id); 3199 PORT_LOCK(pi); 3200 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3201 PORT_UNLOCK(pi); 3202 3203 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3204done: 3205 if (rc != 0) 3206 cxgbe_uninit_synchronized(pi); 3207 3208 return (rc); 3209} 3210 3211/* 3212 * Idempotent. 3213 */ 3214static int 3215cxgbe_uninit_synchronized(struct port_info *pi) 3216{ 3217 struct adapter *sc = pi->adapter; 3218 struct ifnet *ifp = pi->ifp; 3219 int rc, i; 3220 struct sge_txq *txq; 3221 3222 ASSERT_SYNCHRONIZED_OP(sc); 3223 3224 /* 3225 * Disable the VI so that all its data in either direction is discarded 3226 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3227 * tick) intact as the TP can deliver negative advice or data that it's 3228 * holding in its RAM (for an offloaded connection) even after the VI is 3229 * disabled. 3230 */ 3231 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3232 if (rc) { 3233 if_printf(ifp, "disable_vi failed: %d\n", rc); 3234 return (rc); 3235 } 3236 3237 for_each_txq(pi, i, txq) { 3238 TXQ_LOCK(txq); 3239 txq->eq.flags &= ~EQ_ENABLED; 3240 TXQ_UNLOCK(txq); 3241 } 3242 3243 clrbit(&sc->open_device_map, pi->port_id); 3244 PORT_LOCK(pi); 3245 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3246 PORT_UNLOCK(pi); 3247 3248 pi->link_cfg.link_ok = 0; 3249 pi->link_cfg.speed = 0; 3250 pi->linkdnrc = -1; 3251 t4_os_link_changed(sc, pi->port_id, 0, -1); 3252 3253 return (0); 3254} 3255 3256/* 3257 * It is ok for this function to fail midway and return right away. t4_detach 3258 * will walk the entire sc->irq list and clean up whatever is valid. 3259 */ 3260static int 3261setup_intr_handlers(struct adapter *sc) 3262{ 3263 int rc, rid, p, q; 3264 char s[8]; 3265 struct irq *irq; 3266 struct port_info *pi; 3267 struct sge_rxq *rxq; 3268#ifdef TCP_OFFLOAD 3269 struct sge_ofld_rxq *ofld_rxq; 3270#endif 3271#ifdef DEV_NETMAP 3272 struct sge_nm_rxq *nm_rxq; 3273#endif 3274 3275 /* 3276 * Setup interrupts. 3277 */ 3278 irq = &sc->irq[0]; 3279 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3280 if (sc->intr_count == 1) 3281 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3282 3283 /* Multiple interrupts. */ 3284 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3285 ("%s: too few intr.", __func__)); 3286 3287 /* The first one is always error intr */ 3288 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3289 if (rc != 0) 3290 return (rc); 3291 irq++; 3292 rid++; 3293 3294 /* The second one is always the firmware event queue */ 3295 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3296 if (rc != 0) 3297 return (rc); 3298 irq++; 3299 rid++; 3300 3301 for_each_port(sc, p) { 3302 pi = sc->port[p]; 3303 3304 if (pi->flags & INTR_RXQ) { 3305 for_each_rxq(pi, q, rxq) { 3306 snprintf(s, sizeof(s), "%d.%d", p, q); 3307 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3308 s); 3309 if (rc != 0) 3310 return (rc); 3311 irq++; 3312 rid++; 3313 } 3314 } 3315#ifdef TCP_OFFLOAD 3316 if (pi->flags & INTR_OFLD_RXQ) { 3317 for_each_ofld_rxq(pi, q, ofld_rxq) { 3318 snprintf(s, sizeof(s), "%d,%d", p, q); 3319 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3320 ofld_rxq, s); 3321 if (rc != 0) 3322 return (rc); 3323 irq++; 3324 rid++; 3325 } 3326 } 3327#endif 3328#ifdef DEV_NETMAP 3329 if (pi->flags & INTR_NM_RXQ) { 3330 for_each_nm_rxq(pi, q, nm_rxq) { 3331 snprintf(s, sizeof(s), "%d-%d", p, q); 3332 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3333 nm_rxq, s); 3334 if (rc != 0) 3335 return (rc); 3336 irq++; 3337 rid++; 3338 } 3339 } 3340#endif 3341 } 3342 MPASS(irq == &sc->irq[sc->intr_count]); 3343 3344 return (0); 3345} 3346 3347int 3348adapter_full_init(struct adapter *sc) 3349{ 3350 int rc, i; 3351 3352 ASSERT_SYNCHRONIZED_OP(sc); 3353 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3354 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3355 ("%s: FULL_INIT_DONE already", __func__)); 3356 3357 /* 3358 * queues that belong to the adapter (not any particular port). 3359 */ 3360 rc = t4_setup_adapter_queues(sc); 3361 if (rc != 0) 3362 goto done; 3363 3364 for (i = 0; i < nitems(sc->tq); i++) { 3365 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3366 taskqueue_thread_enqueue, &sc->tq[i]); 3367 if (sc->tq[i] == NULL) { 3368 device_printf(sc->dev, 3369 "failed to allocate task queue %d\n", i); 3370 rc = ENOMEM; 3371 goto done; 3372 } 3373 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3374 device_get_nameunit(sc->dev), i); 3375 } 3376 3377 t4_intr_enable(sc); 3378 sc->flags |= FULL_INIT_DONE; 3379done: 3380 if (rc != 0) 3381 adapter_full_uninit(sc); 3382 3383 return (rc); 3384} 3385 3386int 3387adapter_full_uninit(struct adapter *sc) 3388{ 3389 int i; 3390 3391 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3392 3393 t4_teardown_adapter_queues(sc); 3394 3395 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3396 taskqueue_free(sc->tq[i]); 3397 sc->tq[i] = NULL; 3398 } 3399 3400 sc->flags &= ~FULL_INIT_DONE; 3401 3402 return (0); 3403} 3404 3405int 3406port_full_init(struct port_info *pi) 3407{ 3408 struct adapter *sc = pi->adapter; 3409 struct ifnet *ifp = pi->ifp; 3410 uint16_t *rss; 3411 struct sge_rxq *rxq; 3412 int rc, i, j; 3413 3414 ASSERT_SYNCHRONIZED_OP(sc); 3415 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3416 ("%s: PORT_INIT_DONE already", __func__)); 3417 3418 sysctl_ctx_init(&pi->ctx); 3419 pi->flags |= PORT_SYSCTL_CTX; 3420 3421 /* 3422 * Allocate tx/rx/fl queues for this port. 3423 */ 3424 rc = t4_setup_port_queues(pi); 3425 if (rc != 0) 3426 goto done; /* error message displayed already */ 3427 3428 /* 3429 * Setup RSS for this port. Save a copy of the RSS table for later use. 3430 */ 3431 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3432 for (i = 0; i < pi->rss_size;) { 3433 for_each_rxq(pi, j, rxq) { 3434 rss[i++] = rxq->iq.abs_id; 3435 if (i == pi->rss_size) 3436 break; 3437 } 3438 } 3439 3440 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3441 pi->rss_size); 3442 if (rc != 0) { 3443 if_printf(ifp, "rss_config failed: %d\n", rc); 3444 goto done; 3445 } 3446 3447 pi->rss = rss; 3448 pi->flags |= PORT_INIT_DONE; 3449done: 3450 if (rc != 0) 3451 port_full_uninit(pi); 3452 3453 return (rc); 3454} 3455 3456/* 3457 * Idempotent. 3458 */ 3459int 3460port_full_uninit(struct port_info *pi) 3461{ 3462 struct adapter *sc = pi->adapter; 3463 int i; 3464 struct sge_rxq *rxq; 3465 struct sge_txq *txq; 3466#ifdef TCP_OFFLOAD 3467 struct sge_ofld_rxq *ofld_rxq; 3468 struct sge_wrq *ofld_txq; 3469#endif 3470 3471 if (pi->flags & PORT_INIT_DONE) { 3472 3473 /* Need to quiesce queues. */ 3474 3475 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3476 3477 for_each_txq(pi, i, txq) { 3478 quiesce_txq(sc, txq); 3479 } 3480 3481#ifdef TCP_OFFLOAD 3482 for_each_ofld_txq(pi, i, ofld_txq) { 3483 quiesce_wrq(sc, ofld_txq); 3484 } 3485#endif 3486 3487 for_each_rxq(pi, i, rxq) { 3488 quiesce_iq(sc, &rxq->iq); 3489 quiesce_fl(sc, &rxq->fl); 3490 } 3491 3492#ifdef TCP_OFFLOAD 3493 for_each_ofld_rxq(pi, i, ofld_rxq) { 3494 quiesce_iq(sc, &ofld_rxq->iq); 3495 quiesce_fl(sc, &ofld_rxq->fl); 3496 } 3497#endif 3498 free(pi->rss, M_CXGBE); 3499 } 3500 3501 t4_teardown_port_queues(pi); 3502 pi->flags &= ~PORT_INIT_DONE; 3503 3504 return (0); 3505} 3506 3507static void 3508quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3509{ 3510 struct sge_eq *eq = &txq->eq; 3511 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3512 3513 (void) sc; /* unused */ 3514 3515#ifdef INVARIANTS 3516 TXQ_LOCK(txq); 3517 MPASS((eq->flags & EQ_ENABLED) == 0); 3518 TXQ_UNLOCK(txq); 3519#endif 3520 3521 /* Wait for the mp_ring to empty. */ 3522 while (!mp_ring_is_idle(txq->r)) { 3523 mp_ring_check_drainage(txq->r, 0); 3524 pause("rquiesce", 1); 3525 } 3526 3527 /* Then wait for the hardware to finish. */ 3528 while (spg->cidx != htobe16(eq->pidx)) 3529 pause("equiesce", 1); 3530 3531 /* Finally, wait for the driver to reclaim all descriptors. */ 3532 while (eq->cidx != eq->pidx) 3533 pause("dquiesce", 1); 3534} 3535 3536static void 3537quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3538{ 3539 3540 /* XXXTX */ 3541} 3542 3543static void 3544quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3545{ 3546 (void) sc; /* unused */ 3547 3548 /* Synchronize with the interrupt handler */ 3549 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3550 pause("iqfree", 1); 3551} 3552 3553static void 3554quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3555{ 3556 mtx_lock(&sc->sfl_lock); 3557 FL_LOCK(fl); 3558 fl->flags |= FL_DOOMED; 3559 FL_UNLOCK(fl); 3560 mtx_unlock(&sc->sfl_lock); 3561 3562 callout_drain(&sc->sfl_callout); 3563 KASSERT((fl->flags & FL_STARVING) == 0, 3564 ("%s: still starving", __func__)); 3565} 3566 3567static int 3568t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3569 driver_intr_t *handler, void *arg, char *name) 3570{ 3571 int rc; 3572 3573 irq->rid = rid; 3574 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3575 RF_SHAREABLE | RF_ACTIVE); 3576 if (irq->res == NULL) { 3577 device_printf(sc->dev, 3578 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3579 return (ENOMEM); 3580 } 3581 3582 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3583 NULL, handler, arg, &irq->tag); 3584 if (rc != 0) { 3585 device_printf(sc->dev, 3586 "failed to setup interrupt for rid %d, name %s: %d\n", 3587 rid, name, rc); 3588 } else if (name) 3589 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3590 3591 return (rc); 3592} 3593 3594static int 3595t4_free_irq(struct adapter *sc, struct irq *irq) 3596{ 3597 if (irq->tag) 3598 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3599 if (irq->res) 3600 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3601 3602 bzero(irq, sizeof(*irq)); 3603 3604 return (0); 3605} 3606 3607static void 3608reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3609 unsigned int end) 3610{ 3611 uint32_t *p = (uint32_t *)(buf + start); 3612 3613 for ( ; start <= end; start += sizeof(uint32_t)) 3614 *p++ = t4_read_reg(sc, start); 3615} 3616 3617static void 3618t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3619{ 3620 int i, n; 3621 const unsigned int *reg_ranges; 3622 static const unsigned int t4_reg_ranges[] = { 3623 0x1008, 0x1108, 3624 0x1180, 0x11b4, 3625 0x11fc, 0x123c, 3626 0x1300, 0x173c, 3627 0x1800, 0x18fc, 3628 0x3000, 0x30d8, 3629 0x30e0, 0x5924, 3630 0x5960, 0x59d4, 3631 0x5a00, 0x5af8, 3632 0x6000, 0x6098, 3633 0x6100, 0x6150, 3634 0x6200, 0x6208, 3635 0x6240, 0x6248, 3636 0x6280, 0x6338, 3637 0x6370, 0x638c, 3638 0x6400, 0x643c, 3639 0x6500, 0x6524, 3640 0x6a00, 0x6a38, 3641 0x6a60, 0x6a78, 3642 0x6b00, 0x6b84, 3643 0x6bf0, 0x6c84, 3644 0x6cf0, 0x6d84, 3645 0x6df0, 0x6e84, 3646 0x6ef0, 0x6f84, 3647 0x6ff0, 0x7084, 3648 0x70f0, 0x7184, 3649 0x71f0, 0x7284, 3650 0x72f0, 0x7384, 3651 0x73f0, 0x7450, 3652 0x7500, 0x7530, 3653 0x7600, 0x761c, 3654 0x7680, 0x76cc, 3655 0x7700, 0x7798, 3656 0x77c0, 0x77fc, 3657 0x7900, 0x79fc, 3658 0x7b00, 0x7c38, 3659 0x7d00, 0x7efc, 3660 0x8dc0, 0x8e1c, 3661 0x8e30, 0x8e78, 3662 0x8ea0, 0x8f6c, 3663 0x8fc0, 0x9074, 3664 0x90fc, 0x90fc, 3665 0x9400, 0x9458, 3666 0x9600, 0x96bc, 3667 0x9800, 0x9808, 3668 0x9820, 0x983c, 3669 0x9850, 0x9864, 3670 0x9c00, 0x9c6c, 3671 0x9c80, 0x9cec, 3672 0x9d00, 0x9d6c, 3673 0x9d80, 0x9dec, 3674 0x9e00, 0x9e6c, 3675 0x9e80, 0x9eec, 3676 0x9f00, 0x9f6c, 3677 0x9f80, 0x9fec, 3678 0xd004, 0xd03c, 3679 0xdfc0, 0xdfe0, 3680 0xe000, 0xea7c, 3681 0xf000, 0x11110, 3682 0x11118, 0x11190, 3683 0x19040, 0x1906c, 3684 0x19078, 0x19080, 3685 0x1908c, 0x19124, 3686 0x19150, 0x191b0, 3687 0x191d0, 0x191e8, 3688 0x19238, 0x1924c, 3689 0x193f8, 0x19474, 3690 0x19490, 0x194f8, 3691 0x19800, 0x19f30, 3692 0x1a000, 0x1a06c, 3693 0x1a0b0, 0x1a120, 3694 0x1a128, 0x1a138, 3695 0x1a190, 0x1a1c4, 3696 0x1a1fc, 0x1a1fc, 3697 0x1e040, 0x1e04c, 3698 0x1e284, 0x1e28c, 3699 0x1e2c0, 0x1e2c0, 3700 0x1e2e0, 0x1e2e0, 3701 0x1e300, 0x1e384, 3702 0x1e3c0, 0x1e3c8, 3703 0x1e440, 0x1e44c, 3704 0x1e684, 0x1e68c, 3705 0x1e6c0, 0x1e6c0, 3706 0x1e6e0, 0x1e6e0, 3707 0x1e700, 0x1e784, 3708 0x1e7c0, 0x1e7c8, 3709 0x1e840, 0x1e84c, 3710 0x1ea84, 0x1ea8c, 3711 0x1eac0, 0x1eac0, 3712 0x1eae0, 0x1eae0, 3713 0x1eb00, 0x1eb84, 3714 0x1ebc0, 0x1ebc8, 3715 0x1ec40, 0x1ec4c, 3716 0x1ee84, 0x1ee8c, 3717 0x1eec0, 0x1eec0, 3718 0x1eee0, 0x1eee0, 3719 0x1ef00, 0x1ef84, 3720 0x1efc0, 0x1efc8, 3721 0x1f040, 0x1f04c, 3722 0x1f284, 0x1f28c, 3723 0x1f2c0, 0x1f2c0, 3724 0x1f2e0, 0x1f2e0, 3725 0x1f300, 0x1f384, 3726 0x1f3c0, 0x1f3c8, 3727 0x1f440, 0x1f44c, 3728 0x1f684, 0x1f68c, 3729 0x1f6c0, 0x1f6c0, 3730 0x1f6e0, 0x1f6e0, 3731 0x1f700, 0x1f784, 3732 0x1f7c0, 0x1f7c8, 3733 0x1f840, 0x1f84c, 3734 0x1fa84, 0x1fa8c, 3735 0x1fac0, 0x1fac0, 3736 0x1fae0, 0x1fae0, 3737 0x1fb00, 0x1fb84, 3738 0x1fbc0, 0x1fbc8, 3739 0x1fc40, 0x1fc4c, 3740 0x1fe84, 0x1fe8c, 3741 0x1fec0, 0x1fec0, 3742 0x1fee0, 0x1fee0, 3743 0x1ff00, 0x1ff84, 3744 0x1ffc0, 0x1ffc8, 3745 0x20000, 0x2002c, 3746 0x20100, 0x2013c, 3747 0x20190, 0x201c8, 3748 0x20200, 0x20318, 3749 0x20400, 0x20528, 3750 0x20540, 0x20614, 3751 0x21000, 0x21040, 3752 0x2104c, 0x21060, 3753 0x210c0, 0x210ec, 3754 0x21200, 0x21268, 3755 0x21270, 0x21284, 3756 0x212fc, 0x21388, 3757 0x21400, 0x21404, 3758 0x21500, 0x21518, 3759 0x2152c, 0x2153c, 3760 0x21550, 0x21554, 3761 0x21600, 0x21600, 3762 0x21608, 0x21628, 3763 0x21630, 0x2163c, 3764 0x21700, 0x2171c, 3765 0x21780, 0x2178c, 3766 0x21800, 0x21c38, 3767 0x21c80, 0x21d7c, 3768 0x21e00, 0x21e04, 3769 0x22000, 0x2202c, 3770 0x22100, 0x2213c, 3771 0x22190, 0x221c8, 3772 0x22200, 0x22318, 3773 0x22400, 0x22528, 3774 0x22540, 0x22614, 3775 0x23000, 0x23040, 3776 0x2304c, 0x23060, 3777 0x230c0, 0x230ec, 3778 0x23200, 0x23268, 3779 0x23270, 0x23284, 3780 0x232fc, 0x23388, 3781 0x23400, 0x23404, 3782 0x23500, 0x23518, 3783 0x2352c, 0x2353c, 3784 0x23550, 0x23554, 3785 0x23600, 0x23600, 3786 0x23608, 0x23628, 3787 0x23630, 0x2363c, 3788 0x23700, 0x2371c, 3789 0x23780, 0x2378c, 3790 0x23800, 0x23c38, 3791 0x23c80, 0x23d7c, 3792 0x23e00, 0x23e04, 3793 0x24000, 0x2402c, 3794 0x24100, 0x2413c, 3795 0x24190, 0x241c8, 3796 0x24200, 0x24318, 3797 0x24400, 0x24528, 3798 0x24540, 0x24614, 3799 0x25000, 0x25040, 3800 0x2504c, 0x25060, 3801 0x250c0, 0x250ec, 3802 0x25200, 0x25268, 3803 0x25270, 0x25284, 3804 0x252fc, 0x25388, 3805 0x25400, 0x25404, 3806 0x25500, 0x25518, 3807 0x2552c, 0x2553c, 3808 0x25550, 0x25554, 3809 0x25600, 0x25600, 3810 0x25608, 0x25628, 3811 0x25630, 0x2563c, 3812 0x25700, 0x2571c, 3813 0x25780, 0x2578c, 3814 0x25800, 0x25c38, 3815 0x25c80, 0x25d7c, 3816 0x25e00, 0x25e04, 3817 0x26000, 0x2602c, 3818 0x26100, 0x2613c, 3819 0x26190, 0x261c8, 3820 0x26200, 0x26318, 3821 0x26400, 0x26528, 3822 0x26540, 0x26614, 3823 0x27000, 0x27040, 3824 0x2704c, 0x27060, 3825 0x270c0, 0x270ec, 3826 0x27200, 0x27268, 3827 0x27270, 0x27284, 3828 0x272fc, 0x27388, 3829 0x27400, 0x27404, 3830 0x27500, 0x27518, 3831 0x2752c, 0x2753c, 3832 0x27550, 0x27554, 3833 0x27600, 0x27600, 3834 0x27608, 0x27628, 3835 0x27630, 0x2763c, 3836 0x27700, 0x2771c, 3837 0x27780, 0x2778c, 3838 0x27800, 0x27c38, 3839 0x27c80, 0x27d7c, 3840 0x27e00, 0x27e04 3841 }; 3842 static const unsigned int t5_reg_ranges[] = { 3843 0x1008, 0x1148, 3844 0x1180, 0x11b4, 3845 0x11fc, 0x123c, 3846 0x1280, 0x173c, 3847 0x1800, 0x18fc, 3848 0x3000, 0x3028, 3849 0x3060, 0x30d8, 3850 0x30e0, 0x30fc, 3851 0x3140, 0x357c, 3852 0x35a8, 0x35cc, 3853 0x35ec, 0x35ec, 3854 0x3600, 0x5624, 3855 0x56cc, 0x575c, 3856 0x580c, 0x5814, 3857 0x5890, 0x58bc, 3858 0x5940, 0x59dc, 3859 0x59fc, 0x5a18, 3860 0x5a60, 0x5a9c, 3861 0x5b94, 0x5bfc, 3862 0x6000, 0x6040, 3863 0x6058, 0x614c, 3864 0x7700, 0x7798, 3865 0x77c0, 0x78fc, 3866 0x7b00, 0x7c54, 3867 0x7d00, 0x7efc, 3868 0x8dc0, 0x8de0, 3869 0x8df8, 0x8e84, 3870 0x8ea0, 0x8f84, 3871 0x8fc0, 0x90f8, 3872 0x9400, 0x9470, 3873 0x9600, 0x96f4, 3874 0x9800, 0x9808, 3875 0x9820, 0x983c, 3876 0x9850, 0x9864, 3877 0x9c00, 0x9c6c, 3878 0x9c80, 0x9cec, 3879 0x9d00, 0x9d6c, 3880 0x9d80, 0x9dec, 3881 0x9e00, 0x9e6c, 3882 0x9e80, 0x9eec, 3883 0x9f00, 0x9f6c, 3884 0x9f80, 0xa020, 3885 0xd004, 0xd03c, 3886 0xdfc0, 0xdfe0, 3887 0xe000, 0x11088, 3888 0x1109c, 0x11110, 3889 0x11118, 0x1117c, 3890 0x11190, 0x11204, 3891 0x19040, 0x1906c, 3892 0x19078, 0x19080, 3893 0x1908c, 0x19124, 3894 0x19150, 0x191b0, 3895 0x191d0, 0x191e8, 3896 0x19238, 0x19290, 3897 0x193f8, 0x19474, 3898 0x19490, 0x194cc, 3899 0x194f0, 0x194f8, 3900 0x19c00, 0x19c60, 3901 0x19c94, 0x19e10, 3902 0x19e50, 0x19f34, 3903 0x19f40, 0x19f50, 3904 0x19f90, 0x19fe4, 3905 0x1a000, 0x1a06c, 3906 0x1a0b0, 0x1a120, 3907 0x1a128, 0x1a138, 3908 0x1a190, 0x1a1c4, 3909 0x1a1fc, 0x1a1fc, 3910 0x1e008, 0x1e00c, 3911 0x1e040, 0x1e04c, 3912 0x1e284, 0x1e290, 3913 0x1e2c0, 0x1e2c0, 3914 0x1e2e0, 0x1e2e0, 3915 0x1e300, 0x1e384, 3916 0x1e3c0, 0x1e3c8, 3917 0x1e408, 0x1e40c, 3918 0x1e440, 0x1e44c, 3919 0x1e684, 0x1e690, 3920 0x1e6c0, 0x1e6c0, 3921 0x1e6e0, 0x1e6e0, 3922 0x1e700, 0x1e784, 3923 0x1e7c0, 0x1e7c8, 3924 0x1e808, 0x1e80c, 3925 0x1e840, 0x1e84c, 3926 0x1ea84, 0x1ea90, 3927 0x1eac0, 0x1eac0, 3928 0x1eae0, 0x1eae0, 3929 0x1eb00, 0x1eb84, 3930 0x1ebc0, 0x1ebc8, 3931 0x1ec08, 0x1ec0c, 3932 0x1ec40, 0x1ec4c, 3933 0x1ee84, 0x1ee90, 3934 0x1eec0, 0x1eec0, 3935 0x1eee0, 0x1eee0, 3936 0x1ef00, 0x1ef84, 3937 0x1efc0, 0x1efc8, 3938 0x1f008, 0x1f00c, 3939 0x1f040, 0x1f04c, 3940 0x1f284, 0x1f290, 3941 0x1f2c0, 0x1f2c0, 3942 0x1f2e0, 0x1f2e0, 3943 0x1f300, 0x1f384, 3944 0x1f3c0, 0x1f3c8, 3945 0x1f408, 0x1f40c, 3946 0x1f440, 0x1f44c, 3947 0x1f684, 0x1f690, 3948 0x1f6c0, 0x1f6c0, 3949 0x1f6e0, 0x1f6e0, 3950 0x1f700, 0x1f784, 3951 0x1f7c0, 0x1f7c8, 3952 0x1f808, 0x1f80c, 3953 0x1f840, 0x1f84c, 3954 0x1fa84, 0x1fa90, 3955 0x1fac0, 0x1fac0, 3956 0x1fae0, 0x1fae0, 3957 0x1fb00, 0x1fb84, 3958 0x1fbc0, 0x1fbc8, 3959 0x1fc08, 0x1fc0c, 3960 0x1fc40, 0x1fc4c, 3961 0x1fe84, 0x1fe90, 3962 0x1fec0, 0x1fec0, 3963 0x1fee0, 0x1fee0, 3964 0x1ff00, 0x1ff84, 3965 0x1ffc0, 0x1ffc8, 3966 0x30000, 0x30030, 3967 0x30100, 0x30144, 3968 0x30190, 0x301d0, 3969 0x30200, 0x30318, 3970 0x30400, 0x3052c, 3971 0x30540, 0x3061c, 3972 0x30800, 0x30834, 3973 0x308c0, 0x30908, 3974 0x30910, 0x309ac, 3975 0x30a00, 0x30a2c, 3976 0x30a44, 0x30a50, 3977 0x30a74, 0x30c24, 3978 0x30d00, 0x30d00, 3979 0x30d08, 0x30d14, 3980 0x30d1c, 0x30d20, 3981 0x30d3c, 0x30d50, 3982 0x31200, 0x3120c, 3983 0x31220, 0x31220, 3984 0x31240, 0x31240, 3985 0x31600, 0x3160c, 3986 0x31a00, 0x31a1c, 3987 0x31e00, 0x31e20, 3988 0x31e38, 0x31e3c, 3989 0x31e80, 0x31e80, 3990 0x31e88, 0x31ea8, 3991 0x31eb0, 0x31eb4, 3992 0x31ec8, 0x31ed4, 3993 0x31fb8, 0x32004, 3994 0x32200, 0x32200, 3995 0x32208, 0x32240, 3996 0x32248, 0x32280, 3997 0x32288, 0x322c0, 3998 0x322c8, 0x322fc, 3999 0x32600, 0x32630, 4000 0x32a00, 0x32abc, 4001 0x32b00, 0x32b70, 4002 0x33000, 0x33048, 4003 0x33060, 0x3309c, 4004 0x330f0, 0x33148, 4005 0x33160, 0x3319c, 4006 0x331f0, 0x332e4, 4007 0x332f8, 0x333e4, 4008 0x333f8, 0x33448, 4009 0x33460, 0x3349c, 4010 0x334f0, 0x33548, 4011 0x33560, 0x3359c, 4012 0x335f0, 0x336e4, 4013 0x336f8, 0x337e4, 4014 0x337f8, 0x337fc, 4015 0x33814, 0x33814, 4016 0x3382c, 0x3382c, 4017 0x33880, 0x3388c, 4018 0x338e8, 0x338ec, 4019 0x33900, 0x33948, 4020 0x33960, 0x3399c, 4021 0x339f0, 0x33ae4, 4022 0x33af8, 0x33b10, 4023 0x33b28, 0x33b28, 4024 0x33b3c, 0x33b50, 4025 0x33bf0, 0x33c10, 4026 0x33c28, 0x33c28, 4027 0x33c3c, 0x33c50, 4028 0x33cf0, 0x33cfc, 4029 0x34000, 0x34030, 4030 0x34100, 0x34144, 4031 0x34190, 0x341d0, 4032 0x34200, 0x34318, 4033 0x34400, 0x3452c, 4034 0x34540, 0x3461c, 4035 0x34800, 0x34834, 4036 0x348c0, 0x34908, 4037 0x34910, 0x349ac, 4038 0x34a00, 0x34a2c, 4039 0x34a44, 0x34a50, 4040 0x34a74, 0x34c24, 4041 0x34d00, 0x34d00, 4042 0x34d08, 0x34d14, 4043 0x34d1c, 0x34d20, 4044 0x34d3c, 0x34d50, 4045 0x35200, 0x3520c, 4046 0x35220, 0x35220, 4047 0x35240, 0x35240, 4048 0x35600, 0x3560c, 4049 0x35a00, 0x35a1c, 4050 0x35e00, 0x35e20, 4051 0x35e38, 0x35e3c, 4052 0x35e80, 0x35e80, 4053 0x35e88, 0x35ea8, 4054 0x35eb0, 0x35eb4, 4055 0x35ec8, 0x35ed4, 4056 0x35fb8, 0x36004, 4057 0x36200, 0x36200, 4058 0x36208, 0x36240, 4059 0x36248, 0x36280, 4060 0x36288, 0x362c0, 4061 0x362c8, 0x362fc, 4062 0x36600, 0x36630, 4063 0x36a00, 0x36abc, 4064 0x36b00, 0x36b70, 4065 0x37000, 0x37048, 4066 0x37060, 0x3709c, 4067 0x370f0, 0x37148, 4068 0x37160, 0x3719c, 4069 0x371f0, 0x372e4, 4070 0x372f8, 0x373e4, 4071 0x373f8, 0x37448, 4072 0x37460, 0x3749c, 4073 0x374f0, 0x37548, 4074 0x37560, 0x3759c, 4075 0x375f0, 0x376e4, 4076 0x376f8, 0x377e4, 4077 0x377f8, 0x377fc, 4078 0x37814, 0x37814, 4079 0x3782c, 0x3782c, 4080 0x37880, 0x3788c, 4081 0x378e8, 0x378ec, 4082 0x37900, 0x37948, 4083 0x37960, 0x3799c, 4084 0x379f0, 0x37ae4, 4085 0x37af8, 0x37b10, 4086 0x37b28, 0x37b28, 4087 0x37b3c, 0x37b50, 4088 0x37bf0, 0x37c10, 4089 0x37c28, 0x37c28, 4090 0x37c3c, 0x37c50, 4091 0x37cf0, 0x37cfc, 4092 0x38000, 0x38030, 4093 0x38100, 0x38144, 4094 0x38190, 0x381d0, 4095 0x38200, 0x38318, 4096 0x38400, 0x3852c, 4097 0x38540, 0x3861c, 4098 0x38800, 0x38834, 4099 0x388c0, 0x38908, 4100 0x38910, 0x389ac, 4101 0x38a00, 0x38a2c, 4102 0x38a44, 0x38a50, 4103 0x38a74, 0x38c24, 4104 0x38d00, 0x38d00, 4105 0x38d08, 0x38d14, 4106 0x38d1c, 0x38d20, 4107 0x38d3c, 0x38d50, 4108 0x39200, 0x3920c, 4109 0x39220, 0x39220, 4110 0x39240, 0x39240, 4111 0x39600, 0x3960c, 4112 0x39a00, 0x39a1c, 4113 0x39e00, 0x39e20, 4114 0x39e38, 0x39e3c, 4115 0x39e80, 0x39e80, 4116 0x39e88, 0x39ea8, 4117 0x39eb0, 0x39eb4, 4118 0x39ec8, 0x39ed4, 4119 0x39fb8, 0x3a004, 4120 0x3a200, 0x3a200, 4121 0x3a208, 0x3a240, 4122 0x3a248, 0x3a280, 4123 0x3a288, 0x3a2c0, 4124 0x3a2c8, 0x3a2fc, 4125 0x3a600, 0x3a630, 4126 0x3aa00, 0x3aabc, 4127 0x3ab00, 0x3ab70, 4128 0x3b000, 0x3b048, 4129 0x3b060, 0x3b09c, 4130 0x3b0f0, 0x3b148, 4131 0x3b160, 0x3b19c, 4132 0x3b1f0, 0x3b2e4, 4133 0x3b2f8, 0x3b3e4, 4134 0x3b3f8, 0x3b448, 4135 0x3b460, 0x3b49c, 4136 0x3b4f0, 0x3b548, 4137 0x3b560, 0x3b59c, 4138 0x3b5f0, 0x3b6e4, 4139 0x3b6f8, 0x3b7e4, 4140 0x3b7f8, 0x3b7fc, 4141 0x3b814, 0x3b814, 4142 0x3b82c, 0x3b82c, 4143 0x3b880, 0x3b88c, 4144 0x3b8e8, 0x3b8ec, 4145 0x3b900, 0x3b948, 4146 0x3b960, 0x3b99c, 4147 0x3b9f0, 0x3bae4, 4148 0x3baf8, 0x3bb10, 4149 0x3bb28, 0x3bb28, 4150 0x3bb3c, 0x3bb50, 4151 0x3bbf0, 0x3bc10, 4152 0x3bc28, 0x3bc28, 4153 0x3bc3c, 0x3bc50, 4154 0x3bcf0, 0x3bcfc, 4155 0x3c000, 0x3c030, 4156 0x3c100, 0x3c144, 4157 0x3c190, 0x3c1d0, 4158 0x3c200, 0x3c318, 4159 0x3c400, 0x3c52c, 4160 0x3c540, 0x3c61c, 4161 0x3c800, 0x3c834, 4162 0x3c8c0, 0x3c908, 4163 0x3c910, 0x3c9ac, 4164 0x3ca00, 0x3ca2c, 4165 0x3ca44, 0x3ca50, 4166 0x3ca74, 0x3cc24, 4167 0x3cd00, 0x3cd00, 4168 0x3cd08, 0x3cd14, 4169 0x3cd1c, 0x3cd20, 4170 0x3cd3c, 0x3cd50, 4171 0x3d200, 0x3d20c, 4172 0x3d220, 0x3d220, 4173 0x3d240, 0x3d240, 4174 0x3d600, 0x3d60c, 4175 0x3da00, 0x3da1c, 4176 0x3de00, 0x3de20, 4177 0x3de38, 0x3de3c, 4178 0x3de80, 0x3de80, 4179 0x3de88, 0x3dea8, 4180 0x3deb0, 0x3deb4, 4181 0x3dec8, 0x3ded4, 4182 0x3dfb8, 0x3e004, 4183 0x3e200, 0x3e200, 4184 0x3e208, 0x3e240, 4185 0x3e248, 0x3e280, 4186 0x3e288, 0x3e2c0, 4187 0x3e2c8, 0x3e2fc, 4188 0x3e600, 0x3e630, 4189 0x3ea00, 0x3eabc, 4190 0x3eb00, 0x3eb70, 4191 0x3f000, 0x3f048, 4192 0x3f060, 0x3f09c, 4193 0x3f0f0, 0x3f148, 4194 0x3f160, 0x3f19c, 4195 0x3f1f0, 0x3f2e4, 4196 0x3f2f8, 0x3f3e4, 4197 0x3f3f8, 0x3f448, 4198 0x3f460, 0x3f49c, 4199 0x3f4f0, 0x3f548, 4200 0x3f560, 0x3f59c, 4201 0x3f5f0, 0x3f6e4, 4202 0x3f6f8, 0x3f7e4, 4203 0x3f7f8, 0x3f7fc, 4204 0x3f814, 0x3f814, 4205 0x3f82c, 0x3f82c, 4206 0x3f880, 0x3f88c, 4207 0x3f8e8, 0x3f8ec, 4208 0x3f900, 0x3f948, 4209 0x3f960, 0x3f99c, 4210 0x3f9f0, 0x3fae4, 4211 0x3faf8, 0x3fb10, 4212 0x3fb28, 0x3fb28, 4213 0x3fb3c, 0x3fb50, 4214 0x3fbf0, 0x3fc10, 4215 0x3fc28, 0x3fc28, 4216 0x3fc3c, 0x3fc50, 4217 0x3fcf0, 0x3fcfc, 4218 0x40000, 0x4000c, 4219 0x40040, 0x40068, 4220 0x4007c, 0x40144, 4221 0x40180, 0x4018c, 4222 0x40200, 0x40298, 4223 0x402ac, 0x4033c, 4224 0x403f8, 0x403fc, 4225 0x41304, 0x413c4, 4226 0x41400, 0x4141c, 4227 0x41480, 0x414d0, 4228 0x44000, 0x44078, 4229 0x440c0, 0x44278, 4230 0x442c0, 0x44478, 4231 0x444c0, 0x44678, 4232 0x446c0, 0x44878, 4233 0x448c0, 0x449fc, 4234 0x45000, 0x45068, 4235 0x45080, 0x45084, 4236 0x450a0, 0x450b0, 4237 0x45200, 0x45268, 4238 0x45280, 0x45284, 4239 0x452a0, 0x452b0, 4240 0x460c0, 0x460e4, 4241 0x47000, 0x4708c, 4242 0x47200, 0x47250, 4243 0x47400, 0x47420, 4244 0x47600, 0x47618, 4245 0x47800, 0x47814, 4246 0x48000, 0x4800c, 4247 0x48040, 0x48068, 4248 0x4807c, 0x48144, 4249 0x48180, 0x4818c, 4250 0x48200, 0x48298, 4251 0x482ac, 0x4833c, 4252 0x483f8, 0x483fc, 4253 0x49304, 0x493c4, 4254 0x49400, 0x4941c, 4255 0x49480, 0x494d0, 4256 0x4c000, 0x4c078, 4257 0x4c0c0, 0x4c278, 4258 0x4c2c0, 0x4c478, 4259 0x4c4c0, 0x4c678, 4260 0x4c6c0, 0x4c878, 4261 0x4c8c0, 0x4c9fc, 4262 0x4d000, 0x4d068, 4263 0x4d080, 0x4d084, 4264 0x4d0a0, 0x4d0b0, 4265 0x4d200, 0x4d268, 4266 0x4d280, 0x4d284, 4267 0x4d2a0, 0x4d2b0, 4268 0x4e0c0, 0x4e0e4, 4269 0x4f000, 0x4f08c, 4270 0x4f200, 0x4f250, 4271 0x4f400, 0x4f420, 4272 0x4f600, 0x4f618, 4273 0x4f800, 0x4f814, 4274 0x50000, 0x500cc, 4275 0x50400, 0x50400, 4276 0x50800, 0x508cc, 4277 0x50c00, 0x50c00, 4278 0x51000, 0x5101c, 4279 0x51300, 0x51308, 4280 }; 4281 4282 if (is_t4(sc)) { 4283 reg_ranges = &t4_reg_ranges[0]; 4284 n = nitems(t4_reg_ranges); 4285 } else { 4286 reg_ranges = &t5_reg_ranges[0]; 4287 n = nitems(t5_reg_ranges); 4288 } 4289 4290 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4291 for (i = 0; i < n; i += 2) 4292 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4293} 4294 4295static void 4296cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4297{ 4298 struct ifnet *ifp = pi->ifp; 4299 struct sge_txq *txq; 4300 int i, drops; 4301 struct port_stats *s = &pi->stats; 4302 struct timeval tv; 4303 const struct timeval interval = {0, 250000}; /* 250ms */ 4304 4305 getmicrotime(&tv); 4306 timevalsub(&tv, &interval); 4307 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4308 return; 4309 4310 t4_get_port_stats(sc, pi->tx_chan, s); 4311 4312 ifp->if_opackets = s->tx_frames - s->tx_pause; 4313 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4314 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4315 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4316 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4317 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4318 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4319 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4320 s->rx_trunc3; 4321 for (i = 0; i < NCHAN; i++) { 4322 if (pi->rx_chan_map & (1 << i)) { 4323 uint32_t v; 4324 4325 mtx_lock(&sc->regwin_lock); 4326 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4327 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4328 mtx_unlock(&sc->regwin_lock); 4329 ifp->if_iqdrops += v; 4330 } 4331 } 4332 4333 drops = s->tx_drop; 4334 for_each_txq(pi, i, txq) 4335 drops += counter_u64_fetch(txq->r->drops); 4336 ifp->if_snd.ifq_drops = drops; 4337 4338 ifp->if_oerrors = s->tx_error_frames; 4339 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4340 s->rx_fcs_err + s->rx_len_err; 4341 4342 getmicrotime(&pi->last_refreshed); 4343} 4344 4345static void 4346cxgbe_tick(void *arg) 4347{ 4348 struct port_info *pi = arg; 4349 struct adapter *sc = pi->adapter; 4350 struct ifnet *ifp = pi->ifp; 4351 4352 PORT_LOCK(pi); 4353 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4354 PORT_UNLOCK(pi); 4355 return; /* without scheduling another callout */ 4356 } 4357 4358 cxgbe_refresh_stats(sc, pi); 4359 4360 callout_schedule(&pi->tick, hz); 4361 PORT_UNLOCK(pi); 4362} 4363 4364static void 4365cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4366{ 4367 struct ifnet *vlan; 4368 4369 if (arg != ifp || ifp->if_type != IFT_ETHER) 4370 return; 4371 4372 vlan = VLAN_DEVAT(ifp, vid); 4373 VLAN_SETCOOKIE(vlan, ifp); 4374} 4375 4376static int 4377cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4378{ 4379 4380#ifdef INVARIANTS 4381 panic("%s: opcode 0x%02x on iq %p with payload %p", 4382 __func__, rss->opcode, iq, m); 4383#else 4384 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4385 __func__, rss->opcode, iq, m); 4386 m_freem(m); 4387#endif 4388 return (EDOOFUS); 4389} 4390 4391int 4392t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4393{ 4394 uintptr_t *loc, new; 4395 4396 if (opcode >= nitems(sc->cpl_handler)) 4397 return (EINVAL); 4398 4399 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4400 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4401 atomic_store_rel_ptr(loc, new); 4402 4403 return (0); 4404} 4405 4406static int 4407an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4408{ 4409 4410#ifdef INVARIANTS 4411 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4412#else 4413 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4414 __func__, iq, ctrl); 4415#endif 4416 return (EDOOFUS); 4417} 4418 4419int 4420t4_register_an_handler(struct adapter *sc, an_handler_t h) 4421{ 4422 uintptr_t *loc, new; 4423 4424 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4425 loc = (uintptr_t *) &sc->an_handler; 4426 atomic_store_rel_ptr(loc, new); 4427 4428 return (0); 4429} 4430 4431static int 4432fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4433{ 4434 const struct cpl_fw6_msg *cpl = 4435 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4436 4437#ifdef INVARIANTS 4438 panic("%s: fw_msg type %d", __func__, cpl->type); 4439#else 4440 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4441#endif 4442 return (EDOOFUS); 4443} 4444 4445int 4446t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4447{ 4448 uintptr_t *loc, new; 4449 4450 if (type >= nitems(sc->fw_msg_handler)) 4451 return (EINVAL); 4452 4453 /* 4454 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4455 * handler dispatch table. Reject any attempt to install a handler for 4456 * this subtype. 4457 */ 4458 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4459 return (EINVAL); 4460 4461 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4462 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4463 atomic_store_rel_ptr(loc, new); 4464 4465 return (0); 4466} 4467 4468static int 4469t4_sysctls(struct adapter *sc) 4470{ 4471 struct sysctl_ctx_list *ctx; 4472 struct sysctl_oid *oid; 4473 struct sysctl_oid_list *children, *c0; 4474 static char *caps[] = { 4475 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4476 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4477 "\6HASHFILTER\7ETHOFLD", 4478 "\20\1TOE", /* caps[2] toecaps */ 4479 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4480 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4481 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4482 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4483 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4484 "\4PO_INITIAOR\5PO_TARGET" 4485 }; 4486 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4487 4488 ctx = device_get_sysctl_ctx(sc->dev); 4489 4490 /* 4491 * dev.t4nex.X. 4492 */ 4493 oid = device_get_sysctl_tree(sc->dev); 4494 c0 = children = SYSCTL_CHILDREN(oid); 4495 4496 sc->sc_do_rxcopy = 1; 4497 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4498 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4499 4500 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4501 sc->params.nports, "# of ports"); 4502 4503 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4504 NULL, chip_rev(sc), "chip hardware revision"); 4505 4506 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4507 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4508 4509 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4510 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4511 4512 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4513 sc->cfcsum, "config file checksum"); 4514 4515 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4516 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4517 sysctl_bitfield, "A", "available doorbells"); 4518 4519 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4520 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4521 sysctl_bitfield, "A", "available link capabilities"); 4522 4523 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4524 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4525 sysctl_bitfield, "A", "available NIC capabilities"); 4526 4527 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4528 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4529 sysctl_bitfield, "A", "available TCP offload capabilities"); 4530 4531 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4532 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4533 sysctl_bitfield, "A", "available RDMA capabilities"); 4534 4535 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4536 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4537 sysctl_bitfield, "A", "available iSCSI capabilities"); 4538 4539 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4540 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4541 sysctl_bitfield, "A", "available FCoE capabilities"); 4542 4543 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4544 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4545 4546 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4547 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4548 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4549 "interrupt holdoff timer values (us)"); 4550 4551 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4552 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4553 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4554 "interrupt holdoff packet counter values"); 4555 4556 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4557 NULL, sc->tids.nftids, "number of filters"); 4558 4559 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4560 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4561 "chip temperature (in Celsius)"); 4562 4563 t4_sge_sysctls(sc, ctx, children); 4564 4565 sc->lro_timeout = 100; 4566 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4567 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4568 4569 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 4570 &sc->debug_flags, 0, "flags to enable runtime debugging"); 4571 4572#ifdef SBUF_DRAIN 4573 /* 4574 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4575 */ 4576 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4577 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4578 "logs and miscellaneous information"); 4579 children = SYSCTL_CHILDREN(oid); 4580 4581 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4582 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4583 sysctl_cctrl, "A", "congestion control"); 4584 4585 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4586 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4587 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4588 4589 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4590 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4591 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4592 4593 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4594 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4595 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4596 4597 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4598 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4599 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4600 4601 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4602 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4603 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4604 4605 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4606 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4607 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4608 4609 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4610 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4611 sysctl_cim_la, "A", "CIM logic analyzer"); 4612 4613 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4614 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4615 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4616 4617 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4618 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4619 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4620 4621 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4622 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4623 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4624 4625 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4626 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4627 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4628 4629 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4630 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4631 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4632 4633 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4634 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4635 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4636 4637 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4638 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4639 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4640 4641 if (is_t5(sc)) { 4642 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4643 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4644 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4645 4646 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4647 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4648 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4649 } 4650 4651 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4652 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4653 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4654 4655 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4656 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4657 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4658 4659 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4660 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4661 sysctl_cpl_stats, "A", "CPL statistics"); 4662 4663 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4664 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4665 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4666 4667 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4668 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4669 sysctl_devlog, "A", "firmware's device log"); 4670 4671 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4672 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4673 sysctl_fcoe_stats, "A", "FCoE statistics"); 4674 4675 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4676 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4677 sysctl_hw_sched, "A", "hardware scheduler "); 4678 4679 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4680 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4681 sysctl_l2t, "A", "hardware L2 table"); 4682 4683 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4684 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4685 sysctl_lb_stats, "A", "loopback statistics"); 4686 4687 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4688 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4689 sysctl_meminfo, "A", "memory regions"); 4690 4691 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4692 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4693 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4694 4695 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4696 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4697 sysctl_path_mtus, "A", "path MTUs"); 4698 4699 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4700 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4701 sysctl_pm_stats, "A", "PM statistics"); 4702 4703 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4704 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4705 sysctl_rdma_stats, "A", "RDMA statistics"); 4706 4707 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4708 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4709 sysctl_tcp_stats, "A", "TCP statistics"); 4710 4711 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4712 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4713 sysctl_tids, "A", "TID information"); 4714 4715 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4716 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4717 sysctl_tp_err_stats, "A", "TP error statistics"); 4718 4719 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4720 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4721 sysctl_tp_la, "A", "TP logic analyzer"); 4722 4723 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4724 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4725 sysctl_tx_rate, "A", "Tx rate"); 4726 4727 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4728 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4729 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4730 4731 if (is_t5(sc)) { 4732 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4733 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4734 sysctl_wcwr_stats, "A", "write combined work requests"); 4735 } 4736#endif 4737 4738#ifdef TCP_OFFLOAD 4739 if (is_offload(sc)) { 4740 /* 4741 * dev.t4nex.X.toe. 4742 */ 4743 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4744 NULL, "TOE parameters"); 4745 children = SYSCTL_CHILDREN(oid); 4746 4747 sc->tt.sndbuf = 256 * 1024; 4748 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4749 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4750 4751 sc->tt.ddp = 0; 4752 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4753 &sc->tt.ddp, 0, "DDP allowed"); 4754 4755 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4756 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4757 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4758 4759 sc->tt.ddp_thres = 4760 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4761 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4762 &sc->tt.ddp_thres, 0, "DDP threshold"); 4763 4764 sc->tt.rx_coalesce = 1; 4765 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4766 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4767 4768 sc->tt.tx_align = 1; 4769 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4770 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4771 } 4772#endif 4773 4774 4775 return (0); 4776} 4777 4778static int 4779cxgbe_sysctls(struct port_info *pi) 4780{ 4781 struct sysctl_ctx_list *ctx; 4782 struct sysctl_oid *oid; 4783 struct sysctl_oid_list *children; 4784 struct adapter *sc = pi->adapter; 4785 4786 ctx = device_get_sysctl_ctx(pi->dev); 4787 4788 /* 4789 * dev.cxgbe.X. 4790 */ 4791 oid = device_get_sysctl_tree(pi->dev); 4792 children = SYSCTL_CHILDREN(oid); 4793 4794 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4795 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4796 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4797 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4798 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4799 "PHY temperature (in Celsius)"); 4800 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4801 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4802 "PHY firmware version"); 4803 } 4804 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4805 &pi->nrxq, 0, "# of rx queues"); 4806 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4807 &pi->ntxq, 0, "# of tx queues"); 4808 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4809 &pi->first_rxq, 0, "index of first rx queue"); 4810 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4811 &pi->first_txq, 0, "index of first tx queue"); 4812 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4813 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4814 "Reserve queue 0 for non-flowid packets"); 4815 4816#ifdef TCP_OFFLOAD 4817 if (is_offload(sc)) { 4818 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4819 &pi->nofldrxq, 0, 4820 "# of rx queues for offloaded TCP connections"); 4821 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4822 &pi->nofldtxq, 0, 4823 "# of tx queues for offloaded TCP connections"); 4824 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4825 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4826 "index of first TOE rx queue"); 4827 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4828 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4829 "index of first TOE tx queue"); 4830 } 4831#endif 4832#ifdef DEV_NETMAP 4833 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4834 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4835 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4836 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4837 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4838 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4839 "index of first netmap rx queue"); 4840 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4841 CTLFLAG_RD, &pi->first_nm_txq, 0, 4842 "index of first netmap tx queue"); 4843#endif 4844 4845 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4846 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4847 "holdoff timer index"); 4848 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4849 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4850 "holdoff packet counter index"); 4851 4852 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4853 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4854 "rx queue size"); 4855 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4856 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4857 "tx queue size"); 4858 4859 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4860 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4861 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4862 4863 /* 4864 * dev.cxgbe.X.stats. 4865 */ 4866 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4867 NULL, "port statistics"); 4868 children = SYSCTL_CHILDREN(oid); 4869 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4870 &pi->tx_parse_error, 0, 4871 "# of tx packets with invalid length or # of segments"); 4872 4873#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4874 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4875 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4876 sysctl_handle_t4_reg64, "QU", desc) 4877 4878 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4879 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4880 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4881 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4882 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4883 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4884 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4885 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4886 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4887 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4888 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4889 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4890 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4891 "# of tx frames in this range", 4892 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4893 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4894 "# of tx frames in this range", 4895 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4896 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4897 "# of tx frames in this range", 4898 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4899 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4900 "# of tx frames in this range", 4901 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4902 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4903 "# of tx frames in this range", 4904 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4905 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4906 "# of tx frames in this range", 4907 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4908 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4909 "# of tx frames in this range", 4910 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4911 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4912 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4913 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4914 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4915 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4916 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4917 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4918 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4919 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4920 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4921 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4922 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4923 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4924 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4925 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4926 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4927 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4928 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4929 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4930 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4931 4932 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4933 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4934 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4935 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4936 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4937 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4938 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4939 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4940 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4941 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4942 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4943 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4944 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4945 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4946 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4947 "# of frames received with bad FCS", 4948 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4949 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4950 "# of frames received with length error", 4951 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4952 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4953 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4954 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4955 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4956 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4957 "# of rx frames in this range", 4958 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4959 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4960 "# of rx frames in this range", 4961 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4962 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4963 "# of rx frames in this range", 4964 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4965 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4966 "# of rx frames in this range", 4967 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4968 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4969 "# of rx frames in this range", 4970 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4971 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4972 "# of rx frames in this range", 4973 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4974 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4975 "# of rx frames in this range", 4976 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4977 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4978 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4979 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4980 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4981 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4982 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4983 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4984 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4985 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4986 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4987 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4988 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4989 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4990 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4991 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4992 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4993 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4994 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4995 4996#undef SYSCTL_ADD_T4_REG64 4997 4998#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4999 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5000 &pi->stats.name, desc) 5001 5002 /* We get these from port_stats and they may be stale by upto 1s */ 5003 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5004 "# drops due to buffer-group 0 overflows"); 5005 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5006 "# drops due to buffer-group 1 overflows"); 5007 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5008 "# drops due to buffer-group 2 overflows"); 5009 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5010 "# drops due to buffer-group 3 overflows"); 5011 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5012 "# of buffer-group 0 truncated packets"); 5013 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5014 "# of buffer-group 1 truncated packets"); 5015 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5016 "# of buffer-group 2 truncated packets"); 5017 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5018 "# of buffer-group 3 truncated packets"); 5019 5020#undef SYSCTL_ADD_T4_PORTSTAT 5021 5022 return (0); 5023} 5024 5025static int 5026sysctl_int_array(SYSCTL_HANDLER_ARGS) 5027{ 5028 int rc, *i, space = 0; 5029 struct sbuf sb; 5030 5031 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5032 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5033 if (space) 5034 sbuf_printf(&sb, " "); 5035 sbuf_printf(&sb, "%d", *i); 5036 space = 1; 5037 } 5038 sbuf_finish(&sb); 5039 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5040 sbuf_delete(&sb); 5041 return (rc); 5042} 5043 5044static int 5045sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5046{ 5047 int rc; 5048 struct sbuf *sb; 5049 5050 rc = sysctl_wire_old_buffer(req, 0); 5051 if (rc != 0) 5052 return(rc); 5053 5054 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5055 if (sb == NULL) 5056 return (ENOMEM); 5057 5058 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5059 rc = sbuf_finish(sb); 5060 sbuf_delete(sb); 5061 5062 return (rc); 5063} 5064 5065static int 5066sysctl_btphy(SYSCTL_HANDLER_ARGS) 5067{ 5068 struct port_info *pi = arg1; 5069 int op = arg2; 5070 struct adapter *sc = pi->adapter; 5071 u_int v; 5072 int rc; 5073 5074 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5075 if (rc) 5076 return (rc); 5077 /* XXX: magic numbers */ 5078 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5079 &v); 5080 end_synchronized_op(sc, 0); 5081 if (rc) 5082 return (rc); 5083 if (op == 0) 5084 v /= 256; 5085 5086 rc = sysctl_handle_int(oidp, &v, 0, req); 5087 return (rc); 5088} 5089 5090static int 5091sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5092{ 5093 struct port_info *pi = arg1; 5094 int rc, val; 5095 5096 val = pi->rsrv_noflowq; 5097 rc = sysctl_handle_int(oidp, &val, 0, req); 5098 if (rc != 0 || req->newptr == NULL) 5099 return (rc); 5100 5101 if ((val >= 1) && (pi->ntxq > 1)) 5102 pi->rsrv_noflowq = 1; 5103 else 5104 pi->rsrv_noflowq = 0; 5105 5106 return (rc); 5107} 5108 5109static int 5110sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5111{ 5112 struct port_info *pi = arg1; 5113 struct adapter *sc = pi->adapter; 5114 int idx, rc, i; 5115 struct sge_rxq *rxq; 5116#ifdef TCP_OFFLOAD 5117 struct sge_ofld_rxq *ofld_rxq; 5118#endif 5119 uint8_t v; 5120 5121 idx = pi->tmr_idx; 5122 5123 rc = sysctl_handle_int(oidp, &idx, 0, req); 5124 if (rc != 0 || req->newptr == NULL) 5125 return (rc); 5126 5127 if (idx < 0 || idx >= SGE_NTIMERS) 5128 return (EINVAL); 5129 5130 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5131 "t4tmr"); 5132 if (rc) 5133 return (rc); 5134 5135 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5136 for_each_rxq(pi, i, rxq) { 5137#ifdef atomic_store_rel_8 5138 atomic_store_rel_8(&rxq->iq.intr_params, v); 5139#else 5140 rxq->iq.intr_params = v; 5141#endif 5142 } 5143#ifdef TCP_OFFLOAD 5144 for_each_ofld_rxq(pi, i, ofld_rxq) { 5145#ifdef atomic_store_rel_8 5146 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5147#else 5148 ofld_rxq->iq.intr_params = v; 5149#endif 5150 } 5151#endif 5152 pi->tmr_idx = idx; 5153 5154 end_synchronized_op(sc, LOCK_HELD); 5155 return (0); 5156} 5157 5158static int 5159sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5160{ 5161 struct port_info *pi = arg1; 5162 struct adapter *sc = pi->adapter; 5163 int idx, rc; 5164 5165 idx = pi->pktc_idx; 5166 5167 rc = sysctl_handle_int(oidp, &idx, 0, req); 5168 if (rc != 0 || req->newptr == NULL) 5169 return (rc); 5170 5171 if (idx < -1 || idx >= SGE_NCOUNTERS) 5172 return (EINVAL); 5173 5174 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5175 "t4pktc"); 5176 if (rc) 5177 return (rc); 5178 5179 if (pi->flags & PORT_INIT_DONE) 5180 rc = EBUSY; /* cannot be changed once the queues are created */ 5181 else 5182 pi->pktc_idx = idx; 5183 5184 end_synchronized_op(sc, LOCK_HELD); 5185 return (rc); 5186} 5187 5188static int 5189sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5190{ 5191 struct port_info *pi = arg1; 5192 struct adapter *sc = pi->adapter; 5193 int qsize, rc; 5194 5195 qsize = pi->qsize_rxq; 5196 5197 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5198 if (rc != 0 || req->newptr == NULL) 5199 return (rc); 5200 5201 if (qsize < 128 || (qsize & 7)) 5202 return (EINVAL); 5203 5204 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5205 "t4rxqs"); 5206 if (rc) 5207 return (rc); 5208 5209 if (pi->flags & PORT_INIT_DONE) 5210 rc = EBUSY; /* cannot be changed once the queues are created */ 5211 else 5212 pi->qsize_rxq = qsize; 5213 5214 end_synchronized_op(sc, LOCK_HELD); 5215 return (rc); 5216} 5217 5218static int 5219sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5220{ 5221 struct port_info *pi = arg1; 5222 struct adapter *sc = pi->adapter; 5223 int qsize, rc; 5224 5225 qsize = pi->qsize_txq; 5226 5227 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5228 if (rc != 0 || req->newptr == NULL) 5229 return (rc); 5230 5231 if (qsize < 128 || qsize > 65536) 5232 return (EINVAL); 5233 5234 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5235 "t4txqs"); 5236 if (rc) 5237 return (rc); 5238 5239 if (pi->flags & PORT_INIT_DONE) 5240 rc = EBUSY; /* cannot be changed once the queues are created */ 5241 else 5242 pi->qsize_txq = qsize; 5243 5244 end_synchronized_op(sc, LOCK_HELD); 5245 return (rc); 5246} 5247 5248static int 5249sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5250{ 5251 struct port_info *pi = arg1; 5252 struct adapter *sc = pi->adapter; 5253 struct link_config *lc = &pi->link_cfg; 5254 int rc; 5255 5256 if (req->newptr == NULL) { 5257 struct sbuf *sb; 5258 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5259 5260 rc = sysctl_wire_old_buffer(req, 0); 5261 if (rc != 0) 5262 return(rc); 5263 5264 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5265 if (sb == NULL) 5266 return (ENOMEM); 5267 5268 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5269 rc = sbuf_finish(sb); 5270 sbuf_delete(sb); 5271 } else { 5272 char s[2]; 5273 int n; 5274 5275 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5276 s[1] = 0; 5277 5278 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5279 if (rc != 0) 5280 return(rc); 5281 5282 if (s[1] != 0) 5283 return (EINVAL); 5284 if (s[0] < '0' || s[0] > '9') 5285 return (EINVAL); /* not a number */ 5286 n = s[0] - '0'; 5287 if (n & ~(PAUSE_TX | PAUSE_RX)) 5288 return (EINVAL); /* some other bit is set too */ 5289 5290 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5291 if (rc) 5292 return (rc); 5293 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5294 int link_ok = lc->link_ok; 5295 5296 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5297 lc->requested_fc |= n; 5298 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5299 lc->link_ok = link_ok; /* restore */ 5300 } 5301 end_synchronized_op(sc, 0); 5302 } 5303 5304 return (rc); 5305} 5306 5307static int 5308sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5309{ 5310 struct adapter *sc = arg1; 5311 int reg = arg2; 5312 uint64_t val; 5313 5314 val = t4_read_reg64(sc, reg); 5315 5316 return (sysctl_handle_64(oidp, &val, 0, req)); 5317} 5318 5319static int 5320sysctl_temperature(SYSCTL_HANDLER_ARGS) 5321{ 5322 struct adapter *sc = arg1; 5323 int rc, t; 5324 uint32_t param, val; 5325 5326 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5327 if (rc) 5328 return (rc); 5329 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5330 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5331 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5332 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5333 end_synchronized_op(sc, 0); 5334 if (rc) 5335 return (rc); 5336 5337 /* unknown is returned as 0 but we display -1 in that case */ 5338 t = val == 0 ? -1 : val; 5339 5340 rc = sysctl_handle_int(oidp, &t, 0, req); 5341 return (rc); 5342} 5343 5344#ifdef SBUF_DRAIN 5345static int 5346sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5347{ 5348 struct adapter *sc = arg1; 5349 struct sbuf *sb; 5350 int rc, i; 5351 uint16_t incr[NMTUS][NCCTRL_WIN]; 5352 static const char *dec_fac[] = { 5353 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5354 "0.9375" 5355 }; 5356 5357 rc = sysctl_wire_old_buffer(req, 0); 5358 if (rc != 0) 5359 return (rc); 5360 5361 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5362 if (sb == NULL) 5363 return (ENOMEM); 5364 5365 t4_read_cong_tbl(sc, incr); 5366 5367 for (i = 0; i < NCCTRL_WIN; ++i) { 5368 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5369 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5370 incr[5][i], incr[6][i], incr[7][i]); 5371 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5372 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5373 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5374 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5375 } 5376 5377 rc = sbuf_finish(sb); 5378 sbuf_delete(sb); 5379 5380 return (rc); 5381} 5382 5383static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5384 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5385 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5386 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5387}; 5388 5389static int 5390sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5391{ 5392 struct adapter *sc = arg1; 5393 struct sbuf *sb; 5394 int rc, i, n, qid = arg2; 5395 uint32_t *buf, *p; 5396 char *qtype; 5397 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5398 5399 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5400 ("%s: bad qid %d\n", __func__, qid)); 5401 5402 if (qid < CIM_NUM_IBQ) { 5403 /* inbound queue */ 5404 qtype = "IBQ"; 5405 n = 4 * CIM_IBQ_SIZE; 5406 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5407 rc = t4_read_cim_ibq(sc, qid, buf, n); 5408 } else { 5409 /* outbound queue */ 5410 qtype = "OBQ"; 5411 qid -= CIM_NUM_IBQ; 5412 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5413 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5414 rc = t4_read_cim_obq(sc, qid, buf, n); 5415 } 5416 5417 if (rc < 0) { 5418 rc = -rc; 5419 goto done; 5420 } 5421 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5422 5423 rc = sysctl_wire_old_buffer(req, 0); 5424 if (rc != 0) 5425 goto done; 5426 5427 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5428 if (sb == NULL) { 5429 rc = ENOMEM; 5430 goto done; 5431 } 5432 5433 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5434 for (i = 0, p = buf; i < n; i += 16, p += 4) 5435 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5436 p[2], p[3]); 5437 5438 rc = sbuf_finish(sb); 5439 sbuf_delete(sb); 5440done: 5441 free(buf, M_CXGBE); 5442 return (rc); 5443} 5444 5445static int 5446sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5447{ 5448 struct adapter *sc = arg1; 5449 u_int cfg; 5450 struct sbuf *sb; 5451 uint32_t *buf, *p; 5452 int rc; 5453 5454 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5455 if (rc != 0) 5456 return (rc); 5457 5458 rc = sysctl_wire_old_buffer(req, 0); 5459 if (rc != 0) 5460 return (rc); 5461 5462 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5463 if (sb == NULL) 5464 return (ENOMEM); 5465 5466 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5467 M_ZERO | M_WAITOK); 5468 5469 rc = -t4_cim_read_la(sc, buf, NULL); 5470 if (rc != 0) 5471 goto done; 5472 5473 sbuf_printf(sb, "Status Data PC%s", 5474 cfg & F_UPDBGLACAPTPCONLY ? "" : 5475 " LS0Stat LS0Addr LS0Data"); 5476 5477 KASSERT((sc->params.cim_la_size & 7) == 0, 5478 ("%s: p will walk off the end of buf", __func__)); 5479 5480 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5481 if (cfg & F_UPDBGLACAPTPCONLY) { 5482 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5483 p[6], p[7]); 5484 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5485 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5486 p[4] & 0xff, p[5] >> 8); 5487 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5488 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5489 p[1] & 0xf, p[2] >> 4); 5490 } else { 5491 sbuf_printf(sb, 5492 "\n %02x %x%07x %x%07x %08x %08x " 5493 "%08x%08x%08x%08x", 5494 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5495 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5496 p[6], p[7]); 5497 } 5498 } 5499 5500 rc = sbuf_finish(sb); 5501 sbuf_delete(sb); 5502done: 5503 free(buf, M_CXGBE); 5504 return (rc); 5505} 5506 5507static int 5508sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5509{ 5510 struct adapter *sc = arg1; 5511 u_int i; 5512 struct sbuf *sb; 5513 uint32_t *buf, *p; 5514 int rc; 5515 5516 rc = sysctl_wire_old_buffer(req, 0); 5517 if (rc != 0) 5518 return (rc); 5519 5520 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5521 if (sb == NULL) 5522 return (ENOMEM); 5523 5524 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5525 M_ZERO | M_WAITOK); 5526 5527 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5528 p = buf; 5529 5530 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5531 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5532 p[1], p[0]); 5533 } 5534 5535 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5536 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5537 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5538 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5539 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5540 (p[1] >> 2) | ((p[2] & 3) << 30), 5541 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5542 p[0] & 1); 5543 } 5544 5545 rc = sbuf_finish(sb); 5546 sbuf_delete(sb); 5547 free(buf, M_CXGBE); 5548 return (rc); 5549} 5550 5551static int 5552sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5553{ 5554 struct adapter *sc = arg1; 5555 u_int i; 5556 struct sbuf *sb; 5557 uint32_t *buf, *p; 5558 int rc; 5559 5560 rc = sysctl_wire_old_buffer(req, 0); 5561 if (rc != 0) 5562 return (rc); 5563 5564 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5565 if (sb == NULL) 5566 return (ENOMEM); 5567 5568 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5569 M_ZERO | M_WAITOK); 5570 5571 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5572 p = buf; 5573 5574 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5575 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5576 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5577 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5578 p[4], p[3], p[2], p[1], p[0]); 5579 } 5580 5581 sbuf_printf(sb, "\n\nCntl ID Data"); 5582 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5583 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5584 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5585 } 5586 5587 rc = sbuf_finish(sb); 5588 sbuf_delete(sb); 5589 free(buf, M_CXGBE); 5590 return (rc); 5591} 5592 5593static int 5594sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5595{ 5596 struct adapter *sc = arg1; 5597 struct sbuf *sb; 5598 int rc, i; 5599 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5600 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5601 uint16_t thres[CIM_NUM_IBQ]; 5602 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5603 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5604 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5605 5606 if (is_t4(sc)) { 5607 cim_num_obq = CIM_NUM_OBQ; 5608 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5609 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5610 } else { 5611 cim_num_obq = CIM_NUM_OBQ_T5; 5612 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5613 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5614 } 5615 nq = CIM_NUM_IBQ + cim_num_obq; 5616 5617 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5618 if (rc == 0) 5619 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5620 if (rc != 0) 5621 return (rc); 5622 5623 t4_read_cimq_cfg(sc, base, size, thres); 5624 5625 rc = sysctl_wire_old_buffer(req, 0); 5626 if (rc != 0) 5627 return (rc); 5628 5629 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5630 if (sb == NULL) 5631 return (ENOMEM); 5632 5633 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5634 5635 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5636 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5637 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5638 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5639 G_QUEREMFLITS(p[2]) * 16); 5640 for ( ; i < nq; i++, p += 4, wr += 2) 5641 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5642 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5643 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5644 G_QUEREMFLITS(p[2]) * 16); 5645 5646 rc = sbuf_finish(sb); 5647 sbuf_delete(sb); 5648 5649 return (rc); 5650} 5651 5652static int 5653sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5654{ 5655 struct adapter *sc = arg1; 5656 struct sbuf *sb; 5657 int rc; 5658 struct tp_cpl_stats stats; 5659 5660 rc = sysctl_wire_old_buffer(req, 0); 5661 if (rc != 0) 5662 return (rc); 5663 5664 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5665 if (sb == NULL) 5666 return (ENOMEM); 5667 5668 t4_tp_get_cpl_stats(sc, &stats); 5669 5670 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5671 "channel 3\n"); 5672 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5673 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5674 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5675 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5676 5677 rc = sbuf_finish(sb); 5678 sbuf_delete(sb); 5679 5680 return (rc); 5681} 5682 5683static int 5684sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5685{ 5686 struct adapter *sc = arg1; 5687 struct sbuf *sb; 5688 int rc; 5689 struct tp_usm_stats stats; 5690 5691 rc = sysctl_wire_old_buffer(req, 0); 5692 if (rc != 0) 5693 return(rc); 5694 5695 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5696 if (sb == NULL) 5697 return (ENOMEM); 5698 5699 t4_get_usm_stats(sc, &stats); 5700 5701 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5702 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5703 sbuf_printf(sb, "Drops: %u", stats.drops); 5704 5705 rc = sbuf_finish(sb); 5706 sbuf_delete(sb); 5707 5708 return (rc); 5709} 5710 5711const char *devlog_level_strings[] = { 5712 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5713 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5714 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5715 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5716 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5717 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5718}; 5719 5720const char *devlog_facility_strings[] = { 5721 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5722 [FW_DEVLOG_FACILITY_CF] = "CF", 5723 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5724 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5725 [FW_DEVLOG_FACILITY_RES] = "RES", 5726 [FW_DEVLOG_FACILITY_HW] = "HW", 5727 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5728 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5729 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5730 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5731 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5732 [FW_DEVLOG_FACILITY_VI] = "VI", 5733 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5734 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5735 [FW_DEVLOG_FACILITY_TM] = "TM", 5736 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5737 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5738 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5739 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5740 [FW_DEVLOG_FACILITY_RI] = "RI", 5741 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5742 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5743 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5744 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5745}; 5746 5747static int 5748sysctl_devlog(SYSCTL_HANDLER_ARGS) 5749{ 5750 struct adapter *sc = arg1; 5751 struct devlog_params *dparams = &sc->params.devlog; 5752 struct fw_devlog_e *buf, *e; 5753 int i, j, rc, nentries, first = 0, m; 5754 struct sbuf *sb; 5755 uint64_t ftstamp = UINT64_MAX; 5756 5757 if (dparams->start == 0) { 5758 dparams->memtype = FW_MEMTYPE_EDC0; 5759 dparams->start = 0x84000; 5760 dparams->size = 32768; 5761 } 5762 5763 nentries = dparams->size / sizeof(struct fw_devlog_e); 5764 5765 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5766 if (buf == NULL) 5767 return (ENOMEM); 5768 5769 m = fwmtype_to_hwmtype(dparams->memtype); 5770 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5771 if (rc != 0) 5772 goto done; 5773 5774 for (i = 0; i < nentries; i++) { 5775 e = &buf[i]; 5776 5777 if (e->timestamp == 0) 5778 break; /* end */ 5779 5780 e->timestamp = be64toh(e->timestamp); 5781 e->seqno = be32toh(e->seqno); 5782 for (j = 0; j < 8; j++) 5783 e->params[j] = be32toh(e->params[j]); 5784 5785 if (e->timestamp < ftstamp) { 5786 ftstamp = e->timestamp; 5787 first = i; 5788 } 5789 } 5790 5791 if (buf[first].timestamp == 0) 5792 goto done; /* nothing in the log */ 5793 5794 rc = sysctl_wire_old_buffer(req, 0); 5795 if (rc != 0) 5796 goto done; 5797 5798 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5799 if (sb == NULL) { 5800 rc = ENOMEM; 5801 goto done; 5802 } 5803 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5804 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5805 5806 i = first; 5807 do { 5808 e = &buf[i]; 5809 if (e->timestamp == 0) 5810 break; /* end */ 5811 5812 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5813 e->seqno, e->timestamp, 5814 (e->level < nitems(devlog_level_strings) ? 5815 devlog_level_strings[e->level] : "UNKNOWN"), 5816 (e->facility < nitems(devlog_facility_strings) ? 5817 devlog_facility_strings[e->facility] : "UNKNOWN")); 5818 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5819 e->params[2], e->params[3], e->params[4], 5820 e->params[5], e->params[6], e->params[7]); 5821 5822 if (++i == nentries) 5823 i = 0; 5824 } while (i != first); 5825 5826 rc = sbuf_finish(sb); 5827 sbuf_delete(sb); 5828done: 5829 free(buf, M_CXGBE); 5830 return (rc); 5831} 5832 5833static int 5834sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5835{ 5836 struct adapter *sc = arg1; 5837 struct sbuf *sb; 5838 int rc; 5839 struct tp_fcoe_stats stats[4]; 5840 5841 rc = sysctl_wire_old_buffer(req, 0); 5842 if (rc != 0) 5843 return (rc); 5844 5845 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5846 if (sb == NULL) 5847 return (ENOMEM); 5848 5849 t4_get_fcoe_stats(sc, 0, &stats[0]); 5850 t4_get_fcoe_stats(sc, 1, &stats[1]); 5851 t4_get_fcoe_stats(sc, 2, &stats[2]); 5852 t4_get_fcoe_stats(sc, 3, &stats[3]); 5853 5854 sbuf_printf(sb, " channel 0 channel 1 " 5855 "channel 2 channel 3\n"); 5856 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5857 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5858 stats[3].octetsDDP); 5859 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5860 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5861 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5862 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5863 stats[3].framesDrop); 5864 5865 rc = sbuf_finish(sb); 5866 sbuf_delete(sb); 5867 5868 return (rc); 5869} 5870 5871static int 5872sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5873{ 5874 struct adapter *sc = arg1; 5875 struct sbuf *sb; 5876 int rc, i; 5877 unsigned int map, kbps, ipg, mode; 5878 unsigned int pace_tab[NTX_SCHED]; 5879 5880 rc = sysctl_wire_old_buffer(req, 0); 5881 if (rc != 0) 5882 return (rc); 5883 5884 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5885 if (sb == NULL) 5886 return (ENOMEM); 5887 5888 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5889 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5890 t4_read_pace_tbl(sc, pace_tab); 5891 5892 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5893 "Class IPG (0.1 ns) Flow IPG (us)"); 5894 5895 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5896 t4_get_tx_sched(sc, i, &kbps, &ipg); 5897 sbuf_printf(sb, "\n %u %-5s %u ", i, 5898 (mode & (1 << i)) ? "flow" : "class", map & 3); 5899 if (kbps) 5900 sbuf_printf(sb, "%9u ", kbps); 5901 else 5902 sbuf_printf(sb, " disabled "); 5903 5904 if (ipg) 5905 sbuf_printf(sb, "%13u ", ipg); 5906 else 5907 sbuf_printf(sb, " disabled "); 5908 5909 if (pace_tab[i]) 5910 sbuf_printf(sb, "%10u", pace_tab[i]); 5911 else 5912 sbuf_printf(sb, " disabled"); 5913 } 5914 5915 rc = sbuf_finish(sb); 5916 sbuf_delete(sb); 5917 5918 return (rc); 5919} 5920 5921static int 5922sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5923{ 5924 struct adapter *sc = arg1; 5925 struct sbuf *sb; 5926 int rc, i, j; 5927 uint64_t *p0, *p1; 5928 struct lb_port_stats s[2]; 5929 static const char *stat_name[] = { 5930 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5931 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5932 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5933 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5934 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5935 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5936 "BG2FramesTrunc:", "BG3FramesTrunc:" 5937 }; 5938 5939 rc = sysctl_wire_old_buffer(req, 0); 5940 if (rc != 0) 5941 return (rc); 5942 5943 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5944 if (sb == NULL) 5945 return (ENOMEM); 5946 5947 memset(s, 0, sizeof(s)); 5948 5949 for (i = 0; i < 4; i += 2) { 5950 t4_get_lb_stats(sc, i, &s[0]); 5951 t4_get_lb_stats(sc, i + 1, &s[1]); 5952 5953 p0 = &s[0].octets; 5954 p1 = &s[1].octets; 5955 sbuf_printf(sb, "%s Loopback %u" 5956 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5957 5958 for (j = 0; j < nitems(stat_name); j++) 5959 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5960 *p0++, *p1++); 5961 } 5962 5963 rc = sbuf_finish(sb); 5964 sbuf_delete(sb); 5965 5966 return (rc); 5967} 5968 5969static int 5970sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5971{ 5972 int rc = 0; 5973 struct port_info *pi = arg1; 5974 struct sbuf *sb; 5975 static const char *linkdnreasons[] = { 5976 "non-specific", "remote fault", "autoneg failed", "reserved3", 5977 "PHY overheated", "unknown", "rx los", "reserved7" 5978 }; 5979 5980 rc = sysctl_wire_old_buffer(req, 0); 5981 if (rc != 0) 5982 return(rc); 5983 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5984 if (sb == NULL) 5985 return (ENOMEM); 5986 5987 if (pi->linkdnrc < 0) 5988 sbuf_printf(sb, "n/a"); 5989 else if (pi->linkdnrc < nitems(linkdnreasons)) 5990 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5991 else 5992 sbuf_printf(sb, "%d", pi->linkdnrc); 5993 5994 rc = sbuf_finish(sb); 5995 sbuf_delete(sb); 5996 5997 return (rc); 5998} 5999 6000struct mem_desc { 6001 unsigned int base; 6002 unsigned int limit; 6003 unsigned int idx; 6004}; 6005 6006static int 6007mem_desc_cmp(const void *a, const void *b) 6008{ 6009 return ((const struct mem_desc *)a)->base - 6010 ((const struct mem_desc *)b)->base; 6011} 6012 6013static void 6014mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6015 unsigned int to) 6016{ 6017 unsigned int size; 6018 6019 size = to - from + 1; 6020 if (size == 0) 6021 return; 6022 6023 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6024 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6025} 6026 6027static int 6028sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6029{ 6030 struct adapter *sc = arg1; 6031 struct sbuf *sb; 6032 int rc, i, n; 6033 uint32_t lo, hi, used, alloc; 6034 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6035 static const char *region[] = { 6036 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6037 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6038 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6039 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6040 "RQUDP region:", "PBL region:", "TXPBL region:", 6041 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6042 "On-chip queues:" 6043 }; 6044 struct mem_desc avail[4]; 6045 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6046 struct mem_desc *md = mem; 6047 6048 rc = sysctl_wire_old_buffer(req, 0); 6049 if (rc != 0) 6050 return (rc); 6051 6052 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6053 if (sb == NULL) 6054 return (ENOMEM); 6055 6056 for (i = 0; i < nitems(mem); i++) { 6057 mem[i].limit = 0; 6058 mem[i].idx = i; 6059 } 6060 6061 /* Find and sort the populated memory ranges */ 6062 i = 0; 6063 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6064 if (lo & F_EDRAM0_ENABLE) { 6065 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6066 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6067 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6068 avail[i].idx = 0; 6069 i++; 6070 } 6071 if (lo & F_EDRAM1_ENABLE) { 6072 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6073 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6074 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6075 avail[i].idx = 1; 6076 i++; 6077 } 6078 if (lo & F_EXT_MEM_ENABLE) { 6079 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6080 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6081 avail[i].limit = avail[i].base + 6082 (G_EXT_MEM_SIZE(hi) << 20); 6083 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6084 i++; 6085 } 6086 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6087 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6088 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6089 avail[i].limit = avail[i].base + 6090 (G_EXT_MEM1_SIZE(hi) << 20); 6091 avail[i].idx = 4; 6092 i++; 6093 } 6094 if (!i) /* no memory available */ 6095 return 0; 6096 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6097 6098 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6099 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6100 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6101 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6102 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6103 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6104 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6105 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6106 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6107 6108 /* the next few have explicit upper bounds */ 6109 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6110 md->limit = md->base - 1 + 6111 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6112 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6113 md++; 6114 6115 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6116 md->limit = md->base - 1 + 6117 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6118 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6119 md++; 6120 6121 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6122 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6123 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6124 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6125 } else { 6126 md->base = 0; 6127 md->idx = nitems(region); /* hide it */ 6128 } 6129 md++; 6130 6131#define ulp_region(reg) \ 6132 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6133 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6134 6135 ulp_region(RX_ISCSI); 6136 ulp_region(RX_TDDP); 6137 ulp_region(TX_TPT); 6138 ulp_region(RX_STAG); 6139 ulp_region(RX_RQ); 6140 ulp_region(RX_RQUDP); 6141 ulp_region(RX_PBL); 6142 ulp_region(TX_PBL); 6143#undef ulp_region 6144 6145 md->base = 0; 6146 md->idx = nitems(region); 6147 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6148 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6149 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6150 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6151 } 6152 md++; 6153 6154 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6155 md->limit = md->base + sc->tids.ntids - 1; 6156 md++; 6157 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6158 md->limit = md->base + sc->tids.ntids - 1; 6159 md++; 6160 6161 md->base = sc->vres.ocq.start; 6162 if (sc->vres.ocq.size) 6163 md->limit = md->base + sc->vres.ocq.size - 1; 6164 else 6165 md->idx = nitems(region); /* hide it */ 6166 md++; 6167 6168 /* add any address-space holes, there can be up to 3 */ 6169 for (n = 0; n < i - 1; n++) 6170 if (avail[n].limit < avail[n + 1].base) 6171 (md++)->base = avail[n].limit; 6172 if (avail[n].limit) 6173 (md++)->base = avail[n].limit; 6174 6175 n = md - mem; 6176 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6177 6178 for (lo = 0; lo < i; lo++) 6179 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6180 avail[lo].limit - 1); 6181 6182 sbuf_printf(sb, "\n"); 6183 for (i = 0; i < n; i++) { 6184 if (mem[i].idx >= nitems(region)) 6185 continue; /* skip holes */ 6186 if (!mem[i].limit) 6187 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6188 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6189 mem[i].limit); 6190 } 6191 6192 sbuf_printf(sb, "\n"); 6193 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6194 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6195 mem_region_show(sb, "uP RAM:", lo, hi); 6196 6197 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6198 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6199 mem_region_show(sb, "uP Extmem2:", lo, hi); 6200 6201 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6202 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6203 G_PMRXMAXPAGE(lo), 6204 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6205 (lo & F_PMRXNUMCHN) ? 2 : 1); 6206 6207 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6208 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6209 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6210 G_PMTXMAXPAGE(lo), 6211 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6212 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6213 sbuf_printf(sb, "%u p-structs\n", 6214 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6215 6216 for (i = 0; i < 4; i++) { 6217 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6218 if (is_t4(sc)) { 6219 used = G_USED(lo); 6220 alloc = G_ALLOC(lo); 6221 } else { 6222 used = G_T5_USED(lo); 6223 alloc = G_T5_ALLOC(lo); 6224 } 6225 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6226 i, used, alloc); 6227 } 6228 for (i = 0; i < 4; i++) { 6229 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6230 if (is_t4(sc)) { 6231 used = G_USED(lo); 6232 alloc = G_ALLOC(lo); 6233 } else { 6234 used = G_T5_USED(lo); 6235 alloc = G_T5_ALLOC(lo); 6236 } 6237 sbuf_printf(sb, 6238 "\nLoopback %d using %u pages out of %u allocated", 6239 i, used, alloc); 6240 } 6241 6242 rc = sbuf_finish(sb); 6243 sbuf_delete(sb); 6244 6245 return (rc); 6246} 6247 6248static inline void 6249tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6250{ 6251 *mask = x | y; 6252 y = htobe64(y); 6253 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6254} 6255 6256static int 6257sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6258{ 6259 struct adapter *sc = arg1; 6260 struct sbuf *sb; 6261 int rc, i, n; 6262 6263 rc = sysctl_wire_old_buffer(req, 0); 6264 if (rc != 0) 6265 return (rc); 6266 6267 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6268 if (sb == NULL) 6269 return (ENOMEM); 6270 6271 sbuf_printf(sb, 6272 "Idx Ethernet address Mask Vld Ports PF" 6273 " VF Replication P0 P1 P2 P3 ML"); 6274 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6275 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6276 for (i = 0; i < n; i++) { 6277 uint64_t tcamx, tcamy, mask; 6278 uint32_t cls_lo, cls_hi; 6279 uint8_t addr[ETHER_ADDR_LEN]; 6280 6281 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6282 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6283 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6284 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6285 6286 if (tcamx & tcamy) 6287 continue; 6288 6289 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6290 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6291 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6292 addr[3], addr[4], addr[5], (uintmax_t)mask, 6293 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6294 G_PORTMAP(cls_hi), G_PF(cls_lo), 6295 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6296 6297 if (cls_lo & F_REPLICATE) { 6298 struct fw_ldst_cmd ldst_cmd; 6299 6300 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6301 ldst_cmd.op_to_addrspace = 6302 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6303 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6304 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6305 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6306 ldst_cmd.u.mps.rplc.fid_idx = 6307 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6308 V_FW_LDST_CMD_IDX(i)); 6309 6310 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6311 "t4mps"); 6312 if (rc) 6313 break; 6314 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6315 sizeof(ldst_cmd), &ldst_cmd); 6316 end_synchronized_op(sc, 0); 6317 6318 if (rc != 0) { 6319 sbuf_printf(sb, 6320 " ------------ error %3u ------------", rc); 6321 rc = 0; 6322 } else { 6323 sbuf_printf(sb, " %08x %08x %08x %08x", 6324 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6325 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6326 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6327 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6328 } 6329 } else 6330 sbuf_printf(sb, "%36s", ""); 6331 6332 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6333 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6334 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6335 } 6336 6337 if (rc) 6338 (void) sbuf_finish(sb); 6339 else 6340 rc = sbuf_finish(sb); 6341 sbuf_delete(sb); 6342 6343 return (rc); 6344} 6345 6346static int 6347sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6348{ 6349 struct adapter *sc = arg1; 6350 struct sbuf *sb; 6351 int rc; 6352 uint16_t mtus[NMTUS]; 6353 6354 rc = sysctl_wire_old_buffer(req, 0); 6355 if (rc != 0) 6356 return (rc); 6357 6358 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6359 if (sb == NULL) 6360 return (ENOMEM); 6361 6362 t4_read_mtu_tbl(sc, mtus, NULL); 6363 6364 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6365 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6366 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6367 mtus[14], mtus[15]); 6368 6369 rc = sbuf_finish(sb); 6370 sbuf_delete(sb); 6371 6372 return (rc); 6373} 6374 6375static int 6376sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6377{ 6378 struct adapter *sc = arg1; 6379 struct sbuf *sb; 6380 int rc, i; 6381 uint32_t cnt[PM_NSTATS]; 6382 uint64_t cyc[PM_NSTATS]; 6383 static const char *rx_stats[] = { 6384 "Read:", "Write bypass:", "Write mem:", "Flush:" 6385 }; 6386 static const char *tx_stats[] = { 6387 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6388 }; 6389 6390 rc = sysctl_wire_old_buffer(req, 0); 6391 if (rc != 0) 6392 return (rc); 6393 6394 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6395 if (sb == NULL) 6396 return (ENOMEM); 6397 6398 t4_pmtx_get_stats(sc, cnt, cyc); 6399 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6400 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6401 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6402 cyc[i]); 6403 6404 t4_pmrx_get_stats(sc, cnt, cyc); 6405 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6406 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6407 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6408 cyc[i]); 6409 6410 rc = sbuf_finish(sb); 6411 sbuf_delete(sb); 6412 6413 return (rc); 6414} 6415 6416static int 6417sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6418{ 6419 struct adapter *sc = arg1; 6420 struct sbuf *sb; 6421 int rc; 6422 struct tp_rdma_stats stats; 6423 6424 rc = sysctl_wire_old_buffer(req, 0); 6425 if (rc != 0) 6426 return (rc); 6427 6428 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6429 if (sb == NULL) 6430 return (ENOMEM); 6431 6432 t4_tp_get_rdma_stats(sc, &stats); 6433 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6434 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6435 6436 rc = sbuf_finish(sb); 6437 sbuf_delete(sb); 6438 6439 return (rc); 6440} 6441 6442static int 6443sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6444{ 6445 struct adapter *sc = arg1; 6446 struct sbuf *sb; 6447 int rc; 6448 struct tp_tcp_stats v4, v6; 6449 6450 rc = sysctl_wire_old_buffer(req, 0); 6451 if (rc != 0) 6452 return (rc); 6453 6454 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6455 if (sb == NULL) 6456 return (ENOMEM); 6457 6458 t4_tp_get_tcp_stats(sc, &v4, &v6); 6459 sbuf_printf(sb, 6460 " IP IPv6\n"); 6461 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6462 v4.tcpOutRsts, v6.tcpOutRsts); 6463 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6464 v4.tcpInSegs, v6.tcpInSegs); 6465 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6466 v4.tcpOutSegs, v6.tcpOutSegs); 6467 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6468 v4.tcpRetransSegs, v6.tcpRetransSegs); 6469 6470 rc = sbuf_finish(sb); 6471 sbuf_delete(sb); 6472 6473 return (rc); 6474} 6475 6476static int 6477sysctl_tids(SYSCTL_HANDLER_ARGS) 6478{ 6479 struct adapter *sc = arg1; 6480 struct sbuf *sb; 6481 int rc; 6482 struct tid_info *t = &sc->tids; 6483 6484 rc = sysctl_wire_old_buffer(req, 0); 6485 if (rc != 0) 6486 return (rc); 6487 6488 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6489 if (sb == NULL) 6490 return (ENOMEM); 6491 6492 if (t->natids) { 6493 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6494 t->atids_in_use); 6495 } 6496 6497 if (t->ntids) { 6498 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6499 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6500 6501 if (b) { 6502 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6503 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6504 t->ntids - 1); 6505 } else { 6506 sbuf_printf(sb, "TID range: %u-%u", 6507 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6508 t->ntids - 1); 6509 } 6510 } else 6511 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6512 sbuf_printf(sb, ", in use: %u\n", 6513 atomic_load_acq_int(&t->tids_in_use)); 6514 } 6515 6516 if (t->nstids) { 6517 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6518 t->stid_base + t->nstids - 1, t->stids_in_use); 6519 } 6520 6521 if (t->nftids) { 6522 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6523 t->ftid_base + t->nftids - 1); 6524 } 6525 6526 if (t->netids) { 6527 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6528 t->etid_base + t->netids - 1); 6529 } 6530 6531 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6532 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6533 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6534 6535 rc = sbuf_finish(sb); 6536 sbuf_delete(sb); 6537 6538 return (rc); 6539} 6540 6541static int 6542sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6543{ 6544 struct adapter *sc = arg1; 6545 struct sbuf *sb; 6546 int rc; 6547 struct tp_err_stats stats; 6548 6549 rc = sysctl_wire_old_buffer(req, 0); 6550 if (rc != 0) 6551 return (rc); 6552 6553 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6554 if (sb == NULL) 6555 return (ENOMEM); 6556 6557 t4_tp_get_err_stats(sc, &stats); 6558 6559 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6560 "channel 3\n"); 6561 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6562 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6563 stats.macInErrs[3]); 6564 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6565 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6566 stats.hdrInErrs[3]); 6567 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6568 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6569 stats.tcpInErrs[3]); 6570 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6571 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6572 stats.tcp6InErrs[3]); 6573 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6574 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6575 stats.tnlCongDrops[3]); 6576 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6577 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6578 stats.tnlTxDrops[3]); 6579 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6580 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6581 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6582 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6583 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6584 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6585 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6586 stats.ofldNoNeigh, stats.ofldCongDefer); 6587 6588 rc = sbuf_finish(sb); 6589 sbuf_delete(sb); 6590 6591 return (rc); 6592} 6593 6594struct field_desc { 6595 const char *name; 6596 u_int start; 6597 u_int width; 6598}; 6599 6600static void 6601field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6602{ 6603 char buf[32]; 6604 int line_size = 0; 6605 6606 while (f->name) { 6607 uint64_t mask = (1ULL << f->width) - 1; 6608 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6609 ((uintmax_t)v >> f->start) & mask); 6610 6611 if (line_size + len >= 79) { 6612 line_size = 8; 6613 sbuf_printf(sb, "\n "); 6614 } 6615 sbuf_printf(sb, "%s ", buf); 6616 line_size += len + 1; 6617 f++; 6618 } 6619 sbuf_printf(sb, "\n"); 6620} 6621 6622static struct field_desc tp_la0[] = { 6623 { "RcfOpCodeOut", 60, 4 }, 6624 { "State", 56, 4 }, 6625 { "WcfState", 52, 4 }, 6626 { "RcfOpcSrcOut", 50, 2 }, 6627 { "CRxError", 49, 1 }, 6628 { "ERxError", 48, 1 }, 6629 { "SanityFailed", 47, 1 }, 6630 { "SpuriousMsg", 46, 1 }, 6631 { "FlushInputMsg", 45, 1 }, 6632 { "FlushInputCpl", 44, 1 }, 6633 { "RssUpBit", 43, 1 }, 6634 { "RssFilterHit", 42, 1 }, 6635 { "Tid", 32, 10 }, 6636 { "InitTcb", 31, 1 }, 6637 { "LineNumber", 24, 7 }, 6638 { "Emsg", 23, 1 }, 6639 { "EdataOut", 22, 1 }, 6640 { "Cmsg", 21, 1 }, 6641 { "CdataOut", 20, 1 }, 6642 { "EreadPdu", 19, 1 }, 6643 { "CreadPdu", 18, 1 }, 6644 { "TunnelPkt", 17, 1 }, 6645 { "RcfPeerFin", 16, 1 }, 6646 { "RcfReasonOut", 12, 4 }, 6647 { "TxCchannel", 10, 2 }, 6648 { "RcfTxChannel", 8, 2 }, 6649 { "RxEchannel", 6, 2 }, 6650 { "RcfRxChannel", 5, 1 }, 6651 { "RcfDataOutSrdy", 4, 1 }, 6652 { "RxDvld", 3, 1 }, 6653 { "RxOoDvld", 2, 1 }, 6654 { "RxCongestion", 1, 1 }, 6655 { "TxCongestion", 0, 1 }, 6656 { NULL } 6657}; 6658 6659static struct field_desc tp_la1[] = { 6660 { "CplCmdIn", 56, 8 }, 6661 { "CplCmdOut", 48, 8 }, 6662 { "ESynOut", 47, 1 }, 6663 { "EAckOut", 46, 1 }, 6664 { "EFinOut", 45, 1 }, 6665 { "ERstOut", 44, 1 }, 6666 { "SynIn", 43, 1 }, 6667 { "AckIn", 42, 1 }, 6668 { "FinIn", 41, 1 }, 6669 { "RstIn", 40, 1 }, 6670 { "DataIn", 39, 1 }, 6671 { "DataInVld", 38, 1 }, 6672 { "PadIn", 37, 1 }, 6673 { "RxBufEmpty", 36, 1 }, 6674 { "RxDdp", 35, 1 }, 6675 { "RxFbCongestion", 34, 1 }, 6676 { "TxFbCongestion", 33, 1 }, 6677 { "TxPktSumSrdy", 32, 1 }, 6678 { "RcfUlpType", 28, 4 }, 6679 { "Eread", 27, 1 }, 6680 { "Ebypass", 26, 1 }, 6681 { "Esave", 25, 1 }, 6682 { "Static0", 24, 1 }, 6683 { "Cread", 23, 1 }, 6684 { "Cbypass", 22, 1 }, 6685 { "Csave", 21, 1 }, 6686 { "CPktOut", 20, 1 }, 6687 { "RxPagePoolFull", 18, 2 }, 6688 { "RxLpbkPkt", 17, 1 }, 6689 { "TxLpbkPkt", 16, 1 }, 6690 { "RxVfValid", 15, 1 }, 6691 { "SynLearned", 14, 1 }, 6692 { "SetDelEntry", 13, 1 }, 6693 { "SetInvEntry", 12, 1 }, 6694 { "CpcmdDvld", 11, 1 }, 6695 { "CpcmdSave", 10, 1 }, 6696 { "RxPstructsFull", 8, 2 }, 6697 { "EpcmdDvld", 7, 1 }, 6698 { "EpcmdFlush", 6, 1 }, 6699 { "EpcmdTrimPrefix", 5, 1 }, 6700 { "EpcmdTrimPostfix", 4, 1 }, 6701 { "ERssIp4Pkt", 3, 1 }, 6702 { "ERssIp6Pkt", 2, 1 }, 6703 { "ERssTcpUdpPkt", 1, 1 }, 6704 { "ERssFceFipPkt", 0, 1 }, 6705 { NULL } 6706}; 6707 6708static struct field_desc tp_la2[] = { 6709 { "CplCmdIn", 56, 8 }, 6710 { "MpsVfVld", 55, 1 }, 6711 { "MpsPf", 52, 3 }, 6712 { "MpsVf", 44, 8 }, 6713 { "SynIn", 43, 1 }, 6714 { "AckIn", 42, 1 }, 6715 { "FinIn", 41, 1 }, 6716 { "RstIn", 40, 1 }, 6717 { "DataIn", 39, 1 }, 6718 { "DataInVld", 38, 1 }, 6719 { "PadIn", 37, 1 }, 6720 { "RxBufEmpty", 36, 1 }, 6721 { "RxDdp", 35, 1 }, 6722 { "RxFbCongestion", 34, 1 }, 6723 { "TxFbCongestion", 33, 1 }, 6724 { "TxPktSumSrdy", 32, 1 }, 6725 { "RcfUlpType", 28, 4 }, 6726 { "Eread", 27, 1 }, 6727 { "Ebypass", 26, 1 }, 6728 { "Esave", 25, 1 }, 6729 { "Static0", 24, 1 }, 6730 { "Cread", 23, 1 }, 6731 { "Cbypass", 22, 1 }, 6732 { "Csave", 21, 1 }, 6733 { "CPktOut", 20, 1 }, 6734 { "RxPagePoolFull", 18, 2 }, 6735 { "RxLpbkPkt", 17, 1 }, 6736 { "TxLpbkPkt", 16, 1 }, 6737 { "RxVfValid", 15, 1 }, 6738 { "SynLearned", 14, 1 }, 6739 { "SetDelEntry", 13, 1 }, 6740 { "SetInvEntry", 12, 1 }, 6741 { "CpcmdDvld", 11, 1 }, 6742 { "CpcmdSave", 10, 1 }, 6743 { "RxPstructsFull", 8, 2 }, 6744 { "EpcmdDvld", 7, 1 }, 6745 { "EpcmdFlush", 6, 1 }, 6746 { "EpcmdTrimPrefix", 5, 1 }, 6747 { "EpcmdTrimPostfix", 4, 1 }, 6748 { "ERssIp4Pkt", 3, 1 }, 6749 { "ERssIp6Pkt", 2, 1 }, 6750 { "ERssTcpUdpPkt", 1, 1 }, 6751 { "ERssFceFipPkt", 0, 1 }, 6752 { NULL } 6753}; 6754 6755static void 6756tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6757{ 6758 6759 field_desc_show(sb, *p, tp_la0); 6760} 6761 6762static void 6763tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6764{ 6765 6766 if (idx) 6767 sbuf_printf(sb, "\n"); 6768 field_desc_show(sb, p[0], tp_la0); 6769 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6770 field_desc_show(sb, p[1], tp_la0); 6771} 6772 6773static void 6774tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6775{ 6776 6777 if (idx) 6778 sbuf_printf(sb, "\n"); 6779 field_desc_show(sb, p[0], tp_la0); 6780 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6781 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6782} 6783 6784static int 6785sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6786{ 6787 struct adapter *sc = arg1; 6788 struct sbuf *sb; 6789 uint64_t *buf, *p; 6790 int rc; 6791 u_int i, inc; 6792 void (*show_func)(struct sbuf *, uint64_t *, int); 6793 6794 rc = sysctl_wire_old_buffer(req, 0); 6795 if (rc != 0) 6796 return (rc); 6797 6798 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6799 if (sb == NULL) 6800 return (ENOMEM); 6801 6802 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6803 6804 t4_tp_read_la(sc, buf, NULL); 6805 p = buf; 6806 6807 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6808 case 2: 6809 inc = 2; 6810 show_func = tp_la_show2; 6811 break; 6812 case 3: 6813 inc = 2; 6814 show_func = tp_la_show3; 6815 break; 6816 default: 6817 inc = 1; 6818 show_func = tp_la_show; 6819 } 6820 6821 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6822 (*show_func)(sb, p, i); 6823 6824 rc = sbuf_finish(sb); 6825 sbuf_delete(sb); 6826 free(buf, M_CXGBE); 6827 return (rc); 6828} 6829 6830static int 6831sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6832{ 6833 struct adapter *sc = arg1; 6834 struct sbuf *sb; 6835 int rc; 6836 u64 nrate[NCHAN], orate[NCHAN]; 6837 6838 rc = sysctl_wire_old_buffer(req, 0); 6839 if (rc != 0) 6840 return (rc); 6841 6842 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6843 if (sb == NULL) 6844 return (ENOMEM); 6845 6846 t4_get_chan_txrate(sc, nrate, orate); 6847 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6848 "channel 3\n"); 6849 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6850 nrate[0], nrate[1], nrate[2], nrate[3]); 6851 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6852 orate[0], orate[1], orate[2], orate[3]); 6853 6854 rc = sbuf_finish(sb); 6855 sbuf_delete(sb); 6856 6857 return (rc); 6858} 6859 6860static int 6861sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6862{ 6863 struct adapter *sc = arg1; 6864 struct sbuf *sb; 6865 uint32_t *buf, *p; 6866 int rc, i; 6867 6868 rc = sysctl_wire_old_buffer(req, 0); 6869 if (rc != 0) 6870 return (rc); 6871 6872 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6873 if (sb == NULL) 6874 return (ENOMEM); 6875 6876 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6877 M_ZERO | M_WAITOK); 6878 6879 t4_ulprx_read_la(sc, buf); 6880 p = buf; 6881 6882 sbuf_printf(sb, " Pcmd Type Message" 6883 " Data"); 6884 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6885 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6886 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6887 } 6888 6889 rc = sbuf_finish(sb); 6890 sbuf_delete(sb); 6891 free(buf, M_CXGBE); 6892 return (rc); 6893} 6894 6895static int 6896sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6897{ 6898 struct adapter *sc = arg1; 6899 struct sbuf *sb; 6900 int rc, v; 6901 6902 rc = sysctl_wire_old_buffer(req, 0); 6903 if (rc != 0) 6904 return (rc); 6905 6906 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6907 if (sb == NULL) 6908 return (ENOMEM); 6909 6910 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6911 if (G_STATSOURCE_T5(v) == 7) { 6912 if (G_STATMODE(v) == 0) { 6913 sbuf_printf(sb, "total %d, incomplete %d", 6914 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6915 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6916 } else if (G_STATMODE(v) == 1) { 6917 sbuf_printf(sb, "total %d, data overflow %d", 6918 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6919 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6920 } 6921 } 6922 rc = sbuf_finish(sb); 6923 sbuf_delete(sb); 6924 6925 return (rc); 6926} 6927#endif 6928 6929static uint32_t 6930fconf_to_mode(uint32_t fconf) 6931{ 6932 uint32_t mode; 6933 6934 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6935 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6936 6937 if (fconf & F_FRAGMENTATION) 6938 mode |= T4_FILTER_IP_FRAGMENT; 6939 6940 if (fconf & F_MPSHITTYPE) 6941 mode |= T4_FILTER_MPS_HIT_TYPE; 6942 6943 if (fconf & F_MACMATCH) 6944 mode |= T4_FILTER_MAC_IDX; 6945 6946 if (fconf & F_ETHERTYPE) 6947 mode |= T4_FILTER_ETH_TYPE; 6948 6949 if (fconf & F_PROTOCOL) 6950 mode |= T4_FILTER_IP_PROTO; 6951 6952 if (fconf & F_TOS) 6953 mode |= T4_FILTER_IP_TOS; 6954 6955 if (fconf & F_VLAN) 6956 mode |= T4_FILTER_VLAN; 6957 6958 if (fconf & F_VNIC_ID) 6959 mode |= T4_FILTER_VNIC; 6960 6961 if (fconf & F_PORT) 6962 mode |= T4_FILTER_PORT; 6963 6964 if (fconf & F_FCOE) 6965 mode |= T4_FILTER_FCoE; 6966 6967 return (mode); 6968} 6969 6970static uint32_t 6971mode_to_fconf(uint32_t mode) 6972{ 6973 uint32_t fconf = 0; 6974 6975 if (mode & T4_FILTER_IP_FRAGMENT) 6976 fconf |= F_FRAGMENTATION; 6977 6978 if (mode & T4_FILTER_MPS_HIT_TYPE) 6979 fconf |= F_MPSHITTYPE; 6980 6981 if (mode & T4_FILTER_MAC_IDX) 6982 fconf |= F_MACMATCH; 6983 6984 if (mode & T4_FILTER_ETH_TYPE) 6985 fconf |= F_ETHERTYPE; 6986 6987 if (mode & T4_FILTER_IP_PROTO) 6988 fconf |= F_PROTOCOL; 6989 6990 if (mode & T4_FILTER_IP_TOS) 6991 fconf |= F_TOS; 6992 6993 if (mode & T4_FILTER_VLAN) 6994 fconf |= F_VLAN; 6995 6996 if (mode & T4_FILTER_VNIC) 6997 fconf |= F_VNIC_ID; 6998 6999 if (mode & T4_FILTER_PORT) 7000 fconf |= F_PORT; 7001 7002 if (mode & T4_FILTER_FCoE) 7003 fconf |= F_FCOE; 7004 7005 return (fconf); 7006} 7007 7008static uint32_t 7009fspec_to_fconf(struct t4_filter_specification *fs) 7010{ 7011 uint32_t fconf = 0; 7012 7013 if (fs->val.frag || fs->mask.frag) 7014 fconf |= F_FRAGMENTATION; 7015 7016 if (fs->val.matchtype || fs->mask.matchtype) 7017 fconf |= F_MPSHITTYPE; 7018 7019 if (fs->val.macidx || fs->mask.macidx) 7020 fconf |= F_MACMATCH; 7021 7022 if (fs->val.ethtype || fs->mask.ethtype) 7023 fconf |= F_ETHERTYPE; 7024 7025 if (fs->val.proto || fs->mask.proto) 7026 fconf |= F_PROTOCOL; 7027 7028 if (fs->val.tos || fs->mask.tos) 7029 fconf |= F_TOS; 7030 7031 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7032 fconf |= F_VLAN; 7033 7034 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7035 fconf |= F_VNIC_ID; 7036 7037 if (fs->val.iport || fs->mask.iport) 7038 fconf |= F_PORT; 7039 7040 if (fs->val.fcoe || fs->mask.fcoe) 7041 fconf |= F_FCOE; 7042 7043 return (fconf); 7044} 7045 7046static int 7047get_filter_mode(struct adapter *sc, uint32_t *mode) 7048{ 7049 int rc; 7050 uint32_t fconf; 7051 7052 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7053 "t4getfm"); 7054 if (rc) 7055 return (rc); 7056 7057 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7058 A_TP_VLAN_PRI_MAP); 7059 7060 if (sc->params.tp.vlan_pri_map != fconf) { 7061 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7062 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7063 fconf); 7064 } 7065 7066 *mode = fconf_to_mode(fconf); 7067 7068 end_synchronized_op(sc, LOCK_HELD); 7069 return (0); 7070} 7071 7072static int 7073set_filter_mode(struct adapter *sc, uint32_t mode) 7074{ 7075 uint32_t fconf; 7076 int rc; 7077 7078 fconf = mode_to_fconf(mode); 7079 7080 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7081 "t4setfm"); 7082 if (rc) 7083 return (rc); 7084 7085 if (sc->tids.ftids_in_use > 0) { 7086 rc = EBUSY; 7087 goto done; 7088 } 7089 7090#ifdef TCP_OFFLOAD 7091 if (uld_active(sc, ULD_TOM)) { 7092 rc = EBUSY; 7093 goto done; 7094 } 7095#endif 7096 7097 rc = -t4_set_filter_mode(sc, fconf); 7098done: 7099 end_synchronized_op(sc, LOCK_HELD); 7100 return (rc); 7101} 7102 7103static inline uint64_t 7104get_filter_hits(struct adapter *sc, uint32_t fid) 7105{ 7106 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7107 uint64_t hits; 7108 7109 memwin_info(sc, 0, &mw_base, NULL); 7110 off = position_memwin(sc, 0, 7111 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7112 if (is_t4(sc)) { 7113 hits = t4_read_reg64(sc, mw_base + off + 16); 7114 hits = be64toh(hits); 7115 } else { 7116 hits = t4_read_reg(sc, mw_base + off + 24); 7117 hits = be32toh(hits); 7118 } 7119 7120 return (hits); 7121} 7122 7123static int 7124get_filter(struct adapter *sc, struct t4_filter *t) 7125{ 7126 int i, rc, nfilters = sc->tids.nftids; 7127 struct filter_entry *f; 7128 7129 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7130 "t4getf"); 7131 if (rc) 7132 return (rc); 7133 7134 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7135 t->idx >= nfilters) { 7136 t->idx = 0xffffffff; 7137 goto done; 7138 } 7139 7140 f = &sc->tids.ftid_tab[t->idx]; 7141 for (i = t->idx; i < nfilters; i++, f++) { 7142 if (f->valid) { 7143 t->idx = i; 7144 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7145 t->smtidx = f->smtidx; 7146 if (f->fs.hitcnts) 7147 t->hits = get_filter_hits(sc, t->idx); 7148 else 7149 t->hits = UINT64_MAX; 7150 t->fs = f->fs; 7151 7152 goto done; 7153 } 7154 } 7155 7156 t->idx = 0xffffffff; 7157done: 7158 end_synchronized_op(sc, LOCK_HELD); 7159 return (0); 7160} 7161 7162static int 7163set_filter(struct adapter *sc, struct t4_filter *t) 7164{ 7165 unsigned int nfilters, nports; 7166 struct filter_entry *f; 7167 int i, rc; 7168 7169 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7170 if (rc) 7171 return (rc); 7172 7173 nfilters = sc->tids.nftids; 7174 nports = sc->params.nports; 7175 7176 if (nfilters == 0) { 7177 rc = ENOTSUP; 7178 goto done; 7179 } 7180 7181 if (!(sc->flags & FULL_INIT_DONE)) { 7182 rc = EAGAIN; 7183 goto done; 7184 } 7185 7186 if (t->idx >= nfilters) { 7187 rc = EINVAL; 7188 goto done; 7189 } 7190 7191 /* Validate against the global filter mode */ 7192 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7193 sc->params.tp.vlan_pri_map) { 7194 rc = E2BIG; 7195 goto done; 7196 } 7197 7198 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7199 rc = EINVAL; 7200 goto done; 7201 } 7202 7203 if (t->fs.val.iport >= nports) { 7204 rc = EINVAL; 7205 goto done; 7206 } 7207 7208 /* Can't specify an iq if not steering to it */ 7209 if (!t->fs.dirsteer && t->fs.iq) { 7210 rc = EINVAL; 7211 goto done; 7212 } 7213 7214 /* IPv6 filter idx must be 4 aligned */ 7215 if (t->fs.type == 1 && 7216 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7217 rc = EINVAL; 7218 goto done; 7219 } 7220 7221 if (sc->tids.ftid_tab == NULL) { 7222 KASSERT(sc->tids.ftids_in_use == 0, 7223 ("%s: no memory allocated but filters_in_use > 0", 7224 __func__)); 7225 7226 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7227 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7228 if (sc->tids.ftid_tab == NULL) { 7229 rc = ENOMEM; 7230 goto done; 7231 } 7232 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7233 } 7234 7235 for (i = 0; i < 4; i++) { 7236 f = &sc->tids.ftid_tab[t->idx + i]; 7237 7238 if (f->pending || f->valid) { 7239 rc = EBUSY; 7240 goto done; 7241 } 7242 if (f->locked) { 7243 rc = EPERM; 7244 goto done; 7245 } 7246 7247 if (t->fs.type == 0) 7248 break; 7249 } 7250 7251 f = &sc->tids.ftid_tab[t->idx]; 7252 f->fs = t->fs; 7253 7254 rc = set_filter_wr(sc, t->idx); 7255done: 7256 end_synchronized_op(sc, 0); 7257 7258 if (rc == 0) { 7259 mtx_lock(&sc->tids.ftid_lock); 7260 for (;;) { 7261 if (f->pending == 0) { 7262 rc = f->valid ? 0 : EIO; 7263 break; 7264 } 7265 7266 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7267 PCATCH, "t4setfw", 0)) { 7268 rc = EINPROGRESS; 7269 break; 7270 } 7271 } 7272 mtx_unlock(&sc->tids.ftid_lock); 7273 } 7274 return (rc); 7275} 7276 7277static int 7278del_filter(struct adapter *sc, struct t4_filter *t) 7279{ 7280 unsigned int nfilters; 7281 struct filter_entry *f; 7282 int rc; 7283 7284 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7285 if (rc) 7286 return (rc); 7287 7288 nfilters = sc->tids.nftids; 7289 7290 if (nfilters == 0) { 7291 rc = ENOTSUP; 7292 goto done; 7293 } 7294 7295 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7296 t->idx >= nfilters) { 7297 rc = EINVAL; 7298 goto done; 7299 } 7300 7301 if (!(sc->flags & FULL_INIT_DONE)) { 7302 rc = EAGAIN; 7303 goto done; 7304 } 7305 7306 f = &sc->tids.ftid_tab[t->idx]; 7307 7308 if (f->pending) { 7309 rc = EBUSY; 7310 goto done; 7311 } 7312 if (f->locked) { 7313 rc = EPERM; 7314 goto done; 7315 } 7316 7317 if (f->valid) { 7318 t->fs = f->fs; /* extra info for the caller */ 7319 rc = del_filter_wr(sc, t->idx); 7320 } 7321 7322done: 7323 end_synchronized_op(sc, 0); 7324 7325 if (rc == 0) { 7326 mtx_lock(&sc->tids.ftid_lock); 7327 for (;;) { 7328 if (f->pending == 0) { 7329 rc = f->valid ? EIO : 0; 7330 break; 7331 } 7332 7333 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7334 PCATCH, "t4delfw", 0)) { 7335 rc = EINPROGRESS; 7336 break; 7337 } 7338 } 7339 mtx_unlock(&sc->tids.ftid_lock); 7340 } 7341 7342 return (rc); 7343} 7344 7345static void 7346clear_filter(struct filter_entry *f) 7347{ 7348 if (f->l2t) 7349 t4_l2t_release(f->l2t); 7350 7351 bzero(f, sizeof (*f)); 7352} 7353 7354static int 7355set_filter_wr(struct adapter *sc, int fidx) 7356{ 7357 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7358 struct fw_filter_wr *fwr; 7359 unsigned int ftid; 7360 struct wrq_cookie cookie; 7361 7362 ASSERT_SYNCHRONIZED_OP(sc); 7363 7364 if (f->fs.newdmac || f->fs.newvlan) { 7365 /* This filter needs an L2T entry; allocate one. */ 7366 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7367 if (f->l2t == NULL) 7368 return (EAGAIN); 7369 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7370 f->fs.dmac)) { 7371 t4_l2t_release(f->l2t); 7372 f->l2t = NULL; 7373 return (ENOMEM); 7374 } 7375 } 7376 7377 ftid = sc->tids.ftid_base + fidx; 7378 7379 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7380 if (fwr == NULL) 7381 return (ENOMEM); 7382 bzero(fwr, sizeof(*fwr)); 7383 7384 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7385 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7386 fwr->tid_to_iq = 7387 htobe32(V_FW_FILTER_WR_TID(ftid) | 7388 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7389 V_FW_FILTER_WR_NOREPLY(0) | 7390 V_FW_FILTER_WR_IQ(f->fs.iq)); 7391 fwr->del_filter_to_l2tix = 7392 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7393 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7394 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7395 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7396 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7397 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7398 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7399 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7400 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7401 f->fs.newvlan == VLAN_REWRITE) | 7402 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7403 f->fs.newvlan == VLAN_REWRITE) | 7404 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7405 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7406 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7407 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7408 fwr->ethtype = htobe16(f->fs.val.ethtype); 7409 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7410 fwr->frag_to_ovlan_vldm = 7411 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7412 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7413 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7414 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7415 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7416 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7417 fwr->smac_sel = 0; 7418 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7419 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7420 fwr->maci_to_matchtypem = 7421 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7422 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7423 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7424 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7425 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7426 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7427 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7428 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7429 fwr->ptcl = f->fs.val.proto; 7430 fwr->ptclm = f->fs.mask.proto; 7431 fwr->ttyp = f->fs.val.tos; 7432 fwr->ttypm = f->fs.mask.tos; 7433 fwr->ivlan = htobe16(f->fs.val.vlan); 7434 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7435 fwr->ovlan = htobe16(f->fs.val.vnic); 7436 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7437 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7438 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7439 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7440 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7441 fwr->lp = htobe16(f->fs.val.dport); 7442 fwr->lpm = htobe16(f->fs.mask.dport); 7443 fwr->fp = htobe16(f->fs.val.sport); 7444 fwr->fpm = htobe16(f->fs.mask.sport); 7445 if (f->fs.newsmac) 7446 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7447 7448 f->pending = 1; 7449 sc->tids.ftids_in_use++; 7450 7451 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7452 return (0); 7453} 7454 7455static int 7456del_filter_wr(struct adapter *sc, int fidx) 7457{ 7458 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7459 struct fw_filter_wr *fwr; 7460 unsigned int ftid; 7461 struct wrq_cookie cookie; 7462 7463 ftid = sc->tids.ftid_base + fidx; 7464 7465 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7466 if (fwr == NULL) 7467 return (ENOMEM); 7468 bzero(fwr, sizeof (*fwr)); 7469 7470 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7471 7472 f->pending = 1; 7473 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7474 return (0); 7475} 7476 7477int 7478t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7479{ 7480 struct adapter *sc = iq->adapter; 7481 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7482 unsigned int idx = GET_TID(rpl); 7483 unsigned int rc; 7484 struct filter_entry *f; 7485 7486 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7487 rss->opcode)); 7488 7489 if (is_ftid(sc, idx)) { 7490 7491 idx -= sc->tids.ftid_base; 7492 f = &sc->tids.ftid_tab[idx]; 7493 rc = G_COOKIE(rpl->cookie); 7494 7495 mtx_lock(&sc->tids.ftid_lock); 7496 if (rc == FW_FILTER_WR_FLT_ADDED) { 7497 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7498 __func__, idx)); 7499 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7500 f->pending = 0; /* asynchronous setup completed */ 7501 f->valid = 1; 7502 } else { 7503 if (rc != FW_FILTER_WR_FLT_DELETED) { 7504 /* Add or delete failed, display an error */ 7505 log(LOG_ERR, 7506 "filter %u setup failed with error %u\n", 7507 idx, rc); 7508 } 7509 7510 clear_filter(f); 7511 sc->tids.ftids_in_use--; 7512 } 7513 wakeup(&sc->tids.ftid_tab); 7514 mtx_unlock(&sc->tids.ftid_lock); 7515 } 7516 7517 return (0); 7518} 7519 7520static int 7521get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7522{ 7523 int rc; 7524 7525 if (cntxt->cid > M_CTXTQID) 7526 return (EINVAL); 7527 7528 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7529 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7530 return (EINVAL); 7531 7532 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7533 if (rc) 7534 return (rc); 7535 7536 if (sc->flags & FW_OK) { 7537 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7538 &cntxt->data[0]); 7539 if (rc == 0) 7540 goto done; 7541 } 7542 7543 /* 7544 * Read via firmware failed or wasn't even attempted. Read directly via 7545 * the backdoor. 7546 */ 7547 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7548done: 7549 end_synchronized_op(sc, 0); 7550 return (rc); 7551} 7552 7553static int 7554load_fw(struct adapter *sc, struct t4_data *fw) 7555{ 7556 int rc; 7557 uint8_t *fw_data; 7558 7559 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7560 if (rc) 7561 return (rc); 7562 7563 if (sc->flags & FULL_INIT_DONE) { 7564 rc = EBUSY; 7565 goto done; 7566 } 7567 7568 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7569 if (fw_data == NULL) { 7570 rc = ENOMEM; 7571 goto done; 7572 } 7573 7574 rc = copyin(fw->data, fw_data, fw->len); 7575 if (rc == 0) 7576 rc = -t4_load_fw(sc, fw_data, fw->len); 7577 7578 free(fw_data, M_CXGBE); 7579done: 7580 end_synchronized_op(sc, 0); 7581 return (rc); 7582} 7583 7584static int 7585read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7586{ 7587 uint32_t addr, off, remaining, i, n; 7588 uint32_t *buf, *b; 7589 uint32_t mw_base, mw_aperture; 7590 int rc; 7591 uint8_t *dst; 7592 7593 rc = validate_mem_range(sc, mr->addr, mr->len); 7594 if (rc != 0) 7595 return (rc); 7596 7597 memwin_info(sc, win, &mw_base, &mw_aperture); 7598 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7599 addr = mr->addr; 7600 remaining = mr->len; 7601 dst = (void *)mr->data; 7602 7603 while (remaining) { 7604 off = position_memwin(sc, win, addr); 7605 7606 /* number of bytes that we'll copy in the inner loop */ 7607 n = min(remaining, mw_aperture - off); 7608 for (i = 0; i < n; i += 4) 7609 *b++ = t4_read_reg(sc, mw_base + off + i); 7610 7611 rc = copyout(buf, dst, n); 7612 if (rc != 0) 7613 break; 7614 7615 b = buf; 7616 dst += n; 7617 remaining -= n; 7618 addr += n; 7619 } 7620 7621 free(buf, M_CXGBE); 7622 return (rc); 7623} 7624 7625static int 7626read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7627{ 7628 int rc; 7629 7630 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7631 return (EINVAL); 7632 7633 if (i2cd->len > sizeof(i2cd->data)) 7634 return (EFBIG); 7635 7636 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7637 if (rc) 7638 return (rc); 7639 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7640 i2cd->offset, i2cd->len, &i2cd->data[0]); 7641 end_synchronized_op(sc, 0); 7642 7643 return (rc); 7644} 7645 7646static int 7647in_range(int val, int lo, int hi) 7648{ 7649 7650 return (val < 0 || (val <= hi && val >= lo)); 7651} 7652 7653static int 7654set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7655{ 7656 int fw_subcmd, fw_type, rc; 7657 7658 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7659 if (rc) 7660 return (rc); 7661 7662 if (!(sc->flags & FULL_INIT_DONE)) { 7663 rc = EAGAIN; 7664 goto done; 7665 } 7666 7667 /* 7668 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7669 * sub-command and type are in common locations.) 7670 */ 7671 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7672 fw_subcmd = FW_SCHED_SC_CONFIG; 7673 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7674 fw_subcmd = FW_SCHED_SC_PARAMS; 7675 else { 7676 rc = EINVAL; 7677 goto done; 7678 } 7679 if (p->type == SCHED_CLASS_TYPE_PACKET) 7680 fw_type = FW_SCHED_TYPE_PKTSCHED; 7681 else { 7682 rc = EINVAL; 7683 goto done; 7684 } 7685 7686 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7687 /* Vet our parameters ..*/ 7688 if (p->u.config.minmax < 0) { 7689 rc = EINVAL; 7690 goto done; 7691 } 7692 7693 /* And pass the request to the firmware ...*/ 7694 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7695 goto done; 7696 } 7697 7698 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7699 int fw_level; 7700 int fw_mode; 7701 int fw_rateunit; 7702 int fw_ratemode; 7703 7704 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7705 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7706 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7707 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7708 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7709 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7710 else { 7711 rc = EINVAL; 7712 goto done; 7713 } 7714 7715 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7716 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7717 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7718 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7719 else { 7720 rc = EINVAL; 7721 goto done; 7722 } 7723 7724 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7725 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7726 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7727 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7728 else { 7729 rc = EINVAL; 7730 goto done; 7731 } 7732 7733 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7734 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7735 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7736 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7737 else { 7738 rc = EINVAL; 7739 goto done; 7740 } 7741 7742 /* Vet our parameters ... */ 7743 if (!in_range(p->u.params.channel, 0, 3) || 7744 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7745 !in_range(p->u.params.minrate, 0, 10000000) || 7746 !in_range(p->u.params.maxrate, 0, 10000000) || 7747 !in_range(p->u.params.weight, 0, 100)) { 7748 rc = ERANGE; 7749 goto done; 7750 } 7751 7752 /* 7753 * Translate any unset parameters into the firmware's 7754 * nomenclature and/or fail the call if the parameters 7755 * are required ... 7756 */ 7757 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7758 p->u.params.channel < 0 || p->u.params.cl < 0) { 7759 rc = EINVAL; 7760 goto done; 7761 } 7762 if (p->u.params.minrate < 0) 7763 p->u.params.minrate = 0; 7764 if (p->u.params.maxrate < 0) { 7765 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7766 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7767 rc = EINVAL; 7768 goto done; 7769 } else 7770 p->u.params.maxrate = 0; 7771 } 7772 if (p->u.params.weight < 0) { 7773 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7774 rc = EINVAL; 7775 goto done; 7776 } else 7777 p->u.params.weight = 0; 7778 } 7779 if (p->u.params.pktsize < 0) { 7780 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7781 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7782 rc = EINVAL; 7783 goto done; 7784 } else 7785 p->u.params.pktsize = 0; 7786 } 7787 7788 /* See what the firmware thinks of the request ... */ 7789 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7790 fw_rateunit, fw_ratemode, p->u.params.channel, 7791 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7792 p->u.params.weight, p->u.params.pktsize, 1); 7793 goto done; 7794 } 7795 7796 rc = EINVAL; 7797done: 7798 end_synchronized_op(sc, 0); 7799 return (rc); 7800} 7801 7802static int 7803set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7804{ 7805 struct port_info *pi = NULL; 7806 struct sge_txq *txq; 7807 uint32_t fw_mnem, fw_queue, fw_class; 7808 int i, rc; 7809 7810 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7811 if (rc) 7812 return (rc); 7813 7814 if (!(sc->flags & FULL_INIT_DONE)) { 7815 rc = EAGAIN; 7816 goto done; 7817 } 7818 7819 if (p->port >= sc->params.nports) { 7820 rc = EINVAL; 7821 goto done; 7822 } 7823 7824 pi = sc->port[p->port]; 7825 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7826 rc = EINVAL; 7827 goto done; 7828 } 7829 7830 /* 7831 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7832 * Scheduling Class in this case). 7833 */ 7834 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7835 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7836 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7837 7838 /* 7839 * If op.queue is non-negative, then we're only changing the scheduling 7840 * on a single specified TX queue. 7841 */ 7842 if (p->queue >= 0) { 7843 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7844 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7845 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7846 &fw_class); 7847 goto done; 7848 } 7849 7850 /* 7851 * Change the scheduling on all the TX queues for the 7852 * interface. 7853 */ 7854 for_each_txq(pi, i, txq) { 7855 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7856 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7857 &fw_class); 7858 if (rc) 7859 goto done; 7860 } 7861 7862 rc = 0; 7863done: 7864 end_synchronized_op(sc, 0); 7865 return (rc); 7866} 7867 7868int 7869t4_os_find_pci_capability(struct adapter *sc, int cap) 7870{ 7871 int i; 7872 7873 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7874} 7875 7876int 7877t4_os_pci_save_state(struct adapter *sc) 7878{ 7879 device_t dev; 7880 struct pci_devinfo *dinfo; 7881 7882 dev = sc->dev; 7883 dinfo = device_get_ivars(dev); 7884 7885 pci_cfg_save(dev, dinfo, 0); 7886 return (0); 7887} 7888 7889int 7890t4_os_pci_restore_state(struct adapter *sc) 7891{ 7892 device_t dev; 7893 struct pci_devinfo *dinfo; 7894 7895 dev = sc->dev; 7896 dinfo = device_get_ivars(dev); 7897 7898 pci_cfg_restore(dev, dinfo); 7899 return (0); 7900} 7901 7902void 7903t4_os_portmod_changed(const struct adapter *sc, int idx) 7904{ 7905 struct port_info *pi = sc->port[idx]; 7906 static const char *mod_str[] = { 7907 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7908 }; 7909 7910 build_medialist(pi, &pi->media); 7911#ifdef DEV_NETMAP 7912 build_medialist(pi, &pi->nm_media); 7913#endif 7914 7915 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7916 if_printf(pi->ifp, "transceiver unplugged.\n"); 7917 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7918 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7919 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7920 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7921 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7922 if_printf(pi->ifp, "%s transceiver inserted.\n", 7923 mod_str[pi->mod_type]); 7924 } else { 7925 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7926 pi->mod_type); 7927 } 7928} 7929 7930void 7931t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7932{ 7933 struct port_info *pi = sc->port[idx]; 7934 struct ifnet *ifp = pi->ifp; 7935 7936 if (link_stat) { 7937 pi->linkdnrc = -1; 7938 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7939 if_link_state_change(ifp, LINK_STATE_UP); 7940 } else { 7941 if (reason >= 0) 7942 pi->linkdnrc = reason; 7943 if_link_state_change(ifp, LINK_STATE_DOWN); 7944 } 7945} 7946 7947void 7948t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7949{ 7950 struct adapter *sc; 7951 7952 sx_slock(&t4_list_lock); 7953 SLIST_FOREACH(sc, &t4_list, link) { 7954 /* 7955 * func should not make any assumptions about what state sc is 7956 * in - the only guarantee is that sc->sc_lock is a valid lock. 7957 */ 7958 func(sc, arg); 7959 } 7960 sx_sunlock(&t4_list_lock); 7961} 7962 7963static int 7964t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7965{ 7966 return (0); 7967} 7968 7969static int 7970t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7971{ 7972 return (0); 7973} 7974 7975static int 7976t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7977 struct thread *td) 7978{ 7979 int rc; 7980 struct adapter *sc = dev->si_drv1; 7981 7982 rc = priv_check(td, PRIV_DRIVER); 7983 if (rc != 0) 7984 return (rc); 7985 7986 switch (cmd) { 7987 case CHELSIO_T4_GETREG: { 7988 struct t4_reg *edata = (struct t4_reg *)data; 7989 7990 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7991 return (EFAULT); 7992 7993 if (edata->size == 4) 7994 edata->val = t4_read_reg(sc, edata->addr); 7995 else if (edata->size == 8) 7996 edata->val = t4_read_reg64(sc, edata->addr); 7997 else 7998 return (EINVAL); 7999 8000 break; 8001 } 8002 case CHELSIO_T4_SETREG: { 8003 struct t4_reg *edata = (struct t4_reg *)data; 8004 8005 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8006 return (EFAULT); 8007 8008 if (edata->size == 4) { 8009 if (edata->val & 0xffffffff00000000) 8010 return (EINVAL); 8011 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8012 } else if (edata->size == 8) 8013 t4_write_reg64(sc, edata->addr, edata->val); 8014 else 8015 return (EINVAL); 8016 break; 8017 } 8018 case CHELSIO_T4_REGDUMP: { 8019 struct t4_regdump *regs = (struct t4_regdump *)data; 8020 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8021 uint8_t *buf; 8022 8023 if (regs->len < reglen) { 8024 regs->len = reglen; /* hint to the caller */ 8025 return (ENOBUFS); 8026 } 8027 8028 regs->len = reglen; 8029 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8030 t4_get_regs(sc, regs, buf); 8031 rc = copyout(buf, regs->data, reglen); 8032 free(buf, M_CXGBE); 8033 break; 8034 } 8035 case CHELSIO_T4_GET_FILTER_MODE: 8036 rc = get_filter_mode(sc, (uint32_t *)data); 8037 break; 8038 case CHELSIO_T4_SET_FILTER_MODE: 8039 rc = set_filter_mode(sc, *(uint32_t *)data); 8040 break; 8041 case CHELSIO_T4_GET_FILTER: 8042 rc = get_filter(sc, (struct t4_filter *)data); 8043 break; 8044 case CHELSIO_T4_SET_FILTER: 8045 rc = set_filter(sc, (struct t4_filter *)data); 8046 break; 8047 case CHELSIO_T4_DEL_FILTER: 8048 rc = del_filter(sc, (struct t4_filter *)data); 8049 break; 8050 case CHELSIO_T4_GET_SGE_CONTEXT: 8051 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8052 break; 8053 case CHELSIO_T4_LOAD_FW: 8054 rc = load_fw(sc, (struct t4_data *)data); 8055 break; 8056 case CHELSIO_T4_GET_MEM: 8057 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8058 break; 8059 case CHELSIO_T4_GET_I2C: 8060 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8061 break; 8062 case CHELSIO_T4_CLEAR_STATS: { 8063 int i; 8064 u_int port_id = *(uint32_t *)data; 8065 struct port_info *pi; 8066 8067 if (port_id >= sc->params.nports) 8068 return (EINVAL); 8069 pi = sc->port[port_id]; 8070 8071 /* MAC stats */ 8072 t4_clr_port_stats(sc, pi->tx_chan); 8073 pi->tx_parse_error = 0; 8074 8075 if (pi->flags & PORT_INIT_DONE) { 8076 struct sge_rxq *rxq; 8077 struct sge_txq *txq; 8078 struct sge_wrq *wrq; 8079 8080 for_each_rxq(pi, i, rxq) { 8081#if defined(INET) || defined(INET6) 8082 rxq->lro.lro_queued = 0; 8083 rxq->lro.lro_flushed = 0; 8084#endif 8085 rxq->rxcsum = 0; 8086 rxq->vlan_extraction = 0; 8087 } 8088 8089 for_each_txq(pi, i, txq) { 8090 txq->txcsum = 0; 8091 txq->tso_wrs = 0; 8092 txq->vlan_insertion = 0; 8093 txq->imm_wrs = 0; 8094 txq->sgl_wrs = 0; 8095 txq->txpkt_wrs = 0; 8096 txq->txpkts0_wrs = 0; 8097 txq->txpkts1_wrs = 0; 8098 txq->txpkts0_pkts = 0; 8099 txq->txpkts1_pkts = 0; 8100 mp_ring_reset_stats(txq->r); 8101 } 8102 8103#ifdef TCP_OFFLOAD 8104 /* nothing to clear for each ofld_rxq */ 8105 8106 for_each_ofld_txq(pi, i, wrq) { 8107 wrq->tx_wrs_direct = 0; 8108 wrq->tx_wrs_copied = 0; 8109 } 8110#endif 8111 wrq = &sc->sge.ctrlq[pi->port_id]; 8112 wrq->tx_wrs_direct = 0; 8113 wrq->tx_wrs_copied = 0; 8114 } 8115 break; 8116 } 8117 case CHELSIO_T4_SCHED_CLASS: 8118 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8119 break; 8120 case CHELSIO_T4_SCHED_QUEUE: 8121 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8122 break; 8123 case CHELSIO_T4_GET_TRACER: 8124 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8125 break; 8126 case CHELSIO_T4_SET_TRACER: 8127 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8128 break; 8129 default: 8130 rc = EINVAL; 8131 } 8132 8133 return (rc); 8134} 8135 8136#ifdef TCP_OFFLOAD 8137void 8138t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8139 const unsigned int *pgsz_order) 8140{ 8141 struct port_info *pi = ifp->if_softc; 8142 struct adapter *sc = pi->adapter; 8143 8144 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8145 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8146 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8147 V_HPZ3(pgsz_order[3])); 8148} 8149 8150static int 8151toe_capability(struct port_info *pi, int enable) 8152{ 8153 int rc; 8154 struct adapter *sc = pi->adapter; 8155 8156 ASSERT_SYNCHRONIZED_OP(sc); 8157 8158 if (!is_offload(sc)) 8159 return (ENODEV); 8160 8161 if (enable) { 8162 /* 8163 * We need the port's queues around so that we're able to send 8164 * and receive CPLs to/from the TOE even if the ifnet for this 8165 * port has never been UP'd administratively. 8166 */ 8167 if (!(pi->flags & PORT_INIT_DONE)) { 8168 rc = cxgbe_init_synchronized(pi); 8169 if (rc) 8170 return (rc); 8171 } 8172 8173 if (isset(&sc->offload_map, pi->port_id)) 8174 return (0); 8175 8176 if (!uld_active(sc, ULD_TOM)) { 8177 rc = t4_activate_uld(sc, ULD_TOM); 8178 if (rc == EAGAIN) { 8179 log(LOG_WARNING, 8180 "You must kldload t4_tom.ko before trying " 8181 "to enable TOE on a cxgbe interface.\n"); 8182 } 8183 if (rc != 0) 8184 return (rc); 8185 KASSERT(sc->tom_softc != NULL, 8186 ("%s: TOM activated but softc NULL", __func__)); 8187 KASSERT(uld_active(sc, ULD_TOM), 8188 ("%s: TOM activated but flag not set", __func__)); 8189 } 8190 8191 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8192 if (!uld_active(sc, ULD_IWARP)) 8193 (void) t4_activate_uld(sc, ULD_IWARP); 8194 if (!uld_active(sc, ULD_ISCSI)) 8195 (void) t4_activate_uld(sc, ULD_ISCSI); 8196 8197 setbit(&sc->offload_map, pi->port_id); 8198 } else { 8199 if (!isset(&sc->offload_map, pi->port_id)) 8200 return (0); 8201 8202 KASSERT(uld_active(sc, ULD_TOM), 8203 ("%s: TOM never initialized?", __func__)); 8204 clrbit(&sc->offload_map, pi->port_id); 8205 } 8206 8207 return (0); 8208} 8209 8210/* 8211 * Add an upper layer driver to the global list. 8212 */ 8213int 8214t4_register_uld(struct uld_info *ui) 8215{ 8216 int rc = 0; 8217 struct uld_info *u; 8218 8219 sx_xlock(&t4_uld_list_lock); 8220 SLIST_FOREACH(u, &t4_uld_list, link) { 8221 if (u->uld_id == ui->uld_id) { 8222 rc = EEXIST; 8223 goto done; 8224 } 8225 } 8226 8227 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8228 ui->refcount = 0; 8229done: 8230 sx_xunlock(&t4_uld_list_lock); 8231 return (rc); 8232} 8233 8234int 8235t4_unregister_uld(struct uld_info *ui) 8236{ 8237 int rc = EINVAL; 8238 struct uld_info *u; 8239 8240 sx_xlock(&t4_uld_list_lock); 8241 8242 SLIST_FOREACH(u, &t4_uld_list, link) { 8243 if (u == ui) { 8244 if (ui->refcount > 0) { 8245 rc = EBUSY; 8246 goto done; 8247 } 8248 8249 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8250 rc = 0; 8251 goto done; 8252 } 8253 } 8254done: 8255 sx_xunlock(&t4_uld_list_lock); 8256 return (rc); 8257} 8258 8259int 8260t4_activate_uld(struct adapter *sc, int id) 8261{ 8262 int rc; 8263 struct uld_info *ui; 8264 8265 ASSERT_SYNCHRONIZED_OP(sc); 8266 8267 if (id < 0 || id > ULD_MAX) 8268 return (EINVAL); 8269 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8270 8271 sx_slock(&t4_uld_list_lock); 8272 8273 SLIST_FOREACH(ui, &t4_uld_list, link) { 8274 if (ui->uld_id == id) { 8275 if (!(sc->flags & FULL_INIT_DONE)) { 8276 rc = adapter_full_init(sc); 8277 if (rc != 0) 8278 break; 8279 } 8280 8281 rc = ui->activate(sc); 8282 if (rc == 0) { 8283 setbit(&sc->active_ulds, id); 8284 ui->refcount++; 8285 } 8286 break; 8287 } 8288 } 8289 8290 sx_sunlock(&t4_uld_list_lock); 8291 8292 return (rc); 8293} 8294 8295int 8296t4_deactivate_uld(struct adapter *sc, int id) 8297{ 8298 int rc; 8299 struct uld_info *ui; 8300 8301 ASSERT_SYNCHRONIZED_OP(sc); 8302 8303 if (id < 0 || id > ULD_MAX) 8304 return (EINVAL); 8305 rc = ENXIO; 8306 8307 sx_slock(&t4_uld_list_lock); 8308 8309 SLIST_FOREACH(ui, &t4_uld_list, link) { 8310 if (ui->uld_id == id) { 8311 rc = ui->deactivate(sc); 8312 if (rc == 0) { 8313 clrbit(&sc->active_ulds, id); 8314 ui->refcount--; 8315 } 8316 break; 8317 } 8318 } 8319 8320 sx_sunlock(&t4_uld_list_lock); 8321 8322 return (rc); 8323} 8324 8325int 8326uld_active(struct adapter *sc, int uld_id) 8327{ 8328 8329 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8330 8331 return (isset(&sc->active_ulds, uld_id)); 8332} 8333#endif 8334 8335/* 8336 * Come up with reasonable defaults for some of the tunables, provided they're 8337 * not set by the user (in which case we'll use the values as is). 8338 */ 8339static void 8340tweak_tunables(void) 8341{ 8342 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8343 8344 if (t4_ntxq10g < 1) 8345 t4_ntxq10g = min(nc, NTXQ_10G); 8346 8347 if (t4_ntxq1g < 1) 8348 t4_ntxq1g = min(nc, NTXQ_1G); 8349 8350 if (t4_nrxq10g < 1) 8351 t4_nrxq10g = min(nc, NRXQ_10G); 8352 8353 if (t4_nrxq1g < 1) 8354 t4_nrxq1g = min(nc, NRXQ_1G); 8355 8356#ifdef TCP_OFFLOAD 8357 if (t4_nofldtxq10g < 1) 8358 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8359 8360 if (t4_nofldtxq1g < 1) 8361 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8362 8363 if (t4_nofldrxq10g < 1) 8364 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8365 8366 if (t4_nofldrxq1g < 1) 8367 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8368 8369 if (t4_toecaps_allowed == -1) 8370 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8371#else 8372 if (t4_toecaps_allowed == -1) 8373 t4_toecaps_allowed = 0; 8374#endif 8375 8376#ifdef DEV_NETMAP 8377 if (t4_nnmtxq10g < 1) 8378 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8379 8380 if (t4_nnmtxq1g < 1) 8381 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8382 8383 if (t4_nnmrxq10g < 1) 8384 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8385 8386 if (t4_nnmrxq1g < 1) 8387 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8388#endif 8389 8390 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8391 t4_tmr_idx_10g = TMR_IDX_10G; 8392 8393 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8394 t4_pktc_idx_10g = PKTC_IDX_10G; 8395 8396 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8397 t4_tmr_idx_1g = TMR_IDX_1G; 8398 8399 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8400 t4_pktc_idx_1g = PKTC_IDX_1G; 8401 8402 if (t4_qsize_txq < 128) 8403 t4_qsize_txq = 128; 8404 8405 if (t4_qsize_rxq < 128) 8406 t4_qsize_rxq = 128; 8407 while (t4_qsize_rxq & 7) 8408 t4_qsize_rxq++; 8409 8410 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8411} 8412 8413static struct sx mlu; /* mod load unload */ 8414SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8415 8416static int 8417mod_event(module_t mod, int cmd, void *arg) 8418{ 8419 int rc = 0; 8420 static int loaded = 0; 8421 8422 switch (cmd) { 8423 case MOD_LOAD: 8424 sx_xlock(&mlu); 8425 if (loaded++ == 0) { 8426 t4_sge_modload(); 8427 sx_init(&t4_list_lock, "T4/T5 adapters"); 8428 SLIST_INIT(&t4_list); 8429#ifdef TCP_OFFLOAD 8430 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8431 SLIST_INIT(&t4_uld_list); 8432#endif 8433 t4_tracer_modload(); 8434 tweak_tunables(); 8435 } 8436 sx_xunlock(&mlu); 8437 break; 8438 8439 case MOD_UNLOAD: 8440 sx_xlock(&mlu); 8441 if (--loaded == 0) { 8442 int tries; 8443 8444 sx_slock(&t4_list_lock); 8445 if (!SLIST_EMPTY(&t4_list)) { 8446 rc = EBUSY; 8447 sx_sunlock(&t4_list_lock); 8448 goto done_unload; 8449 } 8450#ifdef TCP_OFFLOAD 8451 sx_slock(&t4_uld_list_lock); 8452 if (!SLIST_EMPTY(&t4_uld_list)) { 8453 rc = EBUSY; 8454 sx_sunlock(&t4_uld_list_lock); 8455 sx_sunlock(&t4_list_lock); 8456 goto done_unload; 8457 } 8458#endif 8459 tries = 0; 8460 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8461 uprintf("%ju clusters with custom free routine " 8462 "still is use.\n", t4_sge_extfree_refs()); 8463 pause("t4unload", 2 * hz); 8464 } 8465#ifdef TCP_OFFLOAD 8466 sx_sunlock(&t4_uld_list_lock); 8467#endif 8468 sx_sunlock(&t4_list_lock); 8469 8470 if (t4_sge_extfree_refs() == 0) { 8471 t4_tracer_modunload(); 8472#ifdef TCP_OFFLOAD 8473 sx_destroy(&t4_uld_list_lock); 8474#endif 8475 sx_destroy(&t4_list_lock); 8476 t4_sge_modunload(); 8477 loaded = 0; 8478 } else { 8479 rc = EBUSY; 8480 loaded++; /* undo earlier decrement */ 8481 } 8482 } 8483done_unload: 8484 sx_xunlock(&mlu); 8485 break; 8486 } 8487 8488 return (rc); 8489} 8490 8491static devclass_t t4_devclass, t5_devclass; 8492static devclass_t cxgbe_devclass, cxl_devclass; 8493 8494DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8495MODULE_VERSION(t4nex, 1); 8496MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8497 8498DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8499MODULE_VERSION(t5nex, 1); 8500MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8501 8502DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8503MODULE_VERSION(cxgbe, 1); 8504 8505DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8506MODULE_VERSION(cxl, 1); 8507