t4_main.c revision 281259
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 281259 2015-04-08 03:55:02Z np $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/module.h> 40#include <sys/malloc.h> 41#include <sys/queue.h> 42#include <sys/taskqueue.h> 43#include <sys/pciio.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46#include <dev/pci/pci_private.h> 47#include <sys/firmware.h> 48#include <sys/sbuf.h> 49#include <sys/smp.h> 50#include <sys/socket.h> 51#include <sys/sockio.h> 52#include <sys/sysctl.h> 53#include <net/ethernet.h> 54#include <net/if.h> 55#include <net/if_types.h> 56#include <net/if_dl.h> 57#include <net/if_vlan_var.h> 58#if defined(__i386__) || defined(__amd64__) 59#include <vm/vm.h> 60#include <vm/pmap.h> 61#endif 62 63#include "common/common.h" 64#include "common/t4_msg.h" 65#include "common/t4_regs.h" 66#include "common/t4_regs_values.h" 67#include "t4_ioctl.h" 68#include "t4_l2t.h" 69 70/* T4 bus driver interface */ 71static int t4_probe(device_t); 72static int t4_attach(device_t); 73static int t4_detach(device_t); 74static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80}; 81static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85}; 86 87 88/* T4 port (cxgbe) interface */ 89static int cxgbe_probe(device_t); 90static int cxgbe_attach(device_t); 91static int cxgbe_detach(device_t); 92static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97}; 98static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102}; 103 104static d_ioctl_t t4_ioctl; 105static d_open_t t4_open; 106static d_close_t t4_close; 107 108static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115}; 116 117/* T5 bus driver interface */ 118static int t5_probe(device_t); 119static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125}; 126static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130}; 131 132 133/* T5 port (cxl) interface */ 134static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138}; 139 140static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147}; 148 149/* ifnet + media interface */ 150static void cxgbe_init(void *); 151static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153static void cxgbe_qflush(struct ifnet *); 154static int cxgbe_media_change(struct ifnet *); 155static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159/* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163static struct sx t4_list_lock; 164SLIST_HEAD(, adapter) t4_list; 165#ifdef TCP_OFFLOAD 166static struct sx t4_uld_list_lock; 167SLIST_HEAD(, uld_info) t4_uld_list; 168#endif 169 170/* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181/* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184#define NTXQ_10G 16 185static int t4_ntxq10g = -1; 186TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188#define NRXQ_10G 8 189static int t4_nrxq10g = -1; 190TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192#define NTXQ_1G 4 193static int t4_ntxq1g = -1; 194TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196#define NRXQ_1G 2 197static int t4_nrxq1g = -1; 198TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200static int t4_rsrv_noflowq = 0; 201TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 202 203#ifdef TCP_OFFLOAD 204#define NOFLDTXQ_10G 8 205static int t4_nofldtxq10g = -1; 206TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 207 208#define NOFLDRXQ_10G 2 209static int t4_nofldrxq10g = -1; 210TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 211 212#define NOFLDTXQ_1G 2 213static int t4_nofldtxq1g = -1; 214TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 215 216#define NOFLDRXQ_1G 1 217static int t4_nofldrxq1g = -1; 218TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 219#endif 220 221#ifdef DEV_NETMAP 222#define NNMTXQ_10G 2 223static int t4_nnmtxq10g = -1; 224TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 225 226#define NNMRXQ_10G 2 227static int t4_nnmrxq10g = -1; 228TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 229 230#define NNMTXQ_1G 1 231static int t4_nnmtxq1g = -1; 232TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 233 234#define NNMRXQ_1G 1 235static int t4_nnmrxq1g = -1; 236TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 237#endif 238 239/* 240 * Holdoff parameters for 10G and 1G ports. 241 */ 242#define TMR_IDX_10G 1 243static int t4_tmr_idx_10g = TMR_IDX_10G; 244TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 245 246#define PKTC_IDX_10G (-1) 247static int t4_pktc_idx_10g = PKTC_IDX_10G; 248TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 249 250#define TMR_IDX_1G 1 251static int t4_tmr_idx_1g = TMR_IDX_1G; 252TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 253 254#define PKTC_IDX_1G (-1) 255static int t4_pktc_idx_1g = PKTC_IDX_1G; 256TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 257 258/* 259 * Size (# of entries) of each tx and rx queue. 260 */ 261static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 262TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 263 264static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 265TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 266 267/* 268 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 269 */ 270static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 271TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 272 273/* 274 * Configuration file. 275 */ 276#define DEFAULT_CF "default" 277#define FLASH_CF "flash" 278#define UWIRE_CF "uwire" 279#define FPGA_CF "fpga" 280static char t4_cfg_file[32] = DEFAULT_CF; 281TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 282 283/* 284 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 285 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 286 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 287 * mark or when signalled to do so, 0 to never emit PAUSE. 288 */ 289static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 290TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 291 292/* 293 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 294 * encouraged respectively). 295 */ 296static unsigned int t4_fw_install = 1; 297TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 298 299/* 300 * ASIC features that will be used. Disable the ones you don't want so that the 301 * chip resources aren't wasted on features that will not be used. 302 */ 303static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 304TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 305 306static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 307TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 308 309static int t4_toecaps_allowed = -1; 310TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 311 312static int t4_rdmacaps_allowed = 0; 313TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 314 315static int t4_iscsicaps_allowed = 0; 316TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 317 318static int t4_fcoecaps_allowed = 0; 319TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 320 321static int t5_write_combine = 0; 322TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 323 324struct intrs_and_queues { 325 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 326 uint16_t nirq; /* Total # of vectors */ 327 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 328 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 329 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 330 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 331 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 332 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 333 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 334#ifdef TCP_OFFLOAD 335 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 336 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 337 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 338 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 339#endif 340#ifdef DEV_NETMAP 341 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 342 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 343 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 344 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 345#endif 346}; 347 348struct filter_entry { 349 uint32_t valid:1; /* filter allocated and valid */ 350 uint32_t locked:1; /* filter is administratively locked */ 351 uint32_t pending:1; /* filter action is pending firmware reply */ 352 uint32_t smtidx:8; /* Source MAC Table index for smac */ 353 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 354 355 struct t4_filter_specification fs; 356}; 357 358static int map_bars_0_and_4(struct adapter *); 359static int map_bar_2(struct adapter *); 360static void setup_memwin(struct adapter *); 361static int validate_mem_range(struct adapter *, uint32_t, int); 362static int fwmtype_to_hwmtype(int); 363static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 364 uint32_t *); 365static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 366static uint32_t position_memwin(struct adapter *, int, uint32_t); 367static int cfg_itype_and_nqueues(struct adapter *, int, int, 368 struct intrs_and_queues *); 369static int prep_firmware(struct adapter *); 370static int partition_resources(struct adapter *, const struct firmware *, 371 const char *); 372static int get_params__pre_init(struct adapter *); 373static int get_params__post_init(struct adapter *); 374static int set_params__post_init(struct adapter *); 375static void t4_set_desc(struct adapter *); 376static void build_medialist(struct port_info *, struct ifmedia *); 377static int cxgbe_init_synchronized(struct port_info *); 378static int cxgbe_uninit_synchronized(struct port_info *); 379static int setup_intr_handlers(struct adapter *); 380static void quiesce_eq(struct adapter *, struct sge_eq *); 381static void quiesce_iq(struct adapter *, struct sge_iq *); 382static void quiesce_fl(struct adapter *, struct sge_fl *); 383static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 384 driver_intr_t *, void *, char *); 385static int t4_free_irq(struct adapter *, struct irq *); 386static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 387 unsigned int); 388static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 389static void cxgbe_tick(void *); 390static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 391static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 392 struct mbuf *); 393static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 394static int fw_msg_not_handled(struct adapter *, const __be64 *); 395static int t4_sysctls(struct adapter *); 396static int cxgbe_sysctls(struct port_info *); 397static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 398static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 399static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 400static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 401static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 402static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 403static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 404static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 405static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 406static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 407static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 408#ifdef SBUF_DRAIN 409static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 410static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 411static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 412static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 413static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 414static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 415static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 416static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 417static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 418static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 419static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 420static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 421static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 422static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 423static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 424static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 425static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 426static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 427static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 428static int sysctl_tids(SYSCTL_HANDLER_ARGS); 429static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 430static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 431static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 432static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 433static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 434#endif 435static inline void txq_start(struct ifnet *, struct sge_txq *); 436static uint32_t fconf_to_mode(uint32_t); 437static uint32_t mode_to_fconf(uint32_t); 438static uint32_t fspec_to_fconf(struct t4_filter_specification *); 439static int get_filter_mode(struct adapter *, uint32_t *); 440static int set_filter_mode(struct adapter *, uint32_t); 441static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 442static int get_filter(struct adapter *, struct t4_filter *); 443static int set_filter(struct adapter *, struct t4_filter *); 444static int del_filter(struct adapter *, struct t4_filter *); 445static void clear_filter(struct filter_entry *); 446static int set_filter_wr(struct adapter *, int); 447static int del_filter_wr(struct adapter *, int); 448static int get_sge_context(struct adapter *, struct t4_sge_context *); 449static int load_fw(struct adapter *, struct t4_data *); 450static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 451static int read_i2c(struct adapter *, struct t4_i2c_data *); 452static int set_sched_class(struct adapter *, struct t4_sched_params *); 453static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 454#ifdef TCP_OFFLOAD 455static int toe_capability(struct port_info *, int); 456#endif 457static int mod_event(module_t, int, void *); 458 459struct { 460 uint16_t device; 461 char *desc; 462} t4_pciids[] = { 463 {0xa000, "Chelsio Terminator 4 FPGA"}, 464 {0x4400, "Chelsio T440-dbg"}, 465 {0x4401, "Chelsio T420-CR"}, 466 {0x4402, "Chelsio T422-CR"}, 467 {0x4403, "Chelsio T440-CR"}, 468 {0x4404, "Chelsio T420-BCH"}, 469 {0x4405, "Chelsio T440-BCH"}, 470 {0x4406, "Chelsio T440-CH"}, 471 {0x4407, "Chelsio T420-SO"}, 472 {0x4408, "Chelsio T420-CX"}, 473 {0x4409, "Chelsio T420-BT"}, 474 {0x440a, "Chelsio T404-BT"}, 475 {0x440e, "Chelsio T440-LP-CR"}, 476}, t5_pciids[] = { 477 {0xb000, "Chelsio Terminator 5 FPGA"}, 478 {0x5400, "Chelsio T580-dbg"}, 479 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 480 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 481 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 482 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 483 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 484 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 485 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 486 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 487 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 488 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 489 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 490 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 491 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 492#ifdef notyet 493 {0x5404, "Chelsio T520-BCH"}, 494 {0x5405, "Chelsio T540-BCH"}, 495 {0x5406, "Chelsio T540-CH"}, 496 {0x5408, "Chelsio T520-CX"}, 497 {0x540b, "Chelsio B520-SR"}, 498 {0x540c, "Chelsio B504-BT"}, 499 {0x540f, "Chelsio Amsterdam"}, 500 {0x5413, "Chelsio T580-CHR"}, 501#endif 502}; 503 504#ifdef TCP_OFFLOAD 505/* 506 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 507 * exactly the same for both rxq and ofld_rxq. 508 */ 509CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 510CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 511#endif 512 513/* No easy way to include t4_msg.h before adapter.h so we check this way */ 514CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 515CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 516 517CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 518 519static int 520t4_probe(device_t dev) 521{ 522 int i; 523 uint16_t v = pci_get_vendor(dev); 524 uint16_t d = pci_get_device(dev); 525 uint8_t f = pci_get_function(dev); 526 527 if (v != PCI_VENDOR_ID_CHELSIO) 528 return (ENXIO); 529 530 /* Attach only to PF0 of the FPGA */ 531 if (d == 0xa000 && f != 0) 532 return (ENXIO); 533 534 for (i = 0; i < nitems(t4_pciids); i++) { 535 if (d == t4_pciids[i].device) { 536 device_set_desc(dev, t4_pciids[i].desc); 537 return (BUS_PROBE_DEFAULT); 538 } 539 } 540 541 return (ENXIO); 542} 543 544static int 545t5_probe(device_t dev) 546{ 547 int i; 548 uint16_t v = pci_get_vendor(dev); 549 uint16_t d = pci_get_device(dev); 550 uint8_t f = pci_get_function(dev); 551 552 if (v != PCI_VENDOR_ID_CHELSIO) 553 return (ENXIO); 554 555 /* Attach only to PF0 of the FPGA */ 556 if (d == 0xb000 && f != 0) 557 return (ENXIO); 558 559 for (i = 0; i < nitems(t5_pciids); i++) { 560 if (d == t5_pciids[i].device) { 561 device_set_desc(dev, t5_pciids[i].desc); 562 return (BUS_PROBE_DEFAULT); 563 } 564 } 565 566 return (ENXIO); 567} 568 569static int 570t4_attach(device_t dev) 571{ 572 struct adapter *sc; 573 int rc = 0, i, n10g, n1g, rqidx, tqidx; 574 struct intrs_and_queues iaq; 575 struct sge *s; 576#ifdef TCP_OFFLOAD 577 int ofld_rqidx, ofld_tqidx; 578#endif 579#ifdef DEV_NETMAP 580 int nm_rqidx, nm_tqidx; 581#endif 582 583 sc = device_get_softc(dev); 584 sc->dev = dev; 585 586 pci_enable_busmaster(dev); 587 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 588 uint32_t v; 589 590 pci_set_max_read_req(dev, 4096); 591 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 592 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 593 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 594 595 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 596 } 597 598 sc->traceq = -1; 599 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 600 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 601 device_get_nameunit(dev)); 602 603 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 604 device_get_nameunit(dev)); 605 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 606 sx_xlock(&t4_list_lock); 607 SLIST_INSERT_HEAD(&t4_list, sc, link); 608 sx_xunlock(&t4_list_lock); 609 610 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 611 TAILQ_INIT(&sc->sfl); 612 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 613 614 rc = map_bars_0_and_4(sc); 615 if (rc != 0) 616 goto done; /* error message displayed already */ 617 618 /* 619 * This is the real PF# to which we're attaching. Works from within PCI 620 * passthrough environments too, where pci_get_function() could return a 621 * different PF# depending on the passthrough configuration. We need to 622 * use the real PF# in all our communication with the firmware. 623 */ 624 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 625 sc->mbox = sc->pf; 626 627 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 628 sc->an_handler = an_not_handled; 629 for (i = 0; i < nitems(sc->cpl_handler); i++) 630 sc->cpl_handler[i] = cpl_not_handled; 631 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 632 sc->fw_msg_handler[i] = fw_msg_not_handled; 633 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 634 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 635 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 636 t4_init_sge_cpl_handlers(sc); 637 638 /* Prepare the adapter for operation */ 639 rc = -t4_prep_adapter(sc); 640 if (rc != 0) { 641 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 642 goto done; 643 } 644 645 /* 646 * Do this really early, with the memory windows set up even before the 647 * character device. The userland tool's register i/o and mem read 648 * will work even in "recovery mode". 649 */ 650 setup_memwin(sc); 651 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 652 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 653 device_get_nameunit(dev)); 654 if (sc->cdev == NULL) 655 device_printf(dev, "failed to create nexus char device.\n"); 656 else 657 sc->cdev->si_drv1 = sc; 658 659 /* Go no further if recovery mode has been requested. */ 660 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 661 device_printf(dev, "recovery mode.\n"); 662 goto done; 663 } 664 665 /* Prepare the firmware for operation */ 666 rc = prep_firmware(sc); 667 if (rc != 0) 668 goto done; /* error message displayed already */ 669 670 rc = get_params__post_init(sc); 671 if (rc != 0) 672 goto done; /* error message displayed already */ 673 674 rc = set_params__post_init(sc); 675 if (rc != 0) 676 goto done; /* error message displayed already */ 677 678 rc = map_bar_2(sc); 679 if (rc != 0) 680 goto done; /* error message displayed already */ 681 682 rc = t4_create_dma_tag(sc); 683 if (rc != 0) 684 goto done; /* error message displayed already */ 685 686 /* 687 * First pass over all the ports - allocate VIs and initialize some 688 * basic parameters like mac address, port type, etc. We also figure 689 * out whether a port is 10G or 1G and use that information when 690 * calculating how many interrupts to attempt to allocate. 691 */ 692 n10g = n1g = 0; 693 for_each_port(sc, i) { 694 struct port_info *pi; 695 696 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 697 sc->port[i] = pi; 698 699 /* These must be set before t4_port_init */ 700 pi->adapter = sc; 701 pi->port_id = i; 702 703 /* Allocate the vi and initialize parameters like mac addr */ 704 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 705 if (rc != 0) { 706 device_printf(dev, "unable to initialize port %d: %d\n", 707 i, rc); 708 free(pi, M_CXGBE); 709 sc->port[i] = NULL; 710 goto done; 711 } 712 713 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 714 pi->link_cfg.requested_fc |= t4_pause_settings; 715 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 716 pi->link_cfg.fc |= t4_pause_settings; 717 718 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 719 if (rc != 0) { 720 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 721 free(pi, M_CXGBE); 722 sc->port[i] = NULL; 723 goto done; 724 } 725 726 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 727 device_get_nameunit(dev), i); 728 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 729 sc->chan_map[pi->tx_chan] = i; 730 731 if (is_10G_port(pi) || is_40G_port(pi)) { 732 n10g++; 733 pi->tmr_idx = t4_tmr_idx_10g; 734 pi->pktc_idx = t4_pktc_idx_10g; 735 } else { 736 n1g++; 737 pi->tmr_idx = t4_tmr_idx_1g; 738 pi->pktc_idx = t4_pktc_idx_1g; 739 } 740 741 pi->xact_addr_filt = -1; 742 pi->linkdnrc = -1; 743 744 pi->qsize_rxq = t4_qsize_rxq; 745 pi->qsize_txq = t4_qsize_txq; 746 747 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 748 if (pi->dev == NULL) { 749 device_printf(dev, 750 "failed to add device for port %d.\n", i); 751 rc = ENXIO; 752 goto done; 753 } 754 device_set_softc(pi->dev, pi); 755 } 756 757 /* 758 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 759 */ 760 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 761 if (rc != 0) 762 goto done; /* error message displayed already */ 763 764 sc->intr_type = iaq.intr_type; 765 sc->intr_count = iaq.nirq; 766 767 s = &sc->sge; 768 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 769 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 770 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 771 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 772 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 773#ifdef TCP_OFFLOAD 774 if (is_offload(sc)) { 775 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 776 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 777 s->neq += s->nofldtxq + s->nofldrxq; 778 s->niq += s->nofldrxq; 779 780 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 781 M_CXGBE, M_ZERO | M_WAITOK); 782 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 783 M_CXGBE, M_ZERO | M_WAITOK); 784 } 785#endif 786#ifdef DEV_NETMAP 787 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 788 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 789 s->neq += s->nnmtxq + s->nnmrxq; 790 s->niq += s->nnmrxq; 791 792 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 793 M_CXGBE, M_ZERO | M_WAITOK); 794 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 795 M_CXGBE, M_ZERO | M_WAITOK); 796#endif 797 798 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 799 M_ZERO | M_WAITOK); 800 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 801 M_ZERO | M_WAITOK); 802 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 803 M_ZERO | M_WAITOK); 804 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 805 M_ZERO | M_WAITOK); 806 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 807 M_ZERO | M_WAITOK); 808 809 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 810 M_ZERO | M_WAITOK); 811 812 t4_init_l2t(sc, M_WAITOK); 813 814 /* 815 * Second pass over the ports. This time we know the number of rx and 816 * tx queues that each port should get. 817 */ 818 rqidx = tqidx = 0; 819#ifdef TCP_OFFLOAD 820 ofld_rqidx = ofld_tqidx = 0; 821#endif 822#ifdef DEV_NETMAP 823 nm_rqidx = nm_tqidx = 0; 824#endif 825 for_each_port(sc, i) { 826 struct port_info *pi = sc->port[i]; 827 828 if (pi == NULL) 829 continue; 830 831 pi->first_rxq = rqidx; 832 pi->first_txq = tqidx; 833 if (is_10G_port(pi) || is_40G_port(pi)) { 834 pi->flags |= iaq.intr_flags_10g; 835 pi->nrxq = iaq.nrxq10g; 836 pi->ntxq = iaq.ntxq10g; 837 } else { 838 pi->flags |= iaq.intr_flags_1g; 839 pi->nrxq = iaq.nrxq1g; 840 pi->ntxq = iaq.ntxq1g; 841 } 842 843 if (pi->ntxq > 1) 844 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 845 else 846 pi->rsrv_noflowq = 0; 847 848 rqidx += pi->nrxq; 849 tqidx += pi->ntxq; 850#ifdef TCP_OFFLOAD 851 if (is_offload(sc)) { 852 pi->first_ofld_rxq = ofld_rqidx; 853 pi->first_ofld_txq = ofld_tqidx; 854 if (is_10G_port(pi) || is_40G_port(pi)) { 855 pi->nofldrxq = iaq.nofldrxq10g; 856 pi->nofldtxq = iaq.nofldtxq10g; 857 } else { 858 pi->nofldrxq = iaq.nofldrxq1g; 859 pi->nofldtxq = iaq.nofldtxq1g; 860 } 861 ofld_rqidx += pi->nofldrxq; 862 ofld_tqidx += pi->nofldtxq; 863 } 864#endif 865#ifdef DEV_NETMAP 866 pi->first_nm_rxq = nm_rqidx; 867 pi->first_nm_txq = nm_tqidx; 868 if (is_10G_port(pi) || is_40G_port(pi)) { 869 pi->nnmrxq = iaq.nnmrxq10g; 870 pi->nnmtxq = iaq.nnmtxq10g; 871 } else { 872 pi->nnmrxq = iaq.nnmrxq1g; 873 pi->nnmtxq = iaq.nnmtxq1g; 874 } 875 nm_rqidx += pi->nnmrxq; 876 nm_tqidx += pi->nnmtxq; 877#endif 878 } 879 880 rc = setup_intr_handlers(sc); 881 if (rc != 0) { 882 device_printf(dev, 883 "failed to setup interrupt handlers: %d\n", rc); 884 goto done; 885 } 886 887 rc = bus_generic_attach(dev); 888 if (rc != 0) { 889 device_printf(dev, 890 "failed to attach all child ports: %d\n", rc); 891 goto done; 892 } 893 894 device_printf(dev, 895 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 896 sc->params.pci.width, sc->params.nports, sc->intr_count, 897 sc->intr_type == INTR_MSIX ? "MSI-X" : 898 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 899 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 900 901 t4_set_desc(sc); 902 903done: 904 if (rc != 0 && sc->cdev) { 905 /* cdev was created and so cxgbetool works; recover that way. */ 906 device_printf(dev, 907 "error during attach, adapter is now in recovery mode.\n"); 908 rc = 0; 909 } 910 911 if (rc != 0) 912 t4_detach(dev); 913 else 914 t4_sysctls(sc); 915 916 return (rc); 917} 918 919/* 920 * Idempotent 921 */ 922static int 923t4_detach(device_t dev) 924{ 925 struct adapter *sc; 926 struct port_info *pi; 927 int i, rc; 928 929 sc = device_get_softc(dev); 930 931 if (sc->flags & FULL_INIT_DONE) 932 t4_intr_disable(sc); 933 934 if (sc->cdev) { 935 destroy_dev(sc->cdev); 936 sc->cdev = NULL; 937 } 938 939 rc = bus_generic_detach(dev); 940 if (rc) { 941 device_printf(dev, 942 "failed to detach child devices: %d\n", rc); 943 return (rc); 944 } 945 946 for (i = 0; i < sc->intr_count; i++) 947 t4_free_irq(sc, &sc->irq[i]); 948 949 for (i = 0; i < MAX_NPORTS; i++) { 950 pi = sc->port[i]; 951 if (pi) { 952 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 953 if (pi->dev) 954 device_delete_child(dev, pi->dev); 955 956 mtx_destroy(&pi->pi_lock); 957 free(pi, M_CXGBE); 958 } 959 } 960 961 if (sc->flags & FULL_INIT_DONE) 962 adapter_full_uninit(sc); 963 964 if (sc->flags & FW_OK) 965 t4_fw_bye(sc, sc->mbox); 966 967 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 968 pci_release_msi(dev); 969 970 if (sc->regs_res) 971 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 972 sc->regs_res); 973 974 if (sc->udbs_res) 975 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 976 sc->udbs_res); 977 978 if (sc->msix_res) 979 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 980 sc->msix_res); 981 982 if (sc->l2t) 983 t4_free_l2t(sc->l2t); 984 985#ifdef TCP_OFFLOAD 986 free(sc->sge.ofld_rxq, M_CXGBE); 987 free(sc->sge.ofld_txq, M_CXGBE); 988#endif 989#ifdef DEV_NETMAP 990 free(sc->sge.nm_rxq, M_CXGBE); 991 free(sc->sge.nm_txq, M_CXGBE); 992#endif 993 free(sc->irq, M_CXGBE); 994 free(sc->sge.rxq, M_CXGBE); 995 free(sc->sge.txq, M_CXGBE); 996 free(sc->sge.ctrlq, M_CXGBE); 997 free(sc->sge.iqmap, M_CXGBE); 998 free(sc->sge.eqmap, M_CXGBE); 999 free(sc->tids.ftid_tab, M_CXGBE); 1000 t4_destroy_dma_tag(sc); 1001 if (mtx_initialized(&sc->sc_lock)) { 1002 sx_xlock(&t4_list_lock); 1003 SLIST_REMOVE(&t4_list, sc, adapter, link); 1004 sx_xunlock(&t4_list_lock); 1005 mtx_destroy(&sc->sc_lock); 1006 } 1007 1008 if (mtx_initialized(&sc->tids.ftid_lock)) 1009 mtx_destroy(&sc->tids.ftid_lock); 1010 if (mtx_initialized(&sc->sfl_lock)) 1011 mtx_destroy(&sc->sfl_lock); 1012 if (mtx_initialized(&sc->ifp_lock)) 1013 mtx_destroy(&sc->ifp_lock); 1014 1015 bzero(sc, sizeof(*sc)); 1016 1017 return (0); 1018} 1019 1020static int 1021cxgbe_probe(device_t dev) 1022{ 1023 char buf[128]; 1024 struct port_info *pi = device_get_softc(dev); 1025 1026 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1027 device_set_desc_copy(dev, buf); 1028 1029 return (BUS_PROBE_DEFAULT); 1030} 1031 1032#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1033 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1034 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1035#define T4_CAP_ENABLE (T4_CAP) 1036 1037static int 1038cxgbe_attach(device_t dev) 1039{ 1040 struct port_info *pi = device_get_softc(dev); 1041 struct ifnet *ifp; 1042 char *s; 1043 int n, o; 1044 1045 /* Allocate an ifnet and set it up */ 1046 ifp = if_alloc(IFT_ETHER); 1047 if (ifp == NULL) { 1048 device_printf(dev, "Cannot allocate ifnet\n"); 1049 return (ENOMEM); 1050 } 1051 pi->ifp = ifp; 1052 ifp->if_softc = pi; 1053 1054 callout_init(&pi->tick, CALLOUT_MPSAFE); 1055 1056 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1057 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1058 1059 ifp->if_init = cxgbe_init; 1060 ifp->if_ioctl = cxgbe_ioctl; 1061 ifp->if_transmit = cxgbe_transmit; 1062 ifp->if_qflush = cxgbe_qflush; 1063 1064 ifp->if_capabilities = T4_CAP; 1065#ifdef TCP_OFFLOAD 1066 if (is_offload(pi->adapter)) 1067 ifp->if_capabilities |= IFCAP_TOE; 1068#endif 1069 ifp->if_capenable = T4_CAP_ENABLE; 1070 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1071 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1072 1073 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1074 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1075 ifp->if_hw_tsomaxsegsize = 65536; 1076 1077 /* Initialize ifmedia for this port */ 1078 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1079 cxgbe_media_status); 1080 build_medialist(pi, &pi->media); 1081 1082 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1083 EVENTHANDLER_PRI_ANY); 1084 1085 ether_ifattach(ifp, pi->hw_addr); 1086 1087 n = 128; 1088 s = malloc(n, M_CXGBE, M_WAITOK); 1089 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1090 MPASS(n > o); 1091#ifdef TCP_OFFLOAD 1092 if (is_offload(pi->adapter)) { 1093 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1094 pi->nofldtxq, pi->nofldrxq); 1095 MPASS(n > o); 1096 } 1097#endif 1098#ifdef DEV_NETMAP 1099 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1100 pi->nnmrxq); 1101 MPASS(n > o); 1102#endif 1103 device_printf(dev, "%s\n", s); 1104 free(s, M_CXGBE); 1105 1106#ifdef DEV_NETMAP 1107 /* nm_media handled here to keep implementation private to this file */ 1108 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1109 cxgbe_media_status); 1110 build_medialist(pi, &pi->nm_media); 1111 create_netmap_ifnet(pi); /* logs errors it something fails */ 1112#endif 1113 cxgbe_sysctls(pi); 1114 1115 return (0); 1116} 1117 1118static int 1119cxgbe_detach(device_t dev) 1120{ 1121 struct port_info *pi = device_get_softc(dev); 1122 struct adapter *sc = pi->adapter; 1123 struct ifnet *ifp = pi->ifp; 1124 1125 /* Tell if_ioctl and if_init that the port is going away */ 1126 ADAPTER_LOCK(sc); 1127 SET_DOOMED(pi); 1128 wakeup(&sc->flags); 1129 while (IS_BUSY(sc)) 1130 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1131 SET_BUSY(sc); 1132#ifdef INVARIANTS 1133 sc->last_op = "t4detach"; 1134 sc->last_op_thr = curthread; 1135#endif 1136 ADAPTER_UNLOCK(sc); 1137 1138 if (pi->flags & HAS_TRACEQ) { 1139 sc->traceq = -1; /* cloner should not create ifnet */ 1140 t4_tracer_port_detach(sc); 1141 } 1142 1143 if (pi->vlan_c) 1144 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1145 1146 PORT_LOCK(pi); 1147 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1148 callout_stop(&pi->tick); 1149 PORT_UNLOCK(pi); 1150 callout_drain(&pi->tick); 1151 1152 /* Let detach proceed even if these fail. */ 1153 cxgbe_uninit_synchronized(pi); 1154 port_full_uninit(pi); 1155 1156 ifmedia_removeall(&pi->media); 1157 ether_ifdetach(pi->ifp); 1158 if_free(pi->ifp); 1159 1160#ifdef DEV_NETMAP 1161 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1162 destroy_netmap_ifnet(pi); 1163#endif 1164 1165 ADAPTER_LOCK(sc); 1166 CLR_BUSY(sc); 1167 wakeup(&sc->flags); 1168 ADAPTER_UNLOCK(sc); 1169 1170 return (0); 1171} 1172 1173static void 1174cxgbe_init(void *arg) 1175{ 1176 struct port_info *pi = arg; 1177 struct adapter *sc = pi->adapter; 1178 1179 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1180 return; 1181 cxgbe_init_synchronized(pi); 1182 end_synchronized_op(sc, 0); 1183} 1184 1185static int 1186cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1187{ 1188 int rc = 0, mtu, flags, can_sleep; 1189 struct port_info *pi = ifp->if_softc; 1190 struct adapter *sc = pi->adapter; 1191 struct ifreq *ifr = (struct ifreq *)data; 1192 uint32_t mask; 1193 1194 switch (cmd) { 1195 case SIOCSIFMTU: 1196 mtu = ifr->ifr_mtu; 1197 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1198 return (EINVAL); 1199 1200 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1201 if (rc) 1202 return (rc); 1203 ifp->if_mtu = mtu; 1204 if (pi->flags & PORT_INIT_DONE) { 1205 t4_update_fl_bufsize(ifp); 1206 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1207 rc = update_mac_settings(ifp, XGMAC_MTU); 1208 } 1209 end_synchronized_op(sc, 0); 1210 break; 1211 1212 case SIOCSIFFLAGS: 1213 can_sleep = 0; 1214redo_sifflags: 1215 rc = begin_synchronized_op(sc, pi, 1216 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1217 if (rc) 1218 return (rc); 1219 1220 if (ifp->if_flags & IFF_UP) { 1221 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1222 flags = pi->if_flags; 1223 if ((ifp->if_flags ^ flags) & 1224 (IFF_PROMISC | IFF_ALLMULTI)) { 1225 if (can_sleep == 1) { 1226 end_synchronized_op(sc, 0); 1227 can_sleep = 0; 1228 goto redo_sifflags; 1229 } 1230 rc = update_mac_settings(ifp, 1231 XGMAC_PROMISC | XGMAC_ALLMULTI); 1232 } 1233 } else { 1234 if (can_sleep == 0) { 1235 end_synchronized_op(sc, LOCK_HELD); 1236 can_sleep = 1; 1237 goto redo_sifflags; 1238 } 1239 rc = cxgbe_init_synchronized(pi); 1240 } 1241 pi->if_flags = ifp->if_flags; 1242 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1243 if (can_sleep == 0) { 1244 end_synchronized_op(sc, LOCK_HELD); 1245 can_sleep = 1; 1246 goto redo_sifflags; 1247 } 1248 rc = cxgbe_uninit_synchronized(pi); 1249 } 1250 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1251 break; 1252 1253 case SIOCADDMULTI: 1254 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1255 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1256 if (rc) 1257 return (rc); 1258 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1259 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1260 end_synchronized_op(sc, LOCK_HELD); 1261 break; 1262 1263 case SIOCSIFCAP: 1264 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1265 if (rc) 1266 return (rc); 1267 1268 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1269 if (mask & IFCAP_TXCSUM) { 1270 ifp->if_capenable ^= IFCAP_TXCSUM; 1271 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1272 1273 if (IFCAP_TSO4 & ifp->if_capenable && 1274 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1275 ifp->if_capenable &= ~IFCAP_TSO4; 1276 if_printf(ifp, 1277 "tso4 disabled due to -txcsum.\n"); 1278 } 1279 } 1280 if (mask & IFCAP_TXCSUM_IPV6) { 1281 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1282 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1283 1284 if (IFCAP_TSO6 & ifp->if_capenable && 1285 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1286 ifp->if_capenable &= ~IFCAP_TSO6; 1287 if_printf(ifp, 1288 "tso6 disabled due to -txcsum6.\n"); 1289 } 1290 } 1291 if (mask & IFCAP_RXCSUM) 1292 ifp->if_capenable ^= IFCAP_RXCSUM; 1293 if (mask & IFCAP_RXCSUM_IPV6) 1294 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1295 1296 /* 1297 * Note that we leave CSUM_TSO alone (it is always set). The 1298 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1299 * sending a TSO request our way, so it's sufficient to toggle 1300 * IFCAP_TSOx only. 1301 */ 1302 if (mask & IFCAP_TSO4) { 1303 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1304 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1305 if_printf(ifp, "enable txcsum first.\n"); 1306 rc = EAGAIN; 1307 goto fail; 1308 } 1309 ifp->if_capenable ^= IFCAP_TSO4; 1310 } 1311 if (mask & IFCAP_TSO6) { 1312 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1313 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1314 if_printf(ifp, "enable txcsum6 first.\n"); 1315 rc = EAGAIN; 1316 goto fail; 1317 } 1318 ifp->if_capenable ^= IFCAP_TSO6; 1319 } 1320 if (mask & IFCAP_LRO) { 1321#if defined(INET) || defined(INET6) 1322 int i; 1323 struct sge_rxq *rxq; 1324 1325 ifp->if_capenable ^= IFCAP_LRO; 1326 for_each_rxq(pi, i, rxq) { 1327 if (ifp->if_capenable & IFCAP_LRO) 1328 rxq->iq.flags |= IQ_LRO_ENABLED; 1329 else 1330 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1331 } 1332#endif 1333 } 1334#ifdef TCP_OFFLOAD 1335 if (mask & IFCAP_TOE) { 1336 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1337 1338 rc = toe_capability(pi, enable); 1339 if (rc != 0) 1340 goto fail; 1341 1342 ifp->if_capenable ^= mask; 1343 } 1344#endif 1345 if (mask & IFCAP_VLAN_HWTAGGING) { 1346 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1347 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1348 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1349 } 1350 if (mask & IFCAP_VLAN_MTU) { 1351 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1352 1353 /* Need to find out how to disable auto-mtu-inflation */ 1354 } 1355 if (mask & IFCAP_VLAN_HWTSO) 1356 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1357 if (mask & IFCAP_VLAN_HWCSUM) 1358 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1359 1360#ifdef VLAN_CAPABILITIES 1361 VLAN_CAPABILITIES(ifp); 1362#endif 1363fail: 1364 end_synchronized_op(sc, 0); 1365 break; 1366 1367 case SIOCSIFMEDIA: 1368 case SIOCGIFMEDIA: 1369 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1370 break; 1371 1372 default: 1373 rc = ether_ioctl(ifp, cmd, data); 1374 } 1375 1376 return (rc); 1377} 1378 1379static int 1380cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1381{ 1382 struct port_info *pi = ifp->if_softc; 1383 struct adapter *sc = pi->adapter; 1384 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1385 struct buf_ring *br; 1386 int rc; 1387 1388 M_ASSERTPKTHDR(m); 1389 1390 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1391 m_freem(m); 1392 return (ENETDOWN); 1393 } 1394 1395 if (m->m_flags & M_FLOWID) 1396 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1397 + pi->rsrv_noflowq); 1398 br = txq->br; 1399 1400 if (TXQ_TRYLOCK(txq) == 0) { 1401 struct sge_eq *eq = &txq->eq; 1402 1403 /* 1404 * It is possible that t4_eth_tx finishes up and releases the 1405 * lock between the TRYLOCK above and the drbr_enqueue here. We 1406 * need to make sure that this mbuf doesn't just sit there in 1407 * the drbr. 1408 */ 1409 1410 rc = drbr_enqueue(ifp, br, m); 1411 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1412 !(eq->flags & EQ_DOOMED)) 1413 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1414 return (rc); 1415 } 1416 1417 /* 1418 * txq->m is the mbuf that is held up due to a temporary shortage of 1419 * resources and it should be put on the wire first. Then what's in 1420 * drbr and finally the mbuf that was just passed in to us. 1421 * 1422 * Return code should indicate the fate of the mbuf that was passed in 1423 * this time. 1424 */ 1425 1426 TXQ_LOCK_ASSERT_OWNED(txq); 1427 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1428 1429 /* Queued for transmission. */ 1430 1431 rc = drbr_enqueue(ifp, br, m); 1432 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1433 (void) t4_eth_tx(ifp, txq, m); 1434 TXQ_UNLOCK(txq); 1435 return (rc); 1436 } 1437 1438 /* Direct transmission. */ 1439 rc = t4_eth_tx(ifp, txq, m); 1440 if (rc != 0 && txq->m) 1441 rc = 0; /* held, will be transmitted soon (hopefully) */ 1442 1443 TXQ_UNLOCK(txq); 1444 return (rc); 1445} 1446 1447static void 1448cxgbe_qflush(struct ifnet *ifp) 1449{ 1450 struct port_info *pi = ifp->if_softc; 1451 struct sge_txq *txq; 1452 int i; 1453 struct mbuf *m; 1454 1455 /* queues do not exist if !PORT_INIT_DONE. */ 1456 if (pi->flags & PORT_INIT_DONE) { 1457 for_each_txq(pi, i, txq) { 1458 TXQ_LOCK(txq); 1459 m_freem(txq->m); 1460 txq->m = NULL; 1461 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1462 m_freem(m); 1463 TXQ_UNLOCK(txq); 1464 } 1465 } 1466 if_qflush(ifp); 1467} 1468 1469static int 1470cxgbe_media_change(struct ifnet *ifp) 1471{ 1472 struct port_info *pi = ifp->if_softc; 1473 1474 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1475 1476 return (EOPNOTSUPP); 1477} 1478 1479static void 1480cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1481{ 1482 struct port_info *pi = ifp->if_softc; 1483 struct ifmedia *media = NULL; 1484 struct ifmedia_entry *cur; 1485 int speed = pi->link_cfg.speed; 1486#ifdef INVARIANTS 1487 int data = (pi->port_type << 8) | pi->mod_type; 1488#endif 1489 1490 if (ifp == pi->ifp) 1491 media = &pi->media; 1492#ifdef DEV_NETMAP 1493 else if (ifp == pi->nm_ifp) 1494 media = &pi->nm_media; 1495#endif 1496 MPASS(media != NULL); 1497 1498 cur = media->ifm_cur; 1499 MPASS(cur->ifm_data == data); 1500 1501 ifmr->ifm_status = IFM_AVALID; 1502 if (!pi->link_cfg.link_ok) 1503 return; 1504 1505 ifmr->ifm_status |= IFM_ACTIVE; 1506 1507 /* active and current will differ iff current media is autoselect. */ 1508 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1509 return; 1510 1511 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1512 if (speed == SPEED_10000) 1513 ifmr->ifm_active |= IFM_10G_T; 1514 else if (speed == SPEED_1000) 1515 ifmr->ifm_active |= IFM_1000_T; 1516 else if (speed == SPEED_100) 1517 ifmr->ifm_active |= IFM_100_TX; 1518 else if (speed == SPEED_10) 1519 ifmr->ifm_active |= IFM_10_T; 1520 else 1521 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1522 speed)); 1523} 1524 1525void 1526t4_fatal_err(struct adapter *sc) 1527{ 1528 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1529 t4_intr_disable(sc); 1530 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1531 device_get_nameunit(sc->dev)); 1532} 1533 1534static int 1535map_bars_0_and_4(struct adapter *sc) 1536{ 1537 sc->regs_rid = PCIR_BAR(0); 1538 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1539 &sc->regs_rid, RF_ACTIVE); 1540 if (sc->regs_res == NULL) { 1541 device_printf(sc->dev, "cannot map registers.\n"); 1542 return (ENXIO); 1543 } 1544 sc->bt = rman_get_bustag(sc->regs_res); 1545 sc->bh = rman_get_bushandle(sc->regs_res); 1546 sc->mmio_len = rman_get_size(sc->regs_res); 1547 setbit(&sc->doorbells, DOORBELL_KDB); 1548 1549 sc->msix_rid = PCIR_BAR(4); 1550 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1551 &sc->msix_rid, RF_ACTIVE); 1552 if (sc->msix_res == NULL) { 1553 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1554 return (ENXIO); 1555 } 1556 1557 return (0); 1558} 1559 1560static int 1561map_bar_2(struct adapter *sc) 1562{ 1563 1564 /* 1565 * T4: only iWARP driver uses the userspace doorbells. There is no need 1566 * to map it if RDMA is disabled. 1567 */ 1568 if (is_t4(sc) && sc->rdmacaps == 0) 1569 return (0); 1570 1571 sc->udbs_rid = PCIR_BAR(2); 1572 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1573 &sc->udbs_rid, RF_ACTIVE); 1574 if (sc->udbs_res == NULL) { 1575 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1576 return (ENXIO); 1577 } 1578 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1579 1580 if (is_t5(sc)) { 1581 setbit(&sc->doorbells, DOORBELL_UDB); 1582#if defined(__i386__) || defined(__amd64__) 1583 if (t5_write_combine) { 1584 int rc; 1585 1586 /* 1587 * Enable write combining on BAR2. This is the 1588 * userspace doorbell BAR and is split into 128B 1589 * (UDBS_SEG_SIZE) doorbell regions, each associated 1590 * with an egress queue. The first 64B has the doorbell 1591 * and the second 64B can be used to submit a tx work 1592 * request with an implicit doorbell. 1593 */ 1594 1595 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1596 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1597 if (rc == 0) { 1598 clrbit(&sc->doorbells, DOORBELL_UDB); 1599 setbit(&sc->doorbells, DOORBELL_WCWR); 1600 setbit(&sc->doorbells, DOORBELL_UDBWC); 1601 } else { 1602 device_printf(sc->dev, 1603 "couldn't enable write combining: %d\n", 1604 rc); 1605 } 1606 1607 t4_write_reg(sc, A_SGE_STAT_CFG, 1608 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1609 } 1610#endif 1611 } 1612 1613 return (0); 1614} 1615 1616static const struct memwin t4_memwin[] = { 1617 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1618 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1619 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1620}; 1621 1622static const struct memwin t5_memwin[] = { 1623 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1624 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1625 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1626}; 1627 1628static void 1629setup_memwin(struct adapter *sc) 1630{ 1631 const struct memwin *mw; 1632 int i, n; 1633 uint32_t bar0; 1634 1635 if (is_t4(sc)) { 1636 /* 1637 * Read low 32b of bar0 indirectly via the hardware backdoor 1638 * mechanism. Works from within PCI passthrough environments 1639 * too, where rman_get_start() can return a different value. We 1640 * need to program the T4 memory window decoders with the actual 1641 * addresses that will be coming across the PCIe link. 1642 */ 1643 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1644 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1645 1646 mw = &t4_memwin[0]; 1647 n = nitems(t4_memwin); 1648 } else { 1649 /* T5 uses the relative offset inside the PCIe BAR */ 1650 bar0 = 0; 1651 1652 mw = &t5_memwin[0]; 1653 n = nitems(t5_memwin); 1654 } 1655 1656 for (i = 0; i < n; i++, mw++) { 1657 t4_write_reg(sc, 1658 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1659 (mw->base + bar0) | V_BIR(0) | 1660 V_WINDOW(ilog2(mw->aperture) - 10)); 1661 } 1662 1663 /* flush */ 1664 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1665} 1666 1667/* 1668 * Verify that the memory range specified by the addr/len pair is valid and lies 1669 * entirely within a single region (EDCx or MCx). 1670 */ 1671static int 1672validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1673{ 1674 uint32_t em, addr_len, maddr, mlen; 1675 1676 /* Memory can only be accessed in naturally aligned 4 byte units */ 1677 if (addr & 3 || len & 3 || len == 0) 1678 return (EINVAL); 1679 1680 /* Enabled memories */ 1681 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1682 if (em & F_EDRAM0_ENABLE) { 1683 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1684 maddr = G_EDRAM0_BASE(addr_len) << 20; 1685 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1686 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1687 addr + len <= maddr + mlen) 1688 return (0); 1689 } 1690 if (em & F_EDRAM1_ENABLE) { 1691 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1692 maddr = G_EDRAM1_BASE(addr_len) << 20; 1693 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1694 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1695 addr + len <= maddr + mlen) 1696 return (0); 1697 } 1698 if (em & F_EXT_MEM_ENABLE) { 1699 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1700 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1701 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1702 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1703 addr + len <= maddr + mlen) 1704 return (0); 1705 } 1706 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1707 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1708 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1709 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1710 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1711 addr + len <= maddr + mlen) 1712 return (0); 1713 } 1714 1715 return (EFAULT); 1716} 1717 1718static int 1719fwmtype_to_hwmtype(int mtype) 1720{ 1721 1722 switch (mtype) { 1723 case FW_MEMTYPE_EDC0: 1724 return (MEM_EDC0); 1725 case FW_MEMTYPE_EDC1: 1726 return (MEM_EDC1); 1727 case FW_MEMTYPE_EXTMEM: 1728 return (MEM_MC0); 1729 case FW_MEMTYPE_EXTMEM1: 1730 return (MEM_MC1); 1731 default: 1732 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1733 } 1734} 1735 1736/* 1737 * Verify that the memory range specified by the memtype/offset/len pair is 1738 * valid and lies entirely within the memtype specified. The global address of 1739 * the start of the range is returned in addr. 1740 */ 1741static int 1742validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1743 uint32_t *addr) 1744{ 1745 uint32_t em, addr_len, maddr, mlen; 1746 1747 /* Memory can only be accessed in naturally aligned 4 byte units */ 1748 if (off & 3 || len & 3 || len == 0) 1749 return (EINVAL); 1750 1751 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1752 switch (fwmtype_to_hwmtype(mtype)) { 1753 case MEM_EDC0: 1754 if (!(em & F_EDRAM0_ENABLE)) 1755 return (EINVAL); 1756 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1757 maddr = G_EDRAM0_BASE(addr_len) << 20; 1758 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1759 break; 1760 case MEM_EDC1: 1761 if (!(em & F_EDRAM1_ENABLE)) 1762 return (EINVAL); 1763 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1764 maddr = G_EDRAM1_BASE(addr_len) << 20; 1765 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1766 break; 1767 case MEM_MC: 1768 if (!(em & F_EXT_MEM_ENABLE)) 1769 return (EINVAL); 1770 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1771 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1772 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1773 break; 1774 case MEM_MC1: 1775 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1776 return (EINVAL); 1777 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1778 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1779 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1780 break; 1781 default: 1782 return (EINVAL); 1783 } 1784 1785 if (mlen > 0 && off < mlen && off + len <= mlen) { 1786 *addr = maddr + off; /* global address */ 1787 return (0); 1788 } 1789 1790 return (EFAULT); 1791} 1792 1793static void 1794memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1795{ 1796 const struct memwin *mw; 1797 1798 if (is_t4(sc)) { 1799 KASSERT(win >= 0 && win < nitems(t4_memwin), 1800 ("%s: incorrect memwin# (%d)", __func__, win)); 1801 mw = &t4_memwin[win]; 1802 } else { 1803 KASSERT(win >= 0 && win < nitems(t5_memwin), 1804 ("%s: incorrect memwin# (%d)", __func__, win)); 1805 mw = &t5_memwin[win]; 1806 } 1807 1808 if (base != NULL) 1809 *base = mw->base; 1810 if (aperture != NULL) 1811 *aperture = mw->aperture; 1812} 1813 1814/* 1815 * Positions the memory window such that it can be used to access the specified 1816 * address in the chip's address space. The return value is the offset of addr 1817 * from the start of the window. 1818 */ 1819static uint32_t 1820position_memwin(struct adapter *sc, int n, uint32_t addr) 1821{ 1822 uint32_t start, pf; 1823 uint32_t reg; 1824 1825 KASSERT(n >= 0 && n <= 3, 1826 ("%s: invalid window %d.", __func__, n)); 1827 KASSERT((addr & 3) == 0, 1828 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1829 1830 if (is_t4(sc)) { 1831 pf = 0; 1832 start = addr & ~0xf; /* start must be 16B aligned */ 1833 } else { 1834 pf = V_PFNUM(sc->pf); 1835 start = addr & ~0x7f; /* start must be 128B aligned */ 1836 } 1837 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1838 1839 t4_write_reg(sc, reg, start | pf); 1840 t4_read_reg(sc, reg); 1841 1842 return (addr - start); 1843} 1844 1845static int 1846cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1847 struct intrs_and_queues *iaq) 1848{ 1849 int rc, itype, navail, nrxq10g, nrxq1g, n; 1850 int nofldrxq10g = 0, nofldrxq1g = 0; 1851 int nnmrxq10g = 0, nnmrxq1g = 0; 1852 1853 bzero(iaq, sizeof(*iaq)); 1854 1855 iaq->ntxq10g = t4_ntxq10g; 1856 iaq->ntxq1g = t4_ntxq1g; 1857 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1858 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1859 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1860#ifdef TCP_OFFLOAD 1861 if (is_offload(sc)) { 1862 iaq->nofldtxq10g = t4_nofldtxq10g; 1863 iaq->nofldtxq1g = t4_nofldtxq1g; 1864 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1865 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1866 } 1867#endif 1868#ifdef DEV_NETMAP 1869 iaq->nnmtxq10g = t4_nnmtxq10g; 1870 iaq->nnmtxq1g = t4_nnmtxq1g; 1871 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1872 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1873#endif 1874 1875 for (itype = INTR_MSIX; itype; itype >>= 1) { 1876 1877 if ((itype & t4_intr_types) == 0) 1878 continue; /* not allowed */ 1879 1880 if (itype == INTR_MSIX) 1881 navail = pci_msix_count(sc->dev); 1882 else if (itype == INTR_MSI) 1883 navail = pci_msi_count(sc->dev); 1884 else 1885 navail = 1; 1886restart: 1887 if (navail == 0) 1888 continue; 1889 1890 iaq->intr_type = itype; 1891 iaq->intr_flags_10g = 0; 1892 iaq->intr_flags_1g = 0; 1893 1894 /* 1895 * Best option: an interrupt vector for errors, one for the 1896 * firmware event queue, and one for every rxq (NIC, TOE, and 1897 * netmap). 1898 */ 1899 iaq->nirq = T4_EXTRA_INTR; 1900 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1901 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1902 if (iaq->nirq <= navail && 1903 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1904 iaq->intr_flags_10g = INTR_ALL; 1905 iaq->intr_flags_1g = INTR_ALL; 1906 goto allocate; 1907 } 1908 1909 /* 1910 * Second best option: a vector for errors, one for the firmware 1911 * event queue, and vectors for either all the NIC rx queues or 1912 * all the TOE rx queues. The queues that don't get vectors 1913 * will forward their interrupts to those that do. 1914 * 1915 * Note: netmap rx queues cannot be created early and so they 1916 * can't be setup to receive forwarded interrupts for others. 1917 */ 1918 iaq->nirq = T4_EXTRA_INTR; 1919 if (nrxq10g >= nofldrxq10g) { 1920 iaq->intr_flags_10g = INTR_RXQ; 1921 iaq->nirq += n10g * nrxq10g; 1922#ifdef DEV_NETMAP 1923 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1924#endif 1925 } else { 1926 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1927 iaq->nirq += n10g * nofldrxq10g; 1928#ifdef DEV_NETMAP 1929 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1930#endif 1931 } 1932 if (nrxq1g >= nofldrxq1g) { 1933 iaq->intr_flags_1g = INTR_RXQ; 1934 iaq->nirq += n1g * nrxq1g; 1935#ifdef DEV_NETMAP 1936 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1937#endif 1938 } else { 1939 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1940 iaq->nirq += n1g * nofldrxq1g; 1941#ifdef DEV_NETMAP 1942 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1943#endif 1944 } 1945 if (iaq->nirq <= navail && 1946 (itype != INTR_MSI || powerof2(iaq->nirq))) 1947 goto allocate; 1948 1949 /* 1950 * Next best option: an interrupt vector for errors, one for the 1951 * firmware event queue, and at least one per port. At this 1952 * point we know we'll have to downsize nrxq and/or nofldrxq 1953 * and/or nnmrxq to fit what's available to us. 1954 */ 1955 iaq->nirq = T4_EXTRA_INTR; 1956 iaq->nirq += n10g + n1g; 1957 if (iaq->nirq <= navail) { 1958 int leftover = navail - iaq->nirq; 1959 1960 if (n10g > 0) { 1961 int target = max(nrxq10g, nofldrxq10g); 1962 1963 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 1964 INTR_RXQ : INTR_OFLD_RXQ; 1965 1966 n = 1; 1967 while (n < target && leftover >= n10g) { 1968 leftover -= n10g; 1969 iaq->nirq += n10g; 1970 n++; 1971 } 1972 iaq->nrxq10g = min(n, nrxq10g); 1973#ifdef TCP_OFFLOAD 1974 iaq->nofldrxq10g = min(n, nofldrxq10g); 1975#endif 1976#ifdef DEV_NETMAP 1977 iaq->nnmrxq10g = min(n, nnmrxq10g); 1978#endif 1979 } 1980 1981 if (n1g > 0) { 1982 int target = max(nrxq1g, nofldrxq1g); 1983 1984 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 1985 INTR_RXQ : INTR_OFLD_RXQ; 1986 1987 n = 1; 1988 while (n < target && leftover >= n1g) { 1989 leftover -= n1g; 1990 iaq->nirq += n1g; 1991 n++; 1992 } 1993 iaq->nrxq1g = min(n, nrxq1g); 1994#ifdef TCP_OFFLOAD 1995 iaq->nofldrxq1g = min(n, nofldrxq1g); 1996#endif 1997#ifdef DEV_NETMAP 1998 iaq->nnmrxq1g = min(n, nnmrxq1g); 1999#endif 2000 } 2001 2002 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2003 goto allocate; 2004 } 2005 2006 /* 2007 * Least desirable option: one interrupt vector for everything. 2008 */ 2009 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2010 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2011#ifdef TCP_OFFLOAD 2012 if (is_offload(sc)) 2013 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2014#endif 2015#ifdef DEV_NETMAP 2016 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2017#endif 2018 2019allocate: 2020 navail = iaq->nirq; 2021 rc = 0; 2022 if (itype == INTR_MSIX) 2023 rc = pci_alloc_msix(sc->dev, &navail); 2024 else if (itype == INTR_MSI) 2025 rc = pci_alloc_msi(sc->dev, &navail); 2026 2027 if (rc == 0) { 2028 if (navail == iaq->nirq) 2029 return (0); 2030 2031 /* 2032 * Didn't get the number requested. Use whatever number 2033 * the kernel is willing to allocate (it's in navail). 2034 */ 2035 device_printf(sc->dev, "fewer vectors than requested, " 2036 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2037 itype, iaq->nirq, navail); 2038 pci_release_msi(sc->dev); 2039 goto restart; 2040 } 2041 2042 device_printf(sc->dev, 2043 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2044 itype, rc, iaq->nirq, navail); 2045 } 2046 2047 device_printf(sc->dev, 2048 "failed to find a usable interrupt type. " 2049 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2050 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2051 2052 return (ENXIO); 2053} 2054 2055#define FW_VERSION(chip) ( \ 2056 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2057 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2058 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2059 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2060#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2061 2062struct fw_info { 2063 uint8_t chip; 2064 char *kld_name; 2065 char *fw_mod_name; 2066 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2067} fw_info[] = { 2068 { 2069 .chip = CHELSIO_T4, 2070 .kld_name = "t4fw_cfg", 2071 .fw_mod_name = "t4fw", 2072 .fw_hdr = { 2073 .chip = FW_HDR_CHIP_T4, 2074 .fw_ver = htobe32_const(FW_VERSION(T4)), 2075 .intfver_nic = FW_INTFVER(T4, NIC), 2076 .intfver_vnic = FW_INTFVER(T4, VNIC), 2077 .intfver_ofld = FW_INTFVER(T4, OFLD), 2078 .intfver_ri = FW_INTFVER(T4, RI), 2079 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2080 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2081 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2082 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2083 }, 2084 }, { 2085 .chip = CHELSIO_T5, 2086 .kld_name = "t5fw_cfg", 2087 .fw_mod_name = "t5fw", 2088 .fw_hdr = { 2089 .chip = FW_HDR_CHIP_T5, 2090 .fw_ver = htobe32_const(FW_VERSION(T5)), 2091 .intfver_nic = FW_INTFVER(T5, NIC), 2092 .intfver_vnic = FW_INTFVER(T5, VNIC), 2093 .intfver_ofld = FW_INTFVER(T5, OFLD), 2094 .intfver_ri = FW_INTFVER(T5, RI), 2095 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2096 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2097 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2098 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2099 }, 2100 } 2101}; 2102 2103static struct fw_info * 2104find_fw_info(int chip) 2105{ 2106 int i; 2107 2108 for (i = 0; i < nitems(fw_info); i++) { 2109 if (fw_info[i].chip == chip) 2110 return (&fw_info[i]); 2111 } 2112 return (NULL); 2113} 2114 2115/* 2116 * Is the given firmware API compatible with the one the driver was compiled 2117 * with? 2118 */ 2119static int 2120fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2121{ 2122 2123 /* short circuit if it's the exact same firmware version */ 2124 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2125 return (1); 2126 2127 /* 2128 * XXX: Is this too conservative? Perhaps I should limit this to the 2129 * features that are supported in the driver. 2130 */ 2131#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2132 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2133 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2134 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2135 return (1); 2136#undef SAME_INTF 2137 2138 return (0); 2139} 2140 2141/* 2142 * The firmware in the KLD is usable, but should it be installed? This routine 2143 * explains itself in detail if it indicates the KLD firmware should be 2144 * installed. 2145 */ 2146static int 2147should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2148{ 2149 const char *reason; 2150 2151 if (!card_fw_usable) { 2152 reason = "incompatible or unusable"; 2153 goto install; 2154 } 2155 2156 if (k > c) { 2157 reason = "older than the version bundled with this driver"; 2158 goto install; 2159 } 2160 2161 if (t4_fw_install == 2 && k != c) { 2162 reason = "different than the version bundled with this driver"; 2163 goto install; 2164 } 2165 2166 return (0); 2167 2168install: 2169 if (t4_fw_install == 0) { 2170 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2171 "but the driver is prohibited from installing a different " 2172 "firmware on the card.\n", 2173 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2174 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2175 2176 return (0); 2177 } 2178 2179 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2180 "installing firmware %u.%u.%u.%u on card.\n", 2181 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2182 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2183 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2184 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2185 2186 return (1); 2187} 2188/* 2189 * Establish contact with the firmware and determine if we are the master driver 2190 * or not, and whether we are responsible for chip initialization. 2191 */ 2192static int 2193prep_firmware(struct adapter *sc) 2194{ 2195 const struct firmware *fw = NULL, *default_cfg; 2196 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2197 enum dev_state state; 2198 struct fw_info *fw_info; 2199 struct fw_hdr *card_fw; /* fw on the card */ 2200 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2201 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2202 against */ 2203 2204 /* Contact firmware. */ 2205 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2206 if (rc < 0 || state == DEV_STATE_ERR) { 2207 rc = -rc; 2208 device_printf(sc->dev, 2209 "failed to connect to the firmware: %d, %d.\n", rc, state); 2210 return (rc); 2211 } 2212 pf = rc; 2213 if (pf == sc->mbox) 2214 sc->flags |= MASTER_PF; 2215 else if (state == DEV_STATE_UNINIT) { 2216 /* 2217 * We didn't get to be the master so we definitely won't be 2218 * configuring the chip. It's a bug if someone else hasn't 2219 * configured it already. 2220 */ 2221 device_printf(sc->dev, "couldn't be master(%d), " 2222 "device not already initialized either(%d).\n", rc, state); 2223 return (EDOOFUS); 2224 } 2225 2226 /* This is the firmware whose headers the driver was compiled against */ 2227 fw_info = find_fw_info(chip_id(sc)); 2228 if (fw_info == NULL) { 2229 device_printf(sc->dev, 2230 "unable to look up firmware information for chip %d.\n", 2231 chip_id(sc)); 2232 return (EINVAL); 2233 } 2234 drv_fw = &fw_info->fw_hdr; 2235 2236 /* 2237 * The firmware KLD contains many modules. The KLD name is also the 2238 * name of the module that contains the default config file. 2239 */ 2240 default_cfg = firmware_get(fw_info->kld_name); 2241 2242 /* Read the header of the firmware on the card */ 2243 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2244 rc = -t4_read_flash(sc, FLASH_FW_START, 2245 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2246 if (rc == 0) 2247 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2248 else { 2249 device_printf(sc->dev, 2250 "Unable to read card's firmware header: %d\n", rc); 2251 card_fw_usable = 0; 2252 } 2253 2254 /* This is the firmware in the KLD */ 2255 fw = firmware_get(fw_info->fw_mod_name); 2256 if (fw != NULL) { 2257 kld_fw = (const void *)fw->data; 2258 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2259 } else { 2260 kld_fw = NULL; 2261 kld_fw_usable = 0; 2262 } 2263 2264 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2265 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2266 /* 2267 * Common case: the firmware on the card is an exact match and 2268 * the KLD is an exact match too, or the KLD is 2269 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2270 * here -- use cxgbetool loadfw if you want to reinstall the 2271 * same firmware as the one on the card. 2272 */ 2273 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2274 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2275 be32toh(card_fw->fw_ver))) { 2276 2277 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2278 if (rc != 0) { 2279 device_printf(sc->dev, 2280 "failed to install firmware: %d\n", rc); 2281 goto done; 2282 } 2283 2284 /* Installed successfully, update the cached header too. */ 2285 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2286 card_fw_usable = 1; 2287 need_fw_reset = 0; /* already reset as part of load_fw */ 2288 } 2289 2290 if (!card_fw_usable) { 2291 uint32_t d, c, k; 2292 2293 d = ntohl(drv_fw->fw_ver); 2294 c = ntohl(card_fw->fw_ver); 2295 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2296 2297 device_printf(sc->dev, "Cannot find a usable firmware: " 2298 "fw_install %d, chip state %d, " 2299 "driver compiled with %d.%d.%d.%d, " 2300 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2301 t4_fw_install, state, 2302 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2303 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2304 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2305 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2306 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2307 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2308 rc = EINVAL; 2309 goto done; 2310 } 2311 2312 /* We're using whatever's on the card and it's known to be good. */ 2313 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2314 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2315 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2316 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2317 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2318 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2319 t4_get_tp_version(sc, &sc->params.tp_vers); 2320 2321 /* Reset device */ 2322 if (need_fw_reset && 2323 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2324 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2325 if (rc != ETIMEDOUT && rc != EIO) 2326 t4_fw_bye(sc, sc->mbox); 2327 goto done; 2328 } 2329 sc->flags |= FW_OK; 2330 2331 rc = get_params__pre_init(sc); 2332 if (rc != 0) 2333 goto done; /* error message displayed already */ 2334 2335 /* Partition adapter resources as specified in the config file. */ 2336 if (state == DEV_STATE_UNINIT) { 2337 2338 KASSERT(sc->flags & MASTER_PF, 2339 ("%s: trying to change chip settings when not master.", 2340 __func__)); 2341 2342 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2343 if (rc != 0) 2344 goto done; /* error message displayed already */ 2345 2346 t4_tweak_chip_settings(sc); 2347 2348 /* get basic stuff going */ 2349 rc = -t4_fw_initialize(sc, sc->mbox); 2350 if (rc != 0) { 2351 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2352 goto done; 2353 } 2354 } else { 2355 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2356 sc->cfcsum = 0; 2357 } 2358 2359done: 2360 free(card_fw, M_CXGBE); 2361 if (fw != NULL) 2362 firmware_put(fw, FIRMWARE_UNLOAD); 2363 if (default_cfg != NULL) 2364 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2365 2366 return (rc); 2367} 2368 2369#define FW_PARAM_DEV(param) \ 2370 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2371 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2372#define FW_PARAM_PFVF(param) \ 2373 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2374 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2375 2376/* 2377 * Partition chip resources for use between various PFs, VFs, etc. 2378 */ 2379static int 2380partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2381 const char *name_prefix) 2382{ 2383 const struct firmware *cfg = NULL; 2384 int rc = 0; 2385 struct fw_caps_config_cmd caps; 2386 uint32_t mtype, moff, finicsum, cfcsum; 2387 2388 /* 2389 * Figure out what configuration file to use. Pick the default config 2390 * file for the card if the user hasn't specified one explicitly. 2391 */ 2392 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2393 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2394 /* Card specific overrides go here. */ 2395 if (pci_get_device(sc->dev) == 0x440a) 2396 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2397 if (is_fpga(sc)) 2398 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2399 } 2400 2401 /* 2402 * We need to load another module if the profile is anything except 2403 * "default" or "flash". 2404 */ 2405 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2406 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2407 char s[32]; 2408 2409 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2410 cfg = firmware_get(s); 2411 if (cfg == NULL) { 2412 if (default_cfg != NULL) { 2413 device_printf(sc->dev, 2414 "unable to load module \"%s\" for " 2415 "configuration profile \"%s\", will use " 2416 "the default config file instead.\n", 2417 s, sc->cfg_file); 2418 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2419 "%s", DEFAULT_CF); 2420 } else { 2421 device_printf(sc->dev, 2422 "unable to load module \"%s\" for " 2423 "configuration profile \"%s\", will use " 2424 "the config file on the card's flash " 2425 "instead.\n", s, sc->cfg_file); 2426 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2427 "%s", FLASH_CF); 2428 } 2429 } 2430 } 2431 2432 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2433 default_cfg == NULL) { 2434 device_printf(sc->dev, 2435 "default config file not available, will use the config " 2436 "file on the card's flash instead.\n"); 2437 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2438 } 2439 2440 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2441 u_int cflen, i, n; 2442 const uint32_t *cfdata; 2443 uint32_t param, val, addr, off, mw_base, mw_aperture; 2444 2445 KASSERT(cfg != NULL || default_cfg != NULL, 2446 ("%s: no config to upload", __func__)); 2447 2448 /* 2449 * Ask the firmware where it wants us to upload the config file. 2450 */ 2451 param = FW_PARAM_DEV(CF); 2452 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2453 if (rc != 0) { 2454 /* No support for config file? Shouldn't happen. */ 2455 device_printf(sc->dev, 2456 "failed to query config file location: %d.\n", rc); 2457 goto done; 2458 } 2459 mtype = G_FW_PARAMS_PARAM_Y(val); 2460 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2461 2462 /* 2463 * XXX: sheer laziness. We deliberately added 4 bytes of 2464 * useless stuffing/comments at the end of the config file so 2465 * it's ok to simply throw away the last remaining bytes when 2466 * the config file is not an exact multiple of 4. This also 2467 * helps with the validate_mt_off_len check. 2468 */ 2469 if (cfg != NULL) { 2470 cflen = cfg->datasize & ~3; 2471 cfdata = cfg->data; 2472 } else { 2473 cflen = default_cfg->datasize & ~3; 2474 cfdata = default_cfg->data; 2475 } 2476 2477 if (cflen > FLASH_CFG_MAX_SIZE) { 2478 device_printf(sc->dev, 2479 "config file too long (%d, max allowed is %d). " 2480 "Will try to use the config on the card, if any.\n", 2481 cflen, FLASH_CFG_MAX_SIZE); 2482 goto use_config_on_flash; 2483 } 2484 2485 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2486 if (rc != 0) { 2487 device_printf(sc->dev, 2488 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2489 "Will try to use the config on the card, if any.\n", 2490 __func__, mtype, moff, cflen, rc); 2491 goto use_config_on_flash; 2492 } 2493 2494 memwin_info(sc, 2, &mw_base, &mw_aperture); 2495 while (cflen) { 2496 off = position_memwin(sc, 2, addr); 2497 n = min(cflen, mw_aperture - off); 2498 for (i = 0; i < n; i += 4) 2499 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2500 cflen -= n; 2501 addr += n; 2502 } 2503 } else { 2504use_config_on_flash: 2505 mtype = FW_MEMTYPE_FLASH; 2506 moff = t4_flash_cfg_addr(sc); 2507 } 2508 2509 bzero(&caps, sizeof(caps)); 2510 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2511 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2512 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2513 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2514 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2515 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2516 if (rc != 0) { 2517 device_printf(sc->dev, 2518 "failed to pre-process config file: %d " 2519 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2520 goto done; 2521 } 2522 2523 finicsum = be32toh(caps.finicsum); 2524 cfcsum = be32toh(caps.cfcsum); 2525 if (finicsum != cfcsum) { 2526 device_printf(sc->dev, 2527 "WARNING: config file checksum mismatch: %08x %08x\n", 2528 finicsum, cfcsum); 2529 } 2530 sc->cfcsum = cfcsum; 2531 2532#define LIMIT_CAPS(x) do { \ 2533 caps.x &= htobe16(t4_##x##_allowed); \ 2534} while (0) 2535 2536 /* 2537 * Let the firmware know what features will (not) be used so it can tune 2538 * things accordingly. 2539 */ 2540 LIMIT_CAPS(linkcaps); 2541 LIMIT_CAPS(niccaps); 2542 LIMIT_CAPS(toecaps); 2543 LIMIT_CAPS(rdmacaps); 2544 LIMIT_CAPS(iscsicaps); 2545 LIMIT_CAPS(fcoecaps); 2546#undef LIMIT_CAPS 2547 2548 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2549 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2550 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2551 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2552 if (rc != 0) { 2553 device_printf(sc->dev, 2554 "failed to process config file: %d.\n", rc); 2555 } 2556done: 2557 if (cfg != NULL) 2558 firmware_put(cfg, FIRMWARE_UNLOAD); 2559 return (rc); 2560} 2561 2562/* 2563 * Retrieve parameters that are needed (or nice to have) very early. 2564 */ 2565static int 2566get_params__pre_init(struct adapter *sc) 2567{ 2568 int rc; 2569 uint32_t param[2], val[2]; 2570 struct fw_devlog_cmd cmd; 2571 struct devlog_params *dlog = &sc->params.devlog; 2572 2573 param[0] = FW_PARAM_DEV(PORTVEC); 2574 param[1] = FW_PARAM_DEV(CCLK); 2575 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2576 if (rc != 0) { 2577 device_printf(sc->dev, 2578 "failed to query parameters (pre_init): %d.\n", rc); 2579 return (rc); 2580 } 2581 2582 sc->params.portvec = val[0]; 2583 sc->params.nports = bitcount32(val[0]); 2584 sc->params.vpd.cclk = val[1]; 2585 2586 /* Read device log parameters. */ 2587 bzero(&cmd, sizeof(cmd)); 2588 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2589 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2590 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2591 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2592 if (rc != 0) { 2593 device_printf(sc->dev, 2594 "failed to get devlog parameters: %d.\n", rc); 2595 bzero(dlog, sizeof (*dlog)); 2596 rc = 0; /* devlog isn't critical for device operation */ 2597 } else { 2598 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2599 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2600 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2601 dlog->size = be32toh(cmd.memsize_devlog); 2602 } 2603 2604 return (rc); 2605} 2606 2607/* 2608 * Retrieve various parameters that are of interest to the driver. The device 2609 * has been initialized by the firmware at this point. 2610 */ 2611static int 2612get_params__post_init(struct adapter *sc) 2613{ 2614 int rc; 2615 uint32_t param[7], val[7]; 2616 struct fw_caps_config_cmd caps; 2617 2618 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2619 param[1] = FW_PARAM_PFVF(EQ_START); 2620 param[2] = FW_PARAM_PFVF(FILTER_START); 2621 param[3] = FW_PARAM_PFVF(FILTER_END); 2622 param[4] = FW_PARAM_PFVF(L2T_START); 2623 param[5] = FW_PARAM_PFVF(L2T_END); 2624 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2625 if (rc != 0) { 2626 device_printf(sc->dev, 2627 "failed to query parameters (post_init): %d.\n", rc); 2628 return (rc); 2629 } 2630 2631 sc->sge.iq_start = val[0]; 2632 sc->sge.eq_start = val[1]; 2633 sc->tids.ftid_base = val[2]; 2634 sc->tids.nftids = val[3] - val[2] + 1; 2635 sc->params.ftid_min = val[2]; 2636 sc->params.ftid_max = val[3]; 2637 sc->vres.l2t.start = val[4]; 2638 sc->vres.l2t.size = val[5] - val[4] + 1; 2639 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2640 ("%s: L2 table size (%u) larger than expected (%u)", 2641 __func__, sc->vres.l2t.size, L2T_SIZE)); 2642 2643 /* get capabilites */ 2644 bzero(&caps, sizeof(caps)); 2645 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2646 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2647 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2648 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2649 if (rc != 0) { 2650 device_printf(sc->dev, 2651 "failed to get card capabilities: %d.\n", rc); 2652 return (rc); 2653 } 2654 2655#define READ_CAPS(x) do { \ 2656 sc->x = htobe16(caps.x); \ 2657} while (0) 2658 READ_CAPS(linkcaps); 2659 READ_CAPS(niccaps); 2660 READ_CAPS(toecaps); 2661 READ_CAPS(rdmacaps); 2662 READ_CAPS(iscsicaps); 2663 READ_CAPS(fcoecaps); 2664 2665 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2666 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2667 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2668 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2669 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2670 if (rc != 0) { 2671 device_printf(sc->dev, 2672 "failed to query NIC parameters: %d.\n", rc); 2673 return (rc); 2674 } 2675 sc->tids.etid_base = val[0]; 2676 sc->params.etid_min = val[0]; 2677 sc->tids.netids = val[1] - val[0] + 1; 2678 sc->params.netids = sc->tids.netids; 2679 sc->params.eo_wr_cred = val[2]; 2680 sc->params.ethoffload = 1; 2681 } 2682 2683 if (sc->toecaps) { 2684 /* query offload-related parameters */ 2685 param[0] = FW_PARAM_DEV(NTID); 2686 param[1] = FW_PARAM_PFVF(SERVER_START); 2687 param[2] = FW_PARAM_PFVF(SERVER_END); 2688 param[3] = FW_PARAM_PFVF(TDDP_START); 2689 param[4] = FW_PARAM_PFVF(TDDP_END); 2690 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2691 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2692 if (rc != 0) { 2693 device_printf(sc->dev, 2694 "failed to query TOE parameters: %d.\n", rc); 2695 return (rc); 2696 } 2697 sc->tids.ntids = val[0]; 2698 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2699 sc->tids.stid_base = val[1]; 2700 sc->tids.nstids = val[2] - val[1] + 1; 2701 sc->vres.ddp.start = val[3]; 2702 sc->vres.ddp.size = val[4] - val[3] + 1; 2703 sc->params.ofldq_wr_cred = val[5]; 2704 sc->params.offload = 1; 2705 } 2706 if (sc->rdmacaps) { 2707 param[0] = FW_PARAM_PFVF(STAG_START); 2708 param[1] = FW_PARAM_PFVF(STAG_END); 2709 param[2] = FW_PARAM_PFVF(RQ_START); 2710 param[3] = FW_PARAM_PFVF(RQ_END); 2711 param[4] = FW_PARAM_PFVF(PBL_START); 2712 param[5] = FW_PARAM_PFVF(PBL_END); 2713 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2714 if (rc != 0) { 2715 device_printf(sc->dev, 2716 "failed to query RDMA parameters(1): %d.\n", rc); 2717 return (rc); 2718 } 2719 sc->vres.stag.start = val[0]; 2720 sc->vres.stag.size = val[1] - val[0] + 1; 2721 sc->vres.rq.start = val[2]; 2722 sc->vres.rq.size = val[3] - val[2] + 1; 2723 sc->vres.pbl.start = val[4]; 2724 sc->vres.pbl.size = val[5] - val[4] + 1; 2725 2726 param[0] = FW_PARAM_PFVF(SQRQ_START); 2727 param[1] = FW_PARAM_PFVF(SQRQ_END); 2728 param[2] = FW_PARAM_PFVF(CQ_START); 2729 param[3] = FW_PARAM_PFVF(CQ_END); 2730 param[4] = FW_PARAM_PFVF(OCQ_START); 2731 param[5] = FW_PARAM_PFVF(OCQ_END); 2732 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2733 if (rc != 0) { 2734 device_printf(sc->dev, 2735 "failed to query RDMA parameters(2): %d.\n", rc); 2736 return (rc); 2737 } 2738 sc->vres.qp.start = val[0]; 2739 sc->vres.qp.size = val[1] - val[0] + 1; 2740 sc->vres.cq.start = val[2]; 2741 sc->vres.cq.size = val[3] - val[2] + 1; 2742 sc->vres.ocq.start = val[4]; 2743 sc->vres.ocq.size = val[5] - val[4] + 1; 2744 } 2745 if (sc->iscsicaps) { 2746 param[0] = FW_PARAM_PFVF(ISCSI_START); 2747 param[1] = FW_PARAM_PFVF(ISCSI_END); 2748 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2749 if (rc != 0) { 2750 device_printf(sc->dev, 2751 "failed to query iSCSI parameters: %d.\n", rc); 2752 return (rc); 2753 } 2754 sc->vres.iscsi.start = val[0]; 2755 sc->vres.iscsi.size = val[1] - val[0] + 1; 2756 } 2757 2758 /* 2759 * We've got the params we wanted to query via the firmware. Now grab 2760 * some others directly from the chip. 2761 */ 2762 rc = t4_read_chip_settings(sc); 2763 2764 return (rc); 2765} 2766 2767static int 2768set_params__post_init(struct adapter *sc) 2769{ 2770 uint32_t param, val; 2771 2772 /* ask for encapsulated CPLs */ 2773 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2774 val = 1; 2775 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2776 2777 return (0); 2778} 2779 2780#undef FW_PARAM_PFVF 2781#undef FW_PARAM_DEV 2782 2783static void 2784t4_set_desc(struct adapter *sc) 2785{ 2786 char buf[128]; 2787 struct adapter_params *p = &sc->params; 2788 2789 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2790 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2791 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2792 2793 device_set_desc_copy(sc->dev, buf); 2794} 2795 2796static void 2797build_medialist(struct port_info *pi, struct ifmedia *media) 2798{ 2799 int data, m; 2800 2801 PORT_LOCK(pi); 2802 2803 ifmedia_removeall(media); 2804 2805 m = IFM_ETHER | IFM_FDX; 2806 data = (pi->port_type << 8) | pi->mod_type; 2807 2808 switch(pi->port_type) { 2809 case FW_PORT_TYPE_BT_XFI: 2810 case FW_PORT_TYPE_BT_XAUI: 2811 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2812 /* fall through */ 2813 2814 case FW_PORT_TYPE_BT_SGMII: 2815 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2816 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2817 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2818 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2819 break; 2820 2821 case FW_PORT_TYPE_CX4: 2822 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2823 ifmedia_set(media, m | IFM_10G_CX4); 2824 break; 2825 2826 case FW_PORT_TYPE_QSFP_10G: 2827 case FW_PORT_TYPE_SFP: 2828 case FW_PORT_TYPE_FIBER_XFI: 2829 case FW_PORT_TYPE_FIBER_XAUI: 2830 switch (pi->mod_type) { 2831 2832 case FW_PORT_MOD_TYPE_LR: 2833 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2834 ifmedia_set(media, m | IFM_10G_LR); 2835 break; 2836 2837 case FW_PORT_MOD_TYPE_SR: 2838 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2839 ifmedia_set(media, m | IFM_10G_SR); 2840 break; 2841 2842 case FW_PORT_MOD_TYPE_LRM: 2843 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2844 ifmedia_set(media, m | IFM_10G_LRM); 2845 break; 2846 2847 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2848 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2849 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2850 ifmedia_set(media, m | IFM_10G_TWINAX); 2851 break; 2852 2853 case FW_PORT_MOD_TYPE_NONE: 2854 m &= ~IFM_FDX; 2855 ifmedia_add(media, m | IFM_NONE, data, NULL); 2856 ifmedia_set(media, m | IFM_NONE); 2857 break; 2858 2859 case FW_PORT_MOD_TYPE_NA: 2860 case FW_PORT_MOD_TYPE_ER: 2861 default: 2862 device_printf(pi->dev, 2863 "unknown port_type (%d), mod_type (%d)\n", 2864 pi->port_type, pi->mod_type); 2865 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2866 ifmedia_set(media, m | IFM_UNKNOWN); 2867 break; 2868 } 2869 break; 2870 2871 case FW_PORT_TYPE_QSFP: 2872 switch (pi->mod_type) { 2873 2874 case FW_PORT_MOD_TYPE_LR: 2875 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2876 ifmedia_set(media, m | IFM_40G_LR4); 2877 break; 2878 2879 case FW_PORT_MOD_TYPE_SR: 2880 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2881 ifmedia_set(media, m | IFM_40G_SR4); 2882 break; 2883 2884 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2885 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2886 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2887 ifmedia_set(media, m | IFM_40G_CR4); 2888 break; 2889 2890 case FW_PORT_MOD_TYPE_NONE: 2891 m &= ~IFM_FDX; 2892 ifmedia_add(media, m | IFM_NONE, data, NULL); 2893 ifmedia_set(media, m | IFM_NONE); 2894 break; 2895 2896 default: 2897 device_printf(pi->dev, 2898 "unknown port_type (%d), mod_type (%d)\n", 2899 pi->port_type, pi->mod_type); 2900 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2901 ifmedia_set(media, m | IFM_UNKNOWN); 2902 break; 2903 } 2904 break; 2905 2906 default: 2907 device_printf(pi->dev, 2908 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2909 pi->mod_type); 2910 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2911 ifmedia_set(media, m | IFM_UNKNOWN); 2912 break; 2913 } 2914 2915 PORT_UNLOCK(pi); 2916} 2917 2918#define FW_MAC_EXACT_CHUNK 7 2919 2920/* 2921 * Program the port's XGMAC based on parameters in ifnet. The caller also 2922 * indicates which parameters should be programmed (the rest are left alone). 2923 */ 2924int 2925update_mac_settings(struct ifnet *ifp, int flags) 2926{ 2927 int rc = 0; 2928 struct port_info *pi = ifp->if_softc; 2929 struct adapter *sc = pi->adapter; 2930 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2931 uint16_t viid = 0xffff; 2932 int16_t *xact_addr_filt = NULL; 2933 2934 ASSERT_SYNCHRONIZED_OP(sc); 2935 KASSERT(flags, ("%s: not told what to update.", __func__)); 2936 2937 if (ifp == pi->ifp) { 2938 viid = pi->viid; 2939 xact_addr_filt = &pi->xact_addr_filt; 2940 } 2941#ifdef DEV_NETMAP 2942 else if (ifp == pi->nm_ifp) { 2943 viid = pi->nm_viid; 2944 xact_addr_filt = &pi->nm_xact_addr_filt; 2945 } 2946#endif 2947 if (flags & XGMAC_MTU) 2948 mtu = ifp->if_mtu; 2949 2950 if (flags & XGMAC_PROMISC) 2951 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2952 2953 if (flags & XGMAC_ALLMULTI) 2954 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2955 2956 if (flags & XGMAC_VLANEX) 2957 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2958 2959 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 2960 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 2961 1, vlanex, false); 2962 if (rc) { 2963 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 2964 rc); 2965 return (rc); 2966 } 2967 } 2968 2969 if (flags & XGMAC_UCADDR) { 2970 uint8_t ucaddr[ETHER_ADDR_LEN]; 2971 2972 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2973 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 2974 true, true); 2975 if (rc < 0) { 2976 rc = -rc; 2977 if_printf(ifp, "change_mac failed: %d\n", rc); 2978 return (rc); 2979 } else { 2980 *xact_addr_filt = rc; 2981 rc = 0; 2982 } 2983 } 2984 2985 if (flags & XGMAC_MCADDRS) { 2986 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2987 int del = 1; 2988 uint64_t hash = 0; 2989 struct ifmultiaddr *ifma; 2990 int i = 0, j; 2991 2992 if_maddr_rlock(ifp); 2993 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2994 if (ifma->ifma_addr->sa_family != AF_LINK) 2995 continue; 2996 mcaddr[i++] = 2997 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2998 2999 if (i == FW_MAC_EXACT_CHUNK) { 3000 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3001 i, mcaddr, NULL, &hash, 0); 3002 if (rc < 0) { 3003 rc = -rc; 3004 for (j = 0; j < i; j++) { 3005 if_printf(ifp, 3006 "failed to add mc address" 3007 " %02x:%02x:%02x:" 3008 "%02x:%02x:%02x rc=%d\n", 3009 mcaddr[j][0], mcaddr[j][1], 3010 mcaddr[j][2], mcaddr[j][3], 3011 mcaddr[j][4], mcaddr[j][5], 3012 rc); 3013 } 3014 goto mcfail; 3015 } 3016 del = 0; 3017 i = 0; 3018 } 3019 } 3020 if (i > 0) { 3021 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3022 mcaddr, NULL, &hash, 0); 3023 if (rc < 0) { 3024 rc = -rc; 3025 for (j = 0; j < i; j++) { 3026 if_printf(ifp, 3027 "failed to add mc address" 3028 " %02x:%02x:%02x:" 3029 "%02x:%02x:%02x rc=%d\n", 3030 mcaddr[j][0], mcaddr[j][1], 3031 mcaddr[j][2], mcaddr[j][3], 3032 mcaddr[j][4], mcaddr[j][5], 3033 rc); 3034 } 3035 goto mcfail; 3036 } 3037 } 3038 3039 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3040 if (rc != 0) 3041 if_printf(ifp, "failed to set mc address hash: %d", rc); 3042mcfail: 3043 if_maddr_runlock(ifp); 3044 } 3045 3046 return (rc); 3047} 3048 3049/* 3050 * {begin|end}_synchronized_op must be called from the same thread. 3051 */ 3052int 3053begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3054 char *wmesg) 3055{ 3056 int rc, pri; 3057 3058#ifdef WITNESS 3059 /* the caller thinks it's ok to sleep, but is it really? */ 3060 if (flags & SLEEP_OK) 3061 pause("t4slptst", 1); 3062#endif 3063 3064 if (INTR_OK) 3065 pri = PCATCH; 3066 else 3067 pri = 0; 3068 3069 ADAPTER_LOCK(sc); 3070 for (;;) { 3071 3072 if (pi && IS_DOOMED(pi)) { 3073 rc = ENXIO; 3074 goto done; 3075 } 3076 3077 if (!IS_BUSY(sc)) { 3078 rc = 0; 3079 break; 3080 } 3081 3082 if (!(flags & SLEEP_OK)) { 3083 rc = EBUSY; 3084 goto done; 3085 } 3086 3087 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3088 rc = EINTR; 3089 goto done; 3090 } 3091 } 3092 3093 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3094 SET_BUSY(sc); 3095#ifdef INVARIANTS 3096 sc->last_op = wmesg; 3097 sc->last_op_thr = curthread; 3098#endif 3099 3100done: 3101 if (!(flags & HOLD_LOCK) || rc) 3102 ADAPTER_UNLOCK(sc); 3103 3104 return (rc); 3105} 3106 3107/* 3108 * {begin|end}_synchronized_op must be called from the same thread. 3109 */ 3110void 3111end_synchronized_op(struct adapter *sc, int flags) 3112{ 3113 3114 if (flags & LOCK_HELD) 3115 ADAPTER_LOCK_ASSERT_OWNED(sc); 3116 else 3117 ADAPTER_LOCK(sc); 3118 3119 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3120 CLR_BUSY(sc); 3121 wakeup(&sc->flags); 3122 ADAPTER_UNLOCK(sc); 3123} 3124 3125static int 3126cxgbe_init_synchronized(struct port_info *pi) 3127{ 3128 struct adapter *sc = pi->adapter; 3129 struct ifnet *ifp = pi->ifp; 3130 int rc = 0; 3131 3132 ASSERT_SYNCHRONIZED_OP(sc); 3133 3134 if (isset(&sc->open_device_map, pi->port_id)) { 3135 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3136 ("mismatch between open_device_map and if_drv_flags")); 3137 return (0); /* already running */ 3138 } 3139 3140 if (!(sc->flags & FULL_INIT_DONE) && 3141 ((rc = adapter_full_init(sc)) != 0)) 3142 return (rc); /* error message displayed already */ 3143 3144 if (!(pi->flags & PORT_INIT_DONE) && 3145 ((rc = port_full_init(pi)) != 0)) 3146 return (rc); /* error message displayed already */ 3147 3148 rc = update_mac_settings(ifp, XGMAC_ALL); 3149 if (rc) 3150 goto done; /* error message displayed already */ 3151 3152 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3153 if (rc != 0) { 3154 if_printf(ifp, "enable_vi failed: %d\n", rc); 3155 goto done; 3156 } 3157 3158 /* 3159 * The first iq of the first port to come up is used for tracing. 3160 */ 3161 if (sc->traceq < 0) { 3162 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3163 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3164 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3165 V_QUEUENUMBER(sc->traceq)); 3166 pi->flags |= HAS_TRACEQ; 3167 } 3168 3169 /* all ok */ 3170 setbit(&sc->open_device_map, pi->port_id); 3171 PORT_LOCK(pi); 3172 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3173 PORT_UNLOCK(pi); 3174 3175 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3176done: 3177 if (rc != 0) 3178 cxgbe_uninit_synchronized(pi); 3179 3180 return (rc); 3181} 3182 3183/* 3184 * Idempotent. 3185 */ 3186static int 3187cxgbe_uninit_synchronized(struct port_info *pi) 3188{ 3189 struct adapter *sc = pi->adapter; 3190 struct ifnet *ifp = pi->ifp; 3191 int rc; 3192 3193 ASSERT_SYNCHRONIZED_OP(sc); 3194 3195 /* 3196 * Disable the VI so that all its data in either direction is discarded 3197 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3198 * tick) intact as the TP can deliver negative advice or data that it's 3199 * holding in its RAM (for an offloaded connection) even after the VI is 3200 * disabled. 3201 */ 3202 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3203 if (rc) { 3204 if_printf(ifp, "disable_vi failed: %d\n", rc); 3205 return (rc); 3206 } 3207 3208 clrbit(&sc->open_device_map, pi->port_id); 3209 PORT_LOCK(pi); 3210 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3211 PORT_UNLOCK(pi); 3212 3213 pi->link_cfg.link_ok = 0; 3214 pi->link_cfg.speed = 0; 3215 pi->linkdnrc = -1; 3216 t4_os_link_changed(sc, pi->port_id, 0, -1); 3217 3218 return (0); 3219} 3220 3221/* 3222 * It is ok for this function to fail midway and return right away. t4_detach 3223 * will walk the entire sc->irq list and clean up whatever is valid. 3224 */ 3225static int 3226setup_intr_handlers(struct adapter *sc) 3227{ 3228 int rc, rid, p, q; 3229 char s[8]; 3230 struct irq *irq; 3231 struct port_info *pi; 3232 struct sge_rxq *rxq; 3233#ifdef TCP_OFFLOAD 3234 struct sge_ofld_rxq *ofld_rxq; 3235#endif 3236#ifdef DEV_NETMAP 3237 struct sge_nm_rxq *nm_rxq; 3238#endif 3239 3240 /* 3241 * Setup interrupts. 3242 */ 3243 irq = &sc->irq[0]; 3244 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3245 if (sc->intr_count == 1) 3246 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3247 3248 /* Multiple interrupts. */ 3249 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3250 ("%s: too few intr.", __func__)); 3251 3252 /* The first one is always error intr */ 3253 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3254 if (rc != 0) 3255 return (rc); 3256 irq++; 3257 rid++; 3258 3259 /* The second one is always the firmware event queue */ 3260 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3261 if (rc != 0) 3262 return (rc); 3263 irq++; 3264 rid++; 3265 3266 for_each_port(sc, p) { 3267 pi = sc->port[p]; 3268 3269 if (pi->flags & INTR_RXQ) { 3270 for_each_rxq(pi, q, rxq) { 3271 snprintf(s, sizeof(s), "%d.%d", p, q); 3272 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3273 s); 3274 if (rc != 0) 3275 return (rc); 3276 irq++; 3277 rid++; 3278 } 3279 } 3280#ifdef TCP_OFFLOAD 3281 if (pi->flags & INTR_OFLD_RXQ) { 3282 for_each_ofld_rxq(pi, q, ofld_rxq) { 3283 snprintf(s, sizeof(s), "%d,%d", p, q); 3284 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3285 ofld_rxq, s); 3286 if (rc != 0) 3287 return (rc); 3288 irq++; 3289 rid++; 3290 } 3291 } 3292#endif 3293#ifdef DEV_NETMAP 3294 if (pi->flags & INTR_NM_RXQ) { 3295 for_each_nm_rxq(pi, q, nm_rxq) { 3296 snprintf(s, sizeof(s), "%d-%d", p, q); 3297 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3298 nm_rxq, s); 3299 if (rc != 0) 3300 return (rc); 3301 irq++; 3302 rid++; 3303 } 3304 } 3305#endif 3306 } 3307 MPASS(irq == &sc->irq[sc->intr_count]); 3308 3309 return (0); 3310} 3311 3312int 3313adapter_full_init(struct adapter *sc) 3314{ 3315 int rc, i; 3316 3317 ASSERT_SYNCHRONIZED_OP(sc); 3318 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3319 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3320 ("%s: FULL_INIT_DONE already", __func__)); 3321 3322 /* 3323 * queues that belong to the adapter (not any particular port). 3324 */ 3325 rc = t4_setup_adapter_queues(sc); 3326 if (rc != 0) 3327 goto done; 3328 3329 for (i = 0; i < nitems(sc->tq); i++) { 3330 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3331 taskqueue_thread_enqueue, &sc->tq[i]); 3332 if (sc->tq[i] == NULL) { 3333 device_printf(sc->dev, 3334 "failed to allocate task queue %d\n", i); 3335 rc = ENOMEM; 3336 goto done; 3337 } 3338 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3339 device_get_nameunit(sc->dev), i); 3340 } 3341 3342 t4_intr_enable(sc); 3343 sc->flags |= FULL_INIT_DONE; 3344done: 3345 if (rc != 0) 3346 adapter_full_uninit(sc); 3347 3348 return (rc); 3349} 3350 3351int 3352adapter_full_uninit(struct adapter *sc) 3353{ 3354 int i; 3355 3356 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3357 3358 t4_teardown_adapter_queues(sc); 3359 3360 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3361 taskqueue_free(sc->tq[i]); 3362 sc->tq[i] = NULL; 3363 } 3364 3365 sc->flags &= ~FULL_INIT_DONE; 3366 3367 return (0); 3368} 3369 3370int 3371port_full_init(struct port_info *pi) 3372{ 3373 struct adapter *sc = pi->adapter; 3374 struct ifnet *ifp = pi->ifp; 3375 uint16_t *rss; 3376 struct sge_rxq *rxq; 3377 int rc, i, j; 3378 3379 ASSERT_SYNCHRONIZED_OP(sc); 3380 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3381 ("%s: PORT_INIT_DONE already", __func__)); 3382 3383 sysctl_ctx_init(&pi->ctx); 3384 pi->flags |= PORT_SYSCTL_CTX; 3385 3386 /* 3387 * Allocate tx/rx/fl queues for this port. 3388 */ 3389 rc = t4_setup_port_queues(pi); 3390 if (rc != 0) 3391 goto done; /* error message displayed already */ 3392 3393 /* 3394 * Setup RSS for this port. Save a copy of the RSS table for later use. 3395 */ 3396 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3397 for (i = 0; i < pi->rss_size;) { 3398 for_each_rxq(pi, j, rxq) { 3399 rss[i++] = rxq->iq.abs_id; 3400 if (i == pi->rss_size) 3401 break; 3402 } 3403 } 3404 3405 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3406 pi->rss_size); 3407 if (rc != 0) { 3408 if_printf(ifp, "rss_config failed: %d\n", rc); 3409 goto done; 3410 } 3411 3412 pi->rss = rss; 3413 pi->flags |= PORT_INIT_DONE; 3414done: 3415 if (rc != 0) 3416 port_full_uninit(pi); 3417 3418 return (rc); 3419} 3420 3421/* 3422 * Idempotent. 3423 */ 3424int 3425port_full_uninit(struct port_info *pi) 3426{ 3427 struct adapter *sc = pi->adapter; 3428 int i; 3429 struct sge_rxq *rxq; 3430 struct sge_txq *txq; 3431#ifdef TCP_OFFLOAD 3432 struct sge_ofld_rxq *ofld_rxq; 3433 struct sge_wrq *ofld_txq; 3434#endif 3435 3436 if (pi->flags & PORT_INIT_DONE) { 3437 3438 /* Need to quiesce queues. XXX: ctrl queues? */ 3439 3440 for_each_txq(pi, i, txq) { 3441 quiesce_eq(sc, &txq->eq); 3442 } 3443 3444#ifdef TCP_OFFLOAD 3445 for_each_ofld_txq(pi, i, ofld_txq) { 3446 quiesce_eq(sc, &ofld_txq->eq); 3447 } 3448#endif 3449 3450 for_each_rxq(pi, i, rxq) { 3451 quiesce_iq(sc, &rxq->iq); 3452 quiesce_fl(sc, &rxq->fl); 3453 } 3454 3455#ifdef TCP_OFFLOAD 3456 for_each_ofld_rxq(pi, i, ofld_rxq) { 3457 quiesce_iq(sc, &ofld_rxq->iq); 3458 quiesce_fl(sc, &ofld_rxq->fl); 3459 } 3460#endif 3461 free(pi->rss, M_CXGBE); 3462 } 3463 3464 t4_teardown_port_queues(pi); 3465 pi->flags &= ~PORT_INIT_DONE; 3466 3467 return (0); 3468} 3469 3470static void 3471quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3472{ 3473 EQ_LOCK(eq); 3474 eq->flags |= EQ_DOOMED; 3475 3476 /* 3477 * Wait for the response to a credit flush if one's 3478 * pending. 3479 */ 3480 while (eq->flags & EQ_CRFLUSHED) 3481 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3482 EQ_UNLOCK(eq); 3483 3484 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3485 pause("callout", 10); /* Still iffy */ 3486 3487 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3488} 3489 3490static void 3491quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3492{ 3493 (void) sc; /* unused */ 3494 3495 /* Synchronize with the interrupt handler */ 3496 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3497 pause("iqfree", 1); 3498} 3499 3500static void 3501quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3502{ 3503 mtx_lock(&sc->sfl_lock); 3504 FL_LOCK(fl); 3505 fl->flags |= FL_DOOMED; 3506 FL_UNLOCK(fl); 3507 mtx_unlock(&sc->sfl_lock); 3508 3509 callout_drain(&sc->sfl_callout); 3510 KASSERT((fl->flags & FL_STARVING) == 0, 3511 ("%s: still starving", __func__)); 3512} 3513 3514static int 3515t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3516 driver_intr_t *handler, void *arg, char *name) 3517{ 3518 int rc; 3519 3520 irq->rid = rid; 3521 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3522 RF_SHAREABLE | RF_ACTIVE); 3523 if (irq->res == NULL) { 3524 device_printf(sc->dev, 3525 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3526 return (ENOMEM); 3527 } 3528 3529 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3530 NULL, handler, arg, &irq->tag); 3531 if (rc != 0) { 3532 device_printf(sc->dev, 3533 "failed to setup interrupt for rid %d, name %s: %d\n", 3534 rid, name, rc); 3535 } else if (name) 3536 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3537 3538 return (rc); 3539} 3540 3541static int 3542t4_free_irq(struct adapter *sc, struct irq *irq) 3543{ 3544 if (irq->tag) 3545 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3546 if (irq->res) 3547 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3548 3549 bzero(irq, sizeof(*irq)); 3550 3551 return (0); 3552} 3553 3554static void 3555reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3556 unsigned int end) 3557{ 3558 uint32_t *p = (uint32_t *)(buf + start); 3559 3560 for ( ; start <= end; start += sizeof(uint32_t)) 3561 *p++ = t4_read_reg(sc, start); 3562} 3563 3564static void 3565t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3566{ 3567 int i, n; 3568 const unsigned int *reg_ranges; 3569 static const unsigned int t4_reg_ranges[] = { 3570 0x1008, 0x1108, 3571 0x1180, 0x11b4, 3572 0x11fc, 0x123c, 3573 0x1300, 0x173c, 3574 0x1800, 0x18fc, 3575 0x3000, 0x30d8, 3576 0x30e0, 0x5924, 3577 0x5960, 0x59d4, 3578 0x5a00, 0x5af8, 3579 0x6000, 0x6098, 3580 0x6100, 0x6150, 3581 0x6200, 0x6208, 3582 0x6240, 0x6248, 3583 0x6280, 0x6338, 3584 0x6370, 0x638c, 3585 0x6400, 0x643c, 3586 0x6500, 0x6524, 3587 0x6a00, 0x6a38, 3588 0x6a60, 0x6a78, 3589 0x6b00, 0x6b84, 3590 0x6bf0, 0x6c84, 3591 0x6cf0, 0x6d84, 3592 0x6df0, 0x6e84, 3593 0x6ef0, 0x6f84, 3594 0x6ff0, 0x7084, 3595 0x70f0, 0x7184, 3596 0x71f0, 0x7284, 3597 0x72f0, 0x7384, 3598 0x73f0, 0x7450, 3599 0x7500, 0x7530, 3600 0x7600, 0x761c, 3601 0x7680, 0x76cc, 3602 0x7700, 0x7798, 3603 0x77c0, 0x77fc, 3604 0x7900, 0x79fc, 3605 0x7b00, 0x7c38, 3606 0x7d00, 0x7efc, 3607 0x8dc0, 0x8e1c, 3608 0x8e30, 0x8e78, 3609 0x8ea0, 0x8f6c, 3610 0x8fc0, 0x9074, 3611 0x90fc, 0x90fc, 3612 0x9400, 0x9458, 3613 0x9600, 0x96bc, 3614 0x9800, 0x9808, 3615 0x9820, 0x983c, 3616 0x9850, 0x9864, 3617 0x9c00, 0x9c6c, 3618 0x9c80, 0x9cec, 3619 0x9d00, 0x9d6c, 3620 0x9d80, 0x9dec, 3621 0x9e00, 0x9e6c, 3622 0x9e80, 0x9eec, 3623 0x9f00, 0x9f6c, 3624 0x9f80, 0x9fec, 3625 0xd004, 0xd03c, 3626 0xdfc0, 0xdfe0, 3627 0xe000, 0xea7c, 3628 0xf000, 0x11110, 3629 0x11118, 0x11190, 3630 0x19040, 0x1906c, 3631 0x19078, 0x19080, 3632 0x1908c, 0x19124, 3633 0x19150, 0x191b0, 3634 0x191d0, 0x191e8, 3635 0x19238, 0x1924c, 3636 0x193f8, 0x19474, 3637 0x19490, 0x194f8, 3638 0x19800, 0x19f30, 3639 0x1a000, 0x1a06c, 3640 0x1a0b0, 0x1a120, 3641 0x1a128, 0x1a138, 3642 0x1a190, 0x1a1c4, 3643 0x1a1fc, 0x1a1fc, 3644 0x1e040, 0x1e04c, 3645 0x1e284, 0x1e28c, 3646 0x1e2c0, 0x1e2c0, 3647 0x1e2e0, 0x1e2e0, 3648 0x1e300, 0x1e384, 3649 0x1e3c0, 0x1e3c8, 3650 0x1e440, 0x1e44c, 3651 0x1e684, 0x1e68c, 3652 0x1e6c0, 0x1e6c0, 3653 0x1e6e0, 0x1e6e0, 3654 0x1e700, 0x1e784, 3655 0x1e7c0, 0x1e7c8, 3656 0x1e840, 0x1e84c, 3657 0x1ea84, 0x1ea8c, 3658 0x1eac0, 0x1eac0, 3659 0x1eae0, 0x1eae0, 3660 0x1eb00, 0x1eb84, 3661 0x1ebc0, 0x1ebc8, 3662 0x1ec40, 0x1ec4c, 3663 0x1ee84, 0x1ee8c, 3664 0x1eec0, 0x1eec0, 3665 0x1eee0, 0x1eee0, 3666 0x1ef00, 0x1ef84, 3667 0x1efc0, 0x1efc8, 3668 0x1f040, 0x1f04c, 3669 0x1f284, 0x1f28c, 3670 0x1f2c0, 0x1f2c0, 3671 0x1f2e0, 0x1f2e0, 3672 0x1f300, 0x1f384, 3673 0x1f3c0, 0x1f3c8, 3674 0x1f440, 0x1f44c, 3675 0x1f684, 0x1f68c, 3676 0x1f6c0, 0x1f6c0, 3677 0x1f6e0, 0x1f6e0, 3678 0x1f700, 0x1f784, 3679 0x1f7c0, 0x1f7c8, 3680 0x1f840, 0x1f84c, 3681 0x1fa84, 0x1fa8c, 3682 0x1fac0, 0x1fac0, 3683 0x1fae0, 0x1fae0, 3684 0x1fb00, 0x1fb84, 3685 0x1fbc0, 0x1fbc8, 3686 0x1fc40, 0x1fc4c, 3687 0x1fe84, 0x1fe8c, 3688 0x1fec0, 0x1fec0, 3689 0x1fee0, 0x1fee0, 3690 0x1ff00, 0x1ff84, 3691 0x1ffc0, 0x1ffc8, 3692 0x20000, 0x2002c, 3693 0x20100, 0x2013c, 3694 0x20190, 0x201c8, 3695 0x20200, 0x20318, 3696 0x20400, 0x20528, 3697 0x20540, 0x20614, 3698 0x21000, 0x21040, 3699 0x2104c, 0x21060, 3700 0x210c0, 0x210ec, 3701 0x21200, 0x21268, 3702 0x21270, 0x21284, 3703 0x212fc, 0x21388, 3704 0x21400, 0x21404, 3705 0x21500, 0x21518, 3706 0x2152c, 0x2153c, 3707 0x21550, 0x21554, 3708 0x21600, 0x21600, 3709 0x21608, 0x21628, 3710 0x21630, 0x2163c, 3711 0x21700, 0x2171c, 3712 0x21780, 0x2178c, 3713 0x21800, 0x21c38, 3714 0x21c80, 0x21d7c, 3715 0x21e00, 0x21e04, 3716 0x22000, 0x2202c, 3717 0x22100, 0x2213c, 3718 0x22190, 0x221c8, 3719 0x22200, 0x22318, 3720 0x22400, 0x22528, 3721 0x22540, 0x22614, 3722 0x23000, 0x23040, 3723 0x2304c, 0x23060, 3724 0x230c0, 0x230ec, 3725 0x23200, 0x23268, 3726 0x23270, 0x23284, 3727 0x232fc, 0x23388, 3728 0x23400, 0x23404, 3729 0x23500, 0x23518, 3730 0x2352c, 0x2353c, 3731 0x23550, 0x23554, 3732 0x23600, 0x23600, 3733 0x23608, 0x23628, 3734 0x23630, 0x2363c, 3735 0x23700, 0x2371c, 3736 0x23780, 0x2378c, 3737 0x23800, 0x23c38, 3738 0x23c80, 0x23d7c, 3739 0x23e00, 0x23e04, 3740 0x24000, 0x2402c, 3741 0x24100, 0x2413c, 3742 0x24190, 0x241c8, 3743 0x24200, 0x24318, 3744 0x24400, 0x24528, 3745 0x24540, 0x24614, 3746 0x25000, 0x25040, 3747 0x2504c, 0x25060, 3748 0x250c0, 0x250ec, 3749 0x25200, 0x25268, 3750 0x25270, 0x25284, 3751 0x252fc, 0x25388, 3752 0x25400, 0x25404, 3753 0x25500, 0x25518, 3754 0x2552c, 0x2553c, 3755 0x25550, 0x25554, 3756 0x25600, 0x25600, 3757 0x25608, 0x25628, 3758 0x25630, 0x2563c, 3759 0x25700, 0x2571c, 3760 0x25780, 0x2578c, 3761 0x25800, 0x25c38, 3762 0x25c80, 0x25d7c, 3763 0x25e00, 0x25e04, 3764 0x26000, 0x2602c, 3765 0x26100, 0x2613c, 3766 0x26190, 0x261c8, 3767 0x26200, 0x26318, 3768 0x26400, 0x26528, 3769 0x26540, 0x26614, 3770 0x27000, 0x27040, 3771 0x2704c, 0x27060, 3772 0x270c0, 0x270ec, 3773 0x27200, 0x27268, 3774 0x27270, 0x27284, 3775 0x272fc, 0x27388, 3776 0x27400, 0x27404, 3777 0x27500, 0x27518, 3778 0x2752c, 0x2753c, 3779 0x27550, 0x27554, 3780 0x27600, 0x27600, 3781 0x27608, 0x27628, 3782 0x27630, 0x2763c, 3783 0x27700, 0x2771c, 3784 0x27780, 0x2778c, 3785 0x27800, 0x27c38, 3786 0x27c80, 0x27d7c, 3787 0x27e00, 0x27e04 3788 }; 3789 static const unsigned int t5_reg_ranges[] = { 3790 0x1008, 0x1148, 3791 0x1180, 0x11b4, 3792 0x11fc, 0x123c, 3793 0x1280, 0x173c, 3794 0x1800, 0x18fc, 3795 0x3000, 0x3028, 3796 0x3060, 0x30d8, 3797 0x30e0, 0x30fc, 3798 0x3140, 0x357c, 3799 0x35a8, 0x35cc, 3800 0x35ec, 0x35ec, 3801 0x3600, 0x5624, 3802 0x56cc, 0x575c, 3803 0x580c, 0x5814, 3804 0x5890, 0x58bc, 3805 0x5940, 0x59dc, 3806 0x59fc, 0x5a18, 3807 0x5a60, 0x5a9c, 3808 0x5b94, 0x5bfc, 3809 0x6000, 0x6040, 3810 0x6058, 0x614c, 3811 0x7700, 0x7798, 3812 0x77c0, 0x78fc, 3813 0x7b00, 0x7c54, 3814 0x7d00, 0x7efc, 3815 0x8dc0, 0x8de0, 3816 0x8df8, 0x8e84, 3817 0x8ea0, 0x8f84, 3818 0x8fc0, 0x90f8, 3819 0x9400, 0x9470, 3820 0x9600, 0x96f4, 3821 0x9800, 0x9808, 3822 0x9820, 0x983c, 3823 0x9850, 0x9864, 3824 0x9c00, 0x9c6c, 3825 0x9c80, 0x9cec, 3826 0x9d00, 0x9d6c, 3827 0x9d80, 0x9dec, 3828 0x9e00, 0x9e6c, 3829 0x9e80, 0x9eec, 3830 0x9f00, 0x9f6c, 3831 0x9f80, 0xa020, 3832 0xd004, 0xd03c, 3833 0xdfc0, 0xdfe0, 3834 0xe000, 0x11088, 3835 0x1109c, 0x11110, 3836 0x11118, 0x1117c, 3837 0x11190, 0x11204, 3838 0x19040, 0x1906c, 3839 0x19078, 0x19080, 3840 0x1908c, 0x19124, 3841 0x19150, 0x191b0, 3842 0x191d0, 0x191e8, 3843 0x19238, 0x19290, 3844 0x193f8, 0x19474, 3845 0x19490, 0x194cc, 3846 0x194f0, 0x194f8, 3847 0x19c00, 0x19c60, 3848 0x19c94, 0x19e10, 3849 0x19e50, 0x19f34, 3850 0x19f40, 0x19f50, 3851 0x19f90, 0x19fe4, 3852 0x1a000, 0x1a06c, 3853 0x1a0b0, 0x1a120, 3854 0x1a128, 0x1a138, 3855 0x1a190, 0x1a1c4, 3856 0x1a1fc, 0x1a1fc, 3857 0x1e008, 0x1e00c, 3858 0x1e040, 0x1e04c, 3859 0x1e284, 0x1e290, 3860 0x1e2c0, 0x1e2c0, 3861 0x1e2e0, 0x1e2e0, 3862 0x1e300, 0x1e384, 3863 0x1e3c0, 0x1e3c8, 3864 0x1e408, 0x1e40c, 3865 0x1e440, 0x1e44c, 3866 0x1e684, 0x1e690, 3867 0x1e6c0, 0x1e6c0, 3868 0x1e6e0, 0x1e6e0, 3869 0x1e700, 0x1e784, 3870 0x1e7c0, 0x1e7c8, 3871 0x1e808, 0x1e80c, 3872 0x1e840, 0x1e84c, 3873 0x1ea84, 0x1ea90, 3874 0x1eac0, 0x1eac0, 3875 0x1eae0, 0x1eae0, 3876 0x1eb00, 0x1eb84, 3877 0x1ebc0, 0x1ebc8, 3878 0x1ec08, 0x1ec0c, 3879 0x1ec40, 0x1ec4c, 3880 0x1ee84, 0x1ee90, 3881 0x1eec0, 0x1eec0, 3882 0x1eee0, 0x1eee0, 3883 0x1ef00, 0x1ef84, 3884 0x1efc0, 0x1efc8, 3885 0x1f008, 0x1f00c, 3886 0x1f040, 0x1f04c, 3887 0x1f284, 0x1f290, 3888 0x1f2c0, 0x1f2c0, 3889 0x1f2e0, 0x1f2e0, 3890 0x1f300, 0x1f384, 3891 0x1f3c0, 0x1f3c8, 3892 0x1f408, 0x1f40c, 3893 0x1f440, 0x1f44c, 3894 0x1f684, 0x1f690, 3895 0x1f6c0, 0x1f6c0, 3896 0x1f6e0, 0x1f6e0, 3897 0x1f700, 0x1f784, 3898 0x1f7c0, 0x1f7c8, 3899 0x1f808, 0x1f80c, 3900 0x1f840, 0x1f84c, 3901 0x1fa84, 0x1fa90, 3902 0x1fac0, 0x1fac0, 3903 0x1fae0, 0x1fae0, 3904 0x1fb00, 0x1fb84, 3905 0x1fbc0, 0x1fbc8, 3906 0x1fc08, 0x1fc0c, 3907 0x1fc40, 0x1fc4c, 3908 0x1fe84, 0x1fe90, 3909 0x1fec0, 0x1fec0, 3910 0x1fee0, 0x1fee0, 3911 0x1ff00, 0x1ff84, 3912 0x1ffc0, 0x1ffc8, 3913 0x30000, 0x30030, 3914 0x30100, 0x30144, 3915 0x30190, 0x301d0, 3916 0x30200, 0x30318, 3917 0x30400, 0x3052c, 3918 0x30540, 0x3061c, 3919 0x30800, 0x30834, 3920 0x308c0, 0x30908, 3921 0x30910, 0x309ac, 3922 0x30a00, 0x30a2c, 3923 0x30a44, 0x30a50, 3924 0x30a74, 0x30c24, 3925 0x30d00, 0x30d00, 3926 0x30d08, 0x30d14, 3927 0x30d1c, 0x30d20, 3928 0x30d3c, 0x30d50, 3929 0x31200, 0x3120c, 3930 0x31220, 0x31220, 3931 0x31240, 0x31240, 3932 0x31600, 0x3160c, 3933 0x31a00, 0x31a1c, 3934 0x31e00, 0x31e20, 3935 0x31e38, 0x31e3c, 3936 0x31e80, 0x31e80, 3937 0x31e88, 0x31ea8, 3938 0x31eb0, 0x31eb4, 3939 0x31ec8, 0x31ed4, 3940 0x31fb8, 0x32004, 3941 0x32200, 0x32200, 3942 0x32208, 0x32240, 3943 0x32248, 0x32280, 3944 0x32288, 0x322c0, 3945 0x322c8, 0x322fc, 3946 0x32600, 0x32630, 3947 0x32a00, 0x32abc, 3948 0x32b00, 0x32b70, 3949 0x33000, 0x33048, 3950 0x33060, 0x3309c, 3951 0x330f0, 0x33148, 3952 0x33160, 0x3319c, 3953 0x331f0, 0x332e4, 3954 0x332f8, 0x333e4, 3955 0x333f8, 0x33448, 3956 0x33460, 0x3349c, 3957 0x334f0, 0x33548, 3958 0x33560, 0x3359c, 3959 0x335f0, 0x336e4, 3960 0x336f8, 0x337e4, 3961 0x337f8, 0x337fc, 3962 0x33814, 0x33814, 3963 0x3382c, 0x3382c, 3964 0x33880, 0x3388c, 3965 0x338e8, 0x338ec, 3966 0x33900, 0x33948, 3967 0x33960, 0x3399c, 3968 0x339f0, 0x33ae4, 3969 0x33af8, 0x33b10, 3970 0x33b28, 0x33b28, 3971 0x33b3c, 0x33b50, 3972 0x33bf0, 0x33c10, 3973 0x33c28, 0x33c28, 3974 0x33c3c, 0x33c50, 3975 0x33cf0, 0x33cfc, 3976 0x34000, 0x34030, 3977 0x34100, 0x34144, 3978 0x34190, 0x341d0, 3979 0x34200, 0x34318, 3980 0x34400, 0x3452c, 3981 0x34540, 0x3461c, 3982 0x34800, 0x34834, 3983 0x348c0, 0x34908, 3984 0x34910, 0x349ac, 3985 0x34a00, 0x34a2c, 3986 0x34a44, 0x34a50, 3987 0x34a74, 0x34c24, 3988 0x34d00, 0x34d00, 3989 0x34d08, 0x34d14, 3990 0x34d1c, 0x34d20, 3991 0x34d3c, 0x34d50, 3992 0x35200, 0x3520c, 3993 0x35220, 0x35220, 3994 0x35240, 0x35240, 3995 0x35600, 0x3560c, 3996 0x35a00, 0x35a1c, 3997 0x35e00, 0x35e20, 3998 0x35e38, 0x35e3c, 3999 0x35e80, 0x35e80, 4000 0x35e88, 0x35ea8, 4001 0x35eb0, 0x35eb4, 4002 0x35ec8, 0x35ed4, 4003 0x35fb8, 0x36004, 4004 0x36200, 0x36200, 4005 0x36208, 0x36240, 4006 0x36248, 0x36280, 4007 0x36288, 0x362c0, 4008 0x362c8, 0x362fc, 4009 0x36600, 0x36630, 4010 0x36a00, 0x36abc, 4011 0x36b00, 0x36b70, 4012 0x37000, 0x37048, 4013 0x37060, 0x3709c, 4014 0x370f0, 0x37148, 4015 0x37160, 0x3719c, 4016 0x371f0, 0x372e4, 4017 0x372f8, 0x373e4, 4018 0x373f8, 0x37448, 4019 0x37460, 0x3749c, 4020 0x374f0, 0x37548, 4021 0x37560, 0x3759c, 4022 0x375f0, 0x376e4, 4023 0x376f8, 0x377e4, 4024 0x377f8, 0x377fc, 4025 0x37814, 0x37814, 4026 0x3782c, 0x3782c, 4027 0x37880, 0x3788c, 4028 0x378e8, 0x378ec, 4029 0x37900, 0x37948, 4030 0x37960, 0x3799c, 4031 0x379f0, 0x37ae4, 4032 0x37af8, 0x37b10, 4033 0x37b28, 0x37b28, 4034 0x37b3c, 0x37b50, 4035 0x37bf0, 0x37c10, 4036 0x37c28, 0x37c28, 4037 0x37c3c, 0x37c50, 4038 0x37cf0, 0x37cfc, 4039 0x38000, 0x38030, 4040 0x38100, 0x38144, 4041 0x38190, 0x381d0, 4042 0x38200, 0x38318, 4043 0x38400, 0x3852c, 4044 0x38540, 0x3861c, 4045 0x38800, 0x38834, 4046 0x388c0, 0x38908, 4047 0x38910, 0x389ac, 4048 0x38a00, 0x38a2c, 4049 0x38a44, 0x38a50, 4050 0x38a74, 0x38c24, 4051 0x38d00, 0x38d00, 4052 0x38d08, 0x38d14, 4053 0x38d1c, 0x38d20, 4054 0x38d3c, 0x38d50, 4055 0x39200, 0x3920c, 4056 0x39220, 0x39220, 4057 0x39240, 0x39240, 4058 0x39600, 0x3960c, 4059 0x39a00, 0x39a1c, 4060 0x39e00, 0x39e20, 4061 0x39e38, 0x39e3c, 4062 0x39e80, 0x39e80, 4063 0x39e88, 0x39ea8, 4064 0x39eb0, 0x39eb4, 4065 0x39ec8, 0x39ed4, 4066 0x39fb8, 0x3a004, 4067 0x3a200, 0x3a200, 4068 0x3a208, 0x3a240, 4069 0x3a248, 0x3a280, 4070 0x3a288, 0x3a2c0, 4071 0x3a2c8, 0x3a2fc, 4072 0x3a600, 0x3a630, 4073 0x3aa00, 0x3aabc, 4074 0x3ab00, 0x3ab70, 4075 0x3b000, 0x3b048, 4076 0x3b060, 0x3b09c, 4077 0x3b0f0, 0x3b148, 4078 0x3b160, 0x3b19c, 4079 0x3b1f0, 0x3b2e4, 4080 0x3b2f8, 0x3b3e4, 4081 0x3b3f8, 0x3b448, 4082 0x3b460, 0x3b49c, 4083 0x3b4f0, 0x3b548, 4084 0x3b560, 0x3b59c, 4085 0x3b5f0, 0x3b6e4, 4086 0x3b6f8, 0x3b7e4, 4087 0x3b7f8, 0x3b7fc, 4088 0x3b814, 0x3b814, 4089 0x3b82c, 0x3b82c, 4090 0x3b880, 0x3b88c, 4091 0x3b8e8, 0x3b8ec, 4092 0x3b900, 0x3b948, 4093 0x3b960, 0x3b99c, 4094 0x3b9f0, 0x3bae4, 4095 0x3baf8, 0x3bb10, 4096 0x3bb28, 0x3bb28, 4097 0x3bb3c, 0x3bb50, 4098 0x3bbf0, 0x3bc10, 4099 0x3bc28, 0x3bc28, 4100 0x3bc3c, 0x3bc50, 4101 0x3bcf0, 0x3bcfc, 4102 0x3c000, 0x3c030, 4103 0x3c100, 0x3c144, 4104 0x3c190, 0x3c1d0, 4105 0x3c200, 0x3c318, 4106 0x3c400, 0x3c52c, 4107 0x3c540, 0x3c61c, 4108 0x3c800, 0x3c834, 4109 0x3c8c0, 0x3c908, 4110 0x3c910, 0x3c9ac, 4111 0x3ca00, 0x3ca2c, 4112 0x3ca44, 0x3ca50, 4113 0x3ca74, 0x3cc24, 4114 0x3cd00, 0x3cd00, 4115 0x3cd08, 0x3cd14, 4116 0x3cd1c, 0x3cd20, 4117 0x3cd3c, 0x3cd50, 4118 0x3d200, 0x3d20c, 4119 0x3d220, 0x3d220, 4120 0x3d240, 0x3d240, 4121 0x3d600, 0x3d60c, 4122 0x3da00, 0x3da1c, 4123 0x3de00, 0x3de20, 4124 0x3de38, 0x3de3c, 4125 0x3de80, 0x3de80, 4126 0x3de88, 0x3dea8, 4127 0x3deb0, 0x3deb4, 4128 0x3dec8, 0x3ded4, 4129 0x3dfb8, 0x3e004, 4130 0x3e200, 0x3e200, 4131 0x3e208, 0x3e240, 4132 0x3e248, 0x3e280, 4133 0x3e288, 0x3e2c0, 4134 0x3e2c8, 0x3e2fc, 4135 0x3e600, 0x3e630, 4136 0x3ea00, 0x3eabc, 4137 0x3eb00, 0x3eb70, 4138 0x3f000, 0x3f048, 4139 0x3f060, 0x3f09c, 4140 0x3f0f0, 0x3f148, 4141 0x3f160, 0x3f19c, 4142 0x3f1f0, 0x3f2e4, 4143 0x3f2f8, 0x3f3e4, 4144 0x3f3f8, 0x3f448, 4145 0x3f460, 0x3f49c, 4146 0x3f4f0, 0x3f548, 4147 0x3f560, 0x3f59c, 4148 0x3f5f0, 0x3f6e4, 4149 0x3f6f8, 0x3f7e4, 4150 0x3f7f8, 0x3f7fc, 4151 0x3f814, 0x3f814, 4152 0x3f82c, 0x3f82c, 4153 0x3f880, 0x3f88c, 4154 0x3f8e8, 0x3f8ec, 4155 0x3f900, 0x3f948, 4156 0x3f960, 0x3f99c, 4157 0x3f9f0, 0x3fae4, 4158 0x3faf8, 0x3fb10, 4159 0x3fb28, 0x3fb28, 4160 0x3fb3c, 0x3fb50, 4161 0x3fbf0, 0x3fc10, 4162 0x3fc28, 0x3fc28, 4163 0x3fc3c, 0x3fc50, 4164 0x3fcf0, 0x3fcfc, 4165 0x40000, 0x4000c, 4166 0x40040, 0x40068, 4167 0x4007c, 0x40144, 4168 0x40180, 0x4018c, 4169 0x40200, 0x40298, 4170 0x402ac, 0x4033c, 4171 0x403f8, 0x403fc, 4172 0x41304, 0x413c4, 4173 0x41400, 0x4141c, 4174 0x41480, 0x414d0, 4175 0x44000, 0x44078, 4176 0x440c0, 0x44278, 4177 0x442c0, 0x44478, 4178 0x444c0, 0x44678, 4179 0x446c0, 0x44878, 4180 0x448c0, 0x449fc, 4181 0x45000, 0x45068, 4182 0x45080, 0x45084, 4183 0x450a0, 0x450b0, 4184 0x45200, 0x45268, 4185 0x45280, 0x45284, 4186 0x452a0, 0x452b0, 4187 0x460c0, 0x460e4, 4188 0x47000, 0x4708c, 4189 0x47200, 0x47250, 4190 0x47400, 0x47420, 4191 0x47600, 0x47618, 4192 0x47800, 0x47814, 4193 0x48000, 0x4800c, 4194 0x48040, 0x48068, 4195 0x4807c, 0x48144, 4196 0x48180, 0x4818c, 4197 0x48200, 0x48298, 4198 0x482ac, 0x4833c, 4199 0x483f8, 0x483fc, 4200 0x49304, 0x493c4, 4201 0x49400, 0x4941c, 4202 0x49480, 0x494d0, 4203 0x4c000, 0x4c078, 4204 0x4c0c0, 0x4c278, 4205 0x4c2c0, 0x4c478, 4206 0x4c4c0, 0x4c678, 4207 0x4c6c0, 0x4c878, 4208 0x4c8c0, 0x4c9fc, 4209 0x4d000, 0x4d068, 4210 0x4d080, 0x4d084, 4211 0x4d0a0, 0x4d0b0, 4212 0x4d200, 0x4d268, 4213 0x4d280, 0x4d284, 4214 0x4d2a0, 0x4d2b0, 4215 0x4e0c0, 0x4e0e4, 4216 0x4f000, 0x4f08c, 4217 0x4f200, 0x4f250, 4218 0x4f400, 0x4f420, 4219 0x4f600, 0x4f618, 4220 0x4f800, 0x4f814, 4221 0x50000, 0x500cc, 4222 0x50400, 0x50400, 4223 0x50800, 0x508cc, 4224 0x50c00, 0x50c00, 4225 0x51000, 0x5101c, 4226 0x51300, 0x51308, 4227 }; 4228 4229 if (is_t4(sc)) { 4230 reg_ranges = &t4_reg_ranges[0]; 4231 n = nitems(t4_reg_ranges); 4232 } else { 4233 reg_ranges = &t5_reg_ranges[0]; 4234 n = nitems(t5_reg_ranges); 4235 } 4236 4237 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4238 for (i = 0; i < n; i += 2) 4239 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4240} 4241 4242static void 4243cxgbe_tick(void *arg) 4244{ 4245 struct port_info *pi = arg; 4246 struct adapter *sc = pi->adapter; 4247 struct ifnet *ifp = pi->ifp; 4248 struct sge_txq *txq; 4249 int i, drops; 4250 struct port_stats *s = &pi->stats; 4251 4252 PORT_LOCK(pi); 4253 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4254 PORT_UNLOCK(pi); 4255 return; /* without scheduling another callout */ 4256 } 4257 4258 t4_get_port_stats(sc, pi->tx_chan, s); 4259 4260 ifp->if_opackets = s->tx_frames - s->tx_pause; 4261 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4262 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4263 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4264 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4265 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4266 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4267 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4268 s->rx_trunc3; 4269 for (i = 0; i < 4; i++) { 4270 if (pi->rx_chan_map & (1 << i)) { 4271 uint32_t v; 4272 4273 /* 4274 * XXX: indirect reads from the same ADDR/DATA pair can 4275 * race with each other. 4276 */ 4277 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4278 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4279 ifp->if_iqdrops += v; 4280 } 4281 } 4282 4283 drops = s->tx_drop; 4284 for_each_txq(pi, i, txq) 4285 drops += txq->br->br_drops; 4286 ifp->if_snd.ifq_drops = drops; 4287 4288 ifp->if_oerrors = s->tx_error_frames; 4289 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4290 s->rx_fcs_err + s->rx_len_err; 4291 4292 callout_schedule(&pi->tick, hz); 4293 PORT_UNLOCK(pi); 4294} 4295 4296static void 4297cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4298{ 4299 struct ifnet *vlan; 4300 4301 if (arg != ifp || ifp->if_type != IFT_ETHER) 4302 return; 4303 4304 vlan = VLAN_DEVAT(ifp, vid); 4305 VLAN_SETCOOKIE(vlan, ifp); 4306} 4307 4308static int 4309cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4310{ 4311 4312#ifdef INVARIANTS 4313 panic("%s: opcode 0x%02x on iq %p with payload %p", 4314 __func__, rss->opcode, iq, m); 4315#else 4316 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4317 __func__, rss->opcode, iq, m); 4318 m_freem(m); 4319#endif 4320 return (EDOOFUS); 4321} 4322 4323int 4324t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4325{ 4326 uintptr_t *loc, new; 4327 4328 if (opcode >= nitems(sc->cpl_handler)) 4329 return (EINVAL); 4330 4331 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4332 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4333 atomic_store_rel_ptr(loc, new); 4334 4335 return (0); 4336} 4337 4338static int 4339an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4340{ 4341 4342#ifdef INVARIANTS 4343 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4344#else 4345 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4346 __func__, iq, ctrl); 4347#endif 4348 return (EDOOFUS); 4349} 4350 4351int 4352t4_register_an_handler(struct adapter *sc, an_handler_t h) 4353{ 4354 uintptr_t *loc, new; 4355 4356 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4357 loc = (uintptr_t *) &sc->an_handler; 4358 atomic_store_rel_ptr(loc, new); 4359 4360 return (0); 4361} 4362 4363static int 4364fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4365{ 4366 const struct cpl_fw6_msg *cpl = 4367 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4368 4369#ifdef INVARIANTS 4370 panic("%s: fw_msg type %d", __func__, cpl->type); 4371#else 4372 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4373#endif 4374 return (EDOOFUS); 4375} 4376 4377int 4378t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4379{ 4380 uintptr_t *loc, new; 4381 4382 if (type >= nitems(sc->fw_msg_handler)) 4383 return (EINVAL); 4384 4385 /* 4386 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4387 * handler dispatch table. Reject any attempt to install a handler for 4388 * this subtype. 4389 */ 4390 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4391 return (EINVAL); 4392 4393 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4394 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4395 atomic_store_rel_ptr(loc, new); 4396 4397 return (0); 4398} 4399 4400static int 4401t4_sysctls(struct adapter *sc) 4402{ 4403 struct sysctl_ctx_list *ctx; 4404 struct sysctl_oid *oid; 4405 struct sysctl_oid_list *children, *c0; 4406 static char *caps[] = { 4407 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4408 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4409 "\6HASHFILTER\7ETHOFLD", 4410 "\20\1TOE", /* caps[2] toecaps */ 4411 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4412 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4413 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4414 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4415 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4416 "\4PO_INITIAOR\5PO_TARGET" 4417 }; 4418 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4419 4420 ctx = device_get_sysctl_ctx(sc->dev); 4421 4422 /* 4423 * dev.t4nex.X. 4424 */ 4425 oid = device_get_sysctl_tree(sc->dev); 4426 c0 = children = SYSCTL_CHILDREN(oid); 4427 4428 sc->sc_do_rxcopy = 1; 4429 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4430 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4431 4432 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4433 sc->params.nports, "# of ports"); 4434 4435 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4436 NULL, chip_rev(sc), "chip hardware revision"); 4437 4438 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4439 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4440 4441 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4442 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4443 4444 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4445 sc->cfcsum, "config file checksum"); 4446 4447 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4448 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4449 sysctl_bitfield, "A", "available doorbells"); 4450 4451 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4452 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4453 sysctl_bitfield, "A", "available link capabilities"); 4454 4455 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4456 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4457 sysctl_bitfield, "A", "available NIC capabilities"); 4458 4459 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4460 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4461 sysctl_bitfield, "A", "available TCP offload capabilities"); 4462 4463 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4464 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4465 sysctl_bitfield, "A", "available RDMA capabilities"); 4466 4467 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4468 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4469 sysctl_bitfield, "A", "available iSCSI capabilities"); 4470 4471 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4472 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4473 sysctl_bitfield, "A", "available FCoE capabilities"); 4474 4475 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4476 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4477 4478 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4479 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4480 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4481 "interrupt holdoff timer values (us)"); 4482 4483 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4484 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4485 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4486 "interrupt holdoff packet counter values"); 4487 4488 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4489 NULL, sc->tids.nftids, "number of filters"); 4490 4491 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4492 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4493 "chip temperature (in Celsius)"); 4494 4495 t4_sge_sysctls(sc, ctx, children); 4496 4497 sc->lro_timeout = 100; 4498 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4499 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4500 4501#ifdef SBUF_DRAIN 4502 /* 4503 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4504 */ 4505 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4506 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4507 "logs and miscellaneous information"); 4508 children = SYSCTL_CHILDREN(oid); 4509 4510 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4511 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4512 sysctl_cctrl, "A", "congestion control"); 4513 4514 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4515 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4516 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4517 4518 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4519 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4520 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4521 4522 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4523 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4524 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4525 4526 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4527 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4528 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4529 4530 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4531 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4532 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4533 4534 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4535 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4536 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4537 4538 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4539 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4540 sysctl_cim_la, "A", "CIM logic analyzer"); 4541 4542 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4543 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4544 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4545 4546 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4547 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4548 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4549 4550 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4551 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4552 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4553 4554 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4555 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4556 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4557 4558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4559 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4560 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4561 4562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4563 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4564 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4565 4566 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4567 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4568 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4569 4570 if (is_t5(sc)) { 4571 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4572 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4573 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4574 4575 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4576 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4577 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4578 } 4579 4580 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4581 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4582 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4583 4584 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4585 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4586 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4587 4588 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4589 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4590 sysctl_cpl_stats, "A", "CPL statistics"); 4591 4592 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4593 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4594 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4595 4596 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4597 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4598 sysctl_devlog, "A", "firmware's device log"); 4599 4600 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4601 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4602 sysctl_fcoe_stats, "A", "FCoE statistics"); 4603 4604 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4605 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4606 sysctl_hw_sched, "A", "hardware scheduler "); 4607 4608 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4609 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4610 sysctl_l2t, "A", "hardware L2 table"); 4611 4612 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4613 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4614 sysctl_lb_stats, "A", "loopback statistics"); 4615 4616 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4617 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4618 sysctl_meminfo, "A", "memory regions"); 4619 4620 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4621 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4622 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4623 4624 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4625 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4626 sysctl_path_mtus, "A", "path MTUs"); 4627 4628 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4629 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4630 sysctl_pm_stats, "A", "PM statistics"); 4631 4632 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4633 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4634 sysctl_rdma_stats, "A", "RDMA statistics"); 4635 4636 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4637 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4638 sysctl_tcp_stats, "A", "TCP statistics"); 4639 4640 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4641 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4642 sysctl_tids, "A", "TID information"); 4643 4644 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4645 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4646 sysctl_tp_err_stats, "A", "TP error statistics"); 4647 4648 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4649 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4650 sysctl_tp_la, "A", "TP logic analyzer"); 4651 4652 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4653 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4654 sysctl_tx_rate, "A", "Tx rate"); 4655 4656 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4657 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4658 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4659 4660 if (is_t5(sc)) { 4661 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4662 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4663 sysctl_wcwr_stats, "A", "write combined work requests"); 4664 } 4665#endif 4666 4667#ifdef TCP_OFFLOAD 4668 if (is_offload(sc)) { 4669 /* 4670 * dev.t4nex.X.toe. 4671 */ 4672 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4673 NULL, "TOE parameters"); 4674 children = SYSCTL_CHILDREN(oid); 4675 4676 sc->tt.sndbuf = 256 * 1024; 4677 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4678 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4679 4680 sc->tt.ddp = 0; 4681 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4682 &sc->tt.ddp, 0, "DDP allowed"); 4683 4684 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4685 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4686 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4687 4688 sc->tt.ddp_thres = 4689 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4690 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4691 &sc->tt.ddp_thres, 0, "DDP threshold"); 4692 4693 sc->tt.rx_coalesce = 1; 4694 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4695 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4696 4697 sc->tt.tx_align = 1; 4698 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4699 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4700 } 4701#endif 4702 4703 4704 return (0); 4705} 4706 4707static int 4708cxgbe_sysctls(struct port_info *pi) 4709{ 4710 struct sysctl_ctx_list *ctx; 4711 struct sysctl_oid *oid; 4712 struct sysctl_oid_list *children; 4713 struct adapter *sc = pi->adapter; 4714 4715 ctx = device_get_sysctl_ctx(pi->dev); 4716 4717 /* 4718 * dev.cxgbe.X. 4719 */ 4720 oid = device_get_sysctl_tree(pi->dev); 4721 children = SYSCTL_CHILDREN(oid); 4722 4723 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4724 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4725 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4726 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4727 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4728 "PHY temperature (in Celsius)"); 4729 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4730 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4731 "PHY firmware version"); 4732 } 4733 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4734 &pi->nrxq, 0, "# of rx queues"); 4735 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4736 &pi->ntxq, 0, "# of tx queues"); 4737 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4738 &pi->first_rxq, 0, "index of first rx queue"); 4739 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4740 &pi->first_txq, 0, "index of first tx queue"); 4741 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4742 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4743 "Reserve queue 0 for non-flowid packets"); 4744 4745#ifdef TCP_OFFLOAD 4746 if (is_offload(sc)) { 4747 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4748 &pi->nofldrxq, 0, 4749 "# of rx queues for offloaded TCP connections"); 4750 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4751 &pi->nofldtxq, 0, 4752 "# of tx queues for offloaded TCP connections"); 4753 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4754 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4755 "index of first TOE rx queue"); 4756 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4757 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4758 "index of first TOE tx queue"); 4759 } 4760#endif 4761#ifdef DEV_NETMAP 4762 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4763 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4764 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4765 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4766 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4767 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4768 "index of first netmap rx queue"); 4769 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4770 CTLFLAG_RD, &pi->first_nm_txq, 0, 4771 "index of first netmap tx queue"); 4772#endif 4773 4774 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4775 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4776 "holdoff timer index"); 4777 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4778 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4779 "holdoff packet counter index"); 4780 4781 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4782 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4783 "rx queue size"); 4784 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4785 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4786 "tx queue size"); 4787 4788 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4789 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4790 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4791 4792 /* 4793 * dev.cxgbe.X.stats. 4794 */ 4795 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4796 NULL, "port statistics"); 4797 children = SYSCTL_CHILDREN(oid); 4798 4799#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4800 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4801 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4802 sysctl_handle_t4_reg64, "QU", desc) 4803 4804 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4805 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4806 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4807 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4808 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4809 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4810 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4811 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4812 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4813 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4814 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4815 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4816 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4817 "# of tx frames in this range", 4818 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4819 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4820 "# of tx frames in this range", 4821 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4822 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4823 "# of tx frames in this range", 4824 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4825 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4826 "# of tx frames in this range", 4827 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4828 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4829 "# of tx frames in this range", 4830 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4831 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4832 "# of tx frames in this range", 4833 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4834 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4835 "# of tx frames in this range", 4836 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4837 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4838 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4839 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4840 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4841 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4842 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4843 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4844 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4845 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4846 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4847 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4848 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4849 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4850 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4851 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4852 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4853 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4854 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4855 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4856 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4857 4858 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4859 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4860 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4861 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4862 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4863 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4864 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4865 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4866 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4867 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4868 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4869 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4870 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4871 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4872 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4873 "# of frames received with bad FCS", 4874 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4875 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4876 "# of frames received with length error", 4877 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4878 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4879 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4880 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4881 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4882 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4883 "# of rx frames in this range", 4884 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4885 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4886 "# of rx frames in this range", 4887 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4888 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4889 "# of rx frames in this range", 4890 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4891 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4892 "# of rx frames in this range", 4893 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4894 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4895 "# of rx frames in this range", 4896 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4897 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4898 "# of rx frames in this range", 4899 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4900 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4901 "# of rx frames in this range", 4902 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4903 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4904 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4905 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4906 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4907 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4908 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4909 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4910 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4911 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4912 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4913 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4914 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4915 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4916 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4917 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4918 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4919 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4920 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4921 4922#undef SYSCTL_ADD_T4_REG64 4923 4924#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4925 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4926 &pi->stats.name, desc) 4927 4928 /* We get these from port_stats and they may be stale by upto 1s */ 4929 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4930 "# drops due to buffer-group 0 overflows"); 4931 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4932 "# drops due to buffer-group 1 overflows"); 4933 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4934 "# drops due to buffer-group 2 overflows"); 4935 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4936 "# drops due to buffer-group 3 overflows"); 4937 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4938 "# of buffer-group 0 truncated packets"); 4939 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4940 "# of buffer-group 1 truncated packets"); 4941 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4942 "# of buffer-group 2 truncated packets"); 4943 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4944 "# of buffer-group 3 truncated packets"); 4945 4946#undef SYSCTL_ADD_T4_PORTSTAT 4947 4948 return (0); 4949} 4950 4951static int 4952sysctl_int_array(SYSCTL_HANDLER_ARGS) 4953{ 4954 int rc, *i, space = 0; 4955 struct sbuf sb; 4956 4957 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4958 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 4959 if (space) 4960 sbuf_printf(&sb, " "); 4961 sbuf_printf(&sb, "%d", *i); 4962 space = 1; 4963 } 4964 sbuf_finish(&sb); 4965 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4966 sbuf_delete(&sb); 4967 return (rc); 4968} 4969 4970static int 4971sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4972{ 4973 int rc; 4974 struct sbuf *sb; 4975 4976 rc = sysctl_wire_old_buffer(req, 0); 4977 if (rc != 0) 4978 return(rc); 4979 4980 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4981 if (sb == NULL) 4982 return (ENOMEM); 4983 4984 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4985 rc = sbuf_finish(sb); 4986 sbuf_delete(sb); 4987 4988 return (rc); 4989} 4990 4991static int 4992sysctl_btphy(SYSCTL_HANDLER_ARGS) 4993{ 4994 struct port_info *pi = arg1; 4995 int op = arg2; 4996 struct adapter *sc = pi->adapter; 4997 u_int v; 4998 int rc; 4999 5000 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5001 if (rc) 5002 return (rc); 5003 /* XXX: magic numbers */ 5004 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5005 &v); 5006 end_synchronized_op(sc, 0); 5007 if (rc) 5008 return (rc); 5009 if (op == 0) 5010 v /= 256; 5011 5012 rc = sysctl_handle_int(oidp, &v, 0, req); 5013 return (rc); 5014} 5015 5016static int 5017sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5018{ 5019 struct port_info *pi = arg1; 5020 int rc, val; 5021 5022 val = pi->rsrv_noflowq; 5023 rc = sysctl_handle_int(oidp, &val, 0, req); 5024 if (rc != 0 || req->newptr == NULL) 5025 return (rc); 5026 5027 if ((val >= 1) && (pi->ntxq > 1)) 5028 pi->rsrv_noflowq = 1; 5029 else 5030 pi->rsrv_noflowq = 0; 5031 5032 return (rc); 5033} 5034 5035static int 5036sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5037{ 5038 struct port_info *pi = arg1; 5039 struct adapter *sc = pi->adapter; 5040 int idx, rc, i; 5041 struct sge_rxq *rxq; 5042#ifdef TCP_OFFLOAD 5043 struct sge_ofld_rxq *ofld_rxq; 5044#endif 5045 uint8_t v; 5046 5047 idx = pi->tmr_idx; 5048 5049 rc = sysctl_handle_int(oidp, &idx, 0, req); 5050 if (rc != 0 || req->newptr == NULL) 5051 return (rc); 5052 5053 if (idx < 0 || idx >= SGE_NTIMERS) 5054 return (EINVAL); 5055 5056 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5057 "t4tmr"); 5058 if (rc) 5059 return (rc); 5060 5061 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5062 for_each_rxq(pi, i, rxq) { 5063#ifdef atomic_store_rel_8 5064 atomic_store_rel_8(&rxq->iq.intr_params, v); 5065#else 5066 rxq->iq.intr_params = v; 5067#endif 5068 } 5069#ifdef TCP_OFFLOAD 5070 for_each_ofld_rxq(pi, i, ofld_rxq) { 5071#ifdef atomic_store_rel_8 5072 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5073#else 5074 ofld_rxq->iq.intr_params = v; 5075#endif 5076 } 5077#endif 5078 pi->tmr_idx = idx; 5079 5080 end_synchronized_op(sc, LOCK_HELD); 5081 return (0); 5082} 5083 5084static int 5085sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5086{ 5087 struct port_info *pi = arg1; 5088 struct adapter *sc = pi->adapter; 5089 int idx, rc; 5090 5091 idx = pi->pktc_idx; 5092 5093 rc = sysctl_handle_int(oidp, &idx, 0, req); 5094 if (rc != 0 || req->newptr == NULL) 5095 return (rc); 5096 5097 if (idx < -1 || idx >= SGE_NCOUNTERS) 5098 return (EINVAL); 5099 5100 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5101 "t4pktc"); 5102 if (rc) 5103 return (rc); 5104 5105 if (pi->flags & PORT_INIT_DONE) 5106 rc = EBUSY; /* cannot be changed once the queues are created */ 5107 else 5108 pi->pktc_idx = idx; 5109 5110 end_synchronized_op(sc, LOCK_HELD); 5111 return (rc); 5112} 5113 5114static int 5115sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5116{ 5117 struct port_info *pi = arg1; 5118 struct adapter *sc = pi->adapter; 5119 int qsize, rc; 5120 5121 qsize = pi->qsize_rxq; 5122 5123 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5124 if (rc != 0 || req->newptr == NULL) 5125 return (rc); 5126 5127 if (qsize < 128 || (qsize & 7)) 5128 return (EINVAL); 5129 5130 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5131 "t4rxqs"); 5132 if (rc) 5133 return (rc); 5134 5135 if (pi->flags & PORT_INIT_DONE) 5136 rc = EBUSY; /* cannot be changed once the queues are created */ 5137 else 5138 pi->qsize_rxq = qsize; 5139 5140 end_synchronized_op(sc, LOCK_HELD); 5141 return (rc); 5142} 5143 5144static int 5145sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5146{ 5147 struct port_info *pi = arg1; 5148 struct adapter *sc = pi->adapter; 5149 int qsize, rc; 5150 5151 qsize = pi->qsize_txq; 5152 5153 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5154 if (rc != 0 || req->newptr == NULL) 5155 return (rc); 5156 5157 /* bufring size must be powerof2 */ 5158 if (qsize < 128 || !powerof2(qsize)) 5159 return (EINVAL); 5160 5161 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5162 "t4txqs"); 5163 if (rc) 5164 return (rc); 5165 5166 if (pi->flags & PORT_INIT_DONE) 5167 rc = EBUSY; /* cannot be changed once the queues are created */ 5168 else 5169 pi->qsize_txq = qsize; 5170 5171 end_synchronized_op(sc, LOCK_HELD); 5172 return (rc); 5173} 5174 5175static int 5176sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5177{ 5178 struct port_info *pi = arg1; 5179 struct adapter *sc = pi->adapter; 5180 struct link_config *lc = &pi->link_cfg; 5181 int rc; 5182 5183 if (req->newptr == NULL) { 5184 struct sbuf *sb; 5185 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5186 5187 rc = sysctl_wire_old_buffer(req, 0); 5188 if (rc != 0) 5189 return(rc); 5190 5191 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5192 if (sb == NULL) 5193 return (ENOMEM); 5194 5195 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5196 rc = sbuf_finish(sb); 5197 sbuf_delete(sb); 5198 } else { 5199 char s[2]; 5200 int n; 5201 5202 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5203 s[1] = 0; 5204 5205 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5206 if (rc != 0) 5207 return(rc); 5208 5209 if (s[1] != 0) 5210 return (EINVAL); 5211 if (s[0] < '0' || s[0] > '9') 5212 return (EINVAL); /* not a number */ 5213 n = s[0] - '0'; 5214 if (n & ~(PAUSE_TX | PAUSE_RX)) 5215 return (EINVAL); /* some other bit is set too */ 5216 5217 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5218 if (rc) 5219 return (rc); 5220 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5221 int link_ok = lc->link_ok; 5222 5223 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5224 lc->requested_fc |= n; 5225 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5226 lc->link_ok = link_ok; /* restore */ 5227 } 5228 end_synchronized_op(sc, 0); 5229 } 5230 5231 return (rc); 5232} 5233 5234static int 5235sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5236{ 5237 struct adapter *sc = arg1; 5238 int reg = arg2; 5239 uint64_t val; 5240 5241 val = t4_read_reg64(sc, reg); 5242 5243 return (sysctl_handle_64(oidp, &val, 0, req)); 5244} 5245 5246static int 5247sysctl_temperature(SYSCTL_HANDLER_ARGS) 5248{ 5249 struct adapter *sc = arg1; 5250 int rc, t; 5251 uint32_t param, val; 5252 5253 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5254 if (rc) 5255 return (rc); 5256 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5257 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5258 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5259 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5260 end_synchronized_op(sc, 0); 5261 if (rc) 5262 return (rc); 5263 5264 /* unknown is returned as 0 but we display -1 in that case */ 5265 t = val == 0 ? -1 : val; 5266 5267 rc = sysctl_handle_int(oidp, &t, 0, req); 5268 return (rc); 5269} 5270 5271#ifdef SBUF_DRAIN 5272static int 5273sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5274{ 5275 struct adapter *sc = arg1; 5276 struct sbuf *sb; 5277 int rc, i; 5278 uint16_t incr[NMTUS][NCCTRL_WIN]; 5279 static const char *dec_fac[] = { 5280 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5281 "0.9375" 5282 }; 5283 5284 rc = sysctl_wire_old_buffer(req, 0); 5285 if (rc != 0) 5286 return (rc); 5287 5288 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5289 if (sb == NULL) 5290 return (ENOMEM); 5291 5292 t4_read_cong_tbl(sc, incr); 5293 5294 for (i = 0; i < NCCTRL_WIN; ++i) { 5295 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5296 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5297 incr[5][i], incr[6][i], incr[7][i]); 5298 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5299 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5300 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5301 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5302 } 5303 5304 rc = sbuf_finish(sb); 5305 sbuf_delete(sb); 5306 5307 return (rc); 5308} 5309 5310static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5311 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5312 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5313 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5314}; 5315 5316static int 5317sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5318{ 5319 struct adapter *sc = arg1; 5320 struct sbuf *sb; 5321 int rc, i, n, qid = arg2; 5322 uint32_t *buf, *p; 5323 char *qtype; 5324 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5325 5326 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5327 ("%s: bad qid %d\n", __func__, qid)); 5328 5329 if (qid < CIM_NUM_IBQ) { 5330 /* inbound queue */ 5331 qtype = "IBQ"; 5332 n = 4 * CIM_IBQ_SIZE; 5333 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5334 rc = t4_read_cim_ibq(sc, qid, buf, n); 5335 } else { 5336 /* outbound queue */ 5337 qtype = "OBQ"; 5338 qid -= CIM_NUM_IBQ; 5339 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5340 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5341 rc = t4_read_cim_obq(sc, qid, buf, n); 5342 } 5343 5344 if (rc < 0) { 5345 rc = -rc; 5346 goto done; 5347 } 5348 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5349 5350 rc = sysctl_wire_old_buffer(req, 0); 5351 if (rc != 0) 5352 goto done; 5353 5354 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5355 if (sb == NULL) { 5356 rc = ENOMEM; 5357 goto done; 5358 } 5359 5360 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5361 for (i = 0, p = buf; i < n; i += 16, p += 4) 5362 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5363 p[2], p[3]); 5364 5365 rc = sbuf_finish(sb); 5366 sbuf_delete(sb); 5367done: 5368 free(buf, M_CXGBE); 5369 return (rc); 5370} 5371 5372static int 5373sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5374{ 5375 struct adapter *sc = arg1; 5376 u_int cfg; 5377 struct sbuf *sb; 5378 uint32_t *buf, *p; 5379 int rc; 5380 5381 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5382 if (rc != 0) 5383 return (rc); 5384 5385 rc = sysctl_wire_old_buffer(req, 0); 5386 if (rc != 0) 5387 return (rc); 5388 5389 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5390 if (sb == NULL) 5391 return (ENOMEM); 5392 5393 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5394 M_ZERO | M_WAITOK); 5395 5396 rc = -t4_cim_read_la(sc, buf, NULL); 5397 if (rc != 0) 5398 goto done; 5399 5400 sbuf_printf(sb, "Status Data PC%s", 5401 cfg & F_UPDBGLACAPTPCONLY ? "" : 5402 " LS0Stat LS0Addr LS0Data"); 5403 5404 KASSERT((sc->params.cim_la_size & 7) == 0, 5405 ("%s: p will walk off the end of buf", __func__)); 5406 5407 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5408 if (cfg & F_UPDBGLACAPTPCONLY) { 5409 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5410 p[6], p[7]); 5411 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5412 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5413 p[4] & 0xff, p[5] >> 8); 5414 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5415 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5416 p[1] & 0xf, p[2] >> 4); 5417 } else { 5418 sbuf_printf(sb, 5419 "\n %02x %x%07x %x%07x %08x %08x " 5420 "%08x%08x%08x%08x", 5421 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5422 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5423 p[6], p[7]); 5424 } 5425 } 5426 5427 rc = sbuf_finish(sb); 5428 sbuf_delete(sb); 5429done: 5430 free(buf, M_CXGBE); 5431 return (rc); 5432} 5433 5434static int 5435sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5436{ 5437 struct adapter *sc = arg1; 5438 u_int i; 5439 struct sbuf *sb; 5440 uint32_t *buf, *p; 5441 int rc; 5442 5443 rc = sysctl_wire_old_buffer(req, 0); 5444 if (rc != 0) 5445 return (rc); 5446 5447 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5448 if (sb == NULL) 5449 return (ENOMEM); 5450 5451 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5452 M_ZERO | M_WAITOK); 5453 5454 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5455 p = buf; 5456 5457 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5458 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5459 p[1], p[0]); 5460 } 5461 5462 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5463 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5464 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5465 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5466 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5467 (p[1] >> 2) | ((p[2] & 3) << 30), 5468 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5469 p[0] & 1); 5470 } 5471 5472 rc = sbuf_finish(sb); 5473 sbuf_delete(sb); 5474 free(buf, M_CXGBE); 5475 return (rc); 5476} 5477 5478static int 5479sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5480{ 5481 struct adapter *sc = arg1; 5482 u_int i; 5483 struct sbuf *sb; 5484 uint32_t *buf, *p; 5485 int rc; 5486 5487 rc = sysctl_wire_old_buffer(req, 0); 5488 if (rc != 0) 5489 return (rc); 5490 5491 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5492 if (sb == NULL) 5493 return (ENOMEM); 5494 5495 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5496 M_ZERO | M_WAITOK); 5497 5498 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5499 p = buf; 5500 5501 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5502 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5503 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5504 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5505 p[4], p[3], p[2], p[1], p[0]); 5506 } 5507 5508 sbuf_printf(sb, "\n\nCntl ID Data"); 5509 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5510 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5511 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5512 } 5513 5514 rc = sbuf_finish(sb); 5515 sbuf_delete(sb); 5516 free(buf, M_CXGBE); 5517 return (rc); 5518} 5519 5520static int 5521sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5522{ 5523 struct adapter *sc = arg1; 5524 struct sbuf *sb; 5525 int rc, i; 5526 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5527 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5528 uint16_t thres[CIM_NUM_IBQ]; 5529 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5530 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5531 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5532 5533 if (is_t4(sc)) { 5534 cim_num_obq = CIM_NUM_OBQ; 5535 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5536 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5537 } else { 5538 cim_num_obq = CIM_NUM_OBQ_T5; 5539 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5540 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5541 } 5542 nq = CIM_NUM_IBQ + cim_num_obq; 5543 5544 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5545 if (rc == 0) 5546 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5547 if (rc != 0) 5548 return (rc); 5549 5550 t4_read_cimq_cfg(sc, base, size, thres); 5551 5552 rc = sysctl_wire_old_buffer(req, 0); 5553 if (rc != 0) 5554 return (rc); 5555 5556 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5557 if (sb == NULL) 5558 return (ENOMEM); 5559 5560 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5561 5562 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5563 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5564 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5565 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5566 G_QUEREMFLITS(p[2]) * 16); 5567 for ( ; i < nq; i++, p += 4, wr += 2) 5568 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5569 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5570 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5571 G_QUEREMFLITS(p[2]) * 16); 5572 5573 rc = sbuf_finish(sb); 5574 sbuf_delete(sb); 5575 5576 return (rc); 5577} 5578 5579static int 5580sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5581{ 5582 struct adapter *sc = arg1; 5583 struct sbuf *sb; 5584 int rc; 5585 struct tp_cpl_stats stats; 5586 5587 rc = sysctl_wire_old_buffer(req, 0); 5588 if (rc != 0) 5589 return (rc); 5590 5591 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5592 if (sb == NULL) 5593 return (ENOMEM); 5594 5595 t4_tp_get_cpl_stats(sc, &stats); 5596 5597 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5598 "channel 3\n"); 5599 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5600 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5601 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5602 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5603 5604 rc = sbuf_finish(sb); 5605 sbuf_delete(sb); 5606 5607 return (rc); 5608} 5609 5610static int 5611sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5612{ 5613 struct adapter *sc = arg1; 5614 struct sbuf *sb; 5615 int rc; 5616 struct tp_usm_stats stats; 5617 5618 rc = sysctl_wire_old_buffer(req, 0); 5619 if (rc != 0) 5620 return(rc); 5621 5622 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5623 if (sb == NULL) 5624 return (ENOMEM); 5625 5626 t4_get_usm_stats(sc, &stats); 5627 5628 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5629 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5630 sbuf_printf(sb, "Drops: %u", stats.drops); 5631 5632 rc = sbuf_finish(sb); 5633 sbuf_delete(sb); 5634 5635 return (rc); 5636} 5637 5638const char *devlog_level_strings[] = { 5639 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5640 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5641 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5642 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5643 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5644 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5645}; 5646 5647const char *devlog_facility_strings[] = { 5648 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5649 [FW_DEVLOG_FACILITY_CF] = "CF", 5650 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5651 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5652 [FW_DEVLOG_FACILITY_RES] = "RES", 5653 [FW_DEVLOG_FACILITY_HW] = "HW", 5654 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5655 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5656 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5657 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5658 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5659 [FW_DEVLOG_FACILITY_VI] = "VI", 5660 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5661 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5662 [FW_DEVLOG_FACILITY_TM] = "TM", 5663 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5664 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5665 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5666 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5667 [FW_DEVLOG_FACILITY_RI] = "RI", 5668 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5669 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5670 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5671 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5672}; 5673 5674static int 5675sysctl_devlog(SYSCTL_HANDLER_ARGS) 5676{ 5677 struct adapter *sc = arg1; 5678 struct devlog_params *dparams = &sc->params.devlog; 5679 struct fw_devlog_e *buf, *e; 5680 int i, j, rc, nentries, first = 0, m; 5681 struct sbuf *sb; 5682 uint64_t ftstamp = UINT64_MAX; 5683 5684 if (dparams->start == 0) { 5685 dparams->memtype = FW_MEMTYPE_EDC0; 5686 dparams->start = 0x84000; 5687 dparams->size = 32768; 5688 } 5689 5690 nentries = dparams->size / sizeof(struct fw_devlog_e); 5691 5692 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5693 if (buf == NULL) 5694 return (ENOMEM); 5695 5696 m = fwmtype_to_hwmtype(dparams->memtype); 5697 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5698 if (rc != 0) 5699 goto done; 5700 5701 for (i = 0; i < nentries; i++) { 5702 e = &buf[i]; 5703 5704 if (e->timestamp == 0) 5705 break; /* end */ 5706 5707 e->timestamp = be64toh(e->timestamp); 5708 e->seqno = be32toh(e->seqno); 5709 for (j = 0; j < 8; j++) 5710 e->params[j] = be32toh(e->params[j]); 5711 5712 if (e->timestamp < ftstamp) { 5713 ftstamp = e->timestamp; 5714 first = i; 5715 } 5716 } 5717 5718 if (buf[first].timestamp == 0) 5719 goto done; /* nothing in the log */ 5720 5721 rc = sysctl_wire_old_buffer(req, 0); 5722 if (rc != 0) 5723 goto done; 5724 5725 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5726 if (sb == NULL) { 5727 rc = ENOMEM; 5728 goto done; 5729 } 5730 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5731 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5732 5733 i = first; 5734 do { 5735 e = &buf[i]; 5736 if (e->timestamp == 0) 5737 break; /* end */ 5738 5739 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5740 e->seqno, e->timestamp, 5741 (e->level < nitems(devlog_level_strings) ? 5742 devlog_level_strings[e->level] : "UNKNOWN"), 5743 (e->facility < nitems(devlog_facility_strings) ? 5744 devlog_facility_strings[e->facility] : "UNKNOWN")); 5745 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5746 e->params[2], e->params[3], e->params[4], 5747 e->params[5], e->params[6], e->params[7]); 5748 5749 if (++i == nentries) 5750 i = 0; 5751 } while (i != first); 5752 5753 rc = sbuf_finish(sb); 5754 sbuf_delete(sb); 5755done: 5756 free(buf, M_CXGBE); 5757 return (rc); 5758} 5759 5760static int 5761sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5762{ 5763 struct adapter *sc = arg1; 5764 struct sbuf *sb; 5765 int rc; 5766 struct tp_fcoe_stats stats[4]; 5767 5768 rc = sysctl_wire_old_buffer(req, 0); 5769 if (rc != 0) 5770 return (rc); 5771 5772 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5773 if (sb == NULL) 5774 return (ENOMEM); 5775 5776 t4_get_fcoe_stats(sc, 0, &stats[0]); 5777 t4_get_fcoe_stats(sc, 1, &stats[1]); 5778 t4_get_fcoe_stats(sc, 2, &stats[2]); 5779 t4_get_fcoe_stats(sc, 3, &stats[3]); 5780 5781 sbuf_printf(sb, " channel 0 channel 1 " 5782 "channel 2 channel 3\n"); 5783 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5784 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5785 stats[3].octetsDDP); 5786 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5787 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5788 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5789 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5790 stats[3].framesDrop); 5791 5792 rc = sbuf_finish(sb); 5793 sbuf_delete(sb); 5794 5795 return (rc); 5796} 5797 5798static int 5799sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5800{ 5801 struct adapter *sc = arg1; 5802 struct sbuf *sb; 5803 int rc, i; 5804 unsigned int map, kbps, ipg, mode; 5805 unsigned int pace_tab[NTX_SCHED]; 5806 5807 rc = sysctl_wire_old_buffer(req, 0); 5808 if (rc != 0) 5809 return (rc); 5810 5811 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5812 if (sb == NULL) 5813 return (ENOMEM); 5814 5815 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5816 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5817 t4_read_pace_tbl(sc, pace_tab); 5818 5819 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5820 "Class IPG (0.1 ns) Flow IPG (us)"); 5821 5822 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5823 t4_get_tx_sched(sc, i, &kbps, &ipg); 5824 sbuf_printf(sb, "\n %u %-5s %u ", i, 5825 (mode & (1 << i)) ? "flow" : "class", map & 3); 5826 if (kbps) 5827 sbuf_printf(sb, "%9u ", kbps); 5828 else 5829 sbuf_printf(sb, " disabled "); 5830 5831 if (ipg) 5832 sbuf_printf(sb, "%13u ", ipg); 5833 else 5834 sbuf_printf(sb, " disabled "); 5835 5836 if (pace_tab[i]) 5837 sbuf_printf(sb, "%10u", pace_tab[i]); 5838 else 5839 sbuf_printf(sb, " disabled"); 5840 } 5841 5842 rc = sbuf_finish(sb); 5843 sbuf_delete(sb); 5844 5845 return (rc); 5846} 5847 5848static int 5849sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5850{ 5851 struct adapter *sc = arg1; 5852 struct sbuf *sb; 5853 int rc, i, j; 5854 uint64_t *p0, *p1; 5855 struct lb_port_stats s[2]; 5856 static const char *stat_name[] = { 5857 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5858 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5859 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5860 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5861 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5862 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5863 "BG2FramesTrunc:", "BG3FramesTrunc:" 5864 }; 5865 5866 rc = sysctl_wire_old_buffer(req, 0); 5867 if (rc != 0) 5868 return (rc); 5869 5870 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5871 if (sb == NULL) 5872 return (ENOMEM); 5873 5874 memset(s, 0, sizeof(s)); 5875 5876 for (i = 0; i < 4; i += 2) { 5877 t4_get_lb_stats(sc, i, &s[0]); 5878 t4_get_lb_stats(sc, i + 1, &s[1]); 5879 5880 p0 = &s[0].octets; 5881 p1 = &s[1].octets; 5882 sbuf_printf(sb, "%s Loopback %u" 5883 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5884 5885 for (j = 0; j < nitems(stat_name); j++) 5886 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5887 *p0++, *p1++); 5888 } 5889 5890 rc = sbuf_finish(sb); 5891 sbuf_delete(sb); 5892 5893 return (rc); 5894} 5895 5896static int 5897sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5898{ 5899 int rc = 0; 5900 struct port_info *pi = arg1; 5901 struct sbuf *sb; 5902 static const char *linkdnreasons[] = { 5903 "non-specific", "remote fault", "autoneg failed", "reserved3", 5904 "PHY overheated", "unknown", "rx los", "reserved7" 5905 }; 5906 5907 rc = sysctl_wire_old_buffer(req, 0); 5908 if (rc != 0) 5909 return(rc); 5910 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5911 if (sb == NULL) 5912 return (ENOMEM); 5913 5914 if (pi->linkdnrc < 0) 5915 sbuf_printf(sb, "n/a"); 5916 else if (pi->linkdnrc < nitems(linkdnreasons)) 5917 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5918 else 5919 sbuf_printf(sb, "%d", pi->linkdnrc); 5920 5921 rc = sbuf_finish(sb); 5922 sbuf_delete(sb); 5923 5924 return (rc); 5925} 5926 5927struct mem_desc { 5928 unsigned int base; 5929 unsigned int limit; 5930 unsigned int idx; 5931}; 5932 5933static int 5934mem_desc_cmp(const void *a, const void *b) 5935{ 5936 return ((const struct mem_desc *)a)->base - 5937 ((const struct mem_desc *)b)->base; 5938} 5939 5940static void 5941mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5942 unsigned int to) 5943{ 5944 unsigned int size; 5945 5946 size = to - from + 1; 5947 if (size == 0) 5948 return; 5949 5950 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5951 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5952} 5953 5954static int 5955sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5956{ 5957 struct adapter *sc = arg1; 5958 struct sbuf *sb; 5959 int rc, i, n; 5960 uint32_t lo, hi, used, alloc; 5961 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5962 static const char *region[] = { 5963 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5964 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5965 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5966 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5967 "RQUDP region:", "PBL region:", "TXPBL region:", 5968 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5969 "On-chip queues:" 5970 }; 5971 struct mem_desc avail[4]; 5972 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5973 struct mem_desc *md = mem; 5974 5975 rc = sysctl_wire_old_buffer(req, 0); 5976 if (rc != 0) 5977 return (rc); 5978 5979 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5980 if (sb == NULL) 5981 return (ENOMEM); 5982 5983 for (i = 0; i < nitems(mem); i++) { 5984 mem[i].limit = 0; 5985 mem[i].idx = i; 5986 } 5987 5988 /* Find and sort the populated memory ranges */ 5989 i = 0; 5990 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5991 if (lo & F_EDRAM0_ENABLE) { 5992 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5993 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5994 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5995 avail[i].idx = 0; 5996 i++; 5997 } 5998 if (lo & F_EDRAM1_ENABLE) { 5999 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6000 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6001 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6002 avail[i].idx = 1; 6003 i++; 6004 } 6005 if (lo & F_EXT_MEM_ENABLE) { 6006 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6007 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6008 avail[i].limit = avail[i].base + 6009 (G_EXT_MEM_SIZE(hi) << 20); 6010 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6011 i++; 6012 } 6013 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6014 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6015 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6016 avail[i].limit = avail[i].base + 6017 (G_EXT_MEM1_SIZE(hi) << 20); 6018 avail[i].idx = 4; 6019 i++; 6020 } 6021 if (!i) /* no memory available */ 6022 return 0; 6023 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6024 6025 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6026 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6027 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6028 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6029 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6030 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6031 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6032 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6033 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6034 6035 /* the next few have explicit upper bounds */ 6036 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6037 md->limit = md->base - 1 + 6038 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6039 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6040 md++; 6041 6042 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6043 md->limit = md->base - 1 + 6044 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6045 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6046 md++; 6047 6048 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6049 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6050 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6051 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6052 } else { 6053 md->base = 0; 6054 md->idx = nitems(region); /* hide it */ 6055 } 6056 md++; 6057 6058#define ulp_region(reg) \ 6059 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6060 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6061 6062 ulp_region(RX_ISCSI); 6063 ulp_region(RX_TDDP); 6064 ulp_region(TX_TPT); 6065 ulp_region(RX_STAG); 6066 ulp_region(RX_RQ); 6067 ulp_region(RX_RQUDP); 6068 ulp_region(RX_PBL); 6069 ulp_region(TX_PBL); 6070#undef ulp_region 6071 6072 md->base = 0; 6073 md->idx = nitems(region); 6074 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6075 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6076 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6077 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6078 } 6079 md++; 6080 6081 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6082 md->limit = md->base + sc->tids.ntids - 1; 6083 md++; 6084 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6085 md->limit = md->base + sc->tids.ntids - 1; 6086 md++; 6087 6088 md->base = sc->vres.ocq.start; 6089 if (sc->vres.ocq.size) 6090 md->limit = md->base + sc->vres.ocq.size - 1; 6091 else 6092 md->idx = nitems(region); /* hide it */ 6093 md++; 6094 6095 /* add any address-space holes, there can be up to 3 */ 6096 for (n = 0; n < i - 1; n++) 6097 if (avail[n].limit < avail[n + 1].base) 6098 (md++)->base = avail[n].limit; 6099 if (avail[n].limit) 6100 (md++)->base = avail[n].limit; 6101 6102 n = md - mem; 6103 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6104 6105 for (lo = 0; lo < i; lo++) 6106 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6107 avail[lo].limit - 1); 6108 6109 sbuf_printf(sb, "\n"); 6110 for (i = 0; i < n; i++) { 6111 if (mem[i].idx >= nitems(region)) 6112 continue; /* skip holes */ 6113 if (!mem[i].limit) 6114 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6115 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6116 mem[i].limit); 6117 } 6118 6119 sbuf_printf(sb, "\n"); 6120 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6121 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6122 mem_region_show(sb, "uP RAM:", lo, hi); 6123 6124 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6125 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6126 mem_region_show(sb, "uP Extmem2:", lo, hi); 6127 6128 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6129 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6130 G_PMRXMAXPAGE(lo), 6131 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6132 (lo & F_PMRXNUMCHN) ? 2 : 1); 6133 6134 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6135 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6136 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6137 G_PMTXMAXPAGE(lo), 6138 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6139 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6140 sbuf_printf(sb, "%u p-structs\n", 6141 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6142 6143 for (i = 0; i < 4; i++) { 6144 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6145 if (is_t4(sc)) { 6146 used = G_USED(lo); 6147 alloc = G_ALLOC(lo); 6148 } else { 6149 used = G_T5_USED(lo); 6150 alloc = G_T5_ALLOC(lo); 6151 } 6152 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6153 i, used, alloc); 6154 } 6155 for (i = 0; i < 4; i++) { 6156 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6157 if (is_t4(sc)) { 6158 used = G_USED(lo); 6159 alloc = G_ALLOC(lo); 6160 } else { 6161 used = G_T5_USED(lo); 6162 alloc = G_T5_ALLOC(lo); 6163 } 6164 sbuf_printf(sb, 6165 "\nLoopback %d using %u pages out of %u allocated", 6166 i, used, alloc); 6167 } 6168 6169 rc = sbuf_finish(sb); 6170 sbuf_delete(sb); 6171 6172 return (rc); 6173} 6174 6175static inline void 6176tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6177{ 6178 *mask = x | y; 6179 y = htobe64(y); 6180 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6181} 6182 6183static int 6184sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6185{ 6186 struct adapter *sc = arg1; 6187 struct sbuf *sb; 6188 int rc, i, n; 6189 6190 rc = sysctl_wire_old_buffer(req, 0); 6191 if (rc != 0) 6192 return (rc); 6193 6194 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6195 if (sb == NULL) 6196 return (ENOMEM); 6197 6198 sbuf_printf(sb, 6199 "Idx Ethernet address Mask Vld Ports PF" 6200 " VF Replication P0 P1 P2 P3 ML"); 6201 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6202 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6203 for (i = 0; i < n; i++) { 6204 uint64_t tcamx, tcamy, mask; 6205 uint32_t cls_lo, cls_hi; 6206 uint8_t addr[ETHER_ADDR_LEN]; 6207 6208 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6209 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6210 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6211 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6212 6213 if (tcamx & tcamy) 6214 continue; 6215 6216 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6217 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6218 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6219 addr[3], addr[4], addr[5], (uintmax_t)mask, 6220 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6221 G_PORTMAP(cls_hi), G_PF(cls_lo), 6222 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6223 6224 if (cls_lo & F_REPLICATE) { 6225 struct fw_ldst_cmd ldst_cmd; 6226 6227 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6228 ldst_cmd.op_to_addrspace = 6229 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6230 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6231 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6232 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6233 ldst_cmd.u.mps.fid_ctl = 6234 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6235 V_FW_LDST_CMD_CTL(i)); 6236 6237 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6238 "t4mps"); 6239 if (rc) 6240 break; 6241 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6242 sizeof(ldst_cmd), &ldst_cmd); 6243 end_synchronized_op(sc, 0); 6244 6245 if (rc != 0) { 6246 sbuf_printf(sb, 6247 " ------------ error %3u ------------", rc); 6248 rc = 0; 6249 } else { 6250 sbuf_printf(sb, " %08x %08x %08x %08x", 6251 be32toh(ldst_cmd.u.mps.rplc127_96), 6252 be32toh(ldst_cmd.u.mps.rplc95_64), 6253 be32toh(ldst_cmd.u.mps.rplc63_32), 6254 be32toh(ldst_cmd.u.mps.rplc31_0)); 6255 } 6256 } else 6257 sbuf_printf(sb, "%36s", ""); 6258 6259 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6260 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6261 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6262 } 6263 6264 if (rc) 6265 (void) sbuf_finish(sb); 6266 else 6267 rc = sbuf_finish(sb); 6268 sbuf_delete(sb); 6269 6270 return (rc); 6271} 6272 6273static int 6274sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6275{ 6276 struct adapter *sc = arg1; 6277 struct sbuf *sb; 6278 int rc; 6279 uint16_t mtus[NMTUS]; 6280 6281 rc = sysctl_wire_old_buffer(req, 0); 6282 if (rc != 0) 6283 return (rc); 6284 6285 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6286 if (sb == NULL) 6287 return (ENOMEM); 6288 6289 t4_read_mtu_tbl(sc, mtus, NULL); 6290 6291 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6292 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6293 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6294 mtus[14], mtus[15]); 6295 6296 rc = sbuf_finish(sb); 6297 sbuf_delete(sb); 6298 6299 return (rc); 6300} 6301 6302static int 6303sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6304{ 6305 struct adapter *sc = arg1; 6306 struct sbuf *sb; 6307 int rc, i; 6308 uint32_t cnt[PM_NSTATS]; 6309 uint64_t cyc[PM_NSTATS]; 6310 static const char *rx_stats[] = { 6311 "Read:", "Write bypass:", "Write mem:", "Flush:" 6312 }; 6313 static const char *tx_stats[] = { 6314 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6315 }; 6316 6317 rc = sysctl_wire_old_buffer(req, 0); 6318 if (rc != 0) 6319 return (rc); 6320 6321 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6322 if (sb == NULL) 6323 return (ENOMEM); 6324 6325 t4_pmtx_get_stats(sc, cnt, cyc); 6326 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6327 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6328 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6329 cyc[i]); 6330 6331 t4_pmrx_get_stats(sc, cnt, cyc); 6332 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6333 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6334 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6335 cyc[i]); 6336 6337 rc = sbuf_finish(sb); 6338 sbuf_delete(sb); 6339 6340 return (rc); 6341} 6342 6343static int 6344sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6345{ 6346 struct adapter *sc = arg1; 6347 struct sbuf *sb; 6348 int rc; 6349 struct tp_rdma_stats stats; 6350 6351 rc = sysctl_wire_old_buffer(req, 0); 6352 if (rc != 0) 6353 return (rc); 6354 6355 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6356 if (sb == NULL) 6357 return (ENOMEM); 6358 6359 t4_tp_get_rdma_stats(sc, &stats); 6360 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6361 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6362 6363 rc = sbuf_finish(sb); 6364 sbuf_delete(sb); 6365 6366 return (rc); 6367} 6368 6369static int 6370sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6371{ 6372 struct adapter *sc = arg1; 6373 struct sbuf *sb; 6374 int rc; 6375 struct tp_tcp_stats v4, v6; 6376 6377 rc = sysctl_wire_old_buffer(req, 0); 6378 if (rc != 0) 6379 return (rc); 6380 6381 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6382 if (sb == NULL) 6383 return (ENOMEM); 6384 6385 t4_tp_get_tcp_stats(sc, &v4, &v6); 6386 sbuf_printf(sb, 6387 " IP IPv6\n"); 6388 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6389 v4.tcpOutRsts, v6.tcpOutRsts); 6390 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6391 v4.tcpInSegs, v6.tcpInSegs); 6392 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6393 v4.tcpOutSegs, v6.tcpOutSegs); 6394 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6395 v4.tcpRetransSegs, v6.tcpRetransSegs); 6396 6397 rc = sbuf_finish(sb); 6398 sbuf_delete(sb); 6399 6400 return (rc); 6401} 6402 6403static int 6404sysctl_tids(SYSCTL_HANDLER_ARGS) 6405{ 6406 struct adapter *sc = arg1; 6407 struct sbuf *sb; 6408 int rc; 6409 struct tid_info *t = &sc->tids; 6410 6411 rc = sysctl_wire_old_buffer(req, 0); 6412 if (rc != 0) 6413 return (rc); 6414 6415 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6416 if (sb == NULL) 6417 return (ENOMEM); 6418 6419 if (t->natids) { 6420 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6421 t->atids_in_use); 6422 } 6423 6424 if (t->ntids) { 6425 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6426 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6427 6428 if (b) { 6429 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6430 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6431 t->ntids - 1); 6432 } else { 6433 sbuf_printf(sb, "TID range: %u-%u", 6434 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6435 t->ntids - 1); 6436 } 6437 } else 6438 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6439 sbuf_printf(sb, ", in use: %u\n", 6440 atomic_load_acq_int(&t->tids_in_use)); 6441 } 6442 6443 if (t->nstids) { 6444 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6445 t->stid_base + t->nstids - 1, t->stids_in_use); 6446 } 6447 6448 if (t->nftids) { 6449 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6450 t->ftid_base + t->nftids - 1); 6451 } 6452 6453 if (t->netids) { 6454 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6455 t->etid_base + t->netids - 1); 6456 } 6457 6458 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6459 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6460 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6461 6462 rc = sbuf_finish(sb); 6463 sbuf_delete(sb); 6464 6465 return (rc); 6466} 6467 6468static int 6469sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6470{ 6471 struct adapter *sc = arg1; 6472 struct sbuf *sb; 6473 int rc; 6474 struct tp_err_stats stats; 6475 6476 rc = sysctl_wire_old_buffer(req, 0); 6477 if (rc != 0) 6478 return (rc); 6479 6480 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6481 if (sb == NULL) 6482 return (ENOMEM); 6483 6484 t4_tp_get_err_stats(sc, &stats); 6485 6486 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6487 "channel 3\n"); 6488 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6489 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6490 stats.macInErrs[3]); 6491 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6492 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6493 stats.hdrInErrs[3]); 6494 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6495 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6496 stats.tcpInErrs[3]); 6497 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6498 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6499 stats.tcp6InErrs[3]); 6500 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6501 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6502 stats.tnlCongDrops[3]); 6503 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6504 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6505 stats.tnlTxDrops[3]); 6506 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6507 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6508 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6509 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6510 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6511 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6512 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6513 stats.ofldNoNeigh, stats.ofldCongDefer); 6514 6515 rc = sbuf_finish(sb); 6516 sbuf_delete(sb); 6517 6518 return (rc); 6519} 6520 6521struct field_desc { 6522 const char *name; 6523 u_int start; 6524 u_int width; 6525}; 6526 6527static void 6528field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6529{ 6530 char buf[32]; 6531 int line_size = 0; 6532 6533 while (f->name) { 6534 uint64_t mask = (1ULL << f->width) - 1; 6535 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6536 ((uintmax_t)v >> f->start) & mask); 6537 6538 if (line_size + len >= 79) { 6539 line_size = 8; 6540 sbuf_printf(sb, "\n "); 6541 } 6542 sbuf_printf(sb, "%s ", buf); 6543 line_size += len + 1; 6544 f++; 6545 } 6546 sbuf_printf(sb, "\n"); 6547} 6548 6549static struct field_desc tp_la0[] = { 6550 { "RcfOpCodeOut", 60, 4 }, 6551 { "State", 56, 4 }, 6552 { "WcfState", 52, 4 }, 6553 { "RcfOpcSrcOut", 50, 2 }, 6554 { "CRxError", 49, 1 }, 6555 { "ERxError", 48, 1 }, 6556 { "SanityFailed", 47, 1 }, 6557 { "SpuriousMsg", 46, 1 }, 6558 { "FlushInputMsg", 45, 1 }, 6559 { "FlushInputCpl", 44, 1 }, 6560 { "RssUpBit", 43, 1 }, 6561 { "RssFilterHit", 42, 1 }, 6562 { "Tid", 32, 10 }, 6563 { "InitTcb", 31, 1 }, 6564 { "LineNumber", 24, 7 }, 6565 { "Emsg", 23, 1 }, 6566 { "EdataOut", 22, 1 }, 6567 { "Cmsg", 21, 1 }, 6568 { "CdataOut", 20, 1 }, 6569 { "EreadPdu", 19, 1 }, 6570 { "CreadPdu", 18, 1 }, 6571 { "TunnelPkt", 17, 1 }, 6572 { "RcfPeerFin", 16, 1 }, 6573 { "RcfReasonOut", 12, 4 }, 6574 { "TxCchannel", 10, 2 }, 6575 { "RcfTxChannel", 8, 2 }, 6576 { "RxEchannel", 6, 2 }, 6577 { "RcfRxChannel", 5, 1 }, 6578 { "RcfDataOutSrdy", 4, 1 }, 6579 { "RxDvld", 3, 1 }, 6580 { "RxOoDvld", 2, 1 }, 6581 { "RxCongestion", 1, 1 }, 6582 { "TxCongestion", 0, 1 }, 6583 { NULL } 6584}; 6585 6586static struct field_desc tp_la1[] = { 6587 { "CplCmdIn", 56, 8 }, 6588 { "CplCmdOut", 48, 8 }, 6589 { "ESynOut", 47, 1 }, 6590 { "EAckOut", 46, 1 }, 6591 { "EFinOut", 45, 1 }, 6592 { "ERstOut", 44, 1 }, 6593 { "SynIn", 43, 1 }, 6594 { "AckIn", 42, 1 }, 6595 { "FinIn", 41, 1 }, 6596 { "RstIn", 40, 1 }, 6597 { "DataIn", 39, 1 }, 6598 { "DataInVld", 38, 1 }, 6599 { "PadIn", 37, 1 }, 6600 { "RxBufEmpty", 36, 1 }, 6601 { "RxDdp", 35, 1 }, 6602 { "RxFbCongestion", 34, 1 }, 6603 { "TxFbCongestion", 33, 1 }, 6604 { "TxPktSumSrdy", 32, 1 }, 6605 { "RcfUlpType", 28, 4 }, 6606 { "Eread", 27, 1 }, 6607 { "Ebypass", 26, 1 }, 6608 { "Esave", 25, 1 }, 6609 { "Static0", 24, 1 }, 6610 { "Cread", 23, 1 }, 6611 { "Cbypass", 22, 1 }, 6612 { "Csave", 21, 1 }, 6613 { "CPktOut", 20, 1 }, 6614 { "RxPagePoolFull", 18, 2 }, 6615 { "RxLpbkPkt", 17, 1 }, 6616 { "TxLpbkPkt", 16, 1 }, 6617 { "RxVfValid", 15, 1 }, 6618 { "SynLearned", 14, 1 }, 6619 { "SetDelEntry", 13, 1 }, 6620 { "SetInvEntry", 12, 1 }, 6621 { "CpcmdDvld", 11, 1 }, 6622 { "CpcmdSave", 10, 1 }, 6623 { "RxPstructsFull", 8, 2 }, 6624 { "EpcmdDvld", 7, 1 }, 6625 { "EpcmdFlush", 6, 1 }, 6626 { "EpcmdTrimPrefix", 5, 1 }, 6627 { "EpcmdTrimPostfix", 4, 1 }, 6628 { "ERssIp4Pkt", 3, 1 }, 6629 { "ERssIp6Pkt", 2, 1 }, 6630 { "ERssTcpUdpPkt", 1, 1 }, 6631 { "ERssFceFipPkt", 0, 1 }, 6632 { NULL } 6633}; 6634 6635static struct field_desc tp_la2[] = { 6636 { "CplCmdIn", 56, 8 }, 6637 { "MpsVfVld", 55, 1 }, 6638 { "MpsPf", 52, 3 }, 6639 { "MpsVf", 44, 8 }, 6640 { "SynIn", 43, 1 }, 6641 { "AckIn", 42, 1 }, 6642 { "FinIn", 41, 1 }, 6643 { "RstIn", 40, 1 }, 6644 { "DataIn", 39, 1 }, 6645 { "DataInVld", 38, 1 }, 6646 { "PadIn", 37, 1 }, 6647 { "RxBufEmpty", 36, 1 }, 6648 { "RxDdp", 35, 1 }, 6649 { "RxFbCongestion", 34, 1 }, 6650 { "TxFbCongestion", 33, 1 }, 6651 { "TxPktSumSrdy", 32, 1 }, 6652 { "RcfUlpType", 28, 4 }, 6653 { "Eread", 27, 1 }, 6654 { "Ebypass", 26, 1 }, 6655 { "Esave", 25, 1 }, 6656 { "Static0", 24, 1 }, 6657 { "Cread", 23, 1 }, 6658 { "Cbypass", 22, 1 }, 6659 { "Csave", 21, 1 }, 6660 { "CPktOut", 20, 1 }, 6661 { "RxPagePoolFull", 18, 2 }, 6662 { "RxLpbkPkt", 17, 1 }, 6663 { "TxLpbkPkt", 16, 1 }, 6664 { "RxVfValid", 15, 1 }, 6665 { "SynLearned", 14, 1 }, 6666 { "SetDelEntry", 13, 1 }, 6667 { "SetInvEntry", 12, 1 }, 6668 { "CpcmdDvld", 11, 1 }, 6669 { "CpcmdSave", 10, 1 }, 6670 { "RxPstructsFull", 8, 2 }, 6671 { "EpcmdDvld", 7, 1 }, 6672 { "EpcmdFlush", 6, 1 }, 6673 { "EpcmdTrimPrefix", 5, 1 }, 6674 { "EpcmdTrimPostfix", 4, 1 }, 6675 { "ERssIp4Pkt", 3, 1 }, 6676 { "ERssIp6Pkt", 2, 1 }, 6677 { "ERssTcpUdpPkt", 1, 1 }, 6678 { "ERssFceFipPkt", 0, 1 }, 6679 { NULL } 6680}; 6681 6682static void 6683tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6684{ 6685 6686 field_desc_show(sb, *p, tp_la0); 6687} 6688 6689static void 6690tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6691{ 6692 6693 if (idx) 6694 sbuf_printf(sb, "\n"); 6695 field_desc_show(sb, p[0], tp_la0); 6696 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6697 field_desc_show(sb, p[1], tp_la0); 6698} 6699 6700static void 6701tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6702{ 6703 6704 if (idx) 6705 sbuf_printf(sb, "\n"); 6706 field_desc_show(sb, p[0], tp_la0); 6707 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6708 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6709} 6710 6711static int 6712sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6713{ 6714 struct adapter *sc = arg1; 6715 struct sbuf *sb; 6716 uint64_t *buf, *p; 6717 int rc; 6718 u_int i, inc; 6719 void (*show_func)(struct sbuf *, uint64_t *, int); 6720 6721 rc = sysctl_wire_old_buffer(req, 0); 6722 if (rc != 0) 6723 return (rc); 6724 6725 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6726 if (sb == NULL) 6727 return (ENOMEM); 6728 6729 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6730 6731 t4_tp_read_la(sc, buf, NULL); 6732 p = buf; 6733 6734 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6735 case 2: 6736 inc = 2; 6737 show_func = tp_la_show2; 6738 break; 6739 case 3: 6740 inc = 2; 6741 show_func = tp_la_show3; 6742 break; 6743 default: 6744 inc = 1; 6745 show_func = tp_la_show; 6746 } 6747 6748 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6749 (*show_func)(sb, p, i); 6750 6751 rc = sbuf_finish(sb); 6752 sbuf_delete(sb); 6753 free(buf, M_CXGBE); 6754 return (rc); 6755} 6756 6757static int 6758sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6759{ 6760 struct adapter *sc = arg1; 6761 struct sbuf *sb; 6762 int rc; 6763 u64 nrate[NCHAN], orate[NCHAN]; 6764 6765 rc = sysctl_wire_old_buffer(req, 0); 6766 if (rc != 0) 6767 return (rc); 6768 6769 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6770 if (sb == NULL) 6771 return (ENOMEM); 6772 6773 t4_get_chan_txrate(sc, nrate, orate); 6774 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6775 "channel 3\n"); 6776 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6777 nrate[0], nrate[1], nrate[2], nrate[3]); 6778 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6779 orate[0], orate[1], orate[2], orate[3]); 6780 6781 rc = sbuf_finish(sb); 6782 sbuf_delete(sb); 6783 6784 return (rc); 6785} 6786 6787static int 6788sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6789{ 6790 struct adapter *sc = arg1; 6791 struct sbuf *sb; 6792 uint32_t *buf, *p; 6793 int rc, i; 6794 6795 rc = sysctl_wire_old_buffer(req, 0); 6796 if (rc != 0) 6797 return (rc); 6798 6799 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6800 if (sb == NULL) 6801 return (ENOMEM); 6802 6803 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6804 M_ZERO | M_WAITOK); 6805 6806 t4_ulprx_read_la(sc, buf); 6807 p = buf; 6808 6809 sbuf_printf(sb, " Pcmd Type Message" 6810 " Data"); 6811 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6812 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6813 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6814 } 6815 6816 rc = sbuf_finish(sb); 6817 sbuf_delete(sb); 6818 free(buf, M_CXGBE); 6819 return (rc); 6820} 6821 6822static int 6823sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6824{ 6825 struct adapter *sc = arg1; 6826 struct sbuf *sb; 6827 int rc, v; 6828 6829 rc = sysctl_wire_old_buffer(req, 0); 6830 if (rc != 0) 6831 return (rc); 6832 6833 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6834 if (sb == NULL) 6835 return (ENOMEM); 6836 6837 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6838 if (G_STATSOURCE_T5(v) == 7) { 6839 if (G_STATMODE(v) == 0) { 6840 sbuf_printf(sb, "total %d, incomplete %d", 6841 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6842 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6843 } else if (G_STATMODE(v) == 1) { 6844 sbuf_printf(sb, "total %d, data overflow %d", 6845 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6846 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6847 } 6848 } 6849 rc = sbuf_finish(sb); 6850 sbuf_delete(sb); 6851 6852 return (rc); 6853} 6854#endif 6855 6856static inline void 6857txq_start(struct ifnet *ifp, struct sge_txq *txq) 6858{ 6859 struct buf_ring *br; 6860 struct mbuf *m; 6861 6862 TXQ_LOCK_ASSERT_OWNED(txq); 6863 6864 br = txq->br; 6865 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6866 if (m) 6867 t4_eth_tx(ifp, txq, m); 6868} 6869 6870void 6871t4_tx_callout(void *arg) 6872{ 6873 struct sge_eq *eq = arg; 6874 struct adapter *sc; 6875 6876 if (EQ_TRYLOCK(eq) == 0) 6877 goto reschedule; 6878 6879 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6880 EQ_UNLOCK(eq); 6881reschedule: 6882 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6883 callout_schedule(&eq->tx_callout, 1); 6884 return; 6885 } 6886 6887 EQ_LOCK_ASSERT_OWNED(eq); 6888 6889 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6890 6891 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6892 struct sge_txq *txq = arg; 6893 struct port_info *pi = txq->ifp->if_softc; 6894 6895 sc = pi->adapter; 6896 } else { 6897 struct sge_wrq *wrq = arg; 6898 6899 sc = wrq->adapter; 6900 } 6901 6902 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6903 } 6904 6905 EQ_UNLOCK(eq); 6906} 6907 6908void 6909t4_tx_task(void *arg, int count) 6910{ 6911 struct sge_eq *eq = arg; 6912 6913 EQ_LOCK(eq); 6914 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6915 struct sge_txq *txq = arg; 6916 txq_start(txq->ifp, txq); 6917 } else { 6918 struct sge_wrq *wrq = arg; 6919 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 6920 } 6921 EQ_UNLOCK(eq); 6922} 6923 6924static uint32_t 6925fconf_to_mode(uint32_t fconf) 6926{ 6927 uint32_t mode; 6928 6929 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6930 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6931 6932 if (fconf & F_FRAGMENTATION) 6933 mode |= T4_FILTER_IP_FRAGMENT; 6934 6935 if (fconf & F_MPSHITTYPE) 6936 mode |= T4_FILTER_MPS_HIT_TYPE; 6937 6938 if (fconf & F_MACMATCH) 6939 mode |= T4_FILTER_MAC_IDX; 6940 6941 if (fconf & F_ETHERTYPE) 6942 mode |= T4_FILTER_ETH_TYPE; 6943 6944 if (fconf & F_PROTOCOL) 6945 mode |= T4_FILTER_IP_PROTO; 6946 6947 if (fconf & F_TOS) 6948 mode |= T4_FILTER_IP_TOS; 6949 6950 if (fconf & F_VLAN) 6951 mode |= T4_FILTER_VLAN; 6952 6953 if (fconf & F_VNIC_ID) 6954 mode |= T4_FILTER_VNIC; 6955 6956 if (fconf & F_PORT) 6957 mode |= T4_FILTER_PORT; 6958 6959 if (fconf & F_FCOE) 6960 mode |= T4_FILTER_FCoE; 6961 6962 return (mode); 6963} 6964 6965static uint32_t 6966mode_to_fconf(uint32_t mode) 6967{ 6968 uint32_t fconf = 0; 6969 6970 if (mode & T4_FILTER_IP_FRAGMENT) 6971 fconf |= F_FRAGMENTATION; 6972 6973 if (mode & T4_FILTER_MPS_HIT_TYPE) 6974 fconf |= F_MPSHITTYPE; 6975 6976 if (mode & T4_FILTER_MAC_IDX) 6977 fconf |= F_MACMATCH; 6978 6979 if (mode & T4_FILTER_ETH_TYPE) 6980 fconf |= F_ETHERTYPE; 6981 6982 if (mode & T4_FILTER_IP_PROTO) 6983 fconf |= F_PROTOCOL; 6984 6985 if (mode & T4_FILTER_IP_TOS) 6986 fconf |= F_TOS; 6987 6988 if (mode & T4_FILTER_VLAN) 6989 fconf |= F_VLAN; 6990 6991 if (mode & T4_FILTER_VNIC) 6992 fconf |= F_VNIC_ID; 6993 6994 if (mode & T4_FILTER_PORT) 6995 fconf |= F_PORT; 6996 6997 if (mode & T4_FILTER_FCoE) 6998 fconf |= F_FCOE; 6999 7000 return (fconf); 7001} 7002 7003static uint32_t 7004fspec_to_fconf(struct t4_filter_specification *fs) 7005{ 7006 uint32_t fconf = 0; 7007 7008 if (fs->val.frag || fs->mask.frag) 7009 fconf |= F_FRAGMENTATION; 7010 7011 if (fs->val.matchtype || fs->mask.matchtype) 7012 fconf |= F_MPSHITTYPE; 7013 7014 if (fs->val.macidx || fs->mask.macidx) 7015 fconf |= F_MACMATCH; 7016 7017 if (fs->val.ethtype || fs->mask.ethtype) 7018 fconf |= F_ETHERTYPE; 7019 7020 if (fs->val.proto || fs->mask.proto) 7021 fconf |= F_PROTOCOL; 7022 7023 if (fs->val.tos || fs->mask.tos) 7024 fconf |= F_TOS; 7025 7026 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7027 fconf |= F_VLAN; 7028 7029 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7030 fconf |= F_VNIC_ID; 7031 7032 if (fs->val.iport || fs->mask.iport) 7033 fconf |= F_PORT; 7034 7035 if (fs->val.fcoe || fs->mask.fcoe) 7036 fconf |= F_FCOE; 7037 7038 return (fconf); 7039} 7040 7041static int 7042get_filter_mode(struct adapter *sc, uint32_t *mode) 7043{ 7044 int rc; 7045 uint32_t fconf; 7046 7047 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7048 "t4getfm"); 7049 if (rc) 7050 return (rc); 7051 7052 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7053 A_TP_VLAN_PRI_MAP); 7054 7055 if (sc->params.tp.vlan_pri_map != fconf) { 7056 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7057 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7058 fconf); 7059 } 7060 7061 *mode = fconf_to_mode(fconf); 7062 7063 end_synchronized_op(sc, LOCK_HELD); 7064 return (0); 7065} 7066 7067static int 7068set_filter_mode(struct adapter *sc, uint32_t mode) 7069{ 7070 uint32_t fconf; 7071 int rc; 7072 7073 fconf = mode_to_fconf(mode); 7074 7075 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7076 "t4setfm"); 7077 if (rc) 7078 return (rc); 7079 7080 if (sc->tids.ftids_in_use > 0) { 7081 rc = EBUSY; 7082 goto done; 7083 } 7084 7085#ifdef TCP_OFFLOAD 7086 if (sc->offload_map) { 7087 rc = EBUSY; 7088 goto done; 7089 } 7090#endif 7091 7092 rc = -t4_set_filter_mode(sc, fconf); 7093done: 7094 end_synchronized_op(sc, LOCK_HELD); 7095 return (rc); 7096} 7097 7098static inline uint64_t 7099get_filter_hits(struct adapter *sc, uint32_t fid) 7100{ 7101 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7102 uint64_t hits; 7103 7104 memwin_info(sc, 0, &mw_base, NULL); 7105 off = position_memwin(sc, 0, 7106 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7107 if (is_t4(sc)) { 7108 hits = t4_read_reg64(sc, mw_base + off + 16); 7109 hits = be64toh(hits); 7110 } else { 7111 hits = t4_read_reg(sc, mw_base + off + 24); 7112 hits = be32toh(hits); 7113 } 7114 7115 return (hits); 7116} 7117 7118static int 7119get_filter(struct adapter *sc, struct t4_filter *t) 7120{ 7121 int i, rc, nfilters = sc->tids.nftids; 7122 struct filter_entry *f; 7123 7124 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7125 "t4getf"); 7126 if (rc) 7127 return (rc); 7128 7129 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7130 t->idx >= nfilters) { 7131 t->idx = 0xffffffff; 7132 goto done; 7133 } 7134 7135 f = &sc->tids.ftid_tab[t->idx]; 7136 for (i = t->idx; i < nfilters; i++, f++) { 7137 if (f->valid) { 7138 t->idx = i; 7139 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7140 t->smtidx = f->smtidx; 7141 if (f->fs.hitcnts) 7142 t->hits = get_filter_hits(sc, t->idx); 7143 else 7144 t->hits = UINT64_MAX; 7145 t->fs = f->fs; 7146 7147 goto done; 7148 } 7149 } 7150 7151 t->idx = 0xffffffff; 7152done: 7153 end_synchronized_op(sc, LOCK_HELD); 7154 return (0); 7155} 7156 7157static int 7158set_filter(struct adapter *sc, struct t4_filter *t) 7159{ 7160 unsigned int nfilters, nports; 7161 struct filter_entry *f; 7162 int i, rc; 7163 7164 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7165 if (rc) 7166 return (rc); 7167 7168 nfilters = sc->tids.nftids; 7169 nports = sc->params.nports; 7170 7171 if (nfilters == 0) { 7172 rc = ENOTSUP; 7173 goto done; 7174 } 7175 7176 if (!(sc->flags & FULL_INIT_DONE)) { 7177 rc = EAGAIN; 7178 goto done; 7179 } 7180 7181 if (t->idx >= nfilters) { 7182 rc = EINVAL; 7183 goto done; 7184 } 7185 7186 /* Validate against the global filter mode */ 7187 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7188 sc->params.tp.vlan_pri_map) { 7189 rc = E2BIG; 7190 goto done; 7191 } 7192 7193 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7194 rc = EINVAL; 7195 goto done; 7196 } 7197 7198 if (t->fs.val.iport >= nports) { 7199 rc = EINVAL; 7200 goto done; 7201 } 7202 7203 /* Can't specify an iq if not steering to it */ 7204 if (!t->fs.dirsteer && t->fs.iq) { 7205 rc = EINVAL; 7206 goto done; 7207 } 7208 7209 /* IPv6 filter idx must be 4 aligned */ 7210 if (t->fs.type == 1 && 7211 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7212 rc = EINVAL; 7213 goto done; 7214 } 7215 7216 if (sc->tids.ftid_tab == NULL) { 7217 KASSERT(sc->tids.ftids_in_use == 0, 7218 ("%s: no memory allocated but filters_in_use > 0", 7219 __func__)); 7220 7221 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7222 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7223 if (sc->tids.ftid_tab == NULL) { 7224 rc = ENOMEM; 7225 goto done; 7226 } 7227 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7228 } 7229 7230 for (i = 0; i < 4; i++) { 7231 f = &sc->tids.ftid_tab[t->idx + i]; 7232 7233 if (f->pending || f->valid) { 7234 rc = EBUSY; 7235 goto done; 7236 } 7237 if (f->locked) { 7238 rc = EPERM; 7239 goto done; 7240 } 7241 7242 if (t->fs.type == 0) 7243 break; 7244 } 7245 7246 f = &sc->tids.ftid_tab[t->idx]; 7247 f->fs = t->fs; 7248 7249 rc = set_filter_wr(sc, t->idx); 7250done: 7251 end_synchronized_op(sc, 0); 7252 7253 if (rc == 0) { 7254 mtx_lock(&sc->tids.ftid_lock); 7255 for (;;) { 7256 if (f->pending == 0) { 7257 rc = f->valid ? 0 : EIO; 7258 break; 7259 } 7260 7261 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7262 PCATCH, "t4setfw", 0)) { 7263 rc = EINPROGRESS; 7264 break; 7265 } 7266 } 7267 mtx_unlock(&sc->tids.ftid_lock); 7268 } 7269 return (rc); 7270} 7271 7272static int 7273del_filter(struct adapter *sc, struct t4_filter *t) 7274{ 7275 unsigned int nfilters; 7276 struct filter_entry *f; 7277 int rc; 7278 7279 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7280 if (rc) 7281 return (rc); 7282 7283 nfilters = sc->tids.nftids; 7284 7285 if (nfilters == 0) { 7286 rc = ENOTSUP; 7287 goto done; 7288 } 7289 7290 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7291 t->idx >= nfilters) { 7292 rc = EINVAL; 7293 goto done; 7294 } 7295 7296 if (!(sc->flags & FULL_INIT_DONE)) { 7297 rc = EAGAIN; 7298 goto done; 7299 } 7300 7301 f = &sc->tids.ftid_tab[t->idx]; 7302 7303 if (f->pending) { 7304 rc = EBUSY; 7305 goto done; 7306 } 7307 if (f->locked) { 7308 rc = EPERM; 7309 goto done; 7310 } 7311 7312 if (f->valid) { 7313 t->fs = f->fs; /* extra info for the caller */ 7314 rc = del_filter_wr(sc, t->idx); 7315 } 7316 7317done: 7318 end_synchronized_op(sc, 0); 7319 7320 if (rc == 0) { 7321 mtx_lock(&sc->tids.ftid_lock); 7322 for (;;) { 7323 if (f->pending == 0) { 7324 rc = f->valid ? EIO : 0; 7325 break; 7326 } 7327 7328 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7329 PCATCH, "t4delfw", 0)) { 7330 rc = EINPROGRESS; 7331 break; 7332 } 7333 } 7334 mtx_unlock(&sc->tids.ftid_lock); 7335 } 7336 7337 return (rc); 7338} 7339 7340static void 7341clear_filter(struct filter_entry *f) 7342{ 7343 if (f->l2t) 7344 t4_l2t_release(f->l2t); 7345 7346 bzero(f, sizeof (*f)); 7347} 7348 7349static int 7350set_filter_wr(struct adapter *sc, int fidx) 7351{ 7352 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7353 struct wrqe *wr; 7354 struct fw_filter_wr *fwr; 7355 unsigned int ftid; 7356 7357 ASSERT_SYNCHRONIZED_OP(sc); 7358 7359 if (f->fs.newdmac || f->fs.newvlan) { 7360 /* This filter needs an L2T entry; allocate one. */ 7361 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7362 if (f->l2t == NULL) 7363 return (EAGAIN); 7364 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7365 f->fs.dmac)) { 7366 t4_l2t_release(f->l2t); 7367 f->l2t = NULL; 7368 return (ENOMEM); 7369 } 7370 } 7371 7372 ftid = sc->tids.ftid_base + fidx; 7373 7374 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7375 if (wr == NULL) 7376 return (ENOMEM); 7377 7378 fwr = wrtod(wr); 7379 bzero(fwr, sizeof (*fwr)); 7380 7381 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7382 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7383 fwr->tid_to_iq = 7384 htobe32(V_FW_FILTER_WR_TID(ftid) | 7385 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7386 V_FW_FILTER_WR_NOREPLY(0) | 7387 V_FW_FILTER_WR_IQ(f->fs.iq)); 7388 fwr->del_filter_to_l2tix = 7389 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7390 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7391 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7392 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7393 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7394 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7395 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7396 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7397 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7398 f->fs.newvlan == VLAN_REWRITE) | 7399 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7400 f->fs.newvlan == VLAN_REWRITE) | 7401 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7402 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7403 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7404 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7405 fwr->ethtype = htobe16(f->fs.val.ethtype); 7406 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7407 fwr->frag_to_ovlan_vldm = 7408 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7409 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7410 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7411 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7412 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7413 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7414 fwr->smac_sel = 0; 7415 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7416 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7417 fwr->maci_to_matchtypem = 7418 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7419 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7420 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7421 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7422 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7423 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7424 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7425 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7426 fwr->ptcl = f->fs.val.proto; 7427 fwr->ptclm = f->fs.mask.proto; 7428 fwr->ttyp = f->fs.val.tos; 7429 fwr->ttypm = f->fs.mask.tos; 7430 fwr->ivlan = htobe16(f->fs.val.vlan); 7431 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7432 fwr->ovlan = htobe16(f->fs.val.vnic); 7433 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7434 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7435 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7436 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7437 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7438 fwr->lp = htobe16(f->fs.val.dport); 7439 fwr->lpm = htobe16(f->fs.mask.dport); 7440 fwr->fp = htobe16(f->fs.val.sport); 7441 fwr->fpm = htobe16(f->fs.mask.sport); 7442 if (f->fs.newsmac) 7443 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7444 7445 f->pending = 1; 7446 sc->tids.ftids_in_use++; 7447 7448 t4_wrq_tx(sc, wr); 7449 return (0); 7450} 7451 7452static int 7453del_filter_wr(struct adapter *sc, int fidx) 7454{ 7455 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7456 struct wrqe *wr; 7457 struct fw_filter_wr *fwr; 7458 unsigned int ftid; 7459 7460 ftid = sc->tids.ftid_base + fidx; 7461 7462 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7463 if (wr == NULL) 7464 return (ENOMEM); 7465 fwr = wrtod(wr); 7466 bzero(fwr, sizeof (*fwr)); 7467 7468 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7469 7470 f->pending = 1; 7471 t4_wrq_tx(sc, wr); 7472 return (0); 7473} 7474 7475int 7476t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7477{ 7478 struct adapter *sc = iq->adapter; 7479 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7480 unsigned int idx = GET_TID(rpl); 7481 unsigned int rc; 7482 struct filter_entry *f; 7483 7484 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7485 rss->opcode)); 7486 7487 if (is_ftid(sc, idx)) { 7488 7489 idx -= sc->tids.ftid_base; 7490 f = &sc->tids.ftid_tab[idx]; 7491 rc = G_COOKIE(rpl->cookie); 7492 7493 mtx_lock(&sc->tids.ftid_lock); 7494 if (rc == FW_FILTER_WR_FLT_ADDED) { 7495 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7496 __func__, idx)); 7497 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7498 f->pending = 0; /* asynchronous setup completed */ 7499 f->valid = 1; 7500 } else { 7501 if (rc != FW_FILTER_WR_FLT_DELETED) { 7502 /* Add or delete failed, display an error */ 7503 log(LOG_ERR, 7504 "filter %u setup failed with error %u\n", 7505 idx, rc); 7506 } 7507 7508 clear_filter(f); 7509 sc->tids.ftids_in_use--; 7510 } 7511 wakeup(&sc->tids.ftid_tab); 7512 mtx_unlock(&sc->tids.ftid_lock); 7513 } 7514 7515 return (0); 7516} 7517 7518static int 7519get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7520{ 7521 int rc; 7522 7523 if (cntxt->cid > M_CTXTQID) 7524 return (EINVAL); 7525 7526 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7527 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7528 return (EINVAL); 7529 7530 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7531 if (rc) 7532 return (rc); 7533 7534 if (sc->flags & FW_OK) { 7535 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7536 &cntxt->data[0]); 7537 if (rc == 0) 7538 goto done; 7539 } 7540 7541 /* 7542 * Read via firmware failed or wasn't even attempted. Read directly via 7543 * the backdoor. 7544 */ 7545 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7546done: 7547 end_synchronized_op(sc, 0); 7548 return (rc); 7549} 7550 7551static int 7552load_fw(struct adapter *sc, struct t4_data *fw) 7553{ 7554 int rc; 7555 uint8_t *fw_data; 7556 7557 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7558 if (rc) 7559 return (rc); 7560 7561 if (sc->flags & FULL_INIT_DONE) { 7562 rc = EBUSY; 7563 goto done; 7564 } 7565 7566 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7567 if (fw_data == NULL) { 7568 rc = ENOMEM; 7569 goto done; 7570 } 7571 7572 rc = copyin(fw->data, fw_data, fw->len); 7573 if (rc == 0) 7574 rc = -t4_load_fw(sc, fw_data, fw->len); 7575 7576 free(fw_data, M_CXGBE); 7577done: 7578 end_synchronized_op(sc, 0); 7579 return (rc); 7580} 7581 7582static int 7583read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7584{ 7585 uint32_t addr, off, remaining, i, n; 7586 uint32_t *buf, *b; 7587 uint32_t mw_base, mw_aperture; 7588 int rc; 7589 uint8_t *dst; 7590 7591 rc = validate_mem_range(sc, mr->addr, mr->len); 7592 if (rc != 0) 7593 return (rc); 7594 7595 memwin_info(sc, win, &mw_base, &mw_aperture); 7596 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7597 addr = mr->addr; 7598 remaining = mr->len; 7599 dst = (void *)mr->data; 7600 7601 while (remaining) { 7602 off = position_memwin(sc, win, addr); 7603 7604 /* number of bytes that we'll copy in the inner loop */ 7605 n = min(remaining, mw_aperture - off); 7606 for (i = 0; i < n; i += 4) 7607 *b++ = t4_read_reg(sc, mw_base + off + i); 7608 7609 rc = copyout(buf, dst, n); 7610 if (rc != 0) 7611 break; 7612 7613 b = buf; 7614 dst += n; 7615 remaining -= n; 7616 addr += n; 7617 } 7618 7619 free(buf, M_CXGBE); 7620 return (rc); 7621} 7622 7623static int 7624read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7625{ 7626 int rc; 7627 7628 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7629 return (EINVAL); 7630 7631 if (i2cd->len > sizeof(i2cd->data)) 7632 return (EFBIG); 7633 7634 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7635 if (rc) 7636 return (rc); 7637 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7638 i2cd->offset, i2cd->len, &i2cd->data[0]); 7639 end_synchronized_op(sc, 0); 7640 7641 return (rc); 7642} 7643 7644static int 7645in_range(int val, int lo, int hi) 7646{ 7647 7648 return (val < 0 || (val <= hi && val >= lo)); 7649} 7650 7651static int 7652set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7653{ 7654 int fw_subcmd, fw_type, rc; 7655 7656 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7657 if (rc) 7658 return (rc); 7659 7660 if (!(sc->flags & FULL_INIT_DONE)) { 7661 rc = EAGAIN; 7662 goto done; 7663 } 7664 7665 /* 7666 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7667 * sub-command and type are in common locations.) 7668 */ 7669 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7670 fw_subcmd = FW_SCHED_SC_CONFIG; 7671 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7672 fw_subcmd = FW_SCHED_SC_PARAMS; 7673 else { 7674 rc = EINVAL; 7675 goto done; 7676 } 7677 if (p->type == SCHED_CLASS_TYPE_PACKET) 7678 fw_type = FW_SCHED_TYPE_PKTSCHED; 7679 else { 7680 rc = EINVAL; 7681 goto done; 7682 } 7683 7684 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7685 /* Vet our parameters ..*/ 7686 if (p->u.config.minmax < 0) { 7687 rc = EINVAL; 7688 goto done; 7689 } 7690 7691 /* And pass the request to the firmware ...*/ 7692 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7693 goto done; 7694 } 7695 7696 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7697 int fw_level; 7698 int fw_mode; 7699 int fw_rateunit; 7700 int fw_ratemode; 7701 7702 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7703 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7704 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7705 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7706 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7707 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7708 else { 7709 rc = EINVAL; 7710 goto done; 7711 } 7712 7713 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7714 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7715 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7716 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7717 else { 7718 rc = EINVAL; 7719 goto done; 7720 } 7721 7722 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7723 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7724 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7725 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7726 else { 7727 rc = EINVAL; 7728 goto done; 7729 } 7730 7731 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7732 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7733 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7734 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7735 else { 7736 rc = EINVAL; 7737 goto done; 7738 } 7739 7740 /* Vet our parameters ... */ 7741 if (!in_range(p->u.params.channel, 0, 3) || 7742 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7743 !in_range(p->u.params.minrate, 0, 10000000) || 7744 !in_range(p->u.params.maxrate, 0, 10000000) || 7745 !in_range(p->u.params.weight, 0, 100)) { 7746 rc = ERANGE; 7747 goto done; 7748 } 7749 7750 /* 7751 * Translate any unset parameters into the firmware's 7752 * nomenclature and/or fail the call if the parameters 7753 * are required ... 7754 */ 7755 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7756 p->u.params.channel < 0 || p->u.params.cl < 0) { 7757 rc = EINVAL; 7758 goto done; 7759 } 7760 if (p->u.params.minrate < 0) 7761 p->u.params.minrate = 0; 7762 if (p->u.params.maxrate < 0) { 7763 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7764 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7765 rc = EINVAL; 7766 goto done; 7767 } else 7768 p->u.params.maxrate = 0; 7769 } 7770 if (p->u.params.weight < 0) { 7771 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7772 rc = EINVAL; 7773 goto done; 7774 } else 7775 p->u.params.weight = 0; 7776 } 7777 if (p->u.params.pktsize < 0) { 7778 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7779 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7780 rc = EINVAL; 7781 goto done; 7782 } else 7783 p->u.params.pktsize = 0; 7784 } 7785 7786 /* See what the firmware thinks of the request ... */ 7787 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7788 fw_rateunit, fw_ratemode, p->u.params.channel, 7789 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7790 p->u.params.weight, p->u.params.pktsize, 1); 7791 goto done; 7792 } 7793 7794 rc = EINVAL; 7795done: 7796 end_synchronized_op(sc, 0); 7797 return (rc); 7798} 7799 7800static int 7801set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7802{ 7803 struct port_info *pi = NULL; 7804 struct sge_txq *txq; 7805 uint32_t fw_mnem, fw_queue, fw_class; 7806 int i, rc; 7807 7808 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7809 if (rc) 7810 return (rc); 7811 7812 if (!(sc->flags & FULL_INIT_DONE)) { 7813 rc = EAGAIN; 7814 goto done; 7815 } 7816 7817 if (p->port >= sc->params.nports) { 7818 rc = EINVAL; 7819 goto done; 7820 } 7821 7822 pi = sc->port[p->port]; 7823 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7824 rc = EINVAL; 7825 goto done; 7826 } 7827 7828 /* 7829 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7830 * Scheduling Class in this case). 7831 */ 7832 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7833 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7834 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7835 7836 /* 7837 * If op.queue is non-negative, then we're only changing the scheduling 7838 * on a single specified TX queue. 7839 */ 7840 if (p->queue >= 0) { 7841 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7842 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7843 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7844 &fw_class); 7845 goto done; 7846 } 7847 7848 /* 7849 * Change the scheduling on all the TX queues for the 7850 * interface. 7851 */ 7852 for_each_txq(pi, i, txq) { 7853 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7854 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7855 &fw_class); 7856 if (rc) 7857 goto done; 7858 } 7859 7860 rc = 0; 7861done: 7862 end_synchronized_op(sc, 0); 7863 return (rc); 7864} 7865 7866int 7867t4_os_find_pci_capability(struct adapter *sc, int cap) 7868{ 7869 int i; 7870 7871 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7872} 7873 7874int 7875t4_os_pci_save_state(struct adapter *sc) 7876{ 7877 device_t dev; 7878 struct pci_devinfo *dinfo; 7879 7880 dev = sc->dev; 7881 dinfo = device_get_ivars(dev); 7882 7883 pci_cfg_save(dev, dinfo, 0); 7884 return (0); 7885} 7886 7887int 7888t4_os_pci_restore_state(struct adapter *sc) 7889{ 7890 device_t dev; 7891 struct pci_devinfo *dinfo; 7892 7893 dev = sc->dev; 7894 dinfo = device_get_ivars(dev); 7895 7896 pci_cfg_restore(dev, dinfo); 7897 return (0); 7898} 7899 7900void 7901t4_os_portmod_changed(const struct adapter *sc, int idx) 7902{ 7903 struct port_info *pi = sc->port[idx]; 7904 static const char *mod_str[] = { 7905 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7906 }; 7907 7908 build_medialist(pi, &pi->media); 7909#ifdef DEV_NETMAP 7910 build_medialist(pi, &pi->nm_media); 7911#endif 7912 7913 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7914 if_printf(pi->ifp, "transceiver unplugged.\n"); 7915 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7916 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7917 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7918 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7919 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7920 if_printf(pi->ifp, "%s transceiver inserted.\n", 7921 mod_str[pi->mod_type]); 7922 } else { 7923 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7924 pi->mod_type); 7925 } 7926} 7927 7928void 7929t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7930{ 7931 struct port_info *pi = sc->port[idx]; 7932 struct ifnet *ifp = pi->ifp; 7933 7934 if (link_stat) { 7935 pi->linkdnrc = -1; 7936 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7937 if_link_state_change(ifp, LINK_STATE_UP); 7938 } else { 7939 if (reason >= 0) 7940 pi->linkdnrc = reason; 7941 if_link_state_change(ifp, LINK_STATE_DOWN); 7942 } 7943} 7944 7945void 7946t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7947{ 7948 struct adapter *sc; 7949 7950 sx_slock(&t4_list_lock); 7951 SLIST_FOREACH(sc, &t4_list, link) { 7952 /* 7953 * func should not make any assumptions about what state sc is 7954 * in - the only guarantee is that sc->sc_lock is a valid lock. 7955 */ 7956 func(sc, arg); 7957 } 7958 sx_sunlock(&t4_list_lock); 7959} 7960 7961static int 7962t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7963{ 7964 return (0); 7965} 7966 7967static int 7968t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7969{ 7970 return (0); 7971} 7972 7973static int 7974t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7975 struct thread *td) 7976{ 7977 int rc; 7978 struct adapter *sc = dev->si_drv1; 7979 7980 rc = priv_check(td, PRIV_DRIVER); 7981 if (rc != 0) 7982 return (rc); 7983 7984 switch (cmd) { 7985 case CHELSIO_T4_GETREG: { 7986 struct t4_reg *edata = (struct t4_reg *)data; 7987 7988 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7989 return (EFAULT); 7990 7991 if (edata->size == 4) 7992 edata->val = t4_read_reg(sc, edata->addr); 7993 else if (edata->size == 8) 7994 edata->val = t4_read_reg64(sc, edata->addr); 7995 else 7996 return (EINVAL); 7997 7998 break; 7999 } 8000 case CHELSIO_T4_SETREG: { 8001 struct t4_reg *edata = (struct t4_reg *)data; 8002 8003 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8004 return (EFAULT); 8005 8006 if (edata->size == 4) { 8007 if (edata->val & 0xffffffff00000000) 8008 return (EINVAL); 8009 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8010 } else if (edata->size == 8) 8011 t4_write_reg64(sc, edata->addr, edata->val); 8012 else 8013 return (EINVAL); 8014 break; 8015 } 8016 case CHELSIO_T4_REGDUMP: { 8017 struct t4_regdump *regs = (struct t4_regdump *)data; 8018 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8019 uint8_t *buf; 8020 8021 if (regs->len < reglen) { 8022 regs->len = reglen; /* hint to the caller */ 8023 return (ENOBUFS); 8024 } 8025 8026 regs->len = reglen; 8027 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8028 t4_get_regs(sc, regs, buf); 8029 rc = copyout(buf, regs->data, reglen); 8030 free(buf, M_CXGBE); 8031 break; 8032 } 8033 case CHELSIO_T4_GET_FILTER_MODE: 8034 rc = get_filter_mode(sc, (uint32_t *)data); 8035 break; 8036 case CHELSIO_T4_SET_FILTER_MODE: 8037 rc = set_filter_mode(sc, *(uint32_t *)data); 8038 break; 8039 case CHELSIO_T4_GET_FILTER: 8040 rc = get_filter(sc, (struct t4_filter *)data); 8041 break; 8042 case CHELSIO_T4_SET_FILTER: 8043 rc = set_filter(sc, (struct t4_filter *)data); 8044 break; 8045 case CHELSIO_T4_DEL_FILTER: 8046 rc = del_filter(sc, (struct t4_filter *)data); 8047 break; 8048 case CHELSIO_T4_GET_SGE_CONTEXT: 8049 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8050 break; 8051 case CHELSIO_T4_LOAD_FW: 8052 rc = load_fw(sc, (struct t4_data *)data); 8053 break; 8054 case CHELSIO_T4_GET_MEM: 8055 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8056 break; 8057 case CHELSIO_T4_GET_I2C: 8058 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8059 break; 8060 case CHELSIO_T4_CLEAR_STATS: { 8061 int i; 8062 u_int port_id = *(uint32_t *)data; 8063 struct port_info *pi; 8064 8065 if (port_id >= sc->params.nports) 8066 return (EINVAL); 8067 pi = sc->port[port_id]; 8068 8069 /* MAC stats */ 8070 t4_clr_port_stats(sc, pi->tx_chan); 8071 8072 if (pi->flags & PORT_INIT_DONE) { 8073 struct sge_rxq *rxq; 8074 struct sge_txq *txq; 8075 struct sge_wrq *wrq; 8076 8077 for_each_rxq(pi, i, rxq) { 8078#if defined(INET) || defined(INET6) 8079 rxq->lro.lro_queued = 0; 8080 rxq->lro.lro_flushed = 0; 8081#endif 8082 rxq->rxcsum = 0; 8083 rxq->vlan_extraction = 0; 8084 } 8085 8086 for_each_txq(pi, i, txq) { 8087 txq->txcsum = 0; 8088 txq->tso_wrs = 0; 8089 txq->vlan_insertion = 0; 8090 txq->imm_wrs = 0; 8091 txq->sgl_wrs = 0; 8092 txq->txpkt_wrs = 0; 8093 txq->txpkts_wrs = 0; 8094 txq->txpkts_pkts = 0; 8095 txq->br->br_drops = 0; 8096 txq->no_dmamap = 0; 8097 txq->no_desc = 0; 8098 } 8099 8100#ifdef TCP_OFFLOAD 8101 /* nothing to clear for each ofld_rxq */ 8102 8103 for_each_ofld_txq(pi, i, wrq) { 8104 wrq->tx_wrs = 0; 8105 wrq->no_desc = 0; 8106 } 8107#endif 8108 wrq = &sc->sge.ctrlq[pi->port_id]; 8109 wrq->tx_wrs = 0; 8110 wrq->no_desc = 0; 8111 } 8112 break; 8113 } 8114 case CHELSIO_T4_SCHED_CLASS: 8115 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8116 break; 8117 case CHELSIO_T4_SCHED_QUEUE: 8118 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8119 break; 8120 case CHELSIO_T4_GET_TRACER: 8121 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8122 break; 8123 case CHELSIO_T4_SET_TRACER: 8124 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8125 break; 8126 default: 8127 rc = EINVAL; 8128 } 8129 8130 return (rc); 8131} 8132 8133#ifdef TCP_OFFLOAD 8134void 8135t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8136 const unsigned int *pgsz_order) 8137{ 8138 struct port_info *pi = ifp->if_softc; 8139 struct adapter *sc = pi->adapter; 8140 8141 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8142 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8143 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8144 V_HPZ3(pgsz_order[3])); 8145} 8146 8147static int 8148toe_capability(struct port_info *pi, int enable) 8149{ 8150 int rc; 8151 struct adapter *sc = pi->adapter; 8152 8153 ASSERT_SYNCHRONIZED_OP(sc); 8154 8155 if (!is_offload(sc)) 8156 return (ENODEV); 8157 8158 if (enable) { 8159 /* 8160 * We need the port's queues around so that we're able to send 8161 * and receive CPLs to/from the TOE even if the ifnet for this 8162 * port has never been UP'd administratively. 8163 */ 8164 if (!(pi->flags & PORT_INIT_DONE)) { 8165 rc = cxgbe_init_synchronized(pi); 8166 if (rc) 8167 return (rc); 8168 } 8169 8170 if (isset(&sc->offload_map, pi->port_id)) 8171 return (0); 8172 8173 if (!(sc->flags & TOM_INIT_DONE)) { 8174 rc = t4_activate_uld(sc, ULD_TOM); 8175 if (rc == EAGAIN) { 8176 log(LOG_WARNING, 8177 "You must kldload t4_tom.ko before trying " 8178 "to enable TOE on a cxgbe interface.\n"); 8179 } 8180 if (rc != 0) 8181 return (rc); 8182 KASSERT(sc->tom_softc != NULL, 8183 ("%s: TOM activated but softc NULL", __func__)); 8184 KASSERT(sc->flags & TOM_INIT_DONE, 8185 ("%s: TOM activated but flag not set", __func__)); 8186 } 8187 8188 setbit(&sc->offload_map, pi->port_id); 8189 } else { 8190 if (!isset(&sc->offload_map, pi->port_id)) 8191 return (0); 8192 8193 KASSERT(sc->flags & TOM_INIT_DONE, 8194 ("%s: TOM never initialized?", __func__)); 8195 clrbit(&sc->offload_map, pi->port_id); 8196 } 8197 8198 return (0); 8199} 8200 8201/* 8202 * Add an upper layer driver to the global list. 8203 */ 8204int 8205t4_register_uld(struct uld_info *ui) 8206{ 8207 int rc = 0; 8208 struct uld_info *u; 8209 8210 sx_xlock(&t4_uld_list_lock); 8211 SLIST_FOREACH(u, &t4_uld_list, link) { 8212 if (u->uld_id == ui->uld_id) { 8213 rc = EEXIST; 8214 goto done; 8215 } 8216 } 8217 8218 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8219 ui->refcount = 0; 8220done: 8221 sx_xunlock(&t4_uld_list_lock); 8222 return (rc); 8223} 8224 8225int 8226t4_unregister_uld(struct uld_info *ui) 8227{ 8228 int rc = EINVAL; 8229 struct uld_info *u; 8230 8231 sx_xlock(&t4_uld_list_lock); 8232 8233 SLIST_FOREACH(u, &t4_uld_list, link) { 8234 if (u == ui) { 8235 if (ui->refcount > 0) { 8236 rc = EBUSY; 8237 goto done; 8238 } 8239 8240 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8241 rc = 0; 8242 goto done; 8243 } 8244 } 8245done: 8246 sx_xunlock(&t4_uld_list_lock); 8247 return (rc); 8248} 8249 8250int 8251t4_activate_uld(struct adapter *sc, int id) 8252{ 8253 int rc = EAGAIN; 8254 struct uld_info *ui; 8255 8256 ASSERT_SYNCHRONIZED_OP(sc); 8257 8258 sx_slock(&t4_uld_list_lock); 8259 8260 SLIST_FOREACH(ui, &t4_uld_list, link) { 8261 if (ui->uld_id == id) { 8262 rc = ui->activate(sc); 8263 if (rc == 0) 8264 ui->refcount++; 8265 goto done; 8266 } 8267 } 8268done: 8269 sx_sunlock(&t4_uld_list_lock); 8270 8271 return (rc); 8272} 8273 8274int 8275t4_deactivate_uld(struct adapter *sc, int id) 8276{ 8277 int rc = EINVAL; 8278 struct uld_info *ui; 8279 8280 ASSERT_SYNCHRONIZED_OP(sc); 8281 8282 sx_slock(&t4_uld_list_lock); 8283 8284 SLIST_FOREACH(ui, &t4_uld_list, link) { 8285 if (ui->uld_id == id) { 8286 rc = ui->deactivate(sc); 8287 if (rc == 0) 8288 ui->refcount--; 8289 goto done; 8290 } 8291 } 8292done: 8293 sx_sunlock(&t4_uld_list_lock); 8294 8295 return (rc); 8296} 8297#endif 8298 8299/* 8300 * Come up with reasonable defaults for some of the tunables, provided they're 8301 * not set by the user (in which case we'll use the values as is). 8302 */ 8303static void 8304tweak_tunables(void) 8305{ 8306 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8307 8308 if (t4_ntxq10g < 1) 8309 t4_ntxq10g = min(nc, NTXQ_10G); 8310 8311 if (t4_ntxq1g < 1) 8312 t4_ntxq1g = min(nc, NTXQ_1G); 8313 8314 if (t4_nrxq10g < 1) 8315 t4_nrxq10g = min(nc, NRXQ_10G); 8316 8317 if (t4_nrxq1g < 1) 8318 t4_nrxq1g = min(nc, NRXQ_1G); 8319 8320#ifdef TCP_OFFLOAD 8321 if (t4_nofldtxq10g < 1) 8322 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8323 8324 if (t4_nofldtxq1g < 1) 8325 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8326 8327 if (t4_nofldrxq10g < 1) 8328 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8329 8330 if (t4_nofldrxq1g < 1) 8331 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8332 8333 if (t4_toecaps_allowed == -1) 8334 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8335#else 8336 if (t4_toecaps_allowed == -1) 8337 t4_toecaps_allowed = 0; 8338#endif 8339 8340#ifdef DEV_NETMAP 8341 if (t4_nnmtxq10g < 1) 8342 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8343 8344 if (t4_nnmtxq1g < 1) 8345 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8346 8347 if (t4_nnmrxq10g < 1) 8348 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8349 8350 if (t4_nnmrxq1g < 1) 8351 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8352#endif 8353 8354 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8355 t4_tmr_idx_10g = TMR_IDX_10G; 8356 8357 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8358 t4_pktc_idx_10g = PKTC_IDX_10G; 8359 8360 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8361 t4_tmr_idx_1g = TMR_IDX_1G; 8362 8363 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8364 t4_pktc_idx_1g = PKTC_IDX_1G; 8365 8366 if (t4_qsize_txq < 128) 8367 t4_qsize_txq = 128; 8368 8369 if (t4_qsize_rxq < 128) 8370 t4_qsize_rxq = 128; 8371 while (t4_qsize_rxq & 7) 8372 t4_qsize_rxq++; 8373 8374 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8375} 8376 8377static struct sx mlu; /* mod load unload */ 8378SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8379 8380static int 8381mod_event(module_t mod, int cmd, void *arg) 8382{ 8383 int rc = 0; 8384 static int loaded = 0; 8385 8386 switch (cmd) { 8387 case MOD_LOAD: 8388 sx_xlock(&mlu); 8389 if (loaded++ == 0) { 8390 t4_sge_modload(); 8391 sx_init(&t4_list_lock, "T4/T5 adapters"); 8392 SLIST_INIT(&t4_list); 8393#ifdef TCP_OFFLOAD 8394 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8395 SLIST_INIT(&t4_uld_list); 8396#endif 8397 t4_tracer_modload(); 8398 tweak_tunables(); 8399 } 8400 sx_xunlock(&mlu); 8401 break; 8402 8403 case MOD_UNLOAD: 8404 sx_xlock(&mlu); 8405 if (--loaded == 0) { 8406 int tries; 8407 8408 sx_slock(&t4_list_lock); 8409 if (!SLIST_EMPTY(&t4_list)) { 8410 rc = EBUSY; 8411 sx_sunlock(&t4_list_lock); 8412 goto done_unload; 8413 } 8414#ifdef TCP_OFFLOAD 8415 sx_slock(&t4_uld_list_lock); 8416 if (!SLIST_EMPTY(&t4_uld_list)) { 8417 rc = EBUSY; 8418 sx_sunlock(&t4_uld_list_lock); 8419 sx_sunlock(&t4_list_lock); 8420 goto done_unload; 8421 } 8422#endif 8423 tries = 0; 8424 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8425 uprintf("%ju clusters with custom free routine " 8426 "still is use.\n", t4_sge_extfree_refs()); 8427 pause("t4unload", 2 * hz); 8428 } 8429#ifdef TCP_OFFLOAD 8430 sx_sunlock(&t4_uld_list_lock); 8431#endif 8432 sx_sunlock(&t4_list_lock); 8433 8434 if (t4_sge_extfree_refs() == 0) { 8435 t4_tracer_modunload(); 8436#ifdef TCP_OFFLOAD 8437 sx_destroy(&t4_uld_list_lock); 8438#endif 8439 sx_destroy(&t4_list_lock); 8440 t4_sge_modunload(); 8441 loaded = 0; 8442 } else { 8443 rc = EBUSY; 8444 loaded++; /* undo earlier decrement */ 8445 } 8446 } 8447done_unload: 8448 sx_xunlock(&mlu); 8449 break; 8450 } 8451 8452 return (rc); 8453} 8454 8455static devclass_t t4_devclass, t5_devclass; 8456static devclass_t cxgbe_devclass, cxl_devclass; 8457 8458DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8459MODULE_VERSION(t4nex, 1); 8460MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8461 8462DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8463MODULE_VERSION(t5nex, 1); 8464MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8465 8466DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8467MODULE_VERSION(cxgbe, 1); 8468 8469DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8470MODULE_VERSION(cxl, 1); 8471