t4_main.c revision 281244
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 281244 2015-04-08 00:35:12Z np $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/module.h> 40#include <sys/malloc.h> 41#include <sys/queue.h> 42#include <sys/taskqueue.h> 43#include <sys/pciio.h> 44#include <dev/pci/pcireg.h> 45#include <dev/pci/pcivar.h> 46#include <dev/pci/pci_private.h> 47#include <sys/firmware.h> 48#include <sys/sbuf.h> 49#include <sys/smp.h> 50#include <sys/socket.h> 51#include <sys/sockio.h> 52#include <sys/sysctl.h> 53#include <net/ethernet.h> 54#include <net/if.h> 55#include <net/if_types.h> 56#include <net/if_dl.h> 57#include <net/if_vlan_var.h> 58#if defined(__i386__) || defined(__amd64__) 59#include <vm/vm.h> 60#include <vm/pmap.h> 61#endif 62 63#include "common/common.h" 64#include "common/t4_msg.h" 65#include "common/t4_regs.h" 66#include "common/t4_regs_values.h" 67#include "t4_ioctl.h" 68#include "t4_l2t.h" 69 70/* T4 bus driver interface */ 71static int t4_probe(device_t); 72static int t4_attach(device_t); 73static int t4_detach(device_t); 74static device_method_t t4_methods[] = { 75 DEVMETHOD(device_probe, t4_probe), 76 DEVMETHOD(device_attach, t4_attach), 77 DEVMETHOD(device_detach, t4_detach), 78 79 DEVMETHOD_END 80}; 81static driver_t t4_driver = { 82 "t4nex", 83 t4_methods, 84 sizeof(struct adapter) 85}; 86 87 88/* T4 port (cxgbe) interface */ 89static int cxgbe_probe(device_t); 90static int cxgbe_attach(device_t); 91static int cxgbe_detach(device_t); 92static device_method_t cxgbe_methods[] = { 93 DEVMETHOD(device_probe, cxgbe_probe), 94 DEVMETHOD(device_attach, cxgbe_attach), 95 DEVMETHOD(device_detach, cxgbe_detach), 96 { 0, 0 } 97}; 98static driver_t cxgbe_driver = { 99 "cxgbe", 100 cxgbe_methods, 101 sizeof(struct port_info) 102}; 103 104static d_ioctl_t t4_ioctl; 105static d_open_t t4_open; 106static d_close_t t4_close; 107 108static struct cdevsw t4_cdevsw = { 109 .d_version = D_VERSION, 110 .d_flags = 0, 111 .d_open = t4_open, 112 .d_close = t4_close, 113 .d_ioctl = t4_ioctl, 114 .d_name = "t4nex", 115}; 116 117/* T5 bus driver interface */ 118static int t5_probe(device_t); 119static device_method_t t5_methods[] = { 120 DEVMETHOD(device_probe, t5_probe), 121 DEVMETHOD(device_attach, t4_attach), 122 DEVMETHOD(device_detach, t4_detach), 123 124 DEVMETHOD_END 125}; 126static driver_t t5_driver = { 127 "t5nex", 128 t5_methods, 129 sizeof(struct adapter) 130}; 131 132 133/* T5 port (cxl) interface */ 134static driver_t cxl_driver = { 135 "cxl", 136 cxgbe_methods, 137 sizeof(struct port_info) 138}; 139 140static struct cdevsw t5_cdevsw = { 141 .d_version = D_VERSION, 142 .d_flags = 0, 143 .d_open = t4_open, 144 .d_close = t4_close, 145 .d_ioctl = t4_ioctl, 146 .d_name = "t5nex", 147}; 148 149/* ifnet + media interface */ 150static void cxgbe_init(void *); 151static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152static int cxgbe_transmit(struct ifnet *, struct mbuf *); 153static void cxgbe_qflush(struct ifnet *); 154static int cxgbe_media_change(struct ifnet *); 155static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156 157MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158 159/* 160 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161 * then ADAPTER_LOCK, then t4_uld_list_lock. 162 */ 163static struct sx t4_list_lock; 164SLIST_HEAD(, adapter) t4_list; 165#ifdef TCP_OFFLOAD 166static struct sx t4_uld_list_lock; 167SLIST_HEAD(, uld_info) t4_uld_list; 168#endif 169 170/* 171 * Tunables. See tweak_tunables() too. 172 * 173 * Each tunable is set to a default value here if it's known at compile-time. 174 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175 * provide a reasonable default when the driver is loaded. 176 * 177 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178 * T5 are under hw.cxl. 179 */ 180 181/* 182 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183 */ 184#define NTXQ_10G 16 185static int t4_ntxq10g = -1; 186TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187 188#define NRXQ_10G 8 189static int t4_nrxq10g = -1; 190TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191 192#define NTXQ_1G 4 193static int t4_ntxq1g = -1; 194TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195 196#define NRXQ_1G 2 197static int t4_nrxq1g = -1; 198TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199 200static int t4_rsrv_noflowq = 0; 201TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 202 203#ifdef TCP_OFFLOAD 204#define NOFLDTXQ_10G 8 205static int t4_nofldtxq10g = -1; 206TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 207 208#define NOFLDRXQ_10G 2 209static int t4_nofldrxq10g = -1; 210TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 211 212#define NOFLDTXQ_1G 2 213static int t4_nofldtxq1g = -1; 214TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 215 216#define NOFLDRXQ_1G 1 217static int t4_nofldrxq1g = -1; 218TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 219#endif 220 221#ifdef DEV_NETMAP 222#define NNMTXQ_10G 2 223static int t4_nnmtxq10g = -1; 224TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 225 226#define NNMRXQ_10G 2 227static int t4_nnmrxq10g = -1; 228TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 229 230#define NNMTXQ_1G 1 231static int t4_nnmtxq1g = -1; 232TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 233 234#define NNMRXQ_1G 1 235static int t4_nnmrxq1g = -1; 236TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 237#endif 238 239/* 240 * Holdoff parameters for 10G and 1G ports. 241 */ 242#define TMR_IDX_10G 1 243static int t4_tmr_idx_10g = TMR_IDX_10G; 244TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 245 246#define PKTC_IDX_10G (-1) 247static int t4_pktc_idx_10g = PKTC_IDX_10G; 248TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 249 250#define TMR_IDX_1G 1 251static int t4_tmr_idx_1g = TMR_IDX_1G; 252TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 253 254#define PKTC_IDX_1G (-1) 255static int t4_pktc_idx_1g = PKTC_IDX_1G; 256TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 257 258/* 259 * Size (# of entries) of each tx and rx queue. 260 */ 261static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 262TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 263 264static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 265TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 266 267/* 268 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 269 */ 270static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 271TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 272 273/* 274 * Configuration file. 275 */ 276#define DEFAULT_CF "default" 277#define FLASH_CF "flash" 278#define UWIRE_CF "uwire" 279#define FPGA_CF "fpga" 280static char t4_cfg_file[32] = DEFAULT_CF; 281TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 282 283/* 284 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 285 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 286 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 287 * mark or when signalled to do so, 0 to never emit PAUSE. 288 */ 289static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 290TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 291 292/* 293 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 294 * encouraged respectively). 295 */ 296static unsigned int t4_fw_install = 1; 297TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 298 299/* 300 * ASIC features that will be used. Disable the ones you don't want so that the 301 * chip resources aren't wasted on features that will not be used. 302 */ 303static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 304TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 305 306static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 307TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 308 309static int t4_toecaps_allowed = -1; 310TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 311 312static int t4_rdmacaps_allowed = 0; 313TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 314 315static int t4_iscsicaps_allowed = 0; 316TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 317 318static int t4_fcoecaps_allowed = 0; 319TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 320 321static int t5_write_combine = 0; 322TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 323 324struct intrs_and_queues { 325 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 326 uint16_t nirq; /* Total # of vectors */ 327 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 328 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 329 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 330 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 331 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 332 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 333 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 334#ifdef TCP_OFFLOAD 335 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 336 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 337 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 338 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 339#endif 340#ifdef DEV_NETMAP 341 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 342 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 343 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 344 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 345#endif 346}; 347 348struct filter_entry { 349 uint32_t valid:1; /* filter allocated and valid */ 350 uint32_t locked:1; /* filter is administratively locked */ 351 uint32_t pending:1; /* filter action is pending firmware reply */ 352 uint32_t smtidx:8; /* Source MAC Table index for smac */ 353 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 354 355 struct t4_filter_specification fs; 356}; 357 358static int map_bars_0_and_4(struct adapter *); 359static int map_bar_2(struct adapter *); 360static void setup_memwin(struct adapter *); 361static int validate_mem_range(struct adapter *, uint32_t, int); 362static int fwmtype_to_hwmtype(int); 363static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 364 uint32_t *); 365static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 366static uint32_t position_memwin(struct adapter *, int, uint32_t); 367static int cfg_itype_and_nqueues(struct adapter *, int, int, 368 struct intrs_and_queues *); 369static int prep_firmware(struct adapter *); 370static int partition_resources(struct adapter *, const struct firmware *, 371 const char *); 372static int get_params__pre_init(struct adapter *); 373static int get_params__post_init(struct adapter *); 374static int set_params__post_init(struct adapter *); 375static void t4_set_desc(struct adapter *); 376static void build_medialist(struct port_info *, struct ifmedia *); 377static int cxgbe_init_synchronized(struct port_info *); 378static int cxgbe_uninit_synchronized(struct port_info *); 379static int setup_intr_handlers(struct adapter *); 380static void quiesce_eq(struct adapter *, struct sge_eq *); 381static void quiesce_iq(struct adapter *, struct sge_iq *); 382static void quiesce_fl(struct adapter *, struct sge_fl *); 383static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 384 driver_intr_t *, void *, char *); 385static int t4_free_irq(struct adapter *, struct irq *); 386static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 387 unsigned int); 388static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 389static void cxgbe_tick(void *); 390static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 391static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 392 struct mbuf *); 393static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 394static int fw_msg_not_handled(struct adapter *, const __be64 *); 395static int t4_sysctls(struct adapter *); 396static int cxgbe_sysctls(struct port_info *); 397static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 398static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 399static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 400static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 401static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 402static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 403static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 404static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 405static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 406static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 407static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 408#ifdef SBUF_DRAIN 409static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 410static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 411static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 412static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 413static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 414static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 415static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 416static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 417static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 418static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 419static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 420static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 421static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 422static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 423static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 424static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 425static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 426static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 427static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 428static int sysctl_tids(SYSCTL_HANDLER_ARGS); 429static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 430static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 431static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 432static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 433static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 434#endif 435static inline void txq_start(struct ifnet *, struct sge_txq *); 436static uint32_t fconf_to_mode(uint32_t); 437static uint32_t mode_to_fconf(uint32_t); 438static uint32_t fspec_to_fconf(struct t4_filter_specification *); 439static int get_filter_mode(struct adapter *, uint32_t *); 440static int set_filter_mode(struct adapter *, uint32_t); 441static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 442static int get_filter(struct adapter *, struct t4_filter *); 443static int set_filter(struct adapter *, struct t4_filter *); 444static int del_filter(struct adapter *, struct t4_filter *); 445static void clear_filter(struct filter_entry *); 446static int set_filter_wr(struct adapter *, int); 447static int del_filter_wr(struct adapter *, int); 448static int get_sge_context(struct adapter *, struct t4_sge_context *); 449static int load_fw(struct adapter *, struct t4_data *); 450static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 451static int read_i2c(struct adapter *, struct t4_i2c_data *); 452static int set_sched_class(struct adapter *, struct t4_sched_params *); 453static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 454#ifdef TCP_OFFLOAD 455static int toe_capability(struct port_info *, int); 456#endif 457static int mod_event(module_t, int, void *); 458 459struct { 460 uint16_t device; 461 char *desc; 462} t4_pciids[] = { 463 {0xa000, "Chelsio Terminator 4 FPGA"}, 464 {0x4400, "Chelsio T440-dbg"}, 465 {0x4401, "Chelsio T420-CR"}, 466 {0x4402, "Chelsio T422-CR"}, 467 {0x4403, "Chelsio T440-CR"}, 468 {0x4404, "Chelsio T420-BCH"}, 469 {0x4405, "Chelsio T440-BCH"}, 470 {0x4406, "Chelsio T440-CH"}, 471 {0x4407, "Chelsio T420-SO"}, 472 {0x4408, "Chelsio T420-CX"}, 473 {0x4409, "Chelsio T420-BT"}, 474 {0x440a, "Chelsio T404-BT"}, 475 {0x440e, "Chelsio T440-LP-CR"}, 476}, t5_pciids[] = { 477 {0xb000, "Chelsio Terminator 5 FPGA"}, 478 {0x5400, "Chelsio T580-dbg"}, 479 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 480 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 481 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 482 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 483 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 484 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 485 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 486 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 487 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 488 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 489 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 490 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 491 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 492#ifdef notyet 493 {0x5404, "Chelsio T520-BCH"}, 494 {0x5405, "Chelsio T540-BCH"}, 495 {0x5406, "Chelsio T540-CH"}, 496 {0x5408, "Chelsio T520-CX"}, 497 {0x540b, "Chelsio B520-SR"}, 498 {0x540c, "Chelsio B504-BT"}, 499 {0x540f, "Chelsio Amsterdam"}, 500 {0x5413, "Chelsio T580-CHR"}, 501#endif 502}; 503 504#ifdef TCP_OFFLOAD 505/* 506 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 507 * exactly the same for both rxq and ofld_rxq. 508 */ 509CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 510CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 511#endif 512 513/* No easy way to include t4_msg.h before adapter.h so we check this way */ 514CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 515CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 516 517CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 518 519static int 520t4_probe(device_t dev) 521{ 522 int i; 523 uint16_t v = pci_get_vendor(dev); 524 uint16_t d = pci_get_device(dev); 525 uint8_t f = pci_get_function(dev); 526 527 if (v != PCI_VENDOR_ID_CHELSIO) 528 return (ENXIO); 529 530 /* Attach only to PF0 of the FPGA */ 531 if (d == 0xa000 && f != 0) 532 return (ENXIO); 533 534 for (i = 0; i < nitems(t4_pciids); i++) { 535 if (d == t4_pciids[i].device) { 536 device_set_desc(dev, t4_pciids[i].desc); 537 return (BUS_PROBE_DEFAULT); 538 } 539 } 540 541 return (ENXIO); 542} 543 544static int 545t5_probe(device_t dev) 546{ 547 int i; 548 uint16_t v = pci_get_vendor(dev); 549 uint16_t d = pci_get_device(dev); 550 uint8_t f = pci_get_function(dev); 551 552 if (v != PCI_VENDOR_ID_CHELSIO) 553 return (ENXIO); 554 555 /* Attach only to PF0 of the FPGA */ 556 if (d == 0xb000 && f != 0) 557 return (ENXIO); 558 559 for (i = 0; i < nitems(t5_pciids); i++) { 560 if (d == t5_pciids[i].device) { 561 device_set_desc(dev, t5_pciids[i].desc); 562 return (BUS_PROBE_DEFAULT); 563 } 564 } 565 566 return (ENXIO); 567} 568 569static int 570t4_attach(device_t dev) 571{ 572 struct adapter *sc; 573 int rc = 0, i, n10g, n1g, rqidx, tqidx; 574 struct intrs_and_queues iaq; 575 struct sge *s; 576#ifdef TCP_OFFLOAD 577 int ofld_rqidx, ofld_tqidx; 578#endif 579#ifdef DEV_NETMAP 580 int nm_rqidx, nm_tqidx; 581#endif 582 583 sc = device_get_softc(dev); 584 sc->dev = dev; 585 586 pci_enable_busmaster(dev); 587 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 588 uint32_t v; 589 590 pci_set_max_read_req(dev, 4096); 591 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 592 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 593 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 594 595 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 596 } 597 598 sc->traceq = -1; 599 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 600 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 601 device_get_nameunit(dev)); 602 603 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 604 device_get_nameunit(dev)); 605 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 606 sx_xlock(&t4_list_lock); 607 SLIST_INSERT_HEAD(&t4_list, sc, link); 608 sx_xunlock(&t4_list_lock); 609 610 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 611 TAILQ_INIT(&sc->sfl); 612 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 613 614 rc = map_bars_0_and_4(sc); 615 if (rc != 0) 616 goto done; /* error message displayed already */ 617 618 /* 619 * This is the real PF# to which we're attaching. Works from within PCI 620 * passthrough environments too, where pci_get_function() could return a 621 * different PF# depending on the passthrough configuration. We need to 622 * use the real PF# in all our communication with the firmware. 623 */ 624 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 625 sc->mbox = sc->pf; 626 627 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 628 sc->an_handler = an_not_handled; 629 for (i = 0; i < nitems(sc->cpl_handler); i++) 630 sc->cpl_handler[i] = cpl_not_handled; 631 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 632 sc->fw_msg_handler[i] = fw_msg_not_handled; 633 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 634 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 635 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 636 t4_init_sge_cpl_handlers(sc); 637 638 /* Prepare the adapter for operation */ 639 rc = -t4_prep_adapter(sc); 640 if (rc != 0) { 641 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 642 goto done; 643 } 644 645 /* 646 * Do this really early, with the memory windows set up even before the 647 * character device. The userland tool's register i/o and mem read 648 * will work even in "recovery mode". 649 */ 650 setup_memwin(sc); 651 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 652 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 653 device_get_nameunit(dev)); 654 if (sc->cdev == NULL) 655 device_printf(dev, "failed to create nexus char device.\n"); 656 else 657 sc->cdev->si_drv1 = sc; 658 659 /* Go no further if recovery mode has been requested. */ 660 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 661 device_printf(dev, "recovery mode.\n"); 662 goto done; 663 } 664 665 /* Prepare the firmware for operation */ 666 rc = prep_firmware(sc); 667 if (rc != 0) 668 goto done; /* error message displayed already */ 669 670 rc = get_params__post_init(sc); 671 if (rc != 0) 672 goto done; /* error message displayed already */ 673 674 rc = set_params__post_init(sc); 675 if (rc != 0) 676 goto done; /* error message displayed already */ 677 678 rc = map_bar_2(sc); 679 if (rc != 0) 680 goto done; /* error message displayed already */ 681 682 rc = t4_create_dma_tag(sc); 683 if (rc != 0) 684 goto done; /* error message displayed already */ 685 686 /* 687 * First pass over all the ports - allocate VIs and initialize some 688 * basic parameters like mac address, port type, etc. We also figure 689 * out whether a port is 10G or 1G and use that information when 690 * calculating how many interrupts to attempt to allocate. 691 */ 692 n10g = n1g = 0; 693 for_each_port(sc, i) { 694 struct port_info *pi; 695 696 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 697 sc->port[i] = pi; 698 699 /* These must be set before t4_port_init */ 700 pi->adapter = sc; 701 pi->port_id = i; 702 703 /* Allocate the vi and initialize parameters like mac addr */ 704 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 705 if (rc != 0) { 706 device_printf(dev, "unable to initialize port %d: %d\n", 707 i, rc); 708 free(pi, M_CXGBE); 709 sc->port[i] = NULL; 710 goto done; 711 } 712 713 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 714 pi->link_cfg.requested_fc |= t4_pause_settings; 715 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 716 pi->link_cfg.fc |= t4_pause_settings; 717 718 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 719 if (rc != 0) { 720 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 721 free(pi, M_CXGBE); 722 sc->port[i] = NULL; 723 goto done; 724 } 725 726 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 727 device_get_nameunit(dev), i); 728 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 729 sc->chan_map[pi->tx_chan] = i; 730 731 if (is_10G_port(pi) || is_40G_port(pi)) { 732 n10g++; 733 pi->tmr_idx = t4_tmr_idx_10g; 734 pi->pktc_idx = t4_pktc_idx_10g; 735 } else { 736 n1g++; 737 pi->tmr_idx = t4_tmr_idx_1g; 738 pi->pktc_idx = t4_pktc_idx_1g; 739 } 740 741 pi->xact_addr_filt = -1; 742 pi->linkdnrc = -1; 743 744 pi->qsize_rxq = t4_qsize_rxq; 745 pi->qsize_txq = t4_qsize_txq; 746 747 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 748 if (pi->dev == NULL) { 749 device_printf(dev, 750 "failed to add device for port %d.\n", i); 751 rc = ENXIO; 752 goto done; 753 } 754 device_set_softc(pi->dev, pi); 755 } 756 757 /* 758 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 759 */ 760 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 761 if (rc != 0) 762 goto done; /* error message displayed already */ 763 764 sc->intr_type = iaq.intr_type; 765 sc->intr_count = iaq.nirq; 766 767 s = &sc->sge; 768 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 769 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 770 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 771 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 772 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 773#ifdef TCP_OFFLOAD 774 if (is_offload(sc)) { 775 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 776 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 777 s->neq += s->nofldtxq + s->nofldrxq; 778 s->niq += s->nofldrxq; 779 780 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 781 M_CXGBE, M_ZERO | M_WAITOK); 782 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 783 M_CXGBE, M_ZERO | M_WAITOK); 784 } 785#endif 786#ifdef DEV_NETMAP 787 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 788 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 789 s->neq += s->nnmtxq + s->nnmrxq; 790 s->niq += s->nnmrxq; 791 792 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 793 M_CXGBE, M_ZERO | M_WAITOK); 794 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 795 M_CXGBE, M_ZERO | M_WAITOK); 796#endif 797 798 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 799 M_ZERO | M_WAITOK); 800 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 801 M_ZERO | M_WAITOK); 802 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 803 M_ZERO | M_WAITOK); 804 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 805 M_ZERO | M_WAITOK); 806 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 807 M_ZERO | M_WAITOK); 808 809 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 810 M_ZERO | M_WAITOK); 811 812 t4_init_l2t(sc, M_WAITOK); 813 814 /* 815 * Second pass over the ports. This time we know the number of rx and 816 * tx queues that each port should get. 817 */ 818 rqidx = tqidx = 0; 819#ifdef TCP_OFFLOAD 820 ofld_rqidx = ofld_tqidx = 0; 821#endif 822#ifdef DEV_NETMAP 823 nm_rqidx = nm_tqidx = 0; 824#endif 825 for_each_port(sc, i) { 826 struct port_info *pi = sc->port[i]; 827 828 if (pi == NULL) 829 continue; 830 831 pi->first_rxq = rqidx; 832 pi->first_txq = tqidx; 833 if (is_10G_port(pi) || is_40G_port(pi)) { 834 pi->flags |= iaq.intr_flags_10g; 835 pi->nrxq = iaq.nrxq10g; 836 pi->ntxq = iaq.ntxq10g; 837 } else { 838 pi->flags |= iaq.intr_flags_1g; 839 pi->nrxq = iaq.nrxq1g; 840 pi->ntxq = iaq.ntxq1g; 841 } 842 843 if (pi->ntxq > 1) 844 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 845 else 846 pi->rsrv_noflowq = 0; 847 848 rqidx += pi->nrxq; 849 tqidx += pi->ntxq; 850#ifdef TCP_OFFLOAD 851 if (is_offload(sc)) { 852 pi->first_ofld_rxq = ofld_rqidx; 853 pi->first_ofld_txq = ofld_tqidx; 854 if (is_10G_port(pi) || is_40G_port(pi)) { 855 pi->nofldrxq = iaq.nofldrxq10g; 856 pi->nofldtxq = iaq.nofldtxq10g; 857 } else { 858 pi->nofldrxq = iaq.nofldrxq1g; 859 pi->nofldtxq = iaq.nofldtxq1g; 860 } 861 ofld_rqidx += pi->nofldrxq; 862 ofld_tqidx += pi->nofldtxq; 863 } 864#endif 865#ifdef DEV_NETMAP 866 pi->first_nm_rxq = nm_rqidx; 867 pi->first_nm_txq = nm_tqidx; 868 if (is_10G_port(pi) || is_40G_port(pi)) { 869 pi->nnmrxq = iaq.nnmrxq10g; 870 pi->nnmtxq = iaq.nnmtxq10g; 871 } else { 872 pi->nnmrxq = iaq.nnmrxq1g; 873 pi->nnmtxq = iaq.nnmtxq1g; 874 } 875 nm_rqidx += pi->nnmrxq; 876 nm_tqidx += pi->nnmtxq; 877#endif 878 } 879 880 rc = setup_intr_handlers(sc); 881 if (rc != 0) { 882 device_printf(dev, 883 "failed to setup interrupt handlers: %d\n", rc); 884 goto done; 885 } 886 887 rc = bus_generic_attach(dev); 888 if (rc != 0) { 889 device_printf(dev, 890 "failed to attach all child ports: %d\n", rc); 891 goto done; 892 } 893 894 device_printf(dev, 895 "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 896 sc->params.pci.width, sc->params.nports, sc->intr_count, 897 sc->intr_type == INTR_MSIX ? "MSI-X" : 898 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 899 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 900 901 t4_set_desc(sc); 902 903done: 904 if (rc != 0 && sc->cdev) { 905 /* cdev was created and so cxgbetool works; recover that way. */ 906 device_printf(dev, 907 "error during attach, adapter is now in recovery mode.\n"); 908 rc = 0; 909 } 910 911 if (rc != 0) 912 t4_detach(dev); 913 else 914 t4_sysctls(sc); 915 916 return (rc); 917} 918 919/* 920 * Idempotent 921 */ 922static int 923t4_detach(device_t dev) 924{ 925 struct adapter *sc; 926 struct port_info *pi; 927 int i, rc; 928 929 sc = device_get_softc(dev); 930 931 if (sc->flags & FULL_INIT_DONE) 932 t4_intr_disable(sc); 933 934 if (sc->cdev) { 935 destroy_dev(sc->cdev); 936 sc->cdev = NULL; 937 } 938 939 rc = bus_generic_detach(dev); 940 if (rc) { 941 device_printf(dev, 942 "failed to detach child devices: %d\n", rc); 943 return (rc); 944 } 945 946 for (i = 0; i < sc->intr_count; i++) 947 t4_free_irq(sc, &sc->irq[i]); 948 949 for (i = 0; i < MAX_NPORTS; i++) { 950 pi = sc->port[i]; 951 if (pi) { 952 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 953 if (pi->dev) 954 device_delete_child(dev, pi->dev); 955 956 mtx_destroy(&pi->pi_lock); 957 free(pi, M_CXGBE); 958 } 959 } 960 961 if (sc->flags & FULL_INIT_DONE) 962 adapter_full_uninit(sc); 963 964 if (sc->flags & FW_OK) 965 t4_fw_bye(sc, sc->mbox); 966 967 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 968 pci_release_msi(dev); 969 970 if (sc->regs_res) 971 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 972 sc->regs_res); 973 974 if (sc->udbs_res) 975 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 976 sc->udbs_res); 977 978 if (sc->msix_res) 979 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 980 sc->msix_res); 981 982 if (sc->l2t) 983 t4_free_l2t(sc->l2t); 984 985#ifdef TCP_OFFLOAD 986 free(sc->sge.ofld_rxq, M_CXGBE); 987 free(sc->sge.ofld_txq, M_CXGBE); 988#endif 989#ifdef DEV_NETMAP 990 free(sc->sge.nm_rxq, M_CXGBE); 991 free(sc->sge.nm_txq, M_CXGBE); 992#endif 993 free(sc->irq, M_CXGBE); 994 free(sc->sge.rxq, M_CXGBE); 995 free(sc->sge.txq, M_CXGBE); 996 free(sc->sge.ctrlq, M_CXGBE); 997 free(sc->sge.iqmap, M_CXGBE); 998 free(sc->sge.eqmap, M_CXGBE); 999 free(sc->tids.ftid_tab, M_CXGBE); 1000 t4_destroy_dma_tag(sc); 1001 if (mtx_initialized(&sc->sc_lock)) { 1002 sx_xlock(&t4_list_lock); 1003 SLIST_REMOVE(&t4_list, sc, adapter, link); 1004 sx_xunlock(&t4_list_lock); 1005 mtx_destroy(&sc->sc_lock); 1006 } 1007 1008 if (mtx_initialized(&sc->tids.ftid_lock)) 1009 mtx_destroy(&sc->tids.ftid_lock); 1010 if (mtx_initialized(&sc->sfl_lock)) 1011 mtx_destroy(&sc->sfl_lock); 1012 if (mtx_initialized(&sc->ifp_lock)) 1013 mtx_destroy(&sc->ifp_lock); 1014 1015 bzero(sc, sizeof(*sc)); 1016 1017 return (0); 1018} 1019 1020static int 1021cxgbe_probe(device_t dev) 1022{ 1023 char buf[128]; 1024 struct port_info *pi = device_get_softc(dev); 1025 1026 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1027 device_set_desc_copy(dev, buf); 1028 1029 return (BUS_PROBE_DEFAULT); 1030} 1031 1032#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1033 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1034 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1035#define T4_CAP_ENABLE (T4_CAP) 1036 1037static int 1038cxgbe_attach(device_t dev) 1039{ 1040 struct port_info *pi = device_get_softc(dev); 1041 struct ifnet *ifp; 1042 char *s; 1043 int n, o; 1044 1045 /* Allocate an ifnet and set it up */ 1046 ifp = if_alloc(IFT_ETHER); 1047 if (ifp == NULL) { 1048 device_printf(dev, "Cannot allocate ifnet\n"); 1049 return (ENOMEM); 1050 } 1051 pi->ifp = ifp; 1052 ifp->if_softc = pi; 1053 1054 callout_init(&pi->tick, CALLOUT_MPSAFE); 1055 1056 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1057 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1058 1059 ifp->if_init = cxgbe_init; 1060 ifp->if_ioctl = cxgbe_ioctl; 1061 ifp->if_transmit = cxgbe_transmit; 1062 ifp->if_qflush = cxgbe_qflush; 1063 1064 ifp->if_capabilities = T4_CAP; 1065#ifdef TCP_OFFLOAD 1066 if (is_offload(pi->adapter)) 1067 ifp->if_capabilities |= IFCAP_TOE; 1068#endif 1069 ifp->if_capenable = T4_CAP_ENABLE; 1070 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1071 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1072 1073 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1074 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1075 ifp->if_hw_tsomaxsegsize = 65536; 1076 1077 /* Initialize ifmedia for this port */ 1078 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1079 cxgbe_media_status); 1080 build_medialist(pi, &pi->media); 1081 1082 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1083 EVENTHANDLER_PRI_ANY); 1084 1085 ether_ifattach(ifp, pi->hw_addr); 1086 1087 n = 128; 1088 s = malloc(n, M_CXGBE, M_WAITOK); 1089 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1090 MPASS(n > o); 1091#ifdef TCP_OFFLOAD 1092 if (is_offload(pi->adapter)) { 1093 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1094 pi->nofldtxq, pi->nofldrxq); 1095 MPASS(n > o); 1096 } 1097#endif 1098#ifdef DEV_NETMAP 1099 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1100 pi->nnmrxq); 1101 MPASS(n > o); 1102#endif 1103 device_printf(dev, "%s\n", s); 1104 free(s, M_CXGBE); 1105 1106#ifdef DEV_NETMAP 1107 /* nm_media handled here to keep implementation private to this file */ 1108 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1109 cxgbe_media_status); 1110 build_medialist(pi, &pi->nm_media); 1111 create_netmap_ifnet(pi); /* logs errors it something fails */ 1112#endif 1113 cxgbe_sysctls(pi); 1114 1115 return (0); 1116} 1117 1118static int 1119cxgbe_detach(device_t dev) 1120{ 1121 struct port_info *pi = device_get_softc(dev); 1122 struct adapter *sc = pi->adapter; 1123 struct ifnet *ifp = pi->ifp; 1124 1125 /* Tell if_ioctl and if_init that the port is going away */ 1126 ADAPTER_LOCK(sc); 1127 SET_DOOMED(pi); 1128 wakeup(&sc->flags); 1129 while (IS_BUSY(sc)) 1130 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1131 SET_BUSY(sc); 1132#ifdef INVARIANTS 1133 sc->last_op = "t4detach"; 1134 sc->last_op_thr = curthread; 1135#endif 1136 ADAPTER_UNLOCK(sc); 1137 1138 if (pi->flags & HAS_TRACEQ) { 1139 sc->traceq = -1; /* cloner should not create ifnet */ 1140 t4_tracer_port_detach(sc); 1141 } 1142 1143 if (pi->vlan_c) 1144 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1145 1146 PORT_LOCK(pi); 1147 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1148 callout_stop(&pi->tick); 1149 PORT_UNLOCK(pi); 1150 callout_drain(&pi->tick); 1151 1152 /* Let detach proceed even if these fail. */ 1153 cxgbe_uninit_synchronized(pi); 1154 port_full_uninit(pi); 1155 1156 ifmedia_removeall(&pi->media); 1157 ether_ifdetach(pi->ifp); 1158 if_free(pi->ifp); 1159 1160#ifdef DEV_NETMAP 1161 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1162 destroy_netmap_ifnet(pi); 1163#endif 1164 1165 ADAPTER_LOCK(sc); 1166 CLR_BUSY(sc); 1167 wakeup(&sc->flags); 1168 ADAPTER_UNLOCK(sc); 1169 1170 return (0); 1171} 1172 1173static void 1174cxgbe_init(void *arg) 1175{ 1176 struct port_info *pi = arg; 1177 struct adapter *sc = pi->adapter; 1178 1179 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1180 return; 1181 cxgbe_init_synchronized(pi); 1182 end_synchronized_op(sc, 0); 1183} 1184 1185static int 1186cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1187{ 1188 int rc = 0, mtu, flags, can_sleep; 1189 struct port_info *pi = ifp->if_softc; 1190 struct adapter *sc = pi->adapter; 1191 struct ifreq *ifr = (struct ifreq *)data; 1192 uint32_t mask; 1193 1194 switch (cmd) { 1195 case SIOCSIFMTU: 1196 mtu = ifr->ifr_mtu; 1197 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1198 return (EINVAL); 1199 1200 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1201 if (rc) 1202 return (rc); 1203 ifp->if_mtu = mtu; 1204 if (pi->flags & PORT_INIT_DONE) { 1205 t4_update_fl_bufsize(ifp); 1206 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1207 rc = update_mac_settings(ifp, XGMAC_MTU); 1208 } 1209 end_synchronized_op(sc, 0); 1210 break; 1211 1212 case SIOCSIFFLAGS: 1213 can_sleep = 0; 1214redo_sifflags: 1215 rc = begin_synchronized_op(sc, pi, 1216 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1217 if (rc) 1218 return (rc); 1219 1220 if (ifp->if_flags & IFF_UP) { 1221 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1222 flags = pi->if_flags; 1223 if ((ifp->if_flags ^ flags) & 1224 (IFF_PROMISC | IFF_ALLMULTI)) { 1225 if (can_sleep == 1) { 1226 end_synchronized_op(sc, 0); 1227 can_sleep = 0; 1228 goto redo_sifflags; 1229 } 1230 rc = update_mac_settings(ifp, 1231 XGMAC_PROMISC | XGMAC_ALLMULTI); 1232 } 1233 } else { 1234 if (can_sleep == 0) { 1235 end_synchronized_op(sc, LOCK_HELD); 1236 can_sleep = 1; 1237 goto redo_sifflags; 1238 } 1239 rc = cxgbe_init_synchronized(pi); 1240 } 1241 pi->if_flags = ifp->if_flags; 1242 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1243 if (can_sleep == 0) { 1244 end_synchronized_op(sc, LOCK_HELD); 1245 can_sleep = 1; 1246 goto redo_sifflags; 1247 } 1248 rc = cxgbe_uninit_synchronized(pi); 1249 } 1250 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1251 break; 1252 1253 case SIOCADDMULTI: 1254 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1255 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1256 if (rc) 1257 return (rc); 1258 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1259 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1260 end_synchronized_op(sc, LOCK_HELD); 1261 break; 1262 1263 case SIOCSIFCAP: 1264 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1265 if (rc) 1266 return (rc); 1267 1268 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1269 if (mask & IFCAP_TXCSUM) { 1270 ifp->if_capenable ^= IFCAP_TXCSUM; 1271 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1272 1273 if (IFCAP_TSO4 & ifp->if_capenable && 1274 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1275 ifp->if_capenable &= ~IFCAP_TSO4; 1276 if_printf(ifp, 1277 "tso4 disabled due to -txcsum.\n"); 1278 } 1279 } 1280 if (mask & IFCAP_TXCSUM_IPV6) { 1281 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1282 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1283 1284 if (IFCAP_TSO6 & ifp->if_capenable && 1285 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1286 ifp->if_capenable &= ~IFCAP_TSO6; 1287 if_printf(ifp, 1288 "tso6 disabled due to -txcsum6.\n"); 1289 } 1290 } 1291 if (mask & IFCAP_RXCSUM) 1292 ifp->if_capenable ^= IFCAP_RXCSUM; 1293 if (mask & IFCAP_RXCSUM_IPV6) 1294 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1295 1296 /* 1297 * Note that we leave CSUM_TSO alone (it is always set). The 1298 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1299 * sending a TSO request our way, so it's sufficient to toggle 1300 * IFCAP_TSOx only. 1301 */ 1302 if (mask & IFCAP_TSO4) { 1303 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1304 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1305 if_printf(ifp, "enable txcsum first.\n"); 1306 rc = EAGAIN; 1307 goto fail; 1308 } 1309 ifp->if_capenable ^= IFCAP_TSO4; 1310 } 1311 if (mask & IFCAP_TSO6) { 1312 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1313 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1314 if_printf(ifp, "enable txcsum6 first.\n"); 1315 rc = EAGAIN; 1316 goto fail; 1317 } 1318 ifp->if_capenable ^= IFCAP_TSO6; 1319 } 1320 if (mask & IFCAP_LRO) { 1321#if defined(INET) || defined(INET6) 1322 int i; 1323 struct sge_rxq *rxq; 1324 1325 ifp->if_capenable ^= IFCAP_LRO; 1326 for_each_rxq(pi, i, rxq) { 1327 if (ifp->if_capenable & IFCAP_LRO) 1328 rxq->iq.flags |= IQ_LRO_ENABLED; 1329 else 1330 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1331 } 1332#endif 1333 } 1334#ifdef TCP_OFFLOAD 1335 if (mask & IFCAP_TOE) { 1336 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1337 1338 rc = toe_capability(pi, enable); 1339 if (rc != 0) 1340 goto fail; 1341 1342 ifp->if_capenable ^= mask; 1343 } 1344#endif 1345 if (mask & IFCAP_VLAN_HWTAGGING) { 1346 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1347 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1348 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1349 } 1350 if (mask & IFCAP_VLAN_MTU) { 1351 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1352 1353 /* Need to find out how to disable auto-mtu-inflation */ 1354 } 1355 if (mask & IFCAP_VLAN_HWTSO) 1356 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1357 if (mask & IFCAP_VLAN_HWCSUM) 1358 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1359 1360#ifdef VLAN_CAPABILITIES 1361 VLAN_CAPABILITIES(ifp); 1362#endif 1363fail: 1364 end_synchronized_op(sc, 0); 1365 break; 1366 1367 case SIOCSIFMEDIA: 1368 case SIOCGIFMEDIA: 1369 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1370 break; 1371 1372 default: 1373 rc = ether_ioctl(ifp, cmd, data); 1374 } 1375 1376 return (rc); 1377} 1378 1379static int 1380cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1381{ 1382 struct port_info *pi = ifp->if_softc; 1383 struct adapter *sc = pi->adapter; 1384 struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1385 struct buf_ring *br; 1386 int rc; 1387 1388 M_ASSERTPKTHDR(m); 1389 1390 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1391 m_freem(m); 1392 return (ENETDOWN); 1393 } 1394 1395 if (m->m_flags & M_FLOWID) 1396 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) 1397 + pi->rsrv_noflowq); 1398 br = txq->br; 1399 1400 if (TXQ_TRYLOCK(txq) == 0) { 1401 struct sge_eq *eq = &txq->eq; 1402 1403 /* 1404 * It is possible that t4_eth_tx finishes up and releases the 1405 * lock between the TRYLOCK above and the drbr_enqueue here. We 1406 * need to make sure that this mbuf doesn't just sit there in 1407 * the drbr. 1408 */ 1409 1410 rc = drbr_enqueue(ifp, br, m); 1411 if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1412 !(eq->flags & EQ_DOOMED)) 1413 callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1414 return (rc); 1415 } 1416 1417 /* 1418 * txq->m is the mbuf that is held up due to a temporary shortage of 1419 * resources and it should be put on the wire first. Then what's in 1420 * drbr and finally the mbuf that was just passed in to us. 1421 * 1422 * Return code should indicate the fate of the mbuf that was passed in 1423 * this time. 1424 */ 1425 1426 TXQ_LOCK_ASSERT_OWNED(txq); 1427 if (drbr_needs_enqueue(ifp, br) || txq->m) { 1428 1429 /* Queued for transmission. */ 1430 1431 rc = drbr_enqueue(ifp, br, m); 1432 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1433 (void) t4_eth_tx(ifp, txq, m); 1434 TXQ_UNLOCK(txq); 1435 return (rc); 1436 } 1437 1438 /* Direct transmission. */ 1439 rc = t4_eth_tx(ifp, txq, m); 1440 if (rc != 0 && txq->m) 1441 rc = 0; /* held, will be transmitted soon (hopefully) */ 1442 1443 TXQ_UNLOCK(txq); 1444 return (rc); 1445} 1446 1447static void 1448cxgbe_qflush(struct ifnet *ifp) 1449{ 1450 struct port_info *pi = ifp->if_softc; 1451 struct sge_txq *txq; 1452 int i; 1453 struct mbuf *m; 1454 1455 /* queues do not exist if !PORT_INIT_DONE. */ 1456 if (pi->flags & PORT_INIT_DONE) { 1457 for_each_txq(pi, i, txq) { 1458 TXQ_LOCK(txq); 1459 m_freem(txq->m); 1460 txq->m = NULL; 1461 while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1462 m_freem(m); 1463 TXQ_UNLOCK(txq); 1464 } 1465 } 1466 if_qflush(ifp); 1467} 1468 1469static int 1470cxgbe_media_change(struct ifnet *ifp) 1471{ 1472 struct port_info *pi = ifp->if_softc; 1473 1474 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1475 1476 return (EOPNOTSUPP); 1477} 1478 1479static void 1480cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1481{ 1482 struct port_info *pi = ifp->if_softc; 1483 struct ifmedia *media = NULL; 1484 struct ifmedia_entry *cur; 1485 int speed = pi->link_cfg.speed; 1486#ifdef INVARIANTS 1487 int data = (pi->port_type << 8) | pi->mod_type; 1488#endif 1489 1490 if (ifp == pi->ifp) 1491 media = &pi->media; 1492#ifdef DEV_NETMAP 1493 else if (ifp == pi->nm_ifp) 1494 media = &pi->nm_media; 1495#endif 1496 MPASS(media != NULL); 1497 1498 cur = media->ifm_cur; 1499 MPASS(cur->ifm_data == data); 1500 1501 ifmr->ifm_status = IFM_AVALID; 1502 if (!pi->link_cfg.link_ok) 1503 return; 1504 1505 ifmr->ifm_status |= IFM_ACTIVE; 1506 1507 /* active and current will differ iff current media is autoselect. */ 1508 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1509 return; 1510 1511 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1512 if (speed == SPEED_10000) 1513 ifmr->ifm_active |= IFM_10G_T; 1514 else if (speed == SPEED_1000) 1515 ifmr->ifm_active |= IFM_1000_T; 1516 else if (speed == SPEED_100) 1517 ifmr->ifm_active |= IFM_100_TX; 1518 else if (speed == SPEED_10) 1519 ifmr->ifm_active |= IFM_10_T; 1520 else 1521 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1522 speed)); 1523} 1524 1525void 1526t4_fatal_err(struct adapter *sc) 1527{ 1528 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1529 t4_intr_disable(sc); 1530 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1531 device_get_nameunit(sc->dev)); 1532} 1533 1534static int 1535map_bars_0_and_4(struct adapter *sc) 1536{ 1537 sc->regs_rid = PCIR_BAR(0); 1538 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1539 &sc->regs_rid, RF_ACTIVE); 1540 if (sc->regs_res == NULL) { 1541 device_printf(sc->dev, "cannot map registers.\n"); 1542 return (ENXIO); 1543 } 1544 sc->bt = rman_get_bustag(sc->regs_res); 1545 sc->bh = rman_get_bushandle(sc->regs_res); 1546 sc->mmio_len = rman_get_size(sc->regs_res); 1547 setbit(&sc->doorbells, DOORBELL_KDB); 1548 1549 sc->msix_rid = PCIR_BAR(4); 1550 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1551 &sc->msix_rid, RF_ACTIVE); 1552 if (sc->msix_res == NULL) { 1553 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1554 return (ENXIO); 1555 } 1556 1557 return (0); 1558} 1559 1560static int 1561map_bar_2(struct adapter *sc) 1562{ 1563 1564 /* 1565 * T4: only iWARP driver uses the userspace doorbells. There is no need 1566 * to map it if RDMA is disabled. 1567 */ 1568 if (is_t4(sc) && sc->rdmacaps == 0) 1569 return (0); 1570 1571 sc->udbs_rid = PCIR_BAR(2); 1572 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1573 &sc->udbs_rid, RF_ACTIVE); 1574 if (sc->udbs_res == NULL) { 1575 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1576 return (ENXIO); 1577 } 1578 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1579 1580 if (is_t5(sc)) { 1581 setbit(&sc->doorbells, DOORBELL_UDB); 1582#if defined(__i386__) || defined(__amd64__) 1583 if (t5_write_combine) { 1584 int rc; 1585 1586 /* 1587 * Enable write combining on BAR2. This is the 1588 * userspace doorbell BAR and is split into 128B 1589 * (UDBS_SEG_SIZE) doorbell regions, each associated 1590 * with an egress queue. The first 64B has the doorbell 1591 * and the second 64B can be used to submit a tx work 1592 * request with an implicit doorbell. 1593 */ 1594 1595 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1596 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1597 if (rc == 0) { 1598 clrbit(&sc->doorbells, DOORBELL_UDB); 1599 setbit(&sc->doorbells, DOORBELL_WCWR); 1600 setbit(&sc->doorbells, DOORBELL_UDBWC); 1601 } else { 1602 device_printf(sc->dev, 1603 "couldn't enable write combining: %d\n", 1604 rc); 1605 } 1606 1607 t4_write_reg(sc, A_SGE_STAT_CFG, 1608 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1609 } 1610#endif 1611 } 1612 1613 return (0); 1614} 1615 1616static const struct memwin t4_memwin[] = { 1617 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1618 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1619 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1620}; 1621 1622static const struct memwin t5_memwin[] = { 1623 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1624 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1625 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1626}; 1627 1628static void 1629setup_memwin(struct adapter *sc) 1630{ 1631 const struct memwin *mw; 1632 int i, n; 1633 uint32_t bar0; 1634 1635 if (is_t4(sc)) { 1636 /* 1637 * Read low 32b of bar0 indirectly via the hardware backdoor 1638 * mechanism. Works from within PCI passthrough environments 1639 * too, where rman_get_start() can return a different value. We 1640 * need to program the T4 memory window decoders with the actual 1641 * addresses that will be coming across the PCIe link. 1642 */ 1643 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1644 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1645 1646 mw = &t4_memwin[0]; 1647 n = nitems(t4_memwin); 1648 } else { 1649 /* T5 uses the relative offset inside the PCIe BAR */ 1650 bar0 = 0; 1651 1652 mw = &t5_memwin[0]; 1653 n = nitems(t5_memwin); 1654 } 1655 1656 for (i = 0; i < n; i++, mw++) { 1657 t4_write_reg(sc, 1658 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1659 (mw->base + bar0) | V_BIR(0) | 1660 V_WINDOW(ilog2(mw->aperture) - 10)); 1661 } 1662 1663 /* flush */ 1664 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1665} 1666 1667/* 1668 * Verify that the memory range specified by the addr/len pair is valid and lies 1669 * entirely within a single region (EDCx or MCx). 1670 */ 1671static int 1672validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1673{ 1674 uint32_t em, addr_len, maddr, mlen; 1675 1676 /* Memory can only be accessed in naturally aligned 4 byte units */ 1677 if (addr & 3 || len & 3 || len == 0) 1678 return (EINVAL); 1679 1680 /* Enabled memories */ 1681 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1682 if (em & F_EDRAM0_ENABLE) { 1683 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1684 maddr = G_EDRAM0_BASE(addr_len) << 20; 1685 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1686 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1687 addr + len <= maddr + mlen) 1688 return (0); 1689 } 1690 if (em & F_EDRAM1_ENABLE) { 1691 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1692 maddr = G_EDRAM1_BASE(addr_len) << 20; 1693 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1694 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1695 addr + len <= maddr + mlen) 1696 return (0); 1697 } 1698 if (em & F_EXT_MEM_ENABLE) { 1699 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1700 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1701 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1702 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1703 addr + len <= maddr + mlen) 1704 return (0); 1705 } 1706 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1707 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1708 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1709 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1710 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1711 addr + len <= maddr + mlen) 1712 return (0); 1713 } 1714 1715 return (EFAULT); 1716} 1717 1718static int 1719fwmtype_to_hwmtype(int mtype) 1720{ 1721 1722 switch (mtype) { 1723 case FW_MEMTYPE_EDC0: 1724 return (MEM_EDC0); 1725 case FW_MEMTYPE_EDC1: 1726 return (MEM_EDC1); 1727 case FW_MEMTYPE_EXTMEM: 1728 return (MEM_MC0); 1729 case FW_MEMTYPE_EXTMEM1: 1730 return (MEM_MC1); 1731 default: 1732 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1733 } 1734} 1735 1736/* 1737 * Verify that the memory range specified by the memtype/offset/len pair is 1738 * valid and lies entirely within the memtype specified. The global address of 1739 * the start of the range is returned in addr. 1740 */ 1741static int 1742validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1743 uint32_t *addr) 1744{ 1745 uint32_t em, addr_len, maddr, mlen; 1746 1747 /* Memory can only be accessed in naturally aligned 4 byte units */ 1748 if (off & 3 || len & 3 || len == 0) 1749 return (EINVAL); 1750 1751 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1752 switch (fwmtype_to_hwmtype(mtype)) { 1753 case MEM_EDC0: 1754 if (!(em & F_EDRAM0_ENABLE)) 1755 return (EINVAL); 1756 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1757 maddr = G_EDRAM0_BASE(addr_len) << 20; 1758 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1759 break; 1760 case MEM_EDC1: 1761 if (!(em & F_EDRAM1_ENABLE)) 1762 return (EINVAL); 1763 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1764 maddr = G_EDRAM1_BASE(addr_len) << 20; 1765 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1766 break; 1767 case MEM_MC: 1768 if (!(em & F_EXT_MEM_ENABLE)) 1769 return (EINVAL); 1770 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1771 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1772 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1773 break; 1774 case MEM_MC1: 1775 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1776 return (EINVAL); 1777 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1778 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1779 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1780 break; 1781 default: 1782 return (EINVAL); 1783 } 1784 1785 if (mlen > 0 && off < mlen && off + len <= mlen) { 1786 *addr = maddr + off; /* global address */ 1787 return (0); 1788 } 1789 1790 return (EFAULT); 1791} 1792 1793static void 1794memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1795{ 1796 const struct memwin *mw; 1797 1798 if (is_t4(sc)) { 1799 KASSERT(win >= 0 && win < nitems(t4_memwin), 1800 ("%s: incorrect memwin# (%d)", __func__, win)); 1801 mw = &t4_memwin[win]; 1802 } else { 1803 KASSERT(win >= 0 && win < nitems(t5_memwin), 1804 ("%s: incorrect memwin# (%d)", __func__, win)); 1805 mw = &t5_memwin[win]; 1806 } 1807 1808 if (base != NULL) 1809 *base = mw->base; 1810 if (aperture != NULL) 1811 *aperture = mw->aperture; 1812} 1813 1814/* 1815 * Positions the memory window such that it can be used to access the specified 1816 * address in the chip's address space. The return value is the offset of addr 1817 * from the start of the window. 1818 */ 1819static uint32_t 1820position_memwin(struct adapter *sc, int n, uint32_t addr) 1821{ 1822 uint32_t start, pf; 1823 uint32_t reg; 1824 1825 KASSERT(n >= 0 && n <= 3, 1826 ("%s: invalid window %d.", __func__, n)); 1827 KASSERT((addr & 3) == 0, 1828 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1829 1830 if (is_t4(sc)) { 1831 pf = 0; 1832 start = addr & ~0xf; /* start must be 16B aligned */ 1833 } else { 1834 pf = V_PFNUM(sc->pf); 1835 start = addr & ~0x7f; /* start must be 128B aligned */ 1836 } 1837 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1838 1839 t4_write_reg(sc, reg, start | pf); 1840 t4_read_reg(sc, reg); 1841 1842 return (addr - start); 1843} 1844 1845static int 1846cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1847 struct intrs_and_queues *iaq) 1848{ 1849 int rc, itype, navail, nrxq10g, nrxq1g, n; 1850 int nofldrxq10g = 0, nofldrxq1g = 0; 1851 int nnmrxq10g = 0, nnmrxq1g = 0; 1852 1853 bzero(iaq, sizeof(*iaq)); 1854 1855 iaq->ntxq10g = t4_ntxq10g; 1856 iaq->ntxq1g = t4_ntxq1g; 1857 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1858 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1859 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1860#ifdef TCP_OFFLOAD 1861 if (is_offload(sc)) { 1862 iaq->nofldtxq10g = t4_nofldtxq10g; 1863 iaq->nofldtxq1g = t4_nofldtxq1g; 1864 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1865 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1866 } 1867#endif 1868#ifdef DEV_NETMAP 1869 iaq->nnmtxq10g = t4_nnmtxq10g; 1870 iaq->nnmtxq1g = t4_nnmtxq1g; 1871 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1872 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1873#endif 1874 1875 for (itype = INTR_MSIX; itype; itype >>= 1) { 1876 1877 if ((itype & t4_intr_types) == 0) 1878 continue; /* not allowed */ 1879 1880 if (itype == INTR_MSIX) 1881 navail = pci_msix_count(sc->dev); 1882 else if (itype == INTR_MSI) 1883 navail = pci_msi_count(sc->dev); 1884 else 1885 navail = 1; 1886restart: 1887 if (navail == 0) 1888 continue; 1889 1890 iaq->intr_type = itype; 1891 iaq->intr_flags_10g = 0; 1892 iaq->intr_flags_1g = 0; 1893 1894 /* 1895 * Best option: an interrupt vector for errors, one for the 1896 * firmware event queue, and one for every rxq (NIC, TOE, and 1897 * netmap). 1898 */ 1899 iaq->nirq = T4_EXTRA_INTR; 1900 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1901 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1902 if (iaq->nirq <= navail && 1903 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1904 iaq->intr_flags_10g = INTR_ALL; 1905 iaq->intr_flags_1g = INTR_ALL; 1906 goto allocate; 1907 } 1908 1909 /* 1910 * Second best option: a vector for errors, one for the firmware 1911 * event queue, and vectors for either all the NIC rx queues or 1912 * all the TOE rx queues. The queues that don't get vectors 1913 * will forward their interrupts to those that do. 1914 * 1915 * Note: netmap rx queues cannot be created early and so they 1916 * can't be setup to receive forwarded interrupts for others. 1917 */ 1918 iaq->nirq = T4_EXTRA_INTR; 1919 if (nrxq10g >= nofldrxq10g) { 1920 iaq->intr_flags_10g = INTR_RXQ; 1921 iaq->nirq += n10g * nrxq10g; 1922#ifdef DEV_NETMAP 1923 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1924#endif 1925 } else { 1926 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1927 iaq->nirq += n10g * nofldrxq10g; 1928#ifdef DEV_NETMAP 1929 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1930#endif 1931 } 1932 if (nrxq1g >= nofldrxq1g) { 1933 iaq->intr_flags_1g = INTR_RXQ; 1934 iaq->nirq += n1g * nrxq1g; 1935#ifdef DEV_NETMAP 1936 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1937#endif 1938 } else { 1939 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1940 iaq->nirq += n1g * nofldrxq1g; 1941#ifdef DEV_NETMAP 1942 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1943#endif 1944 } 1945 if (iaq->nirq <= navail && 1946 (itype != INTR_MSI || powerof2(iaq->nirq))) 1947 goto allocate; 1948 1949 /* 1950 * Next best option: an interrupt vector for errors, one for the 1951 * firmware event queue, and at least one per port. At this 1952 * point we know we'll have to downsize nrxq and/or nofldrxq 1953 * and/or nnmrxq to fit what's available to us. 1954 */ 1955 iaq->nirq = T4_EXTRA_INTR; 1956 iaq->nirq += n10g + n1g; 1957 if (iaq->nirq <= navail) { 1958 int leftover = navail - iaq->nirq; 1959 1960 if (n10g > 0) { 1961 int target = max(nrxq10g, nofldrxq10g); 1962 1963 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 1964 INTR_RXQ : INTR_OFLD_RXQ; 1965 1966 n = 1; 1967 while (n < target && leftover >= n10g) { 1968 leftover -= n10g; 1969 iaq->nirq += n10g; 1970 n++; 1971 } 1972 iaq->nrxq10g = min(n, nrxq10g); 1973#ifdef TCP_OFFLOAD 1974 iaq->nofldrxq10g = min(n, nofldrxq10g); 1975#endif 1976#ifdef DEV_NETMAP 1977 iaq->nnmrxq10g = min(n, nnmrxq10g); 1978#endif 1979 } 1980 1981 if (n1g > 0) { 1982 int target = max(nrxq1g, nofldrxq1g); 1983 1984 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 1985 INTR_RXQ : INTR_OFLD_RXQ; 1986 1987 n = 1; 1988 while (n < target && leftover >= n1g) { 1989 leftover -= n1g; 1990 iaq->nirq += n1g; 1991 n++; 1992 } 1993 iaq->nrxq1g = min(n, nrxq1g); 1994#ifdef TCP_OFFLOAD 1995 iaq->nofldrxq1g = min(n, nofldrxq1g); 1996#endif 1997#ifdef DEV_NETMAP 1998 iaq->nnmrxq1g = min(n, nnmrxq1g); 1999#endif 2000 } 2001 2002 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2003 goto allocate; 2004 } 2005 2006 /* 2007 * Least desirable option: one interrupt vector for everything. 2008 */ 2009 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2010 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2011#ifdef TCP_OFFLOAD 2012 if (is_offload(sc)) 2013 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2014#endif 2015#ifdef DEV_NETMAP 2016 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2017#endif 2018 2019allocate: 2020 navail = iaq->nirq; 2021 rc = 0; 2022 if (itype == INTR_MSIX) 2023 rc = pci_alloc_msix(sc->dev, &navail); 2024 else if (itype == INTR_MSI) 2025 rc = pci_alloc_msi(sc->dev, &navail); 2026 2027 if (rc == 0) { 2028 if (navail == iaq->nirq) 2029 return (0); 2030 2031 /* 2032 * Didn't get the number requested. Use whatever number 2033 * the kernel is willing to allocate (it's in navail). 2034 */ 2035 device_printf(sc->dev, "fewer vectors than requested, " 2036 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2037 itype, iaq->nirq, navail); 2038 pci_release_msi(sc->dev); 2039 goto restart; 2040 } 2041 2042 device_printf(sc->dev, 2043 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2044 itype, rc, iaq->nirq, navail); 2045 } 2046 2047 device_printf(sc->dev, 2048 "failed to find a usable interrupt type. " 2049 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2050 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2051 2052 return (ENXIO); 2053} 2054 2055#define FW_VERSION(chip) ( \ 2056 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2057 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2058 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2059 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2060#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2061 2062struct fw_info { 2063 uint8_t chip; 2064 char *kld_name; 2065 char *fw_mod_name; 2066 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2067} fw_info[] = { 2068 { 2069 .chip = CHELSIO_T4, 2070 .kld_name = "t4fw_cfg", 2071 .fw_mod_name = "t4fw", 2072 .fw_hdr = { 2073 .chip = FW_HDR_CHIP_T4, 2074 .fw_ver = htobe32_const(FW_VERSION(T4)), 2075 .intfver_nic = FW_INTFVER(T4, NIC), 2076 .intfver_vnic = FW_INTFVER(T4, VNIC), 2077 .intfver_ofld = FW_INTFVER(T4, OFLD), 2078 .intfver_ri = FW_INTFVER(T4, RI), 2079 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2080 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2081 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2082 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2083 }, 2084 }, { 2085 .chip = CHELSIO_T5, 2086 .kld_name = "t5fw_cfg", 2087 .fw_mod_name = "t5fw", 2088 .fw_hdr = { 2089 .chip = FW_HDR_CHIP_T5, 2090 .fw_ver = htobe32_const(FW_VERSION(T5)), 2091 .intfver_nic = FW_INTFVER(T5, NIC), 2092 .intfver_vnic = FW_INTFVER(T5, VNIC), 2093 .intfver_ofld = FW_INTFVER(T5, OFLD), 2094 .intfver_ri = FW_INTFVER(T5, RI), 2095 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2096 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2097 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2098 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2099 }, 2100 } 2101}; 2102 2103static struct fw_info * 2104find_fw_info(int chip) 2105{ 2106 int i; 2107 2108 for (i = 0; i < nitems(fw_info); i++) { 2109 if (fw_info[i].chip == chip) 2110 return (&fw_info[i]); 2111 } 2112 return (NULL); 2113} 2114 2115/* 2116 * Is the given firmware API compatible with the one the driver was compiled 2117 * with? 2118 */ 2119static int 2120fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2121{ 2122 2123 /* short circuit if it's the exact same firmware version */ 2124 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2125 return (1); 2126 2127 /* 2128 * XXX: Is this too conservative? Perhaps I should limit this to the 2129 * features that are supported in the driver. 2130 */ 2131#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2132 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2133 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2134 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2135 return (1); 2136#undef SAME_INTF 2137 2138 return (0); 2139} 2140 2141/* 2142 * The firmware in the KLD is usable, but should it be installed? This routine 2143 * explains itself in detail if it indicates the KLD firmware should be 2144 * installed. 2145 */ 2146static int 2147should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2148{ 2149 const char *reason; 2150 2151 if (!card_fw_usable) { 2152 reason = "incompatible or unusable"; 2153 goto install; 2154 } 2155 2156 if (k > c) { 2157 reason = "older than the version bundled with this driver"; 2158 goto install; 2159 } 2160 2161 if (t4_fw_install == 2 && k != c) { 2162 reason = "different than the version bundled with this driver"; 2163 goto install; 2164 } 2165 2166 return (0); 2167 2168install: 2169 if (t4_fw_install == 0) { 2170 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2171 "but the driver is prohibited from installing a different " 2172 "firmware on the card.\n", 2173 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2174 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2175 2176 return (0); 2177 } 2178 2179 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2180 "installing firmware %u.%u.%u.%u on card.\n", 2181 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2182 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2183 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2184 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2185 2186 return (1); 2187} 2188/* 2189 * Establish contact with the firmware and determine if we are the master driver 2190 * or not, and whether we are responsible for chip initialization. 2191 */ 2192static int 2193prep_firmware(struct adapter *sc) 2194{ 2195 const struct firmware *fw = NULL, *default_cfg; 2196 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2197 enum dev_state state; 2198 struct fw_info *fw_info; 2199 struct fw_hdr *card_fw; /* fw on the card */ 2200 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2201 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2202 against */ 2203 2204 /* Contact firmware. */ 2205 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2206 if (rc < 0 || state == DEV_STATE_ERR) { 2207 rc = -rc; 2208 device_printf(sc->dev, 2209 "failed to connect to the firmware: %d, %d.\n", rc, state); 2210 return (rc); 2211 } 2212 pf = rc; 2213 if (pf == sc->mbox) 2214 sc->flags |= MASTER_PF; 2215 else if (state == DEV_STATE_UNINIT) { 2216 /* 2217 * We didn't get to be the master so we definitely won't be 2218 * configuring the chip. It's a bug if someone else hasn't 2219 * configured it already. 2220 */ 2221 device_printf(sc->dev, "couldn't be master(%d), " 2222 "device not already initialized either(%d).\n", rc, state); 2223 return (EDOOFUS); 2224 } 2225 2226 /* This is the firmware whose headers the driver was compiled against */ 2227 fw_info = find_fw_info(chip_id(sc)); 2228 if (fw_info == NULL) { 2229 device_printf(sc->dev, 2230 "unable to look up firmware information for chip %d.\n", 2231 chip_id(sc)); 2232 return (EINVAL); 2233 } 2234 drv_fw = &fw_info->fw_hdr; 2235 2236 /* 2237 * The firmware KLD contains many modules. The KLD name is also the 2238 * name of the module that contains the default config file. 2239 */ 2240 default_cfg = firmware_get(fw_info->kld_name); 2241 2242 /* Read the header of the firmware on the card */ 2243 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2244 rc = -t4_read_flash(sc, FLASH_FW_START, 2245 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2246 if (rc == 0) 2247 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2248 else { 2249 device_printf(sc->dev, 2250 "Unable to read card's firmware header: %d\n", rc); 2251 card_fw_usable = 0; 2252 } 2253 2254 /* This is the firmware in the KLD */ 2255 fw = firmware_get(fw_info->fw_mod_name); 2256 if (fw != NULL) { 2257 kld_fw = (const void *)fw->data; 2258 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2259 } else { 2260 kld_fw = NULL; 2261 kld_fw_usable = 0; 2262 } 2263 2264 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2265 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2266 /* 2267 * Common case: the firmware on the card is an exact match and 2268 * the KLD is an exact match too, or the KLD is 2269 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2270 * here -- use cxgbetool loadfw if you want to reinstall the 2271 * same firmware as the one on the card. 2272 */ 2273 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2274 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2275 be32toh(card_fw->fw_ver))) { 2276 2277 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2278 if (rc != 0) { 2279 device_printf(sc->dev, 2280 "failed to install firmware: %d\n", rc); 2281 goto done; 2282 } 2283 2284 /* Installed successfully, update the cached header too. */ 2285 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2286 card_fw_usable = 1; 2287 need_fw_reset = 0; /* already reset as part of load_fw */ 2288 } 2289 2290 if (!card_fw_usable) { 2291 uint32_t d, c, k; 2292 2293 d = ntohl(drv_fw->fw_ver); 2294 c = ntohl(card_fw->fw_ver); 2295 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2296 2297 device_printf(sc->dev, "Cannot find a usable firmware: " 2298 "fw_install %d, chip state %d, " 2299 "driver compiled with %d.%d.%d.%d, " 2300 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2301 t4_fw_install, state, 2302 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2303 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2304 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2305 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2306 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2307 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2308 rc = EINVAL; 2309 goto done; 2310 } 2311 2312 /* We're using whatever's on the card and it's known to be good. */ 2313 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2314 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2315 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2316 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2317 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2318 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2319 t4_get_tp_version(sc, &sc->params.tp_vers); 2320 2321 /* Reset device */ 2322 if (need_fw_reset && 2323 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2324 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2325 if (rc != ETIMEDOUT && rc != EIO) 2326 t4_fw_bye(sc, sc->mbox); 2327 goto done; 2328 } 2329 sc->flags |= FW_OK; 2330 2331 rc = get_params__pre_init(sc); 2332 if (rc != 0) 2333 goto done; /* error message displayed already */ 2334 2335 /* Partition adapter resources as specified in the config file. */ 2336 if (state == DEV_STATE_UNINIT) { 2337 2338 KASSERT(sc->flags & MASTER_PF, 2339 ("%s: trying to change chip settings when not master.", 2340 __func__)); 2341 2342 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2343 if (rc != 0) 2344 goto done; /* error message displayed already */ 2345 2346 t4_tweak_chip_settings(sc); 2347 2348 /* get basic stuff going */ 2349 rc = -t4_fw_initialize(sc, sc->mbox); 2350 if (rc != 0) { 2351 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2352 goto done; 2353 } 2354 } else { 2355 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2356 sc->cfcsum = 0; 2357 } 2358 2359done: 2360 free(card_fw, M_CXGBE); 2361 if (fw != NULL) 2362 firmware_put(fw, FIRMWARE_UNLOAD); 2363 if (default_cfg != NULL) 2364 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2365 2366 return (rc); 2367} 2368 2369#define FW_PARAM_DEV(param) \ 2370 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2371 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2372#define FW_PARAM_PFVF(param) \ 2373 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2374 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2375 2376/* 2377 * Partition chip resources for use between various PFs, VFs, etc. 2378 */ 2379static int 2380partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2381 const char *name_prefix) 2382{ 2383 const struct firmware *cfg = NULL; 2384 int rc = 0; 2385 struct fw_caps_config_cmd caps; 2386 uint32_t mtype, moff, finicsum, cfcsum; 2387 2388 /* 2389 * Figure out what configuration file to use. Pick the default config 2390 * file for the card if the user hasn't specified one explicitly. 2391 */ 2392 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2393 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2394 /* Card specific overrides go here. */ 2395 if (pci_get_device(sc->dev) == 0x440a) 2396 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2397 if (is_fpga(sc)) 2398 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2399 } 2400 2401 /* 2402 * We need to load another module if the profile is anything except 2403 * "default" or "flash". 2404 */ 2405 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2406 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2407 char s[32]; 2408 2409 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2410 cfg = firmware_get(s); 2411 if (cfg == NULL) { 2412 if (default_cfg != NULL) { 2413 device_printf(sc->dev, 2414 "unable to load module \"%s\" for " 2415 "configuration profile \"%s\", will use " 2416 "the default config file instead.\n", 2417 s, sc->cfg_file); 2418 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2419 "%s", DEFAULT_CF); 2420 } else { 2421 device_printf(sc->dev, 2422 "unable to load module \"%s\" for " 2423 "configuration profile \"%s\", will use " 2424 "the config file on the card's flash " 2425 "instead.\n", s, sc->cfg_file); 2426 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2427 "%s", FLASH_CF); 2428 } 2429 } 2430 } 2431 2432 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2433 default_cfg == NULL) { 2434 device_printf(sc->dev, 2435 "default config file not available, will use the config " 2436 "file on the card's flash instead.\n"); 2437 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2438 } 2439 2440 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2441 u_int cflen, i, n; 2442 const uint32_t *cfdata; 2443 uint32_t param, val, addr, off, mw_base, mw_aperture; 2444 2445 KASSERT(cfg != NULL || default_cfg != NULL, 2446 ("%s: no config to upload", __func__)); 2447 2448 /* 2449 * Ask the firmware where it wants us to upload the config file. 2450 */ 2451 param = FW_PARAM_DEV(CF); 2452 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2453 if (rc != 0) { 2454 /* No support for config file? Shouldn't happen. */ 2455 device_printf(sc->dev, 2456 "failed to query config file location: %d.\n", rc); 2457 goto done; 2458 } 2459 mtype = G_FW_PARAMS_PARAM_Y(val); 2460 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2461 2462 /* 2463 * XXX: sheer laziness. We deliberately added 4 bytes of 2464 * useless stuffing/comments at the end of the config file so 2465 * it's ok to simply throw away the last remaining bytes when 2466 * the config file is not an exact multiple of 4. This also 2467 * helps with the validate_mt_off_len check. 2468 */ 2469 if (cfg != NULL) { 2470 cflen = cfg->datasize & ~3; 2471 cfdata = cfg->data; 2472 } else { 2473 cflen = default_cfg->datasize & ~3; 2474 cfdata = default_cfg->data; 2475 } 2476 2477 if (cflen > FLASH_CFG_MAX_SIZE) { 2478 device_printf(sc->dev, 2479 "config file too long (%d, max allowed is %d). " 2480 "Will try to use the config on the card, if any.\n", 2481 cflen, FLASH_CFG_MAX_SIZE); 2482 goto use_config_on_flash; 2483 } 2484 2485 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2486 if (rc != 0) { 2487 device_printf(sc->dev, 2488 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2489 "Will try to use the config on the card, if any.\n", 2490 __func__, mtype, moff, cflen, rc); 2491 goto use_config_on_flash; 2492 } 2493 2494 memwin_info(sc, 2, &mw_base, &mw_aperture); 2495 while (cflen) { 2496 off = position_memwin(sc, 2, addr); 2497 n = min(cflen, mw_aperture - off); 2498 for (i = 0; i < n; i += 4) 2499 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2500 cflen -= n; 2501 addr += n; 2502 } 2503 } else { 2504use_config_on_flash: 2505 mtype = FW_MEMTYPE_FLASH; 2506 moff = t4_flash_cfg_addr(sc); 2507 } 2508 2509 bzero(&caps, sizeof(caps)); 2510 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2511 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2512 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2513 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2514 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2515 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2516 if (rc != 0) { 2517 device_printf(sc->dev, 2518 "failed to pre-process config file: %d " 2519 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2520 goto done; 2521 } 2522 2523 finicsum = be32toh(caps.finicsum); 2524 cfcsum = be32toh(caps.cfcsum); 2525 if (finicsum != cfcsum) { 2526 device_printf(sc->dev, 2527 "WARNING: config file checksum mismatch: %08x %08x\n", 2528 finicsum, cfcsum); 2529 } 2530 sc->cfcsum = cfcsum; 2531 2532#define LIMIT_CAPS(x) do { \ 2533 caps.x &= htobe16(t4_##x##_allowed); \ 2534} while (0) 2535 2536 /* 2537 * Let the firmware know what features will (not) be used so it can tune 2538 * things accordingly. 2539 */ 2540 LIMIT_CAPS(linkcaps); 2541 LIMIT_CAPS(niccaps); 2542 LIMIT_CAPS(toecaps); 2543 LIMIT_CAPS(rdmacaps); 2544 LIMIT_CAPS(iscsicaps); 2545 LIMIT_CAPS(fcoecaps); 2546#undef LIMIT_CAPS 2547 2548 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2549 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2550 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2551 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2552 if (rc != 0) { 2553 device_printf(sc->dev, 2554 "failed to process config file: %d.\n", rc); 2555 } 2556done: 2557 if (cfg != NULL) 2558 firmware_put(cfg, FIRMWARE_UNLOAD); 2559 return (rc); 2560} 2561 2562/* 2563 * Retrieve parameters that are needed (or nice to have) very early. 2564 */ 2565static int 2566get_params__pre_init(struct adapter *sc) 2567{ 2568 int rc; 2569 uint32_t param[2], val[2]; 2570 struct fw_devlog_cmd cmd; 2571 struct devlog_params *dlog = &sc->params.devlog; 2572 2573 param[0] = FW_PARAM_DEV(PORTVEC); 2574 param[1] = FW_PARAM_DEV(CCLK); 2575 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2576 if (rc != 0) { 2577 device_printf(sc->dev, 2578 "failed to query parameters (pre_init): %d.\n", rc); 2579 return (rc); 2580 } 2581 2582 sc->params.portvec = val[0]; 2583 sc->params.nports = bitcount32(val[0]); 2584 sc->params.vpd.cclk = val[1]; 2585 2586 /* Read device log parameters. */ 2587 bzero(&cmd, sizeof(cmd)); 2588 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2589 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2590 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2591 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2592 if (rc != 0) { 2593 device_printf(sc->dev, 2594 "failed to get devlog parameters: %d.\n", rc); 2595 bzero(dlog, sizeof (*dlog)); 2596 rc = 0; /* devlog isn't critical for device operation */ 2597 } else { 2598 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2599 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2600 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2601 dlog->size = be32toh(cmd.memsize_devlog); 2602 } 2603 2604 return (rc); 2605} 2606 2607/* 2608 * Retrieve various parameters that are of interest to the driver. The device 2609 * has been initialized by the firmware at this point. 2610 */ 2611static int 2612get_params__post_init(struct adapter *sc) 2613{ 2614 int rc; 2615 uint32_t param[7], val[7]; 2616 struct fw_caps_config_cmd caps; 2617 2618 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2619 param[1] = FW_PARAM_PFVF(EQ_START); 2620 param[2] = FW_PARAM_PFVF(FILTER_START); 2621 param[3] = FW_PARAM_PFVF(FILTER_END); 2622 param[4] = FW_PARAM_PFVF(L2T_START); 2623 param[5] = FW_PARAM_PFVF(L2T_END); 2624 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2625 if (rc != 0) { 2626 device_printf(sc->dev, 2627 "failed to query parameters (post_init): %d.\n", rc); 2628 return (rc); 2629 } 2630 2631 sc->sge.iq_start = val[0]; 2632 sc->sge.eq_start = val[1]; 2633 sc->tids.ftid_base = val[2]; 2634 sc->tids.nftids = val[3] - val[2] + 1; 2635 sc->params.ftid_min = val[2]; 2636 sc->params.ftid_max = val[3]; 2637 sc->vres.l2t.start = val[4]; 2638 sc->vres.l2t.size = val[5] - val[4] + 1; 2639 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2640 ("%s: L2 table size (%u) larger than expected (%u)", 2641 __func__, sc->vres.l2t.size, L2T_SIZE)); 2642 2643 /* get capabilites */ 2644 bzero(&caps, sizeof(caps)); 2645 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2646 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2647 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2648 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2649 if (rc != 0) { 2650 device_printf(sc->dev, 2651 "failed to get card capabilities: %d.\n", rc); 2652 return (rc); 2653 } 2654 2655#define READ_CAPS(x) do { \ 2656 sc->x = htobe16(caps.x); \ 2657} while (0) 2658 READ_CAPS(linkcaps); 2659 READ_CAPS(niccaps); 2660 READ_CAPS(toecaps); 2661 READ_CAPS(rdmacaps); 2662 READ_CAPS(iscsicaps); 2663 READ_CAPS(fcoecaps); 2664 2665 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2666 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2667 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2668 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2669 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2670 if (rc != 0) { 2671 device_printf(sc->dev, 2672 "failed to query NIC parameters: %d.\n", rc); 2673 return (rc); 2674 } 2675 sc->tids.etid_base = val[0]; 2676 sc->params.etid_min = val[0]; 2677 sc->tids.netids = val[1] - val[0] + 1; 2678 sc->params.netids = sc->tids.netids; 2679 sc->params.eo_wr_cred = val[2]; 2680 sc->params.ethoffload = 1; 2681 } 2682 2683 if (sc->toecaps) { 2684 /* query offload-related parameters */ 2685 param[0] = FW_PARAM_DEV(NTID); 2686 param[1] = FW_PARAM_PFVF(SERVER_START); 2687 param[2] = FW_PARAM_PFVF(SERVER_END); 2688 param[3] = FW_PARAM_PFVF(TDDP_START); 2689 param[4] = FW_PARAM_PFVF(TDDP_END); 2690 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2691 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2692 if (rc != 0) { 2693 device_printf(sc->dev, 2694 "failed to query TOE parameters: %d.\n", rc); 2695 return (rc); 2696 } 2697 sc->tids.ntids = val[0]; 2698 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2699 sc->tids.stid_base = val[1]; 2700 sc->tids.nstids = val[2] - val[1] + 1; 2701 sc->vres.ddp.start = val[3]; 2702 sc->vres.ddp.size = val[4] - val[3] + 1; 2703 sc->params.ofldq_wr_cred = val[5]; 2704 sc->params.offload = 1; 2705 } 2706 if (sc->rdmacaps) { 2707 param[0] = FW_PARAM_PFVF(STAG_START); 2708 param[1] = FW_PARAM_PFVF(STAG_END); 2709 param[2] = FW_PARAM_PFVF(RQ_START); 2710 param[3] = FW_PARAM_PFVF(RQ_END); 2711 param[4] = FW_PARAM_PFVF(PBL_START); 2712 param[5] = FW_PARAM_PFVF(PBL_END); 2713 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2714 if (rc != 0) { 2715 device_printf(sc->dev, 2716 "failed to query RDMA parameters(1): %d.\n", rc); 2717 return (rc); 2718 } 2719 sc->vres.stag.start = val[0]; 2720 sc->vres.stag.size = val[1] - val[0] + 1; 2721 sc->vres.rq.start = val[2]; 2722 sc->vres.rq.size = val[3] - val[2] + 1; 2723 sc->vres.pbl.start = val[4]; 2724 sc->vres.pbl.size = val[5] - val[4] + 1; 2725 2726 param[0] = FW_PARAM_PFVF(SQRQ_START); 2727 param[1] = FW_PARAM_PFVF(SQRQ_END); 2728 param[2] = FW_PARAM_PFVF(CQ_START); 2729 param[3] = FW_PARAM_PFVF(CQ_END); 2730 param[4] = FW_PARAM_PFVF(OCQ_START); 2731 param[5] = FW_PARAM_PFVF(OCQ_END); 2732 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2733 if (rc != 0) { 2734 device_printf(sc->dev, 2735 "failed to query RDMA parameters(2): %d.\n", rc); 2736 return (rc); 2737 } 2738 sc->vres.qp.start = val[0]; 2739 sc->vres.qp.size = val[1] - val[0] + 1; 2740 sc->vres.cq.start = val[2]; 2741 sc->vres.cq.size = val[3] - val[2] + 1; 2742 sc->vres.ocq.start = val[4]; 2743 sc->vres.ocq.size = val[5] - val[4] + 1; 2744 } 2745 if (sc->iscsicaps) { 2746 param[0] = FW_PARAM_PFVF(ISCSI_START); 2747 param[1] = FW_PARAM_PFVF(ISCSI_END); 2748 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2749 if (rc != 0) { 2750 device_printf(sc->dev, 2751 "failed to query iSCSI parameters: %d.\n", rc); 2752 return (rc); 2753 } 2754 sc->vres.iscsi.start = val[0]; 2755 sc->vres.iscsi.size = val[1] - val[0] + 1; 2756 } 2757 2758 /* 2759 * We've got the params we wanted to query via the firmware. Now grab 2760 * some others directly from the chip. 2761 */ 2762 rc = t4_read_chip_settings(sc); 2763 2764 return (rc); 2765} 2766 2767static int 2768set_params__post_init(struct adapter *sc) 2769{ 2770 uint32_t param, val; 2771 2772 /* ask for encapsulated CPLs */ 2773 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2774 val = 1; 2775 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2776 2777 return (0); 2778} 2779 2780#undef FW_PARAM_PFVF 2781#undef FW_PARAM_DEV 2782 2783static void 2784t4_set_desc(struct adapter *sc) 2785{ 2786 char buf[128]; 2787 struct adapter_params *p = &sc->params; 2788 2789 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2790 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2791 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2792 2793 device_set_desc_copy(sc->dev, buf); 2794} 2795 2796static void 2797build_medialist(struct port_info *pi, struct ifmedia *media) 2798{ 2799 int data, m; 2800 2801 PORT_LOCK(pi); 2802 2803 ifmedia_removeall(media); 2804 2805 m = IFM_ETHER | IFM_FDX; 2806 data = (pi->port_type << 8) | pi->mod_type; 2807 2808 switch(pi->port_type) { 2809 case FW_PORT_TYPE_BT_XFI: 2810 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2811 break; 2812 2813 case FW_PORT_TYPE_BT_XAUI: 2814 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2815 /* fall through */ 2816 2817 case FW_PORT_TYPE_BT_SGMII: 2818 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2819 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2820 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2821 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2822 break; 2823 2824 case FW_PORT_TYPE_CX4: 2825 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2826 ifmedia_set(media, m | IFM_10G_CX4); 2827 break; 2828 2829 case FW_PORT_TYPE_QSFP_10G: 2830 case FW_PORT_TYPE_SFP: 2831 case FW_PORT_TYPE_FIBER_XFI: 2832 case FW_PORT_TYPE_FIBER_XAUI: 2833 switch (pi->mod_type) { 2834 2835 case FW_PORT_MOD_TYPE_LR: 2836 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2837 ifmedia_set(media, m | IFM_10G_LR); 2838 break; 2839 2840 case FW_PORT_MOD_TYPE_SR: 2841 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2842 ifmedia_set(media, m | IFM_10G_SR); 2843 break; 2844 2845 case FW_PORT_MOD_TYPE_LRM: 2846 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2847 ifmedia_set(media, m | IFM_10G_LRM); 2848 break; 2849 2850 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2851 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2852 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2853 ifmedia_set(media, m | IFM_10G_TWINAX); 2854 break; 2855 2856 case FW_PORT_MOD_TYPE_NONE: 2857 m &= ~IFM_FDX; 2858 ifmedia_add(media, m | IFM_NONE, data, NULL); 2859 ifmedia_set(media, m | IFM_NONE); 2860 break; 2861 2862 case FW_PORT_MOD_TYPE_NA: 2863 case FW_PORT_MOD_TYPE_ER: 2864 default: 2865 device_printf(pi->dev, 2866 "unknown port_type (%d), mod_type (%d)\n", 2867 pi->port_type, pi->mod_type); 2868 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2869 ifmedia_set(media, m | IFM_UNKNOWN); 2870 break; 2871 } 2872 break; 2873 2874 case FW_PORT_TYPE_QSFP: 2875 switch (pi->mod_type) { 2876 2877 case FW_PORT_MOD_TYPE_LR: 2878 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2879 ifmedia_set(media, m | IFM_40G_LR4); 2880 break; 2881 2882 case FW_PORT_MOD_TYPE_SR: 2883 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2884 ifmedia_set(media, m | IFM_40G_SR4); 2885 break; 2886 2887 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2888 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2889 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2890 ifmedia_set(media, m | IFM_40G_CR4); 2891 break; 2892 2893 case FW_PORT_MOD_TYPE_NONE: 2894 m &= ~IFM_FDX; 2895 ifmedia_add(media, m | IFM_NONE, data, NULL); 2896 ifmedia_set(media, m | IFM_NONE); 2897 break; 2898 2899 default: 2900 device_printf(pi->dev, 2901 "unknown port_type (%d), mod_type (%d)\n", 2902 pi->port_type, pi->mod_type); 2903 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2904 ifmedia_set(media, m | IFM_UNKNOWN); 2905 break; 2906 } 2907 break; 2908 2909 default: 2910 device_printf(pi->dev, 2911 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2912 pi->mod_type); 2913 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2914 ifmedia_set(media, m | IFM_UNKNOWN); 2915 break; 2916 } 2917 2918 PORT_UNLOCK(pi); 2919} 2920 2921#define FW_MAC_EXACT_CHUNK 7 2922 2923/* 2924 * Program the port's XGMAC based on parameters in ifnet. The caller also 2925 * indicates which parameters should be programmed (the rest are left alone). 2926 */ 2927int 2928update_mac_settings(struct ifnet *ifp, int flags) 2929{ 2930 int rc = 0; 2931 struct port_info *pi = ifp->if_softc; 2932 struct adapter *sc = pi->adapter; 2933 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2934 uint16_t viid = 0xffff; 2935 int16_t *xact_addr_filt = NULL; 2936 2937 ASSERT_SYNCHRONIZED_OP(sc); 2938 KASSERT(flags, ("%s: not told what to update.", __func__)); 2939 2940 if (ifp == pi->ifp) { 2941 viid = pi->viid; 2942 xact_addr_filt = &pi->xact_addr_filt; 2943 } 2944#ifdef DEV_NETMAP 2945 else if (ifp == pi->nm_ifp) { 2946 viid = pi->nm_viid; 2947 xact_addr_filt = &pi->nm_xact_addr_filt; 2948 } 2949#endif 2950 if (flags & XGMAC_MTU) 2951 mtu = ifp->if_mtu; 2952 2953 if (flags & XGMAC_PROMISC) 2954 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2955 2956 if (flags & XGMAC_ALLMULTI) 2957 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2958 2959 if (flags & XGMAC_VLANEX) 2960 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2961 2962 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 2963 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 2964 1, vlanex, false); 2965 if (rc) { 2966 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 2967 rc); 2968 return (rc); 2969 } 2970 } 2971 2972 if (flags & XGMAC_UCADDR) { 2973 uint8_t ucaddr[ETHER_ADDR_LEN]; 2974 2975 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2976 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 2977 true, true); 2978 if (rc < 0) { 2979 rc = -rc; 2980 if_printf(ifp, "change_mac failed: %d\n", rc); 2981 return (rc); 2982 } else { 2983 *xact_addr_filt = rc; 2984 rc = 0; 2985 } 2986 } 2987 2988 if (flags & XGMAC_MCADDRS) { 2989 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2990 int del = 1; 2991 uint64_t hash = 0; 2992 struct ifmultiaddr *ifma; 2993 int i = 0, j; 2994 2995 if_maddr_rlock(ifp); 2996 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2997 if (ifma->ifma_addr->sa_family != AF_LINK) 2998 continue; 2999 mcaddr[i++] = 3000 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3001 3002 if (i == FW_MAC_EXACT_CHUNK) { 3003 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3004 i, mcaddr, NULL, &hash, 0); 3005 if (rc < 0) { 3006 rc = -rc; 3007 for (j = 0; j < i; j++) { 3008 if_printf(ifp, 3009 "failed to add mc address" 3010 " %02x:%02x:%02x:" 3011 "%02x:%02x:%02x rc=%d\n", 3012 mcaddr[j][0], mcaddr[j][1], 3013 mcaddr[j][2], mcaddr[j][3], 3014 mcaddr[j][4], mcaddr[j][5], 3015 rc); 3016 } 3017 goto mcfail; 3018 } 3019 del = 0; 3020 i = 0; 3021 } 3022 } 3023 if (i > 0) { 3024 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3025 mcaddr, NULL, &hash, 0); 3026 if (rc < 0) { 3027 rc = -rc; 3028 for (j = 0; j < i; j++) { 3029 if_printf(ifp, 3030 "failed to add mc address" 3031 " %02x:%02x:%02x:" 3032 "%02x:%02x:%02x rc=%d\n", 3033 mcaddr[j][0], mcaddr[j][1], 3034 mcaddr[j][2], mcaddr[j][3], 3035 mcaddr[j][4], mcaddr[j][5], 3036 rc); 3037 } 3038 goto mcfail; 3039 } 3040 } 3041 3042 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3043 if (rc != 0) 3044 if_printf(ifp, "failed to set mc address hash: %d", rc); 3045mcfail: 3046 if_maddr_runlock(ifp); 3047 } 3048 3049 return (rc); 3050} 3051 3052int 3053begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3054 char *wmesg) 3055{ 3056 int rc, pri; 3057 3058#ifdef WITNESS 3059 /* the caller thinks it's ok to sleep, but is it really? */ 3060 if (flags & SLEEP_OK) 3061 pause("t4slptst", 1); 3062#endif 3063 3064 if (INTR_OK) 3065 pri = PCATCH; 3066 else 3067 pri = 0; 3068 3069 ADAPTER_LOCK(sc); 3070 for (;;) { 3071 3072 if (pi && IS_DOOMED(pi)) { 3073 rc = ENXIO; 3074 goto done; 3075 } 3076 3077 if (!IS_BUSY(sc)) { 3078 rc = 0; 3079 break; 3080 } 3081 3082 if (!(flags & SLEEP_OK)) { 3083 rc = EBUSY; 3084 goto done; 3085 } 3086 3087 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3088 rc = EINTR; 3089 goto done; 3090 } 3091 } 3092 3093 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3094 SET_BUSY(sc); 3095#ifdef INVARIANTS 3096 sc->last_op = wmesg; 3097 sc->last_op_thr = curthread; 3098#endif 3099 3100done: 3101 if (!(flags & HOLD_LOCK) || rc) 3102 ADAPTER_UNLOCK(sc); 3103 3104 return (rc); 3105} 3106 3107void 3108end_synchronized_op(struct adapter *sc, int flags) 3109{ 3110 3111 if (flags & LOCK_HELD) 3112 ADAPTER_LOCK_ASSERT_OWNED(sc); 3113 else 3114 ADAPTER_LOCK(sc); 3115 3116 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3117 CLR_BUSY(sc); 3118 wakeup(&sc->flags); 3119 ADAPTER_UNLOCK(sc); 3120} 3121 3122static int 3123cxgbe_init_synchronized(struct port_info *pi) 3124{ 3125 struct adapter *sc = pi->adapter; 3126 struct ifnet *ifp = pi->ifp; 3127 int rc = 0; 3128 3129 ASSERT_SYNCHRONIZED_OP(sc); 3130 3131 if (isset(&sc->open_device_map, pi->port_id)) { 3132 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3133 ("mismatch between open_device_map and if_drv_flags")); 3134 return (0); /* already running */ 3135 } 3136 3137 if (!(sc->flags & FULL_INIT_DONE) && 3138 ((rc = adapter_full_init(sc)) != 0)) 3139 return (rc); /* error message displayed already */ 3140 3141 if (!(pi->flags & PORT_INIT_DONE) && 3142 ((rc = port_full_init(pi)) != 0)) 3143 return (rc); /* error message displayed already */ 3144 3145 rc = update_mac_settings(ifp, XGMAC_ALL); 3146 if (rc) 3147 goto done; /* error message displayed already */ 3148 3149 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3150 if (rc != 0) { 3151 if_printf(ifp, "enable_vi failed: %d\n", rc); 3152 goto done; 3153 } 3154 3155 /* 3156 * The first iq of the first port to come up is used for tracing. 3157 */ 3158 if (sc->traceq < 0) { 3159 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3160 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3161 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3162 V_QUEUENUMBER(sc->traceq)); 3163 pi->flags |= HAS_TRACEQ; 3164 } 3165 3166 /* all ok */ 3167 setbit(&sc->open_device_map, pi->port_id); 3168 PORT_LOCK(pi); 3169 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3170 PORT_UNLOCK(pi); 3171 3172 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3173done: 3174 if (rc != 0) 3175 cxgbe_uninit_synchronized(pi); 3176 3177 return (rc); 3178} 3179 3180/* 3181 * Idempotent. 3182 */ 3183static int 3184cxgbe_uninit_synchronized(struct port_info *pi) 3185{ 3186 struct adapter *sc = pi->adapter; 3187 struct ifnet *ifp = pi->ifp; 3188 int rc; 3189 3190 ASSERT_SYNCHRONIZED_OP(sc); 3191 3192 /* 3193 * Disable the VI so that all its data in either direction is discarded 3194 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3195 * tick) intact as the TP can deliver negative advice or data that it's 3196 * holding in its RAM (for an offloaded connection) even after the VI is 3197 * disabled. 3198 */ 3199 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3200 if (rc) { 3201 if_printf(ifp, "disable_vi failed: %d\n", rc); 3202 return (rc); 3203 } 3204 3205 clrbit(&sc->open_device_map, pi->port_id); 3206 PORT_LOCK(pi); 3207 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3208 PORT_UNLOCK(pi); 3209 3210 pi->link_cfg.link_ok = 0; 3211 pi->link_cfg.speed = 0; 3212 pi->linkdnrc = -1; 3213 t4_os_link_changed(sc, pi->port_id, 0, -1); 3214 3215 return (0); 3216} 3217 3218/* 3219 * It is ok for this function to fail midway and return right away. t4_detach 3220 * will walk the entire sc->irq list and clean up whatever is valid. 3221 */ 3222static int 3223setup_intr_handlers(struct adapter *sc) 3224{ 3225 int rc, rid, p, q; 3226 char s[8]; 3227 struct irq *irq; 3228 struct port_info *pi; 3229 struct sge_rxq *rxq; 3230#ifdef TCP_OFFLOAD 3231 struct sge_ofld_rxq *ofld_rxq; 3232#endif 3233#ifdef DEV_NETMAP 3234 struct sge_nm_rxq *nm_rxq; 3235#endif 3236 3237 /* 3238 * Setup interrupts. 3239 */ 3240 irq = &sc->irq[0]; 3241 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3242 if (sc->intr_count == 1) 3243 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3244 3245 /* Multiple interrupts. */ 3246 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3247 ("%s: too few intr.", __func__)); 3248 3249 /* The first one is always error intr */ 3250 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3251 if (rc != 0) 3252 return (rc); 3253 irq++; 3254 rid++; 3255 3256 /* The second one is always the firmware event queue */ 3257 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3258 if (rc != 0) 3259 return (rc); 3260 irq++; 3261 rid++; 3262 3263 for_each_port(sc, p) { 3264 pi = sc->port[p]; 3265 3266 if (pi->flags & INTR_RXQ) { 3267 for_each_rxq(pi, q, rxq) { 3268 snprintf(s, sizeof(s), "%d.%d", p, q); 3269 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3270 s); 3271 if (rc != 0) 3272 return (rc); 3273 irq++; 3274 rid++; 3275 } 3276 } 3277#ifdef TCP_OFFLOAD 3278 if (pi->flags & INTR_OFLD_RXQ) { 3279 for_each_ofld_rxq(pi, q, ofld_rxq) { 3280 snprintf(s, sizeof(s), "%d,%d", p, q); 3281 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3282 ofld_rxq, s); 3283 if (rc != 0) 3284 return (rc); 3285 irq++; 3286 rid++; 3287 } 3288 } 3289#endif 3290#ifdef DEV_NETMAP 3291 if (pi->flags & INTR_NM_RXQ) { 3292 for_each_nm_rxq(pi, q, nm_rxq) { 3293 snprintf(s, sizeof(s), "%d-%d", p, q); 3294 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3295 nm_rxq, s); 3296 if (rc != 0) 3297 return (rc); 3298 irq++; 3299 rid++; 3300 } 3301 } 3302#endif 3303 } 3304 MPASS(irq == &sc->irq[sc->intr_count]); 3305 3306 return (0); 3307} 3308 3309int 3310adapter_full_init(struct adapter *sc) 3311{ 3312 int rc, i; 3313 3314 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3315 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3316 ("%s: FULL_INIT_DONE already", __func__)); 3317 3318 /* 3319 * queues that belong to the adapter (not any particular port). 3320 */ 3321 rc = t4_setup_adapter_queues(sc); 3322 if (rc != 0) 3323 goto done; 3324 3325 for (i = 0; i < nitems(sc->tq); i++) { 3326 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3327 taskqueue_thread_enqueue, &sc->tq[i]); 3328 if (sc->tq[i] == NULL) { 3329 device_printf(sc->dev, 3330 "failed to allocate task queue %d\n", i); 3331 rc = ENOMEM; 3332 goto done; 3333 } 3334 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3335 device_get_nameunit(sc->dev), i); 3336 } 3337 3338 t4_intr_enable(sc); 3339 sc->flags |= FULL_INIT_DONE; 3340done: 3341 if (rc != 0) 3342 adapter_full_uninit(sc); 3343 3344 return (rc); 3345} 3346 3347int 3348adapter_full_uninit(struct adapter *sc) 3349{ 3350 int i; 3351 3352 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3353 3354 t4_teardown_adapter_queues(sc); 3355 3356 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3357 taskqueue_free(sc->tq[i]); 3358 sc->tq[i] = NULL; 3359 } 3360 3361 sc->flags &= ~FULL_INIT_DONE; 3362 3363 return (0); 3364} 3365 3366int 3367port_full_init(struct port_info *pi) 3368{ 3369 struct adapter *sc = pi->adapter; 3370 struct ifnet *ifp = pi->ifp; 3371 uint16_t *rss; 3372 struct sge_rxq *rxq; 3373 int rc, i, j; 3374 3375 ASSERT_SYNCHRONIZED_OP(sc); 3376 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3377 ("%s: PORT_INIT_DONE already", __func__)); 3378 3379 sysctl_ctx_init(&pi->ctx); 3380 pi->flags |= PORT_SYSCTL_CTX; 3381 3382 /* 3383 * Allocate tx/rx/fl queues for this port. 3384 */ 3385 rc = t4_setup_port_queues(pi); 3386 if (rc != 0) 3387 goto done; /* error message displayed already */ 3388 3389 /* 3390 * Setup RSS for this port. Save a copy of the RSS table for later use. 3391 */ 3392 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3393 for (i = 0; i < pi->rss_size;) { 3394 for_each_rxq(pi, j, rxq) { 3395 rss[i++] = rxq->iq.abs_id; 3396 if (i == pi->rss_size) 3397 break; 3398 } 3399 } 3400 3401 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3402 pi->rss_size); 3403 if (rc != 0) { 3404 if_printf(ifp, "rss_config failed: %d\n", rc); 3405 goto done; 3406 } 3407 3408 pi->rss = rss; 3409 pi->flags |= PORT_INIT_DONE; 3410done: 3411 if (rc != 0) 3412 port_full_uninit(pi); 3413 3414 return (rc); 3415} 3416 3417/* 3418 * Idempotent. 3419 */ 3420int 3421port_full_uninit(struct port_info *pi) 3422{ 3423 struct adapter *sc = pi->adapter; 3424 int i; 3425 struct sge_rxq *rxq; 3426 struct sge_txq *txq; 3427#ifdef TCP_OFFLOAD 3428 struct sge_ofld_rxq *ofld_rxq; 3429 struct sge_wrq *ofld_txq; 3430#endif 3431 3432 if (pi->flags & PORT_INIT_DONE) { 3433 3434 /* Need to quiesce queues. XXX: ctrl queues? */ 3435 3436 for_each_txq(pi, i, txq) { 3437 quiesce_eq(sc, &txq->eq); 3438 } 3439 3440#ifdef TCP_OFFLOAD 3441 for_each_ofld_txq(pi, i, ofld_txq) { 3442 quiesce_eq(sc, &ofld_txq->eq); 3443 } 3444#endif 3445 3446 for_each_rxq(pi, i, rxq) { 3447 quiesce_iq(sc, &rxq->iq); 3448 quiesce_fl(sc, &rxq->fl); 3449 } 3450 3451#ifdef TCP_OFFLOAD 3452 for_each_ofld_rxq(pi, i, ofld_rxq) { 3453 quiesce_iq(sc, &ofld_rxq->iq); 3454 quiesce_fl(sc, &ofld_rxq->fl); 3455 } 3456#endif 3457 free(pi->rss, M_CXGBE); 3458 } 3459 3460 t4_teardown_port_queues(pi); 3461 pi->flags &= ~PORT_INIT_DONE; 3462 3463 return (0); 3464} 3465 3466static void 3467quiesce_eq(struct adapter *sc, struct sge_eq *eq) 3468{ 3469 EQ_LOCK(eq); 3470 eq->flags |= EQ_DOOMED; 3471 3472 /* 3473 * Wait for the response to a credit flush if one's 3474 * pending. 3475 */ 3476 while (eq->flags & EQ_CRFLUSHED) 3477 mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3478 EQ_UNLOCK(eq); 3479 3480 callout_drain(&eq->tx_callout); /* XXX: iffy */ 3481 pause("callout", 10); /* Still iffy */ 3482 3483 taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3484} 3485 3486static void 3487quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3488{ 3489 (void) sc; /* unused */ 3490 3491 /* Synchronize with the interrupt handler */ 3492 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3493 pause("iqfree", 1); 3494} 3495 3496static void 3497quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3498{ 3499 mtx_lock(&sc->sfl_lock); 3500 FL_LOCK(fl); 3501 fl->flags |= FL_DOOMED; 3502 FL_UNLOCK(fl); 3503 mtx_unlock(&sc->sfl_lock); 3504 3505 callout_drain(&sc->sfl_callout); 3506 KASSERT((fl->flags & FL_STARVING) == 0, 3507 ("%s: still starving", __func__)); 3508} 3509 3510static int 3511t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3512 driver_intr_t *handler, void *arg, char *name) 3513{ 3514 int rc; 3515 3516 irq->rid = rid; 3517 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3518 RF_SHAREABLE | RF_ACTIVE); 3519 if (irq->res == NULL) { 3520 device_printf(sc->dev, 3521 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3522 return (ENOMEM); 3523 } 3524 3525 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3526 NULL, handler, arg, &irq->tag); 3527 if (rc != 0) { 3528 device_printf(sc->dev, 3529 "failed to setup interrupt for rid %d, name %s: %d\n", 3530 rid, name, rc); 3531 } else if (name) 3532 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3533 3534 return (rc); 3535} 3536 3537static int 3538t4_free_irq(struct adapter *sc, struct irq *irq) 3539{ 3540 if (irq->tag) 3541 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3542 if (irq->res) 3543 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3544 3545 bzero(irq, sizeof(*irq)); 3546 3547 return (0); 3548} 3549 3550static void 3551reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3552 unsigned int end) 3553{ 3554 uint32_t *p = (uint32_t *)(buf + start); 3555 3556 for ( ; start <= end; start += sizeof(uint32_t)) 3557 *p++ = t4_read_reg(sc, start); 3558} 3559 3560static void 3561t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3562{ 3563 int i, n; 3564 const unsigned int *reg_ranges; 3565 static const unsigned int t4_reg_ranges[] = { 3566 0x1008, 0x1108, 3567 0x1180, 0x11b4, 3568 0x11fc, 0x123c, 3569 0x1300, 0x173c, 3570 0x1800, 0x18fc, 3571 0x3000, 0x30d8, 3572 0x30e0, 0x5924, 3573 0x5960, 0x59d4, 3574 0x5a00, 0x5af8, 3575 0x6000, 0x6098, 3576 0x6100, 0x6150, 3577 0x6200, 0x6208, 3578 0x6240, 0x6248, 3579 0x6280, 0x6338, 3580 0x6370, 0x638c, 3581 0x6400, 0x643c, 3582 0x6500, 0x6524, 3583 0x6a00, 0x6a38, 3584 0x6a60, 0x6a78, 3585 0x6b00, 0x6b84, 3586 0x6bf0, 0x6c84, 3587 0x6cf0, 0x6d84, 3588 0x6df0, 0x6e84, 3589 0x6ef0, 0x6f84, 3590 0x6ff0, 0x7084, 3591 0x70f0, 0x7184, 3592 0x71f0, 0x7284, 3593 0x72f0, 0x7384, 3594 0x73f0, 0x7450, 3595 0x7500, 0x7530, 3596 0x7600, 0x761c, 3597 0x7680, 0x76cc, 3598 0x7700, 0x7798, 3599 0x77c0, 0x77fc, 3600 0x7900, 0x79fc, 3601 0x7b00, 0x7c38, 3602 0x7d00, 0x7efc, 3603 0x8dc0, 0x8e1c, 3604 0x8e30, 0x8e78, 3605 0x8ea0, 0x8f6c, 3606 0x8fc0, 0x9074, 3607 0x90fc, 0x90fc, 3608 0x9400, 0x9458, 3609 0x9600, 0x96bc, 3610 0x9800, 0x9808, 3611 0x9820, 0x983c, 3612 0x9850, 0x9864, 3613 0x9c00, 0x9c6c, 3614 0x9c80, 0x9cec, 3615 0x9d00, 0x9d6c, 3616 0x9d80, 0x9dec, 3617 0x9e00, 0x9e6c, 3618 0x9e80, 0x9eec, 3619 0x9f00, 0x9f6c, 3620 0x9f80, 0x9fec, 3621 0xd004, 0xd03c, 3622 0xdfc0, 0xdfe0, 3623 0xe000, 0xea7c, 3624 0xf000, 0x11110, 3625 0x11118, 0x11190, 3626 0x19040, 0x1906c, 3627 0x19078, 0x19080, 3628 0x1908c, 0x19124, 3629 0x19150, 0x191b0, 3630 0x191d0, 0x191e8, 3631 0x19238, 0x1924c, 3632 0x193f8, 0x19474, 3633 0x19490, 0x194f8, 3634 0x19800, 0x19f30, 3635 0x1a000, 0x1a06c, 3636 0x1a0b0, 0x1a120, 3637 0x1a128, 0x1a138, 3638 0x1a190, 0x1a1c4, 3639 0x1a1fc, 0x1a1fc, 3640 0x1e040, 0x1e04c, 3641 0x1e284, 0x1e28c, 3642 0x1e2c0, 0x1e2c0, 3643 0x1e2e0, 0x1e2e0, 3644 0x1e300, 0x1e384, 3645 0x1e3c0, 0x1e3c8, 3646 0x1e440, 0x1e44c, 3647 0x1e684, 0x1e68c, 3648 0x1e6c0, 0x1e6c0, 3649 0x1e6e0, 0x1e6e0, 3650 0x1e700, 0x1e784, 3651 0x1e7c0, 0x1e7c8, 3652 0x1e840, 0x1e84c, 3653 0x1ea84, 0x1ea8c, 3654 0x1eac0, 0x1eac0, 3655 0x1eae0, 0x1eae0, 3656 0x1eb00, 0x1eb84, 3657 0x1ebc0, 0x1ebc8, 3658 0x1ec40, 0x1ec4c, 3659 0x1ee84, 0x1ee8c, 3660 0x1eec0, 0x1eec0, 3661 0x1eee0, 0x1eee0, 3662 0x1ef00, 0x1ef84, 3663 0x1efc0, 0x1efc8, 3664 0x1f040, 0x1f04c, 3665 0x1f284, 0x1f28c, 3666 0x1f2c0, 0x1f2c0, 3667 0x1f2e0, 0x1f2e0, 3668 0x1f300, 0x1f384, 3669 0x1f3c0, 0x1f3c8, 3670 0x1f440, 0x1f44c, 3671 0x1f684, 0x1f68c, 3672 0x1f6c0, 0x1f6c0, 3673 0x1f6e0, 0x1f6e0, 3674 0x1f700, 0x1f784, 3675 0x1f7c0, 0x1f7c8, 3676 0x1f840, 0x1f84c, 3677 0x1fa84, 0x1fa8c, 3678 0x1fac0, 0x1fac0, 3679 0x1fae0, 0x1fae0, 3680 0x1fb00, 0x1fb84, 3681 0x1fbc0, 0x1fbc8, 3682 0x1fc40, 0x1fc4c, 3683 0x1fe84, 0x1fe8c, 3684 0x1fec0, 0x1fec0, 3685 0x1fee0, 0x1fee0, 3686 0x1ff00, 0x1ff84, 3687 0x1ffc0, 0x1ffc8, 3688 0x20000, 0x2002c, 3689 0x20100, 0x2013c, 3690 0x20190, 0x201c8, 3691 0x20200, 0x20318, 3692 0x20400, 0x20528, 3693 0x20540, 0x20614, 3694 0x21000, 0x21040, 3695 0x2104c, 0x21060, 3696 0x210c0, 0x210ec, 3697 0x21200, 0x21268, 3698 0x21270, 0x21284, 3699 0x212fc, 0x21388, 3700 0x21400, 0x21404, 3701 0x21500, 0x21518, 3702 0x2152c, 0x2153c, 3703 0x21550, 0x21554, 3704 0x21600, 0x21600, 3705 0x21608, 0x21628, 3706 0x21630, 0x2163c, 3707 0x21700, 0x2171c, 3708 0x21780, 0x2178c, 3709 0x21800, 0x21c38, 3710 0x21c80, 0x21d7c, 3711 0x21e00, 0x21e04, 3712 0x22000, 0x2202c, 3713 0x22100, 0x2213c, 3714 0x22190, 0x221c8, 3715 0x22200, 0x22318, 3716 0x22400, 0x22528, 3717 0x22540, 0x22614, 3718 0x23000, 0x23040, 3719 0x2304c, 0x23060, 3720 0x230c0, 0x230ec, 3721 0x23200, 0x23268, 3722 0x23270, 0x23284, 3723 0x232fc, 0x23388, 3724 0x23400, 0x23404, 3725 0x23500, 0x23518, 3726 0x2352c, 0x2353c, 3727 0x23550, 0x23554, 3728 0x23600, 0x23600, 3729 0x23608, 0x23628, 3730 0x23630, 0x2363c, 3731 0x23700, 0x2371c, 3732 0x23780, 0x2378c, 3733 0x23800, 0x23c38, 3734 0x23c80, 0x23d7c, 3735 0x23e00, 0x23e04, 3736 0x24000, 0x2402c, 3737 0x24100, 0x2413c, 3738 0x24190, 0x241c8, 3739 0x24200, 0x24318, 3740 0x24400, 0x24528, 3741 0x24540, 0x24614, 3742 0x25000, 0x25040, 3743 0x2504c, 0x25060, 3744 0x250c0, 0x250ec, 3745 0x25200, 0x25268, 3746 0x25270, 0x25284, 3747 0x252fc, 0x25388, 3748 0x25400, 0x25404, 3749 0x25500, 0x25518, 3750 0x2552c, 0x2553c, 3751 0x25550, 0x25554, 3752 0x25600, 0x25600, 3753 0x25608, 0x25628, 3754 0x25630, 0x2563c, 3755 0x25700, 0x2571c, 3756 0x25780, 0x2578c, 3757 0x25800, 0x25c38, 3758 0x25c80, 0x25d7c, 3759 0x25e00, 0x25e04, 3760 0x26000, 0x2602c, 3761 0x26100, 0x2613c, 3762 0x26190, 0x261c8, 3763 0x26200, 0x26318, 3764 0x26400, 0x26528, 3765 0x26540, 0x26614, 3766 0x27000, 0x27040, 3767 0x2704c, 0x27060, 3768 0x270c0, 0x270ec, 3769 0x27200, 0x27268, 3770 0x27270, 0x27284, 3771 0x272fc, 0x27388, 3772 0x27400, 0x27404, 3773 0x27500, 0x27518, 3774 0x2752c, 0x2753c, 3775 0x27550, 0x27554, 3776 0x27600, 0x27600, 3777 0x27608, 0x27628, 3778 0x27630, 0x2763c, 3779 0x27700, 0x2771c, 3780 0x27780, 0x2778c, 3781 0x27800, 0x27c38, 3782 0x27c80, 0x27d7c, 3783 0x27e00, 0x27e04 3784 }; 3785 static const unsigned int t5_reg_ranges[] = { 3786 0x1008, 0x1148, 3787 0x1180, 0x11b4, 3788 0x11fc, 0x123c, 3789 0x1280, 0x173c, 3790 0x1800, 0x18fc, 3791 0x3000, 0x3028, 3792 0x3060, 0x30d8, 3793 0x30e0, 0x30fc, 3794 0x3140, 0x357c, 3795 0x35a8, 0x35cc, 3796 0x35ec, 0x35ec, 3797 0x3600, 0x5624, 3798 0x56cc, 0x575c, 3799 0x580c, 0x5814, 3800 0x5890, 0x58bc, 3801 0x5940, 0x59dc, 3802 0x59fc, 0x5a18, 3803 0x5a60, 0x5a9c, 3804 0x5b94, 0x5bfc, 3805 0x6000, 0x6040, 3806 0x6058, 0x614c, 3807 0x7700, 0x7798, 3808 0x77c0, 0x78fc, 3809 0x7b00, 0x7c54, 3810 0x7d00, 0x7efc, 3811 0x8dc0, 0x8de0, 3812 0x8df8, 0x8e84, 3813 0x8ea0, 0x8f84, 3814 0x8fc0, 0x90f8, 3815 0x9400, 0x9470, 3816 0x9600, 0x96f4, 3817 0x9800, 0x9808, 3818 0x9820, 0x983c, 3819 0x9850, 0x9864, 3820 0x9c00, 0x9c6c, 3821 0x9c80, 0x9cec, 3822 0x9d00, 0x9d6c, 3823 0x9d80, 0x9dec, 3824 0x9e00, 0x9e6c, 3825 0x9e80, 0x9eec, 3826 0x9f00, 0x9f6c, 3827 0x9f80, 0xa020, 3828 0xd004, 0xd03c, 3829 0xdfc0, 0xdfe0, 3830 0xe000, 0x11088, 3831 0x1109c, 0x11110, 3832 0x11118, 0x1117c, 3833 0x11190, 0x11204, 3834 0x19040, 0x1906c, 3835 0x19078, 0x19080, 3836 0x1908c, 0x19124, 3837 0x19150, 0x191b0, 3838 0x191d0, 0x191e8, 3839 0x19238, 0x19290, 3840 0x193f8, 0x19474, 3841 0x19490, 0x194cc, 3842 0x194f0, 0x194f8, 3843 0x19c00, 0x19c60, 3844 0x19c94, 0x19e10, 3845 0x19e50, 0x19f34, 3846 0x19f40, 0x19f50, 3847 0x19f90, 0x19fe4, 3848 0x1a000, 0x1a06c, 3849 0x1a0b0, 0x1a120, 3850 0x1a128, 0x1a138, 3851 0x1a190, 0x1a1c4, 3852 0x1a1fc, 0x1a1fc, 3853 0x1e008, 0x1e00c, 3854 0x1e040, 0x1e04c, 3855 0x1e284, 0x1e290, 3856 0x1e2c0, 0x1e2c0, 3857 0x1e2e0, 0x1e2e0, 3858 0x1e300, 0x1e384, 3859 0x1e3c0, 0x1e3c8, 3860 0x1e408, 0x1e40c, 3861 0x1e440, 0x1e44c, 3862 0x1e684, 0x1e690, 3863 0x1e6c0, 0x1e6c0, 3864 0x1e6e0, 0x1e6e0, 3865 0x1e700, 0x1e784, 3866 0x1e7c0, 0x1e7c8, 3867 0x1e808, 0x1e80c, 3868 0x1e840, 0x1e84c, 3869 0x1ea84, 0x1ea90, 3870 0x1eac0, 0x1eac0, 3871 0x1eae0, 0x1eae0, 3872 0x1eb00, 0x1eb84, 3873 0x1ebc0, 0x1ebc8, 3874 0x1ec08, 0x1ec0c, 3875 0x1ec40, 0x1ec4c, 3876 0x1ee84, 0x1ee90, 3877 0x1eec0, 0x1eec0, 3878 0x1eee0, 0x1eee0, 3879 0x1ef00, 0x1ef84, 3880 0x1efc0, 0x1efc8, 3881 0x1f008, 0x1f00c, 3882 0x1f040, 0x1f04c, 3883 0x1f284, 0x1f290, 3884 0x1f2c0, 0x1f2c0, 3885 0x1f2e0, 0x1f2e0, 3886 0x1f300, 0x1f384, 3887 0x1f3c0, 0x1f3c8, 3888 0x1f408, 0x1f40c, 3889 0x1f440, 0x1f44c, 3890 0x1f684, 0x1f690, 3891 0x1f6c0, 0x1f6c0, 3892 0x1f6e0, 0x1f6e0, 3893 0x1f700, 0x1f784, 3894 0x1f7c0, 0x1f7c8, 3895 0x1f808, 0x1f80c, 3896 0x1f840, 0x1f84c, 3897 0x1fa84, 0x1fa90, 3898 0x1fac0, 0x1fac0, 3899 0x1fae0, 0x1fae0, 3900 0x1fb00, 0x1fb84, 3901 0x1fbc0, 0x1fbc8, 3902 0x1fc08, 0x1fc0c, 3903 0x1fc40, 0x1fc4c, 3904 0x1fe84, 0x1fe90, 3905 0x1fec0, 0x1fec0, 3906 0x1fee0, 0x1fee0, 3907 0x1ff00, 0x1ff84, 3908 0x1ffc0, 0x1ffc8, 3909 0x30000, 0x30030, 3910 0x30100, 0x30144, 3911 0x30190, 0x301d0, 3912 0x30200, 0x30318, 3913 0x30400, 0x3052c, 3914 0x30540, 0x3061c, 3915 0x30800, 0x30834, 3916 0x308c0, 0x30908, 3917 0x30910, 0x309ac, 3918 0x30a00, 0x30a2c, 3919 0x30a44, 0x30a50, 3920 0x30a74, 0x30c24, 3921 0x30d00, 0x30d00, 3922 0x30d08, 0x30d14, 3923 0x30d1c, 0x30d20, 3924 0x30d3c, 0x30d50, 3925 0x31200, 0x3120c, 3926 0x31220, 0x31220, 3927 0x31240, 0x31240, 3928 0x31600, 0x3160c, 3929 0x31a00, 0x31a1c, 3930 0x31e00, 0x31e20, 3931 0x31e38, 0x31e3c, 3932 0x31e80, 0x31e80, 3933 0x31e88, 0x31ea8, 3934 0x31eb0, 0x31eb4, 3935 0x31ec8, 0x31ed4, 3936 0x31fb8, 0x32004, 3937 0x32200, 0x32200, 3938 0x32208, 0x32240, 3939 0x32248, 0x32280, 3940 0x32288, 0x322c0, 3941 0x322c8, 0x322fc, 3942 0x32600, 0x32630, 3943 0x32a00, 0x32abc, 3944 0x32b00, 0x32b70, 3945 0x33000, 0x33048, 3946 0x33060, 0x3309c, 3947 0x330f0, 0x33148, 3948 0x33160, 0x3319c, 3949 0x331f0, 0x332e4, 3950 0x332f8, 0x333e4, 3951 0x333f8, 0x33448, 3952 0x33460, 0x3349c, 3953 0x334f0, 0x33548, 3954 0x33560, 0x3359c, 3955 0x335f0, 0x336e4, 3956 0x336f8, 0x337e4, 3957 0x337f8, 0x337fc, 3958 0x33814, 0x33814, 3959 0x3382c, 0x3382c, 3960 0x33880, 0x3388c, 3961 0x338e8, 0x338ec, 3962 0x33900, 0x33948, 3963 0x33960, 0x3399c, 3964 0x339f0, 0x33ae4, 3965 0x33af8, 0x33b10, 3966 0x33b28, 0x33b28, 3967 0x33b3c, 0x33b50, 3968 0x33bf0, 0x33c10, 3969 0x33c28, 0x33c28, 3970 0x33c3c, 0x33c50, 3971 0x33cf0, 0x33cfc, 3972 0x34000, 0x34030, 3973 0x34100, 0x34144, 3974 0x34190, 0x341d0, 3975 0x34200, 0x34318, 3976 0x34400, 0x3452c, 3977 0x34540, 0x3461c, 3978 0x34800, 0x34834, 3979 0x348c0, 0x34908, 3980 0x34910, 0x349ac, 3981 0x34a00, 0x34a2c, 3982 0x34a44, 0x34a50, 3983 0x34a74, 0x34c24, 3984 0x34d00, 0x34d00, 3985 0x34d08, 0x34d14, 3986 0x34d1c, 0x34d20, 3987 0x34d3c, 0x34d50, 3988 0x35200, 0x3520c, 3989 0x35220, 0x35220, 3990 0x35240, 0x35240, 3991 0x35600, 0x3560c, 3992 0x35a00, 0x35a1c, 3993 0x35e00, 0x35e20, 3994 0x35e38, 0x35e3c, 3995 0x35e80, 0x35e80, 3996 0x35e88, 0x35ea8, 3997 0x35eb0, 0x35eb4, 3998 0x35ec8, 0x35ed4, 3999 0x35fb8, 0x36004, 4000 0x36200, 0x36200, 4001 0x36208, 0x36240, 4002 0x36248, 0x36280, 4003 0x36288, 0x362c0, 4004 0x362c8, 0x362fc, 4005 0x36600, 0x36630, 4006 0x36a00, 0x36abc, 4007 0x36b00, 0x36b70, 4008 0x37000, 0x37048, 4009 0x37060, 0x3709c, 4010 0x370f0, 0x37148, 4011 0x37160, 0x3719c, 4012 0x371f0, 0x372e4, 4013 0x372f8, 0x373e4, 4014 0x373f8, 0x37448, 4015 0x37460, 0x3749c, 4016 0x374f0, 0x37548, 4017 0x37560, 0x3759c, 4018 0x375f0, 0x376e4, 4019 0x376f8, 0x377e4, 4020 0x377f8, 0x377fc, 4021 0x37814, 0x37814, 4022 0x3782c, 0x3782c, 4023 0x37880, 0x3788c, 4024 0x378e8, 0x378ec, 4025 0x37900, 0x37948, 4026 0x37960, 0x3799c, 4027 0x379f0, 0x37ae4, 4028 0x37af8, 0x37b10, 4029 0x37b28, 0x37b28, 4030 0x37b3c, 0x37b50, 4031 0x37bf0, 0x37c10, 4032 0x37c28, 0x37c28, 4033 0x37c3c, 0x37c50, 4034 0x37cf0, 0x37cfc, 4035 0x38000, 0x38030, 4036 0x38100, 0x38144, 4037 0x38190, 0x381d0, 4038 0x38200, 0x38318, 4039 0x38400, 0x3852c, 4040 0x38540, 0x3861c, 4041 0x38800, 0x38834, 4042 0x388c0, 0x38908, 4043 0x38910, 0x389ac, 4044 0x38a00, 0x38a2c, 4045 0x38a44, 0x38a50, 4046 0x38a74, 0x38c24, 4047 0x38d00, 0x38d00, 4048 0x38d08, 0x38d14, 4049 0x38d1c, 0x38d20, 4050 0x38d3c, 0x38d50, 4051 0x39200, 0x3920c, 4052 0x39220, 0x39220, 4053 0x39240, 0x39240, 4054 0x39600, 0x3960c, 4055 0x39a00, 0x39a1c, 4056 0x39e00, 0x39e20, 4057 0x39e38, 0x39e3c, 4058 0x39e80, 0x39e80, 4059 0x39e88, 0x39ea8, 4060 0x39eb0, 0x39eb4, 4061 0x39ec8, 0x39ed4, 4062 0x39fb8, 0x3a004, 4063 0x3a200, 0x3a200, 4064 0x3a208, 0x3a240, 4065 0x3a248, 0x3a280, 4066 0x3a288, 0x3a2c0, 4067 0x3a2c8, 0x3a2fc, 4068 0x3a600, 0x3a630, 4069 0x3aa00, 0x3aabc, 4070 0x3ab00, 0x3ab70, 4071 0x3b000, 0x3b048, 4072 0x3b060, 0x3b09c, 4073 0x3b0f0, 0x3b148, 4074 0x3b160, 0x3b19c, 4075 0x3b1f0, 0x3b2e4, 4076 0x3b2f8, 0x3b3e4, 4077 0x3b3f8, 0x3b448, 4078 0x3b460, 0x3b49c, 4079 0x3b4f0, 0x3b548, 4080 0x3b560, 0x3b59c, 4081 0x3b5f0, 0x3b6e4, 4082 0x3b6f8, 0x3b7e4, 4083 0x3b7f8, 0x3b7fc, 4084 0x3b814, 0x3b814, 4085 0x3b82c, 0x3b82c, 4086 0x3b880, 0x3b88c, 4087 0x3b8e8, 0x3b8ec, 4088 0x3b900, 0x3b948, 4089 0x3b960, 0x3b99c, 4090 0x3b9f0, 0x3bae4, 4091 0x3baf8, 0x3bb10, 4092 0x3bb28, 0x3bb28, 4093 0x3bb3c, 0x3bb50, 4094 0x3bbf0, 0x3bc10, 4095 0x3bc28, 0x3bc28, 4096 0x3bc3c, 0x3bc50, 4097 0x3bcf0, 0x3bcfc, 4098 0x3c000, 0x3c030, 4099 0x3c100, 0x3c144, 4100 0x3c190, 0x3c1d0, 4101 0x3c200, 0x3c318, 4102 0x3c400, 0x3c52c, 4103 0x3c540, 0x3c61c, 4104 0x3c800, 0x3c834, 4105 0x3c8c0, 0x3c908, 4106 0x3c910, 0x3c9ac, 4107 0x3ca00, 0x3ca2c, 4108 0x3ca44, 0x3ca50, 4109 0x3ca74, 0x3cc24, 4110 0x3cd00, 0x3cd00, 4111 0x3cd08, 0x3cd14, 4112 0x3cd1c, 0x3cd20, 4113 0x3cd3c, 0x3cd50, 4114 0x3d200, 0x3d20c, 4115 0x3d220, 0x3d220, 4116 0x3d240, 0x3d240, 4117 0x3d600, 0x3d60c, 4118 0x3da00, 0x3da1c, 4119 0x3de00, 0x3de20, 4120 0x3de38, 0x3de3c, 4121 0x3de80, 0x3de80, 4122 0x3de88, 0x3dea8, 4123 0x3deb0, 0x3deb4, 4124 0x3dec8, 0x3ded4, 4125 0x3dfb8, 0x3e004, 4126 0x3e200, 0x3e200, 4127 0x3e208, 0x3e240, 4128 0x3e248, 0x3e280, 4129 0x3e288, 0x3e2c0, 4130 0x3e2c8, 0x3e2fc, 4131 0x3e600, 0x3e630, 4132 0x3ea00, 0x3eabc, 4133 0x3eb00, 0x3eb70, 4134 0x3f000, 0x3f048, 4135 0x3f060, 0x3f09c, 4136 0x3f0f0, 0x3f148, 4137 0x3f160, 0x3f19c, 4138 0x3f1f0, 0x3f2e4, 4139 0x3f2f8, 0x3f3e4, 4140 0x3f3f8, 0x3f448, 4141 0x3f460, 0x3f49c, 4142 0x3f4f0, 0x3f548, 4143 0x3f560, 0x3f59c, 4144 0x3f5f0, 0x3f6e4, 4145 0x3f6f8, 0x3f7e4, 4146 0x3f7f8, 0x3f7fc, 4147 0x3f814, 0x3f814, 4148 0x3f82c, 0x3f82c, 4149 0x3f880, 0x3f88c, 4150 0x3f8e8, 0x3f8ec, 4151 0x3f900, 0x3f948, 4152 0x3f960, 0x3f99c, 4153 0x3f9f0, 0x3fae4, 4154 0x3faf8, 0x3fb10, 4155 0x3fb28, 0x3fb28, 4156 0x3fb3c, 0x3fb50, 4157 0x3fbf0, 0x3fc10, 4158 0x3fc28, 0x3fc28, 4159 0x3fc3c, 0x3fc50, 4160 0x3fcf0, 0x3fcfc, 4161 0x40000, 0x4000c, 4162 0x40040, 0x40068, 4163 0x4007c, 0x40144, 4164 0x40180, 0x4018c, 4165 0x40200, 0x40298, 4166 0x402ac, 0x4033c, 4167 0x403f8, 0x403fc, 4168 0x41304, 0x413c4, 4169 0x41400, 0x4141c, 4170 0x41480, 0x414d0, 4171 0x44000, 0x44078, 4172 0x440c0, 0x44278, 4173 0x442c0, 0x44478, 4174 0x444c0, 0x44678, 4175 0x446c0, 0x44878, 4176 0x448c0, 0x449fc, 4177 0x45000, 0x45068, 4178 0x45080, 0x45084, 4179 0x450a0, 0x450b0, 4180 0x45200, 0x45268, 4181 0x45280, 0x45284, 4182 0x452a0, 0x452b0, 4183 0x460c0, 0x460e4, 4184 0x47000, 0x4708c, 4185 0x47200, 0x47250, 4186 0x47400, 0x47420, 4187 0x47600, 0x47618, 4188 0x47800, 0x47814, 4189 0x48000, 0x4800c, 4190 0x48040, 0x48068, 4191 0x4807c, 0x48144, 4192 0x48180, 0x4818c, 4193 0x48200, 0x48298, 4194 0x482ac, 0x4833c, 4195 0x483f8, 0x483fc, 4196 0x49304, 0x493c4, 4197 0x49400, 0x4941c, 4198 0x49480, 0x494d0, 4199 0x4c000, 0x4c078, 4200 0x4c0c0, 0x4c278, 4201 0x4c2c0, 0x4c478, 4202 0x4c4c0, 0x4c678, 4203 0x4c6c0, 0x4c878, 4204 0x4c8c0, 0x4c9fc, 4205 0x4d000, 0x4d068, 4206 0x4d080, 0x4d084, 4207 0x4d0a0, 0x4d0b0, 4208 0x4d200, 0x4d268, 4209 0x4d280, 0x4d284, 4210 0x4d2a0, 0x4d2b0, 4211 0x4e0c0, 0x4e0e4, 4212 0x4f000, 0x4f08c, 4213 0x4f200, 0x4f250, 4214 0x4f400, 0x4f420, 4215 0x4f600, 0x4f618, 4216 0x4f800, 0x4f814, 4217 0x50000, 0x500cc, 4218 0x50400, 0x50400, 4219 0x50800, 0x508cc, 4220 0x50c00, 0x50c00, 4221 0x51000, 0x5101c, 4222 0x51300, 0x51308, 4223 }; 4224 4225 if (is_t4(sc)) { 4226 reg_ranges = &t4_reg_ranges[0]; 4227 n = nitems(t4_reg_ranges); 4228 } else { 4229 reg_ranges = &t5_reg_ranges[0]; 4230 n = nitems(t5_reg_ranges); 4231 } 4232 4233 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4234 for (i = 0; i < n; i += 2) 4235 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4236} 4237 4238static void 4239cxgbe_tick(void *arg) 4240{ 4241 struct port_info *pi = arg; 4242 struct adapter *sc = pi->adapter; 4243 struct ifnet *ifp = pi->ifp; 4244 struct sge_txq *txq; 4245 int i, drops; 4246 struct port_stats *s = &pi->stats; 4247 4248 PORT_LOCK(pi); 4249 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4250 PORT_UNLOCK(pi); 4251 return; /* without scheduling another callout */ 4252 } 4253 4254 t4_get_port_stats(sc, pi->tx_chan, s); 4255 4256 ifp->if_opackets = s->tx_frames - s->tx_pause; 4257 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4258 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4259 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4260 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4261 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4262 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4263 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4264 s->rx_trunc3; 4265 for (i = 0; i < 4; i++) { 4266 if (pi->rx_chan_map & (1 << i)) { 4267 uint32_t v; 4268 4269 /* 4270 * XXX: indirect reads from the same ADDR/DATA pair can 4271 * race with each other. 4272 */ 4273 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4274 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4275 ifp->if_iqdrops += v; 4276 } 4277 } 4278 4279 drops = s->tx_drop; 4280 for_each_txq(pi, i, txq) 4281 drops += txq->br->br_drops; 4282 ifp->if_snd.ifq_drops = drops; 4283 4284 ifp->if_oerrors = s->tx_error_frames; 4285 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4286 s->rx_fcs_err + s->rx_len_err; 4287 4288 callout_schedule(&pi->tick, hz); 4289 PORT_UNLOCK(pi); 4290} 4291 4292static void 4293cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4294{ 4295 struct ifnet *vlan; 4296 4297 if (arg != ifp || ifp->if_type != IFT_ETHER) 4298 return; 4299 4300 vlan = VLAN_DEVAT(ifp, vid); 4301 VLAN_SETCOOKIE(vlan, ifp); 4302} 4303 4304static int 4305cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4306{ 4307 4308#ifdef INVARIANTS 4309 panic("%s: opcode 0x%02x on iq %p with payload %p", 4310 __func__, rss->opcode, iq, m); 4311#else 4312 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4313 __func__, rss->opcode, iq, m); 4314 m_freem(m); 4315#endif 4316 return (EDOOFUS); 4317} 4318 4319int 4320t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4321{ 4322 uintptr_t *loc, new; 4323 4324 if (opcode >= nitems(sc->cpl_handler)) 4325 return (EINVAL); 4326 4327 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4328 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4329 atomic_store_rel_ptr(loc, new); 4330 4331 return (0); 4332} 4333 4334static int 4335an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4336{ 4337 4338#ifdef INVARIANTS 4339 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4340#else 4341 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4342 __func__, iq, ctrl); 4343#endif 4344 return (EDOOFUS); 4345} 4346 4347int 4348t4_register_an_handler(struct adapter *sc, an_handler_t h) 4349{ 4350 uintptr_t *loc, new; 4351 4352 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4353 loc = (uintptr_t *) &sc->an_handler; 4354 atomic_store_rel_ptr(loc, new); 4355 4356 return (0); 4357} 4358 4359static int 4360fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4361{ 4362 const struct cpl_fw6_msg *cpl = 4363 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4364 4365#ifdef INVARIANTS 4366 panic("%s: fw_msg type %d", __func__, cpl->type); 4367#else 4368 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4369#endif 4370 return (EDOOFUS); 4371} 4372 4373int 4374t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4375{ 4376 uintptr_t *loc, new; 4377 4378 if (type >= nitems(sc->fw_msg_handler)) 4379 return (EINVAL); 4380 4381 /* 4382 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4383 * handler dispatch table. Reject any attempt to install a handler for 4384 * this subtype. 4385 */ 4386 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4387 return (EINVAL); 4388 4389 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4390 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4391 atomic_store_rel_ptr(loc, new); 4392 4393 return (0); 4394} 4395 4396static int 4397t4_sysctls(struct adapter *sc) 4398{ 4399 struct sysctl_ctx_list *ctx; 4400 struct sysctl_oid *oid; 4401 struct sysctl_oid_list *children, *c0; 4402 static char *caps[] = { 4403 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4404 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4405 "\6HASHFILTER\7ETHOFLD", 4406 "\20\1TOE", /* caps[2] toecaps */ 4407 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4408 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4409 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4410 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4411 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4412 "\4PO_INITIAOR\5PO_TARGET" 4413 }; 4414 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4415 4416 ctx = device_get_sysctl_ctx(sc->dev); 4417 4418 /* 4419 * dev.t4nex.X. 4420 */ 4421 oid = device_get_sysctl_tree(sc->dev); 4422 c0 = children = SYSCTL_CHILDREN(oid); 4423 4424 sc->sc_do_rxcopy = 1; 4425 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4426 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4427 4428 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4429 sc->params.nports, "# of ports"); 4430 4431 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4432 NULL, chip_rev(sc), "chip hardware revision"); 4433 4434 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4435 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4436 4437 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4438 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4439 4440 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4441 sc->cfcsum, "config file checksum"); 4442 4443 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4444 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4445 sysctl_bitfield, "A", "available doorbells"); 4446 4447 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4448 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4449 sysctl_bitfield, "A", "available link capabilities"); 4450 4451 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4452 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4453 sysctl_bitfield, "A", "available NIC capabilities"); 4454 4455 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4456 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4457 sysctl_bitfield, "A", "available TCP offload capabilities"); 4458 4459 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4460 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4461 sysctl_bitfield, "A", "available RDMA capabilities"); 4462 4463 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4464 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4465 sysctl_bitfield, "A", "available iSCSI capabilities"); 4466 4467 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4468 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4469 sysctl_bitfield, "A", "available FCoE capabilities"); 4470 4471 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4472 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4473 4474 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4475 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4476 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4477 "interrupt holdoff timer values (us)"); 4478 4479 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4480 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4481 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4482 "interrupt holdoff packet counter values"); 4483 4484 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4485 NULL, sc->tids.nftids, "number of filters"); 4486 4487 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4488 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4489 "chip temperature (in Celsius)"); 4490 4491 t4_sge_sysctls(sc, ctx, children); 4492 4493 sc->lro_timeout = 100; 4494 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4495 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4496 4497#ifdef SBUF_DRAIN 4498 /* 4499 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4500 */ 4501 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4502 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4503 "logs and miscellaneous information"); 4504 children = SYSCTL_CHILDREN(oid); 4505 4506 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4507 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4508 sysctl_cctrl, "A", "congestion control"); 4509 4510 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4511 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4512 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4513 4514 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4515 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4516 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4517 4518 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4519 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4520 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4521 4522 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4523 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4524 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4525 4526 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4527 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4528 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4529 4530 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4531 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4532 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4533 4534 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4535 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4536 sysctl_cim_la, "A", "CIM logic analyzer"); 4537 4538 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4539 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4540 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4541 4542 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4543 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4544 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4545 4546 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4547 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4548 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4549 4550 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4551 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4552 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4553 4554 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4555 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4556 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4557 4558 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4559 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4560 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4561 4562 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4563 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4564 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4565 4566 if (is_t5(sc)) { 4567 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4568 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4569 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4570 4571 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4572 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4573 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4574 } 4575 4576 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4577 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4578 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4579 4580 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4581 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4582 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4583 4584 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4585 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4586 sysctl_cpl_stats, "A", "CPL statistics"); 4587 4588 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4589 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4590 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4591 4592 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4593 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4594 sysctl_devlog, "A", "firmware's device log"); 4595 4596 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4597 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4598 sysctl_fcoe_stats, "A", "FCoE statistics"); 4599 4600 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4601 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4602 sysctl_hw_sched, "A", "hardware scheduler "); 4603 4604 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4605 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4606 sysctl_l2t, "A", "hardware L2 table"); 4607 4608 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4609 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4610 sysctl_lb_stats, "A", "loopback statistics"); 4611 4612 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4613 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4614 sysctl_meminfo, "A", "memory regions"); 4615 4616 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4617 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4618 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4619 4620 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4621 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4622 sysctl_path_mtus, "A", "path MTUs"); 4623 4624 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4625 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4626 sysctl_pm_stats, "A", "PM statistics"); 4627 4628 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4629 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4630 sysctl_rdma_stats, "A", "RDMA statistics"); 4631 4632 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4633 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4634 sysctl_tcp_stats, "A", "TCP statistics"); 4635 4636 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4637 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4638 sysctl_tids, "A", "TID information"); 4639 4640 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4641 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4642 sysctl_tp_err_stats, "A", "TP error statistics"); 4643 4644 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4645 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4646 sysctl_tp_la, "A", "TP logic analyzer"); 4647 4648 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4649 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4650 sysctl_tx_rate, "A", "Tx rate"); 4651 4652 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4653 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4654 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4655 4656 if (is_t5(sc)) { 4657 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4658 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4659 sysctl_wcwr_stats, "A", "write combined work requests"); 4660 } 4661#endif 4662 4663#ifdef TCP_OFFLOAD 4664 if (is_offload(sc)) { 4665 /* 4666 * dev.t4nex.X.toe. 4667 */ 4668 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4669 NULL, "TOE parameters"); 4670 children = SYSCTL_CHILDREN(oid); 4671 4672 sc->tt.sndbuf = 256 * 1024; 4673 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4674 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4675 4676 sc->tt.ddp = 0; 4677 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4678 &sc->tt.ddp, 0, "DDP allowed"); 4679 4680 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4681 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4682 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4683 4684 sc->tt.ddp_thres = 4685 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4686 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4687 &sc->tt.ddp_thres, 0, "DDP threshold"); 4688 4689 sc->tt.rx_coalesce = 1; 4690 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4691 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4692 4693 sc->tt.tx_align = 1; 4694 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4695 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4696 } 4697#endif 4698 4699 4700 return (0); 4701} 4702 4703static int 4704cxgbe_sysctls(struct port_info *pi) 4705{ 4706 struct sysctl_ctx_list *ctx; 4707 struct sysctl_oid *oid; 4708 struct sysctl_oid_list *children; 4709 struct adapter *sc = pi->adapter; 4710 4711 ctx = device_get_sysctl_ctx(pi->dev); 4712 4713 /* 4714 * dev.cxgbe.X. 4715 */ 4716 oid = device_get_sysctl_tree(pi->dev); 4717 children = SYSCTL_CHILDREN(oid); 4718 4719 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4720 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4721 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4722 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4723 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4724 "PHY temperature (in Celsius)"); 4725 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4726 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4727 "PHY firmware version"); 4728 } 4729 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4730 &pi->nrxq, 0, "# of rx queues"); 4731 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4732 &pi->ntxq, 0, "# of tx queues"); 4733 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4734 &pi->first_rxq, 0, "index of first rx queue"); 4735 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4736 &pi->first_txq, 0, "index of first tx queue"); 4737 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4738 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4739 "Reserve queue 0 for non-flowid packets"); 4740 4741#ifdef TCP_OFFLOAD 4742 if (is_offload(sc)) { 4743 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4744 &pi->nofldrxq, 0, 4745 "# of rx queues for offloaded TCP connections"); 4746 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4747 &pi->nofldtxq, 0, 4748 "# of tx queues for offloaded TCP connections"); 4749 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4750 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4751 "index of first TOE rx queue"); 4752 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4753 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4754 "index of first TOE tx queue"); 4755 } 4756#endif 4757#ifdef DEV_NETMAP 4758 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4759 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4760 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4761 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4762 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4763 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4764 "index of first netmap rx queue"); 4765 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4766 CTLFLAG_RD, &pi->first_nm_txq, 0, 4767 "index of first netmap tx queue"); 4768#endif 4769 4770 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4771 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4772 "holdoff timer index"); 4773 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4774 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4775 "holdoff packet counter index"); 4776 4777 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4778 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4779 "rx queue size"); 4780 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4781 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4782 "tx queue size"); 4783 4784 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4785 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4786 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4787 4788 /* 4789 * dev.cxgbe.X.stats. 4790 */ 4791 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4792 NULL, "port statistics"); 4793 children = SYSCTL_CHILDREN(oid); 4794 4795#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4796 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4797 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4798 sysctl_handle_t4_reg64, "QU", desc) 4799 4800 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4801 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4802 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4803 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4804 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4805 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4806 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4807 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4808 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4809 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4810 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4811 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4812 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4813 "# of tx frames in this range", 4814 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4815 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4816 "# of tx frames in this range", 4817 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4818 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4819 "# of tx frames in this range", 4820 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4821 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4822 "# of tx frames in this range", 4823 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4824 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4825 "# of tx frames in this range", 4826 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4827 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4828 "# of tx frames in this range", 4829 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4830 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4831 "# of tx frames in this range", 4832 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4833 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4834 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4835 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4836 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4837 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4838 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4839 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4840 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4841 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4842 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4843 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4844 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4845 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4846 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4847 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4848 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4849 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4850 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4851 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4852 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4853 4854 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4855 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4856 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4857 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4858 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4859 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4860 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4861 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4862 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4863 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4864 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4865 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4866 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4867 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4868 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4869 "# of frames received with bad FCS", 4870 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4871 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4872 "# of frames received with length error", 4873 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4874 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4875 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4876 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4877 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4878 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4879 "# of rx frames in this range", 4880 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4881 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4882 "# of rx frames in this range", 4883 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4884 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4885 "# of rx frames in this range", 4886 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4887 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4888 "# of rx frames in this range", 4889 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4890 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4891 "# of rx frames in this range", 4892 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4893 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4894 "# of rx frames in this range", 4895 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4896 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4897 "# of rx frames in this range", 4898 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4899 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4900 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4901 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4902 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4903 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4904 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4905 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4906 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4907 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4908 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4909 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4910 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4911 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4912 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4913 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4914 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4915 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4916 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4917 4918#undef SYSCTL_ADD_T4_REG64 4919 4920#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4921 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4922 &pi->stats.name, desc) 4923 4924 /* We get these from port_stats and they may be stale by upto 1s */ 4925 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4926 "# drops due to buffer-group 0 overflows"); 4927 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4928 "# drops due to buffer-group 1 overflows"); 4929 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4930 "# drops due to buffer-group 2 overflows"); 4931 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4932 "# drops due to buffer-group 3 overflows"); 4933 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4934 "# of buffer-group 0 truncated packets"); 4935 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4936 "# of buffer-group 1 truncated packets"); 4937 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4938 "# of buffer-group 2 truncated packets"); 4939 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4940 "# of buffer-group 3 truncated packets"); 4941 4942#undef SYSCTL_ADD_T4_PORTSTAT 4943 4944 return (0); 4945} 4946 4947static int 4948sysctl_int_array(SYSCTL_HANDLER_ARGS) 4949{ 4950 int rc, *i; 4951 struct sbuf sb; 4952 4953 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4954 for (i = arg1; arg2; arg2 -= sizeof(int), i++) 4955 sbuf_printf(&sb, "%d ", *i); 4956 sbuf_trim(&sb); 4957 sbuf_finish(&sb); 4958 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4959 sbuf_delete(&sb); 4960 return (rc); 4961} 4962 4963static int 4964sysctl_bitfield(SYSCTL_HANDLER_ARGS) 4965{ 4966 int rc; 4967 struct sbuf *sb; 4968 4969 rc = sysctl_wire_old_buffer(req, 0); 4970 if (rc != 0) 4971 return(rc); 4972 4973 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4974 if (sb == NULL) 4975 return (ENOMEM); 4976 4977 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4978 rc = sbuf_finish(sb); 4979 sbuf_delete(sb); 4980 4981 return (rc); 4982} 4983 4984static int 4985sysctl_btphy(SYSCTL_HANDLER_ARGS) 4986{ 4987 struct port_info *pi = arg1; 4988 int op = arg2; 4989 struct adapter *sc = pi->adapter; 4990 u_int v; 4991 int rc; 4992 4993 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 4994 if (rc) 4995 return (rc); 4996 /* XXX: magic numbers */ 4997 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 4998 &v); 4999 end_synchronized_op(sc, 0); 5000 if (rc) 5001 return (rc); 5002 if (op == 0) 5003 v /= 256; 5004 5005 rc = sysctl_handle_int(oidp, &v, 0, req); 5006 return (rc); 5007} 5008 5009static int 5010sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5011{ 5012 struct port_info *pi = arg1; 5013 int rc, val; 5014 5015 val = pi->rsrv_noflowq; 5016 rc = sysctl_handle_int(oidp, &val, 0, req); 5017 if (rc != 0 || req->newptr == NULL) 5018 return (rc); 5019 5020 if ((val >= 1) && (pi->ntxq > 1)) 5021 pi->rsrv_noflowq = 1; 5022 else 5023 pi->rsrv_noflowq = 0; 5024 5025 return (rc); 5026} 5027 5028static int 5029sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5030{ 5031 struct port_info *pi = arg1; 5032 struct adapter *sc = pi->adapter; 5033 int idx, rc, i; 5034 struct sge_rxq *rxq; 5035#ifdef TCP_OFFLOAD 5036 struct sge_ofld_rxq *ofld_rxq; 5037#endif 5038 uint8_t v; 5039 5040 idx = pi->tmr_idx; 5041 5042 rc = sysctl_handle_int(oidp, &idx, 0, req); 5043 if (rc != 0 || req->newptr == NULL) 5044 return (rc); 5045 5046 if (idx < 0 || idx >= SGE_NTIMERS) 5047 return (EINVAL); 5048 5049 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5050 "t4tmr"); 5051 if (rc) 5052 return (rc); 5053 5054 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5055 for_each_rxq(pi, i, rxq) { 5056#ifdef atomic_store_rel_8 5057 atomic_store_rel_8(&rxq->iq.intr_params, v); 5058#else 5059 rxq->iq.intr_params = v; 5060#endif 5061 } 5062#ifdef TCP_OFFLOAD 5063 for_each_ofld_rxq(pi, i, ofld_rxq) { 5064#ifdef atomic_store_rel_8 5065 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5066#else 5067 ofld_rxq->iq.intr_params = v; 5068#endif 5069 } 5070#endif 5071 pi->tmr_idx = idx; 5072 5073 end_synchronized_op(sc, LOCK_HELD); 5074 return (0); 5075} 5076 5077static int 5078sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5079{ 5080 struct port_info *pi = arg1; 5081 struct adapter *sc = pi->adapter; 5082 int idx, rc; 5083 5084 idx = pi->pktc_idx; 5085 5086 rc = sysctl_handle_int(oidp, &idx, 0, req); 5087 if (rc != 0 || req->newptr == NULL) 5088 return (rc); 5089 5090 if (idx < -1 || idx >= SGE_NCOUNTERS) 5091 return (EINVAL); 5092 5093 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5094 "t4pktc"); 5095 if (rc) 5096 return (rc); 5097 5098 if (pi->flags & PORT_INIT_DONE) 5099 rc = EBUSY; /* cannot be changed once the queues are created */ 5100 else 5101 pi->pktc_idx = idx; 5102 5103 end_synchronized_op(sc, LOCK_HELD); 5104 return (rc); 5105} 5106 5107static int 5108sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5109{ 5110 struct port_info *pi = arg1; 5111 struct adapter *sc = pi->adapter; 5112 int qsize, rc; 5113 5114 qsize = pi->qsize_rxq; 5115 5116 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5117 if (rc != 0 || req->newptr == NULL) 5118 return (rc); 5119 5120 if (qsize < 128 || (qsize & 7)) 5121 return (EINVAL); 5122 5123 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5124 "t4rxqs"); 5125 if (rc) 5126 return (rc); 5127 5128 if (pi->flags & PORT_INIT_DONE) 5129 rc = EBUSY; /* cannot be changed once the queues are created */ 5130 else 5131 pi->qsize_rxq = qsize; 5132 5133 end_synchronized_op(sc, LOCK_HELD); 5134 return (rc); 5135} 5136 5137static int 5138sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5139{ 5140 struct port_info *pi = arg1; 5141 struct adapter *sc = pi->adapter; 5142 int qsize, rc; 5143 5144 qsize = pi->qsize_txq; 5145 5146 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5147 if (rc != 0 || req->newptr == NULL) 5148 return (rc); 5149 5150 /* bufring size must be powerof2 */ 5151 if (qsize < 128 || !powerof2(qsize)) 5152 return (EINVAL); 5153 5154 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5155 "t4txqs"); 5156 if (rc) 5157 return (rc); 5158 5159 if (pi->flags & PORT_INIT_DONE) 5160 rc = EBUSY; /* cannot be changed once the queues are created */ 5161 else 5162 pi->qsize_txq = qsize; 5163 5164 end_synchronized_op(sc, LOCK_HELD); 5165 return (rc); 5166} 5167 5168static int 5169sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5170{ 5171 struct port_info *pi = arg1; 5172 struct adapter *sc = pi->adapter; 5173 struct link_config *lc = &pi->link_cfg; 5174 int rc; 5175 5176 if (req->newptr == NULL) { 5177 struct sbuf *sb; 5178 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5179 5180 rc = sysctl_wire_old_buffer(req, 0); 5181 if (rc != 0) 5182 return(rc); 5183 5184 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5185 if (sb == NULL) 5186 return (ENOMEM); 5187 5188 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5189 rc = sbuf_finish(sb); 5190 sbuf_delete(sb); 5191 } else { 5192 char s[2]; 5193 int n; 5194 5195 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5196 s[1] = 0; 5197 5198 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5199 if (rc != 0) 5200 return(rc); 5201 5202 if (s[1] != 0) 5203 return (EINVAL); 5204 if (s[0] < '0' || s[0] > '9') 5205 return (EINVAL); /* not a number */ 5206 n = s[0] - '0'; 5207 if (n & ~(PAUSE_TX | PAUSE_RX)) 5208 return (EINVAL); /* some other bit is set too */ 5209 5210 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5211 if (rc) 5212 return (rc); 5213 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5214 int link_ok = lc->link_ok; 5215 5216 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5217 lc->requested_fc |= n; 5218 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5219 lc->link_ok = link_ok; /* restore */ 5220 } 5221 end_synchronized_op(sc, 0); 5222 } 5223 5224 return (rc); 5225} 5226 5227static int 5228sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5229{ 5230 struct adapter *sc = arg1; 5231 int reg = arg2; 5232 uint64_t val; 5233 5234 val = t4_read_reg64(sc, reg); 5235 5236 return (sysctl_handle_64(oidp, &val, 0, req)); 5237} 5238 5239static int 5240sysctl_temperature(SYSCTL_HANDLER_ARGS) 5241{ 5242 struct adapter *sc = arg1; 5243 int rc, t; 5244 uint32_t param, val; 5245 5246 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5247 if (rc) 5248 return (rc); 5249 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5250 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5251 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5252 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5253 end_synchronized_op(sc, 0); 5254 if (rc) 5255 return (rc); 5256 5257 /* unknown is returned as 0 but we display -1 in that case */ 5258 t = val == 0 ? -1 : val; 5259 5260 rc = sysctl_handle_int(oidp, &t, 0, req); 5261 return (rc); 5262} 5263 5264#ifdef SBUF_DRAIN 5265static int 5266sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5267{ 5268 struct adapter *sc = arg1; 5269 struct sbuf *sb; 5270 int rc, i; 5271 uint16_t incr[NMTUS][NCCTRL_WIN]; 5272 static const char *dec_fac[] = { 5273 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5274 "0.9375" 5275 }; 5276 5277 rc = sysctl_wire_old_buffer(req, 0); 5278 if (rc != 0) 5279 return (rc); 5280 5281 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5282 if (sb == NULL) 5283 return (ENOMEM); 5284 5285 t4_read_cong_tbl(sc, incr); 5286 5287 for (i = 0; i < NCCTRL_WIN; ++i) { 5288 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5289 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5290 incr[5][i], incr[6][i], incr[7][i]); 5291 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5292 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5293 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5294 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5295 } 5296 5297 rc = sbuf_finish(sb); 5298 sbuf_delete(sb); 5299 5300 return (rc); 5301} 5302 5303static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5304 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5305 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5306 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5307}; 5308 5309static int 5310sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5311{ 5312 struct adapter *sc = arg1; 5313 struct sbuf *sb; 5314 int rc, i, n, qid = arg2; 5315 uint32_t *buf, *p; 5316 char *qtype; 5317 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5318 5319 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5320 ("%s: bad qid %d\n", __func__, qid)); 5321 5322 if (qid < CIM_NUM_IBQ) { 5323 /* inbound queue */ 5324 qtype = "IBQ"; 5325 n = 4 * CIM_IBQ_SIZE; 5326 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5327 rc = t4_read_cim_ibq(sc, qid, buf, n); 5328 } else { 5329 /* outbound queue */ 5330 qtype = "OBQ"; 5331 qid -= CIM_NUM_IBQ; 5332 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5333 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5334 rc = t4_read_cim_obq(sc, qid, buf, n); 5335 } 5336 5337 if (rc < 0) { 5338 rc = -rc; 5339 goto done; 5340 } 5341 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5342 5343 rc = sysctl_wire_old_buffer(req, 0); 5344 if (rc != 0) 5345 goto done; 5346 5347 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5348 if (sb == NULL) { 5349 rc = ENOMEM; 5350 goto done; 5351 } 5352 5353 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5354 for (i = 0, p = buf; i < n; i += 16, p += 4) 5355 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5356 p[2], p[3]); 5357 5358 rc = sbuf_finish(sb); 5359 sbuf_delete(sb); 5360done: 5361 free(buf, M_CXGBE); 5362 return (rc); 5363} 5364 5365static int 5366sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5367{ 5368 struct adapter *sc = arg1; 5369 u_int cfg; 5370 struct sbuf *sb; 5371 uint32_t *buf, *p; 5372 int rc; 5373 5374 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5375 if (rc != 0) 5376 return (rc); 5377 5378 rc = sysctl_wire_old_buffer(req, 0); 5379 if (rc != 0) 5380 return (rc); 5381 5382 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5383 if (sb == NULL) 5384 return (ENOMEM); 5385 5386 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5387 M_ZERO | M_WAITOK); 5388 5389 rc = -t4_cim_read_la(sc, buf, NULL); 5390 if (rc != 0) 5391 goto done; 5392 5393 sbuf_printf(sb, "Status Data PC%s", 5394 cfg & F_UPDBGLACAPTPCONLY ? "" : 5395 " LS0Stat LS0Addr LS0Data"); 5396 5397 KASSERT((sc->params.cim_la_size & 7) == 0, 5398 ("%s: p will walk off the end of buf", __func__)); 5399 5400 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5401 if (cfg & F_UPDBGLACAPTPCONLY) { 5402 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5403 p[6], p[7]); 5404 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5405 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5406 p[4] & 0xff, p[5] >> 8); 5407 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5408 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5409 p[1] & 0xf, p[2] >> 4); 5410 } else { 5411 sbuf_printf(sb, 5412 "\n %02x %x%07x %x%07x %08x %08x " 5413 "%08x%08x%08x%08x", 5414 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5415 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5416 p[6], p[7]); 5417 } 5418 } 5419 5420 rc = sbuf_finish(sb); 5421 sbuf_delete(sb); 5422done: 5423 free(buf, M_CXGBE); 5424 return (rc); 5425} 5426 5427static int 5428sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5429{ 5430 struct adapter *sc = arg1; 5431 u_int i; 5432 struct sbuf *sb; 5433 uint32_t *buf, *p; 5434 int rc; 5435 5436 rc = sysctl_wire_old_buffer(req, 0); 5437 if (rc != 0) 5438 return (rc); 5439 5440 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5441 if (sb == NULL) 5442 return (ENOMEM); 5443 5444 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5445 M_ZERO | M_WAITOK); 5446 5447 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5448 p = buf; 5449 5450 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5451 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5452 p[1], p[0]); 5453 } 5454 5455 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5456 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5457 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5458 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5459 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5460 (p[1] >> 2) | ((p[2] & 3) << 30), 5461 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5462 p[0] & 1); 5463 } 5464 5465 rc = sbuf_finish(sb); 5466 sbuf_delete(sb); 5467 free(buf, M_CXGBE); 5468 return (rc); 5469} 5470 5471static int 5472sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5473{ 5474 struct adapter *sc = arg1; 5475 u_int i; 5476 struct sbuf *sb; 5477 uint32_t *buf, *p; 5478 int rc; 5479 5480 rc = sysctl_wire_old_buffer(req, 0); 5481 if (rc != 0) 5482 return (rc); 5483 5484 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5485 if (sb == NULL) 5486 return (ENOMEM); 5487 5488 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5489 M_ZERO | M_WAITOK); 5490 5491 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5492 p = buf; 5493 5494 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5495 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5496 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5497 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5498 p[4], p[3], p[2], p[1], p[0]); 5499 } 5500 5501 sbuf_printf(sb, "\n\nCntl ID Data"); 5502 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5503 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5504 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5505 } 5506 5507 rc = sbuf_finish(sb); 5508 sbuf_delete(sb); 5509 free(buf, M_CXGBE); 5510 return (rc); 5511} 5512 5513static int 5514sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5515{ 5516 struct adapter *sc = arg1; 5517 struct sbuf *sb; 5518 int rc, i; 5519 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5520 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5521 uint16_t thres[CIM_NUM_IBQ]; 5522 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5523 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5524 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5525 5526 if (is_t4(sc)) { 5527 cim_num_obq = CIM_NUM_OBQ; 5528 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5529 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5530 } else { 5531 cim_num_obq = CIM_NUM_OBQ_T5; 5532 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5533 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5534 } 5535 nq = CIM_NUM_IBQ + cim_num_obq; 5536 5537 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5538 if (rc == 0) 5539 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5540 if (rc != 0) 5541 return (rc); 5542 5543 t4_read_cimq_cfg(sc, base, size, thres); 5544 5545 rc = sysctl_wire_old_buffer(req, 0); 5546 if (rc != 0) 5547 return (rc); 5548 5549 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5550 if (sb == NULL) 5551 return (ENOMEM); 5552 5553 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5554 5555 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5556 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5557 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5558 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5559 G_QUEREMFLITS(p[2]) * 16); 5560 for ( ; i < nq; i++, p += 4, wr += 2) 5561 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5562 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5563 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5564 G_QUEREMFLITS(p[2]) * 16); 5565 5566 rc = sbuf_finish(sb); 5567 sbuf_delete(sb); 5568 5569 return (rc); 5570} 5571 5572static int 5573sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5574{ 5575 struct adapter *sc = arg1; 5576 struct sbuf *sb; 5577 int rc; 5578 struct tp_cpl_stats stats; 5579 5580 rc = sysctl_wire_old_buffer(req, 0); 5581 if (rc != 0) 5582 return (rc); 5583 5584 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5585 if (sb == NULL) 5586 return (ENOMEM); 5587 5588 t4_tp_get_cpl_stats(sc, &stats); 5589 5590 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5591 "channel 3\n"); 5592 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5593 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5594 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5595 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5596 5597 rc = sbuf_finish(sb); 5598 sbuf_delete(sb); 5599 5600 return (rc); 5601} 5602 5603static int 5604sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5605{ 5606 struct adapter *sc = arg1; 5607 struct sbuf *sb; 5608 int rc; 5609 struct tp_usm_stats stats; 5610 5611 rc = sysctl_wire_old_buffer(req, 0); 5612 if (rc != 0) 5613 return(rc); 5614 5615 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5616 if (sb == NULL) 5617 return (ENOMEM); 5618 5619 t4_get_usm_stats(sc, &stats); 5620 5621 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5622 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5623 sbuf_printf(sb, "Drops: %u", stats.drops); 5624 5625 rc = sbuf_finish(sb); 5626 sbuf_delete(sb); 5627 5628 return (rc); 5629} 5630 5631const char *devlog_level_strings[] = { 5632 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5633 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5634 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5635 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5636 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5637 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5638}; 5639 5640const char *devlog_facility_strings[] = { 5641 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5642 [FW_DEVLOG_FACILITY_CF] = "CF", 5643 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5644 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5645 [FW_DEVLOG_FACILITY_RES] = "RES", 5646 [FW_DEVLOG_FACILITY_HW] = "HW", 5647 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5648 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5649 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5650 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5651 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5652 [FW_DEVLOG_FACILITY_VI] = "VI", 5653 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5654 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5655 [FW_DEVLOG_FACILITY_TM] = "TM", 5656 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5657 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5658 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5659 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5660 [FW_DEVLOG_FACILITY_RI] = "RI", 5661 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5662 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5663 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5664 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5665}; 5666 5667static int 5668sysctl_devlog(SYSCTL_HANDLER_ARGS) 5669{ 5670 struct adapter *sc = arg1; 5671 struct devlog_params *dparams = &sc->params.devlog; 5672 struct fw_devlog_e *buf, *e; 5673 int i, j, rc, nentries, first = 0, m; 5674 struct sbuf *sb; 5675 uint64_t ftstamp = UINT64_MAX; 5676 5677 if (dparams->start == 0) { 5678 dparams->memtype = FW_MEMTYPE_EDC0; 5679 dparams->start = 0x84000; 5680 dparams->size = 32768; 5681 } 5682 5683 nentries = dparams->size / sizeof(struct fw_devlog_e); 5684 5685 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5686 if (buf == NULL) 5687 return (ENOMEM); 5688 5689 m = fwmtype_to_hwmtype(dparams->memtype); 5690 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5691 if (rc != 0) 5692 goto done; 5693 5694 for (i = 0; i < nentries; i++) { 5695 e = &buf[i]; 5696 5697 if (e->timestamp == 0) 5698 break; /* end */ 5699 5700 e->timestamp = be64toh(e->timestamp); 5701 e->seqno = be32toh(e->seqno); 5702 for (j = 0; j < 8; j++) 5703 e->params[j] = be32toh(e->params[j]); 5704 5705 if (e->timestamp < ftstamp) { 5706 ftstamp = e->timestamp; 5707 first = i; 5708 } 5709 } 5710 5711 if (buf[first].timestamp == 0) 5712 goto done; /* nothing in the log */ 5713 5714 rc = sysctl_wire_old_buffer(req, 0); 5715 if (rc != 0) 5716 goto done; 5717 5718 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5719 if (sb == NULL) { 5720 rc = ENOMEM; 5721 goto done; 5722 } 5723 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5724 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5725 5726 i = first; 5727 do { 5728 e = &buf[i]; 5729 if (e->timestamp == 0) 5730 break; /* end */ 5731 5732 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5733 e->seqno, e->timestamp, 5734 (e->level < nitems(devlog_level_strings) ? 5735 devlog_level_strings[e->level] : "UNKNOWN"), 5736 (e->facility < nitems(devlog_facility_strings) ? 5737 devlog_facility_strings[e->facility] : "UNKNOWN")); 5738 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5739 e->params[2], e->params[3], e->params[4], 5740 e->params[5], e->params[6], e->params[7]); 5741 5742 if (++i == nentries) 5743 i = 0; 5744 } while (i != first); 5745 5746 rc = sbuf_finish(sb); 5747 sbuf_delete(sb); 5748done: 5749 free(buf, M_CXGBE); 5750 return (rc); 5751} 5752 5753static int 5754sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5755{ 5756 struct adapter *sc = arg1; 5757 struct sbuf *sb; 5758 int rc; 5759 struct tp_fcoe_stats stats[4]; 5760 5761 rc = sysctl_wire_old_buffer(req, 0); 5762 if (rc != 0) 5763 return (rc); 5764 5765 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5766 if (sb == NULL) 5767 return (ENOMEM); 5768 5769 t4_get_fcoe_stats(sc, 0, &stats[0]); 5770 t4_get_fcoe_stats(sc, 1, &stats[1]); 5771 t4_get_fcoe_stats(sc, 2, &stats[2]); 5772 t4_get_fcoe_stats(sc, 3, &stats[3]); 5773 5774 sbuf_printf(sb, " channel 0 channel 1 " 5775 "channel 2 channel 3\n"); 5776 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5777 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5778 stats[3].octetsDDP); 5779 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5780 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5781 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5782 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5783 stats[3].framesDrop); 5784 5785 rc = sbuf_finish(sb); 5786 sbuf_delete(sb); 5787 5788 return (rc); 5789} 5790 5791static int 5792sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5793{ 5794 struct adapter *sc = arg1; 5795 struct sbuf *sb; 5796 int rc, i; 5797 unsigned int map, kbps, ipg, mode; 5798 unsigned int pace_tab[NTX_SCHED]; 5799 5800 rc = sysctl_wire_old_buffer(req, 0); 5801 if (rc != 0) 5802 return (rc); 5803 5804 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5805 if (sb == NULL) 5806 return (ENOMEM); 5807 5808 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5809 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5810 t4_read_pace_tbl(sc, pace_tab); 5811 5812 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5813 "Class IPG (0.1 ns) Flow IPG (us)"); 5814 5815 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5816 t4_get_tx_sched(sc, i, &kbps, &ipg); 5817 sbuf_printf(sb, "\n %u %-5s %u ", i, 5818 (mode & (1 << i)) ? "flow" : "class", map & 3); 5819 if (kbps) 5820 sbuf_printf(sb, "%9u ", kbps); 5821 else 5822 sbuf_printf(sb, " disabled "); 5823 5824 if (ipg) 5825 sbuf_printf(sb, "%13u ", ipg); 5826 else 5827 sbuf_printf(sb, " disabled "); 5828 5829 if (pace_tab[i]) 5830 sbuf_printf(sb, "%10u", pace_tab[i]); 5831 else 5832 sbuf_printf(sb, " disabled"); 5833 } 5834 5835 rc = sbuf_finish(sb); 5836 sbuf_delete(sb); 5837 5838 return (rc); 5839} 5840 5841static int 5842sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5843{ 5844 struct adapter *sc = arg1; 5845 struct sbuf *sb; 5846 int rc, i, j; 5847 uint64_t *p0, *p1; 5848 struct lb_port_stats s[2]; 5849 static const char *stat_name[] = { 5850 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5851 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5852 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5853 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5854 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5855 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5856 "BG2FramesTrunc:", "BG3FramesTrunc:" 5857 }; 5858 5859 rc = sysctl_wire_old_buffer(req, 0); 5860 if (rc != 0) 5861 return (rc); 5862 5863 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5864 if (sb == NULL) 5865 return (ENOMEM); 5866 5867 memset(s, 0, sizeof(s)); 5868 5869 for (i = 0; i < 4; i += 2) { 5870 t4_get_lb_stats(sc, i, &s[0]); 5871 t4_get_lb_stats(sc, i + 1, &s[1]); 5872 5873 p0 = &s[0].octets; 5874 p1 = &s[1].octets; 5875 sbuf_printf(sb, "%s Loopback %u" 5876 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5877 5878 for (j = 0; j < nitems(stat_name); j++) 5879 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5880 *p0++, *p1++); 5881 } 5882 5883 rc = sbuf_finish(sb); 5884 sbuf_delete(sb); 5885 5886 return (rc); 5887} 5888 5889static int 5890sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 5891{ 5892 int rc = 0; 5893 struct port_info *pi = arg1; 5894 struct sbuf *sb; 5895 static const char *linkdnreasons[] = { 5896 "non-specific", "remote fault", "autoneg failed", "reserved3", 5897 "PHY overheated", "unknown", "rx los", "reserved7" 5898 }; 5899 5900 rc = sysctl_wire_old_buffer(req, 0); 5901 if (rc != 0) 5902 return(rc); 5903 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 5904 if (sb == NULL) 5905 return (ENOMEM); 5906 5907 if (pi->linkdnrc < 0) 5908 sbuf_printf(sb, "n/a"); 5909 else if (pi->linkdnrc < nitems(linkdnreasons)) 5910 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 5911 else 5912 sbuf_printf(sb, "%d", pi->linkdnrc); 5913 5914 rc = sbuf_finish(sb); 5915 sbuf_delete(sb); 5916 5917 return (rc); 5918} 5919 5920struct mem_desc { 5921 unsigned int base; 5922 unsigned int limit; 5923 unsigned int idx; 5924}; 5925 5926static int 5927mem_desc_cmp(const void *a, const void *b) 5928{ 5929 return ((const struct mem_desc *)a)->base - 5930 ((const struct mem_desc *)b)->base; 5931} 5932 5933static void 5934mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5935 unsigned int to) 5936{ 5937 unsigned int size; 5938 5939 size = to - from + 1; 5940 if (size == 0) 5941 return; 5942 5943 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5944 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5945} 5946 5947static int 5948sysctl_meminfo(SYSCTL_HANDLER_ARGS) 5949{ 5950 struct adapter *sc = arg1; 5951 struct sbuf *sb; 5952 int rc, i, n; 5953 uint32_t lo, hi, used, alloc; 5954 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5955 static const char *region[] = { 5956 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5957 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5958 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5959 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5960 "RQUDP region:", "PBL region:", "TXPBL region:", 5961 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5962 "On-chip queues:" 5963 }; 5964 struct mem_desc avail[4]; 5965 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5966 struct mem_desc *md = mem; 5967 5968 rc = sysctl_wire_old_buffer(req, 0); 5969 if (rc != 0) 5970 return (rc); 5971 5972 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5973 if (sb == NULL) 5974 return (ENOMEM); 5975 5976 for (i = 0; i < nitems(mem); i++) { 5977 mem[i].limit = 0; 5978 mem[i].idx = i; 5979 } 5980 5981 /* Find and sort the populated memory ranges */ 5982 i = 0; 5983 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5984 if (lo & F_EDRAM0_ENABLE) { 5985 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5986 avail[i].base = G_EDRAM0_BASE(hi) << 20; 5987 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5988 avail[i].idx = 0; 5989 i++; 5990 } 5991 if (lo & F_EDRAM1_ENABLE) { 5992 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5993 avail[i].base = G_EDRAM1_BASE(hi) << 20; 5994 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 5995 avail[i].idx = 1; 5996 i++; 5997 } 5998 if (lo & F_EXT_MEM_ENABLE) { 5999 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6000 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6001 avail[i].limit = avail[i].base + 6002 (G_EXT_MEM_SIZE(hi) << 20); 6003 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6004 i++; 6005 } 6006 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6007 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6008 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6009 avail[i].limit = avail[i].base + 6010 (G_EXT_MEM1_SIZE(hi) << 20); 6011 avail[i].idx = 4; 6012 i++; 6013 } 6014 if (!i) /* no memory available */ 6015 return 0; 6016 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6017 6018 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6019 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6020 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6021 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6022 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6023 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6024 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6025 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6026 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6027 6028 /* the next few have explicit upper bounds */ 6029 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6030 md->limit = md->base - 1 + 6031 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6032 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6033 md++; 6034 6035 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6036 md->limit = md->base - 1 + 6037 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6038 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6039 md++; 6040 6041 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6042 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6043 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6044 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6045 } else { 6046 md->base = 0; 6047 md->idx = nitems(region); /* hide it */ 6048 } 6049 md++; 6050 6051#define ulp_region(reg) \ 6052 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6053 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6054 6055 ulp_region(RX_ISCSI); 6056 ulp_region(RX_TDDP); 6057 ulp_region(TX_TPT); 6058 ulp_region(RX_STAG); 6059 ulp_region(RX_RQ); 6060 ulp_region(RX_RQUDP); 6061 ulp_region(RX_PBL); 6062 ulp_region(TX_PBL); 6063#undef ulp_region 6064 6065 md->base = 0; 6066 md->idx = nitems(region); 6067 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6068 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6069 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6070 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6071 } 6072 md++; 6073 6074 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6075 md->limit = md->base + sc->tids.ntids - 1; 6076 md++; 6077 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6078 md->limit = md->base + sc->tids.ntids - 1; 6079 md++; 6080 6081 md->base = sc->vres.ocq.start; 6082 if (sc->vres.ocq.size) 6083 md->limit = md->base + sc->vres.ocq.size - 1; 6084 else 6085 md->idx = nitems(region); /* hide it */ 6086 md++; 6087 6088 /* add any address-space holes, there can be up to 3 */ 6089 for (n = 0; n < i - 1; n++) 6090 if (avail[n].limit < avail[n + 1].base) 6091 (md++)->base = avail[n].limit; 6092 if (avail[n].limit) 6093 (md++)->base = avail[n].limit; 6094 6095 n = md - mem; 6096 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6097 6098 for (lo = 0; lo < i; lo++) 6099 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6100 avail[lo].limit - 1); 6101 6102 sbuf_printf(sb, "\n"); 6103 for (i = 0; i < n; i++) { 6104 if (mem[i].idx >= nitems(region)) 6105 continue; /* skip holes */ 6106 if (!mem[i].limit) 6107 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6108 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6109 mem[i].limit); 6110 } 6111 6112 sbuf_printf(sb, "\n"); 6113 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6114 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6115 mem_region_show(sb, "uP RAM:", lo, hi); 6116 6117 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6118 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6119 mem_region_show(sb, "uP Extmem2:", lo, hi); 6120 6121 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6122 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6123 G_PMRXMAXPAGE(lo), 6124 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6125 (lo & F_PMRXNUMCHN) ? 2 : 1); 6126 6127 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6128 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6129 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6130 G_PMTXMAXPAGE(lo), 6131 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6132 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6133 sbuf_printf(sb, "%u p-structs\n", 6134 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6135 6136 for (i = 0; i < 4; i++) { 6137 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6138 if (is_t4(sc)) { 6139 used = G_USED(lo); 6140 alloc = G_ALLOC(lo); 6141 } else { 6142 used = G_T5_USED(lo); 6143 alloc = G_T5_ALLOC(lo); 6144 } 6145 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6146 i, used, alloc); 6147 } 6148 for (i = 0; i < 4; i++) { 6149 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6150 if (is_t4(sc)) { 6151 used = G_USED(lo); 6152 alloc = G_ALLOC(lo); 6153 } else { 6154 used = G_T5_USED(lo); 6155 alloc = G_T5_ALLOC(lo); 6156 } 6157 sbuf_printf(sb, 6158 "\nLoopback %d using %u pages out of %u allocated", 6159 i, used, alloc); 6160 } 6161 6162 rc = sbuf_finish(sb); 6163 sbuf_delete(sb); 6164 6165 return (rc); 6166} 6167 6168static inline void 6169tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6170{ 6171 *mask = x | y; 6172 y = htobe64(y); 6173 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6174} 6175 6176static int 6177sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6178{ 6179 struct adapter *sc = arg1; 6180 struct sbuf *sb; 6181 int rc, i, n; 6182 6183 rc = sysctl_wire_old_buffer(req, 0); 6184 if (rc != 0) 6185 return (rc); 6186 6187 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6188 if (sb == NULL) 6189 return (ENOMEM); 6190 6191 sbuf_printf(sb, 6192 "Idx Ethernet address Mask Vld Ports PF" 6193 " VF Replication P0 P1 P2 P3 ML"); 6194 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6195 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6196 for (i = 0; i < n; i++) { 6197 uint64_t tcamx, tcamy, mask; 6198 uint32_t cls_lo, cls_hi; 6199 uint8_t addr[ETHER_ADDR_LEN]; 6200 6201 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6202 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6203 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6204 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6205 6206 if (tcamx & tcamy) 6207 continue; 6208 6209 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6210 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6211 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6212 addr[3], addr[4], addr[5], (uintmax_t)mask, 6213 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6214 G_PORTMAP(cls_hi), G_PF(cls_lo), 6215 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6216 6217 if (cls_lo & F_REPLICATE) { 6218 struct fw_ldst_cmd ldst_cmd; 6219 6220 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6221 ldst_cmd.op_to_addrspace = 6222 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6223 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6224 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6225 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6226 ldst_cmd.u.mps.fid_ctl = 6227 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6228 V_FW_LDST_CMD_CTL(i)); 6229 6230 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6231 "t4mps"); 6232 if (rc) 6233 break; 6234 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6235 sizeof(ldst_cmd), &ldst_cmd); 6236 end_synchronized_op(sc, 0); 6237 6238 if (rc != 0) { 6239 sbuf_printf(sb, 6240 " ------------ error %3u ------------", rc); 6241 rc = 0; 6242 } else { 6243 sbuf_printf(sb, " %08x %08x %08x %08x", 6244 be32toh(ldst_cmd.u.mps.rplc127_96), 6245 be32toh(ldst_cmd.u.mps.rplc95_64), 6246 be32toh(ldst_cmd.u.mps.rplc63_32), 6247 be32toh(ldst_cmd.u.mps.rplc31_0)); 6248 } 6249 } else 6250 sbuf_printf(sb, "%36s", ""); 6251 6252 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6253 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6254 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6255 } 6256 6257 if (rc) 6258 (void) sbuf_finish(sb); 6259 else 6260 rc = sbuf_finish(sb); 6261 sbuf_delete(sb); 6262 6263 return (rc); 6264} 6265 6266static int 6267sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6268{ 6269 struct adapter *sc = arg1; 6270 struct sbuf *sb; 6271 int rc; 6272 uint16_t mtus[NMTUS]; 6273 6274 rc = sysctl_wire_old_buffer(req, 0); 6275 if (rc != 0) 6276 return (rc); 6277 6278 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6279 if (sb == NULL) 6280 return (ENOMEM); 6281 6282 t4_read_mtu_tbl(sc, mtus, NULL); 6283 6284 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6285 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6286 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6287 mtus[14], mtus[15]); 6288 6289 rc = sbuf_finish(sb); 6290 sbuf_delete(sb); 6291 6292 return (rc); 6293} 6294 6295static int 6296sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6297{ 6298 struct adapter *sc = arg1; 6299 struct sbuf *sb; 6300 int rc, i; 6301 uint32_t cnt[PM_NSTATS]; 6302 uint64_t cyc[PM_NSTATS]; 6303 static const char *rx_stats[] = { 6304 "Read:", "Write bypass:", "Write mem:", "Flush:" 6305 }; 6306 static const char *tx_stats[] = { 6307 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6308 }; 6309 6310 rc = sysctl_wire_old_buffer(req, 0); 6311 if (rc != 0) 6312 return (rc); 6313 6314 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6315 if (sb == NULL) 6316 return (ENOMEM); 6317 6318 t4_pmtx_get_stats(sc, cnt, cyc); 6319 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6320 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6321 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6322 cyc[i]); 6323 6324 t4_pmrx_get_stats(sc, cnt, cyc); 6325 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6326 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6327 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6328 cyc[i]); 6329 6330 rc = sbuf_finish(sb); 6331 sbuf_delete(sb); 6332 6333 return (rc); 6334} 6335 6336static int 6337sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6338{ 6339 struct adapter *sc = arg1; 6340 struct sbuf *sb; 6341 int rc; 6342 struct tp_rdma_stats stats; 6343 6344 rc = sysctl_wire_old_buffer(req, 0); 6345 if (rc != 0) 6346 return (rc); 6347 6348 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6349 if (sb == NULL) 6350 return (ENOMEM); 6351 6352 t4_tp_get_rdma_stats(sc, &stats); 6353 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6354 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6355 6356 rc = sbuf_finish(sb); 6357 sbuf_delete(sb); 6358 6359 return (rc); 6360} 6361 6362static int 6363sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6364{ 6365 struct adapter *sc = arg1; 6366 struct sbuf *sb; 6367 int rc; 6368 struct tp_tcp_stats v4, v6; 6369 6370 rc = sysctl_wire_old_buffer(req, 0); 6371 if (rc != 0) 6372 return (rc); 6373 6374 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6375 if (sb == NULL) 6376 return (ENOMEM); 6377 6378 t4_tp_get_tcp_stats(sc, &v4, &v6); 6379 sbuf_printf(sb, 6380 " IP IPv6\n"); 6381 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6382 v4.tcpOutRsts, v6.tcpOutRsts); 6383 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6384 v4.tcpInSegs, v6.tcpInSegs); 6385 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6386 v4.tcpOutSegs, v6.tcpOutSegs); 6387 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6388 v4.tcpRetransSegs, v6.tcpRetransSegs); 6389 6390 rc = sbuf_finish(sb); 6391 sbuf_delete(sb); 6392 6393 return (rc); 6394} 6395 6396static int 6397sysctl_tids(SYSCTL_HANDLER_ARGS) 6398{ 6399 struct adapter *sc = arg1; 6400 struct sbuf *sb; 6401 int rc; 6402 struct tid_info *t = &sc->tids; 6403 6404 rc = sysctl_wire_old_buffer(req, 0); 6405 if (rc != 0) 6406 return (rc); 6407 6408 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6409 if (sb == NULL) 6410 return (ENOMEM); 6411 6412 if (t->natids) { 6413 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6414 t->atids_in_use); 6415 } 6416 6417 if (t->ntids) { 6418 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6419 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6420 6421 if (b) { 6422 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6423 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6424 t->ntids - 1); 6425 } else { 6426 sbuf_printf(sb, "TID range: %u-%u", 6427 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6428 t->ntids - 1); 6429 } 6430 } else 6431 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6432 sbuf_printf(sb, ", in use: %u\n", 6433 atomic_load_acq_int(&t->tids_in_use)); 6434 } 6435 6436 if (t->nstids) { 6437 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6438 t->stid_base + t->nstids - 1, t->stids_in_use); 6439 } 6440 6441 if (t->nftids) { 6442 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6443 t->ftid_base + t->nftids - 1); 6444 } 6445 6446 if (t->netids) { 6447 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6448 t->etid_base + t->netids - 1); 6449 } 6450 6451 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6452 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6453 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6454 6455 rc = sbuf_finish(sb); 6456 sbuf_delete(sb); 6457 6458 return (rc); 6459} 6460 6461static int 6462sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6463{ 6464 struct adapter *sc = arg1; 6465 struct sbuf *sb; 6466 int rc; 6467 struct tp_err_stats stats; 6468 6469 rc = sysctl_wire_old_buffer(req, 0); 6470 if (rc != 0) 6471 return (rc); 6472 6473 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6474 if (sb == NULL) 6475 return (ENOMEM); 6476 6477 t4_tp_get_err_stats(sc, &stats); 6478 6479 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6480 "channel 3\n"); 6481 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6482 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6483 stats.macInErrs[3]); 6484 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6485 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6486 stats.hdrInErrs[3]); 6487 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6488 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6489 stats.tcpInErrs[3]); 6490 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6491 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6492 stats.tcp6InErrs[3]); 6493 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6494 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6495 stats.tnlCongDrops[3]); 6496 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6497 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6498 stats.tnlTxDrops[3]); 6499 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6500 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6501 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6502 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6503 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6504 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6505 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6506 stats.ofldNoNeigh, stats.ofldCongDefer); 6507 6508 rc = sbuf_finish(sb); 6509 sbuf_delete(sb); 6510 6511 return (rc); 6512} 6513 6514struct field_desc { 6515 const char *name; 6516 u_int start; 6517 u_int width; 6518}; 6519 6520static void 6521field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6522{ 6523 char buf[32]; 6524 int line_size = 0; 6525 6526 while (f->name) { 6527 uint64_t mask = (1ULL << f->width) - 1; 6528 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6529 ((uintmax_t)v >> f->start) & mask); 6530 6531 if (line_size + len >= 79) { 6532 line_size = 8; 6533 sbuf_printf(sb, "\n "); 6534 } 6535 sbuf_printf(sb, "%s ", buf); 6536 line_size += len + 1; 6537 f++; 6538 } 6539 sbuf_printf(sb, "\n"); 6540} 6541 6542static struct field_desc tp_la0[] = { 6543 { "RcfOpCodeOut", 60, 4 }, 6544 { "State", 56, 4 }, 6545 { "WcfState", 52, 4 }, 6546 { "RcfOpcSrcOut", 50, 2 }, 6547 { "CRxError", 49, 1 }, 6548 { "ERxError", 48, 1 }, 6549 { "SanityFailed", 47, 1 }, 6550 { "SpuriousMsg", 46, 1 }, 6551 { "FlushInputMsg", 45, 1 }, 6552 { "FlushInputCpl", 44, 1 }, 6553 { "RssUpBit", 43, 1 }, 6554 { "RssFilterHit", 42, 1 }, 6555 { "Tid", 32, 10 }, 6556 { "InitTcb", 31, 1 }, 6557 { "LineNumber", 24, 7 }, 6558 { "Emsg", 23, 1 }, 6559 { "EdataOut", 22, 1 }, 6560 { "Cmsg", 21, 1 }, 6561 { "CdataOut", 20, 1 }, 6562 { "EreadPdu", 19, 1 }, 6563 { "CreadPdu", 18, 1 }, 6564 { "TunnelPkt", 17, 1 }, 6565 { "RcfPeerFin", 16, 1 }, 6566 { "RcfReasonOut", 12, 4 }, 6567 { "TxCchannel", 10, 2 }, 6568 { "RcfTxChannel", 8, 2 }, 6569 { "RxEchannel", 6, 2 }, 6570 { "RcfRxChannel", 5, 1 }, 6571 { "RcfDataOutSrdy", 4, 1 }, 6572 { "RxDvld", 3, 1 }, 6573 { "RxOoDvld", 2, 1 }, 6574 { "RxCongestion", 1, 1 }, 6575 { "TxCongestion", 0, 1 }, 6576 { NULL } 6577}; 6578 6579static struct field_desc tp_la1[] = { 6580 { "CplCmdIn", 56, 8 }, 6581 { "CplCmdOut", 48, 8 }, 6582 { "ESynOut", 47, 1 }, 6583 { "EAckOut", 46, 1 }, 6584 { "EFinOut", 45, 1 }, 6585 { "ERstOut", 44, 1 }, 6586 { "SynIn", 43, 1 }, 6587 { "AckIn", 42, 1 }, 6588 { "FinIn", 41, 1 }, 6589 { "RstIn", 40, 1 }, 6590 { "DataIn", 39, 1 }, 6591 { "DataInVld", 38, 1 }, 6592 { "PadIn", 37, 1 }, 6593 { "RxBufEmpty", 36, 1 }, 6594 { "RxDdp", 35, 1 }, 6595 { "RxFbCongestion", 34, 1 }, 6596 { "TxFbCongestion", 33, 1 }, 6597 { "TxPktSumSrdy", 32, 1 }, 6598 { "RcfUlpType", 28, 4 }, 6599 { "Eread", 27, 1 }, 6600 { "Ebypass", 26, 1 }, 6601 { "Esave", 25, 1 }, 6602 { "Static0", 24, 1 }, 6603 { "Cread", 23, 1 }, 6604 { "Cbypass", 22, 1 }, 6605 { "Csave", 21, 1 }, 6606 { "CPktOut", 20, 1 }, 6607 { "RxPagePoolFull", 18, 2 }, 6608 { "RxLpbkPkt", 17, 1 }, 6609 { "TxLpbkPkt", 16, 1 }, 6610 { "RxVfValid", 15, 1 }, 6611 { "SynLearned", 14, 1 }, 6612 { "SetDelEntry", 13, 1 }, 6613 { "SetInvEntry", 12, 1 }, 6614 { "CpcmdDvld", 11, 1 }, 6615 { "CpcmdSave", 10, 1 }, 6616 { "RxPstructsFull", 8, 2 }, 6617 { "EpcmdDvld", 7, 1 }, 6618 { "EpcmdFlush", 6, 1 }, 6619 { "EpcmdTrimPrefix", 5, 1 }, 6620 { "EpcmdTrimPostfix", 4, 1 }, 6621 { "ERssIp4Pkt", 3, 1 }, 6622 { "ERssIp6Pkt", 2, 1 }, 6623 { "ERssTcpUdpPkt", 1, 1 }, 6624 { "ERssFceFipPkt", 0, 1 }, 6625 { NULL } 6626}; 6627 6628static struct field_desc tp_la2[] = { 6629 { "CplCmdIn", 56, 8 }, 6630 { "MpsVfVld", 55, 1 }, 6631 { "MpsPf", 52, 3 }, 6632 { "MpsVf", 44, 8 }, 6633 { "SynIn", 43, 1 }, 6634 { "AckIn", 42, 1 }, 6635 { "FinIn", 41, 1 }, 6636 { "RstIn", 40, 1 }, 6637 { "DataIn", 39, 1 }, 6638 { "DataInVld", 38, 1 }, 6639 { "PadIn", 37, 1 }, 6640 { "RxBufEmpty", 36, 1 }, 6641 { "RxDdp", 35, 1 }, 6642 { "RxFbCongestion", 34, 1 }, 6643 { "TxFbCongestion", 33, 1 }, 6644 { "TxPktSumSrdy", 32, 1 }, 6645 { "RcfUlpType", 28, 4 }, 6646 { "Eread", 27, 1 }, 6647 { "Ebypass", 26, 1 }, 6648 { "Esave", 25, 1 }, 6649 { "Static0", 24, 1 }, 6650 { "Cread", 23, 1 }, 6651 { "Cbypass", 22, 1 }, 6652 { "Csave", 21, 1 }, 6653 { "CPktOut", 20, 1 }, 6654 { "RxPagePoolFull", 18, 2 }, 6655 { "RxLpbkPkt", 17, 1 }, 6656 { "TxLpbkPkt", 16, 1 }, 6657 { "RxVfValid", 15, 1 }, 6658 { "SynLearned", 14, 1 }, 6659 { "SetDelEntry", 13, 1 }, 6660 { "SetInvEntry", 12, 1 }, 6661 { "CpcmdDvld", 11, 1 }, 6662 { "CpcmdSave", 10, 1 }, 6663 { "RxPstructsFull", 8, 2 }, 6664 { "EpcmdDvld", 7, 1 }, 6665 { "EpcmdFlush", 6, 1 }, 6666 { "EpcmdTrimPrefix", 5, 1 }, 6667 { "EpcmdTrimPostfix", 4, 1 }, 6668 { "ERssIp4Pkt", 3, 1 }, 6669 { "ERssIp6Pkt", 2, 1 }, 6670 { "ERssTcpUdpPkt", 1, 1 }, 6671 { "ERssFceFipPkt", 0, 1 }, 6672 { NULL } 6673}; 6674 6675static void 6676tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6677{ 6678 6679 field_desc_show(sb, *p, tp_la0); 6680} 6681 6682static void 6683tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6684{ 6685 6686 if (idx) 6687 sbuf_printf(sb, "\n"); 6688 field_desc_show(sb, p[0], tp_la0); 6689 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6690 field_desc_show(sb, p[1], tp_la0); 6691} 6692 6693static void 6694tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6695{ 6696 6697 if (idx) 6698 sbuf_printf(sb, "\n"); 6699 field_desc_show(sb, p[0], tp_la0); 6700 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6701 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6702} 6703 6704static int 6705sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6706{ 6707 struct adapter *sc = arg1; 6708 struct sbuf *sb; 6709 uint64_t *buf, *p; 6710 int rc; 6711 u_int i, inc; 6712 void (*show_func)(struct sbuf *, uint64_t *, int); 6713 6714 rc = sysctl_wire_old_buffer(req, 0); 6715 if (rc != 0) 6716 return (rc); 6717 6718 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6719 if (sb == NULL) 6720 return (ENOMEM); 6721 6722 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6723 6724 t4_tp_read_la(sc, buf, NULL); 6725 p = buf; 6726 6727 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6728 case 2: 6729 inc = 2; 6730 show_func = tp_la_show2; 6731 break; 6732 case 3: 6733 inc = 2; 6734 show_func = tp_la_show3; 6735 break; 6736 default: 6737 inc = 1; 6738 show_func = tp_la_show; 6739 } 6740 6741 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6742 (*show_func)(sb, p, i); 6743 6744 rc = sbuf_finish(sb); 6745 sbuf_delete(sb); 6746 free(buf, M_CXGBE); 6747 return (rc); 6748} 6749 6750static int 6751sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6752{ 6753 struct adapter *sc = arg1; 6754 struct sbuf *sb; 6755 int rc; 6756 u64 nrate[NCHAN], orate[NCHAN]; 6757 6758 rc = sysctl_wire_old_buffer(req, 0); 6759 if (rc != 0) 6760 return (rc); 6761 6762 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6763 if (sb == NULL) 6764 return (ENOMEM); 6765 6766 t4_get_chan_txrate(sc, nrate, orate); 6767 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6768 "channel 3\n"); 6769 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6770 nrate[0], nrate[1], nrate[2], nrate[3]); 6771 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6772 orate[0], orate[1], orate[2], orate[3]); 6773 6774 rc = sbuf_finish(sb); 6775 sbuf_delete(sb); 6776 6777 return (rc); 6778} 6779 6780static int 6781sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6782{ 6783 struct adapter *sc = arg1; 6784 struct sbuf *sb; 6785 uint32_t *buf, *p; 6786 int rc, i; 6787 6788 rc = sysctl_wire_old_buffer(req, 0); 6789 if (rc != 0) 6790 return (rc); 6791 6792 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6793 if (sb == NULL) 6794 return (ENOMEM); 6795 6796 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6797 M_ZERO | M_WAITOK); 6798 6799 t4_ulprx_read_la(sc, buf); 6800 p = buf; 6801 6802 sbuf_printf(sb, " Pcmd Type Message" 6803 " Data"); 6804 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6805 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6806 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6807 } 6808 6809 rc = sbuf_finish(sb); 6810 sbuf_delete(sb); 6811 free(buf, M_CXGBE); 6812 return (rc); 6813} 6814 6815static int 6816sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6817{ 6818 struct adapter *sc = arg1; 6819 struct sbuf *sb; 6820 int rc, v; 6821 6822 rc = sysctl_wire_old_buffer(req, 0); 6823 if (rc != 0) 6824 return (rc); 6825 6826 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6827 if (sb == NULL) 6828 return (ENOMEM); 6829 6830 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6831 if (G_STATSOURCE_T5(v) == 7) { 6832 if (G_STATMODE(v) == 0) { 6833 sbuf_printf(sb, "total %d, incomplete %d", 6834 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6835 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6836 } else if (G_STATMODE(v) == 1) { 6837 sbuf_printf(sb, "total %d, data overflow %d", 6838 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6839 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6840 } 6841 } 6842 rc = sbuf_finish(sb); 6843 sbuf_delete(sb); 6844 6845 return (rc); 6846} 6847#endif 6848 6849static inline void 6850txq_start(struct ifnet *ifp, struct sge_txq *txq) 6851{ 6852 struct buf_ring *br; 6853 struct mbuf *m; 6854 6855 TXQ_LOCK_ASSERT_OWNED(txq); 6856 6857 br = txq->br; 6858 m = txq->m ? txq->m : drbr_dequeue(ifp, br); 6859 if (m) 6860 t4_eth_tx(ifp, txq, m); 6861} 6862 6863void 6864t4_tx_callout(void *arg) 6865{ 6866 struct sge_eq *eq = arg; 6867 struct adapter *sc; 6868 6869 if (EQ_TRYLOCK(eq) == 0) 6870 goto reschedule; 6871 6872 if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 6873 EQ_UNLOCK(eq); 6874reschedule: 6875 if (__predict_true(!(eq->flags && EQ_DOOMED))) 6876 callout_schedule(&eq->tx_callout, 1); 6877 return; 6878 } 6879 6880 EQ_LOCK_ASSERT_OWNED(eq); 6881 6882 if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 6883 6884 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6885 struct sge_txq *txq = arg; 6886 struct port_info *pi = txq->ifp->if_softc; 6887 6888 sc = pi->adapter; 6889 } else { 6890 struct sge_wrq *wrq = arg; 6891 6892 sc = wrq->adapter; 6893 } 6894 6895 taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 6896 } 6897 6898 EQ_UNLOCK(eq); 6899} 6900 6901void 6902t4_tx_task(void *arg, int count) 6903{ 6904 struct sge_eq *eq = arg; 6905 6906 EQ_LOCK(eq); 6907 if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 6908 struct sge_txq *txq = arg; 6909 txq_start(txq->ifp, txq); 6910 } else { 6911 struct sge_wrq *wrq = arg; 6912 t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 6913 } 6914 EQ_UNLOCK(eq); 6915} 6916 6917static uint32_t 6918fconf_to_mode(uint32_t fconf) 6919{ 6920 uint32_t mode; 6921 6922 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6923 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6924 6925 if (fconf & F_FRAGMENTATION) 6926 mode |= T4_FILTER_IP_FRAGMENT; 6927 6928 if (fconf & F_MPSHITTYPE) 6929 mode |= T4_FILTER_MPS_HIT_TYPE; 6930 6931 if (fconf & F_MACMATCH) 6932 mode |= T4_FILTER_MAC_IDX; 6933 6934 if (fconf & F_ETHERTYPE) 6935 mode |= T4_FILTER_ETH_TYPE; 6936 6937 if (fconf & F_PROTOCOL) 6938 mode |= T4_FILTER_IP_PROTO; 6939 6940 if (fconf & F_TOS) 6941 mode |= T4_FILTER_IP_TOS; 6942 6943 if (fconf & F_VLAN) 6944 mode |= T4_FILTER_VLAN; 6945 6946 if (fconf & F_VNIC_ID) 6947 mode |= T4_FILTER_VNIC; 6948 6949 if (fconf & F_PORT) 6950 mode |= T4_FILTER_PORT; 6951 6952 if (fconf & F_FCOE) 6953 mode |= T4_FILTER_FCoE; 6954 6955 return (mode); 6956} 6957 6958static uint32_t 6959mode_to_fconf(uint32_t mode) 6960{ 6961 uint32_t fconf = 0; 6962 6963 if (mode & T4_FILTER_IP_FRAGMENT) 6964 fconf |= F_FRAGMENTATION; 6965 6966 if (mode & T4_FILTER_MPS_HIT_TYPE) 6967 fconf |= F_MPSHITTYPE; 6968 6969 if (mode & T4_FILTER_MAC_IDX) 6970 fconf |= F_MACMATCH; 6971 6972 if (mode & T4_FILTER_ETH_TYPE) 6973 fconf |= F_ETHERTYPE; 6974 6975 if (mode & T4_FILTER_IP_PROTO) 6976 fconf |= F_PROTOCOL; 6977 6978 if (mode & T4_FILTER_IP_TOS) 6979 fconf |= F_TOS; 6980 6981 if (mode & T4_FILTER_VLAN) 6982 fconf |= F_VLAN; 6983 6984 if (mode & T4_FILTER_VNIC) 6985 fconf |= F_VNIC_ID; 6986 6987 if (mode & T4_FILTER_PORT) 6988 fconf |= F_PORT; 6989 6990 if (mode & T4_FILTER_FCoE) 6991 fconf |= F_FCOE; 6992 6993 return (fconf); 6994} 6995 6996static uint32_t 6997fspec_to_fconf(struct t4_filter_specification *fs) 6998{ 6999 uint32_t fconf = 0; 7000 7001 if (fs->val.frag || fs->mask.frag) 7002 fconf |= F_FRAGMENTATION; 7003 7004 if (fs->val.matchtype || fs->mask.matchtype) 7005 fconf |= F_MPSHITTYPE; 7006 7007 if (fs->val.macidx || fs->mask.macidx) 7008 fconf |= F_MACMATCH; 7009 7010 if (fs->val.ethtype || fs->mask.ethtype) 7011 fconf |= F_ETHERTYPE; 7012 7013 if (fs->val.proto || fs->mask.proto) 7014 fconf |= F_PROTOCOL; 7015 7016 if (fs->val.tos || fs->mask.tos) 7017 fconf |= F_TOS; 7018 7019 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7020 fconf |= F_VLAN; 7021 7022 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7023 fconf |= F_VNIC_ID; 7024 7025 if (fs->val.iport || fs->mask.iport) 7026 fconf |= F_PORT; 7027 7028 if (fs->val.fcoe || fs->mask.fcoe) 7029 fconf |= F_FCOE; 7030 7031 return (fconf); 7032} 7033 7034static int 7035get_filter_mode(struct adapter *sc, uint32_t *mode) 7036{ 7037 int rc; 7038 uint32_t fconf; 7039 7040 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7041 "t4getfm"); 7042 if (rc) 7043 return (rc); 7044 7045 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7046 A_TP_VLAN_PRI_MAP); 7047 7048 if (sc->params.tp.vlan_pri_map != fconf) { 7049 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7050 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7051 fconf); 7052 sc->params.tp.vlan_pri_map = fconf; 7053 } 7054 7055 *mode = fconf_to_mode(sc->params.tp.vlan_pri_map); 7056 7057 end_synchronized_op(sc, LOCK_HELD); 7058 return (0); 7059} 7060 7061static int 7062set_filter_mode(struct adapter *sc, uint32_t mode) 7063{ 7064 uint32_t fconf; 7065 int rc; 7066 7067 fconf = mode_to_fconf(mode); 7068 7069 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7070 "t4setfm"); 7071 if (rc) 7072 return (rc); 7073 7074 if (sc->tids.ftids_in_use > 0) { 7075 rc = EBUSY; 7076 goto done; 7077 } 7078 7079#ifdef TCP_OFFLOAD 7080 if (sc->offload_map) { 7081 rc = EBUSY; 7082 goto done; 7083 } 7084#endif 7085 7086#ifdef notyet 7087 rc = -t4_set_filter_mode(sc, fconf); 7088 if (rc == 0) 7089 sc->filter_mode = fconf; 7090#else 7091 rc = ENOTSUP; 7092#endif 7093 7094done: 7095 end_synchronized_op(sc, LOCK_HELD); 7096 return (rc); 7097} 7098 7099static inline uint64_t 7100get_filter_hits(struct adapter *sc, uint32_t fid) 7101{ 7102 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7103 uint64_t hits; 7104 7105 memwin_info(sc, 0, &mw_base, NULL); 7106 off = position_memwin(sc, 0, 7107 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7108 if (is_t4(sc)) { 7109 hits = t4_read_reg64(sc, mw_base + off + 16); 7110 hits = be64toh(hits); 7111 } else { 7112 hits = t4_read_reg(sc, mw_base + off + 24); 7113 hits = be32toh(hits); 7114 } 7115 7116 return (hits); 7117} 7118 7119static int 7120get_filter(struct adapter *sc, struct t4_filter *t) 7121{ 7122 int i, rc, nfilters = sc->tids.nftids; 7123 struct filter_entry *f; 7124 7125 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7126 "t4getf"); 7127 if (rc) 7128 return (rc); 7129 7130 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7131 t->idx >= nfilters) { 7132 t->idx = 0xffffffff; 7133 goto done; 7134 } 7135 7136 f = &sc->tids.ftid_tab[t->idx]; 7137 for (i = t->idx; i < nfilters; i++, f++) { 7138 if (f->valid) { 7139 t->idx = i; 7140 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7141 t->smtidx = f->smtidx; 7142 if (f->fs.hitcnts) 7143 t->hits = get_filter_hits(sc, t->idx); 7144 else 7145 t->hits = UINT64_MAX; 7146 t->fs = f->fs; 7147 7148 goto done; 7149 } 7150 } 7151 7152 t->idx = 0xffffffff; 7153done: 7154 end_synchronized_op(sc, LOCK_HELD); 7155 return (0); 7156} 7157 7158static int 7159set_filter(struct adapter *sc, struct t4_filter *t) 7160{ 7161 unsigned int nfilters, nports; 7162 struct filter_entry *f; 7163 int i, rc; 7164 7165 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7166 if (rc) 7167 return (rc); 7168 7169 nfilters = sc->tids.nftids; 7170 nports = sc->params.nports; 7171 7172 if (nfilters == 0) { 7173 rc = ENOTSUP; 7174 goto done; 7175 } 7176 7177 if (!(sc->flags & FULL_INIT_DONE)) { 7178 rc = EAGAIN; 7179 goto done; 7180 } 7181 7182 if (t->idx >= nfilters) { 7183 rc = EINVAL; 7184 goto done; 7185 } 7186 7187 /* Validate against the global filter mode */ 7188 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7189 sc->params.tp.vlan_pri_map) { 7190 rc = E2BIG; 7191 goto done; 7192 } 7193 7194 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7195 rc = EINVAL; 7196 goto done; 7197 } 7198 7199 if (t->fs.val.iport >= nports) { 7200 rc = EINVAL; 7201 goto done; 7202 } 7203 7204 /* Can't specify an iq if not steering to it */ 7205 if (!t->fs.dirsteer && t->fs.iq) { 7206 rc = EINVAL; 7207 goto done; 7208 } 7209 7210 /* IPv6 filter idx must be 4 aligned */ 7211 if (t->fs.type == 1 && 7212 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7213 rc = EINVAL; 7214 goto done; 7215 } 7216 7217 if (sc->tids.ftid_tab == NULL) { 7218 KASSERT(sc->tids.ftids_in_use == 0, 7219 ("%s: no memory allocated but filters_in_use > 0", 7220 __func__)); 7221 7222 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7223 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7224 if (sc->tids.ftid_tab == NULL) { 7225 rc = ENOMEM; 7226 goto done; 7227 } 7228 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7229 } 7230 7231 for (i = 0; i < 4; i++) { 7232 f = &sc->tids.ftid_tab[t->idx + i]; 7233 7234 if (f->pending || f->valid) { 7235 rc = EBUSY; 7236 goto done; 7237 } 7238 if (f->locked) { 7239 rc = EPERM; 7240 goto done; 7241 } 7242 7243 if (t->fs.type == 0) 7244 break; 7245 } 7246 7247 f = &sc->tids.ftid_tab[t->idx]; 7248 f->fs = t->fs; 7249 7250 rc = set_filter_wr(sc, t->idx); 7251done: 7252 end_synchronized_op(sc, 0); 7253 7254 if (rc == 0) { 7255 mtx_lock(&sc->tids.ftid_lock); 7256 for (;;) { 7257 if (f->pending == 0) { 7258 rc = f->valid ? 0 : EIO; 7259 break; 7260 } 7261 7262 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7263 PCATCH, "t4setfw", 0)) { 7264 rc = EINPROGRESS; 7265 break; 7266 } 7267 } 7268 mtx_unlock(&sc->tids.ftid_lock); 7269 } 7270 return (rc); 7271} 7272 7273static int 7274del_filter(struct adapter *sc, struct t4_filter *t) 7275{ 7276 unsigned int nfilters; 7277 struct filter_entry *f; 7278 int rc; 7279 7280 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7281 if (rc) 7282 return (rc); 7283 7284 nfilters = sc->tids.nftids; 7285 7286 if (nfilters == 0) { 7287 rc = ENOTSUP; 7288 goto done; 7289 } 7290 7291 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7292 t->idx >= nfilters) { 7293 rc = EINVAL; 7294 goto done; 7295 } 7296 7297 if (!(sc->flags & FULL_INIT_DONE)) { 7298 rc = EAGAIN; 7299 goto done; 7300 } 7301 7302 f = &sc->tids.ftid_tab[t->idx]; 7303 7304 if (f->pending) { 7305 rc = EBUSY; 7306 goto done; 7307 } 7308 if (f->locked) { 7309 rc = EPERM; 7310 goto done; 7311 } 7312 7313 if (f->valid) { 7314 t->fs = f->fs; /* extra info for the caller */ 7315 rc = del_filter_wr(sc, t->idx); 7316 } 7317 7318done: 7319 end_synchronized_op(sc, 0); 7320 7321 if (rc == 0) { 7322 mtx_lock(&sc->tids.ftid_lock); 7323 for (;;) { 7324 if (f->pending == 0) { 7325 rc = f->valid ? EIO : 0; 7326 break; 7327 } 7328 7329 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7330 PCATCH, "t4delfw", 0)) { 7331 rc = EINPROGRESS; 7332 break; 7333 } 7334 } 7335 mtx_unlock(&sc->tids.ftid_lock); 7336 } 7337 7338 return (rc); 7339} 7340 7341static void 7342clear_filter(struct filter_entry *f) 7343{ 7344 if (f->l2t) 7345 t4_l2t_release(f->l2t); 7346 7347 bzero(f, sizeof (*f)); 7348} 7349 7350static int 7351set_filter_wr(struct adapter *sc, int fidx) 7352{ 7353 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7354 struct wrqe *wr; 7355 struct fw_filter_wr *fwr; 7356 unsigned int ftid; 7357 7358 ASSERT_SYNCHRONIZED_OP(sc); 7359 7360 if (f->fs.newdmac || f->fs.newvlan) { 7361 /* This filter needs an L2T entry; allocate one. */ 7362 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7363 if (f->l2t == NULL) 7364 return (EAGAIN); 7365 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7366 f->fs.dmac)) { 7367 t4_l2t_release(f->l2t); 7368 f->l2t = NULL; 7369 return (ENOMEM); 7370 } 7371 } 7372 7373 ftid = sc->tids.ftid_base + fidx; 7374 7375 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7376 if (wr == NULL) 7377 return (ENOMEM); 7378 7379 fwr = wrtod(wr); 7380 bzero(fwr, sizeof (*fwr)); 7381 7382 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7383 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7384 fwr->tid_to_iq = 7385 htobe32(V_FW_FILTER_WR_TID(ftid) | 7386 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7387 V_FW_FILTER_WR_NOREPLY(0) | 7388 V_FW_FILTER_WR_IQ(f->fs.iq)); 7389 fwr->del_filter_to_l2tix = 7390 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7391 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7392 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7393 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7394 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7395 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7396 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7397 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7398 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7399 f->fs.newvlan == VLAN_REWRITE) | 7400 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7401 f->fs.newvlan == VLAN_REWRITE) | 7402 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7403 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7404 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7405 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7406 fwr->ethtype = htobe16(f->fs.val.ethtype); 7407 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7408 fwr->frag_to_ovlan_vldm = 7409 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7410 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7411 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7412 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7413 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7414 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7415 fwr->smac_sel = 0; 7416 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7417 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7418 fwr->maci_to_matchtypem = 7419 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7420 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7421 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7422 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7423 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7424 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7425 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7426 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7427 fwr->ptcl = f->fs.val.proto; 7428 fwr->ptclm = f->fs.mask.proto; 7429 fwr->ttyp = f->fs.val.tos; 7430 fwr->ttypm = f->fs.mask.tos; 7431 fwr->ivlan = htobe16(f->fs.val.vlan); 7432 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7433 fwr->ovlan = htobe16(f->fs.val.vnic); 7434 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7435 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7436 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7437 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7438 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7439 fwr->lp = htobe16(f->fs.val.dport); 7440 fwr->lpm = htobe16(f->fs.mask.dport); 7441 fwr->fp = htobe16(f->fs.val.sport); 7442 fwr->fpm = htobe16(f->fs.mask.sport); 7443 if (f->fs.newsmac) 7444 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7445 7446 f->pending = 1; 7447 sc->tids.ftids_in_use++; 7448 7449 t4_wrq_tx(sc, wr); 7450 return (0); 7451} 7452 7453static int 7454del_filter_wr(struct adapter *sc, int fidx) 7455{ 7456 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7457 struct wrqe *wr; 7458 struct fw_filter_wr *fwr; 7459 unsigned int ftid; 7460 7461 ftid = sc->tids.ftid_base + fidx; 7462 7463 wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 7464 if (wr == NULL) 7465 return (ENOMEM); 7466 fwr = wrtod(wr); 7467 bzero(fwr, sizeof (*fwr)); 7468 7469 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7470 7471 f->pending = 1; 7472 t4_wrq_tx(sc, wr); 7473 return (0); 7474} 7475 7476int 7477t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7478{ 7479 struct adapter *sc = iq->adapter; 7480 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7481 unsigned int idx = GET_TID(rpl); 7482 unsigned int rc; 7483 struct filter_entry *f; 7484 7485 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7486 rss->opcode)); 7487 7488 if (is_ftid(sc, idx)) { 7489 7490 idx -= sc->tids.ftid_base; 7491 f = &sc->tids.ftid_tab[idx]; 7492 rc = G_COOKIE(rpl->cookie); 7493 7494 mtx_lock(&sc->tids.ftid_lock); 7495 if (rc == FW_FILTER_WR_FLT_ADDED) { 7496 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7497 __func__, idx)); 7498 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7499 f->pending = 0; /* asynchronous setup completed */ 7500 f->valid = 1; 7501 } else { 7502 if (rc != FW_FILTER_WR_FLT_DELETED) { 7503 /* Add or delete failed, display an error */ 7504 log(LOG_ERR, 7505 "filter %u setup failed with error %u\n", 7506 idx, rc); 7507 } 7508 7509 clear_filter(f); 7510 sc->tids.ftids_in_use--; 7511 } 7512 wakeup(&sc->tids.ftid_tab); 7513 mtx_unlock(&sc->tids.ftid_lock); 7514 } 7515 7516 return (0); 7517} 7518 7519static int 7520get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7521{ 7522 int rc; 7523 7524 if (cntxt->cid > M_CTXTQID) 7525 return (EINVAL); 7526 7527 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7528 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7529 return (EINVAL); 7530 7531 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7532 if (rc) 7533 return (rc); 7534 7535 if (sc->flags & FW_OK) { 7536 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7537 &cntxt->data[0]); 7538 if (rc == 0) 7539 goto done; 7540 } 7541 7542 /* 7543 * Read via firmware failed or wasn't even attempted. Read directly via 7544 * the backdoor. 7545 */ 7546 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7547done: 7548 end_synchronized_op(sc, 0); 7549 return (rc); 7550} 7551 7552static int 7553load_fw(struct adapter *sc, struct t4_data *fw) 7554{ 7555 int rc; 7556 uint8_t *fw_data; 7557 7558 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7559 if (rc) 7560 return (rc); 7561 7562 if (sc->flags & FULL_INIT_DONE) { 7563 rc = EBUSY; 7564 goto done; 7565 } 7566 7567 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7568 if (fw_data == NULL) { 7569 rc = ENOMEM; 7570 goto done; 7571 } 7572 7573 rc = copyin(fw->data, fw_data, fw->len); 7574 if (rc == 0) 7575 rc = -t4_load_fw(sc, fw_data, fw->len); 7576 7577 free(fw_data, M_CXGBE); 7578done: 7579 end_synchronized_op(sc, 0); 7580 return (rc); 7581} 7582 7583static int 7584read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7585{ 7586 uint32_t addr, off, remaining, i, n; 7587 uint32_t *buf, *b; 7588 uint32_t mw_base, mw_aperture; 7589 int rc; 7590 uint8_t *dst; 7591 7592 rc = validate_mem_range(sc, mr->addr, mr->len); 7593 if (rc != 0) 7594 return (rc); 7595 7596 memwin_info(sc, win, &mw_base, &mw_aperture); 7597 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7598 addr = mr->addr; 7599 remaining = mr->len; 7600 dst = (void *)mr->data; 7601 7602 while (remaining) { 7603 off = position_memwin(sc, win, addr); 7604 7605 /* number of bytes that we'll copy in the inner loop */ 7606 n = min(remaining, mw_aperture - off); 7607 for (i = 0; i < n; i += 4) 7608 *b++ = t4_read_reg(sc, mw_base + off + i); 7609 7610 rc = copyout(buf, dst, n); 7611 if (rc != 0) 7612 break; 7613 7614 b = buf; 7615 dst += n; 7616 remaining -= n; 7617 addr += n; 7618 } 7619 7620 free(buf, M_CXGBE); 7621 return (rc); 7622} 7623 7624static int 7625read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7626{ 7627 int rc; 7628 7629 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7630 return (EINVAL); 7631 7632 if (i2cd->len > sizeof(i2cd->data)) 7633 return (EFBIG); 7634 7635 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7636 if (rc) 7637 return (rc); 7638 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7639 i2cd->offset, i2cd->len, &i2cd->data[0]); 7640 end_synchronized_op(sc, 0); 7641 7642 return (rc); 7643} 7644 7645static int 7646in_range(int val, int lo, int hi) 7647{ 7648 7649 return (val < 0 || (val <= hi && val >= lo)); 7650} 7651 7652static int 7653set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7654{ 7655 int fw_subcmd, fw_type, rc; 7656 7657 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7658 if (rc) 7659 return (rc); 7660 7661 if (!(sc->flags & FULL_INIT_DONE)) { 7662 rc = EAGAIN; 7663 goto done; 7664 } 7665 7666 /* 7667 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7668 * sub-command and type are in common locations.) 7669 */ 7670 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7671 fw_subcmd = FW_SCHED_SC_CONFIG; 7672 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7673 fw_subcmd = FW_SCHED_SC_PARAMS; 7674 else { 7675 rc = EINVAL; 7676 goto done; 7677 } 7678 if (p->type == SCHED_CLASS_TYPE_PACKET) 7679 fw_type = FW_SCHED_TYPE_PKTSCHED; 7680 else { 7681 rc = EINVAL; 7682 goto done; 7683 } 7684 7685 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7686 /* Vet our parameters ..*/ 7687 if (p->u.config.minmax < 0) { 7688 rc = EINVAL; 7689 goto done; 7690 } 7691 7692 /* And pass the request to the firmware ...*/ 7693 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7694 goto done; 7695 } 7696 7697 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7698 int fw_level; 7699 int fw_mode; 7700 int fw_rateunit; 7701 int fw_ratemode; 7702 7703 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7704 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7705 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7706 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7707 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7708 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7709 else { 7710 rc = EINVAL; 7711 goto done; 7712 } 7713 7714 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7715 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7716 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7717 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7718 else { 7719 rc = EINVAL; 7720 goto done; 7721 } 7722 7723 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7724 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7725 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7726 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7727 else { 7728 rc = EINVAL; 7729 goto done; 7730 } 7731 7732 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7733 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7734 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7735 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7736 else { 7737 rc = EINVAL; 7738 goto done; 7739 } 7740 7741 /* Vet our parameters ... */ 7742 if (!in_range(p->u.params.channel, 0, 3) || 7743 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7744 !in_range(p->u.params.minrate, 0, 10000000) || 7745 !in_range(p->u.params.maxrate, 0, 10000000) || 7746 !in_range(p->u.params.weight, 0, 100)) { 7747 rc = ERANGE; 7748 goto done; 7749 } 7750 7751 /* 7752 * Translate any unset parameters into the firmware's 7753 * nomenclature and/or fail the call if the parameters 7754 * are required ... 7755 */ 7756 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7757 p->u.params.channel < 0 || p->u.params.cl < 0) { 7758 rc = EINVAL; 7759 goto done; 7760 } 7761 if (p->u.params.minrate < 0) 7762 p->u.params.minrate = 0; 7763 if (p->u.params.maxrate < 0) { 7764 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7765 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7766 rc = EINVAL; 7767 goto done; 7768 } else 7769 p->u.params.maxrate = 0; 7770 } 7771 if (p->u.params.weight < 0) { 7772 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7773 rc = EINVAL; 7774 goto done; 7775 } else 7776 p->u.params.weight = 0; 7777 } 7778 if (p->u.params.pktsize < 0) { 7779 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7780 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7781 rc = EINVAL; 7782 goto done; 7783 } else 7784 p->u.params.pktsize = 0; 7785 } 7786 7787 /* See what the firmware thinks of the request ... */ 7788 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7789 fw_rateunit, fw_ratemode, p->u.params.channel, 7790 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7791 p->u.params.weight, p->u.params.pktsize, 1); 7792 goto done; 7793 } 7794 7795 rc = EINVAL; 7796done: 7797 end_synchronized_op(sc, 0); 7798 return (rc); 7799} 7800 7801static int 7802set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7803{ 7804 struct port_info *pi = NULL; 7805 struct sge_txq *txq; 7806 uint32_t fw_mnem, fw_queue, fw_class; 7807 int i, rc; 7808 7809 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7810 if (rc) 7811 return (rc); 7812 7813 if (!(sc->flags & FULL_INIT_DONE)) { 7814 rc = EAGAIN; 7815 goto done; 7816 } 7817 7818 if (p->port >= sc->params.nports) { 7819 rc = EINVAL; 7820 goto done; 7821 } 7822 7823 pi = sc->port[p->port]; 7824 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7825 rc = EINVAL; 7826 goto done; 7827 } 7828 7829 /* 7830 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7831 * Scheduling Class in this case). 7832 */ 7833 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7834 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7835 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7836 7837 /* 7838 * If op.queue is non-negative, then we're only changing the scheduling 7839 * on a single specified TX queue. 7840 */ 7841 if (p->queue >= 0) { 7842 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7843 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7844 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7845 &fw_class); 7846 goto done; 7847 } 7848 7849 /* 7850 * Change the scheduling on all the TX queues for the 7851 * interface. 7852 */ 7853 for_each_txq(pi, i, txq) { 7854 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7855 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7856 &fw_class); 7857 if (rc) 7858 goto done; 7859 } 7860 7861 rc = 0; 7862done: 7863 end_synchronized_op(sc, 0); 7864 return (rc); 7865} 7866 7867int 7868t4_os_find_pci_capability(struct adapter *sc, int cap) 7869{ 7870 int i; 7871 7872 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7873} 7874 7875int 7876t4_os_pci_save_state(struct adapter *sc) 7877{ 7878 device_t dev; 7879 struct pci_devinfo *dinfo; 7880 7881 dev = sc->dev; 7882 dinfo = device_get_ivars(dev); 7883 7884 pci_cfg_save(dev, dinfo, 0); 7885 return (0); 7886} 7887 7888int 7889t4_os_pci_restore_state(struct adapter *sc) 7890{ 7891 device_t dev; 7892 struct pci_devinfo *dinfo; 7893 7894 dev = sc->dev; 7895 dinfo = device_get_ivars(dev); 7896 7897 pci_cfg_restore(dev, dinfo); 7898 return (0); 7899} 7900 7901void 7902t4_os_portmod_changed(const struct adapter *sc, int idx) 7903{ 7904 struct port_info *pi = sc->port[idx]; 7905 static const char *mod_str[] = { 7906 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7907 }; 7908 7909 build_medialist(pi, &pi->media); 7910#ifdef DEV_NETMAP 7911 build_medialist(pi, &pi->nm_media); 7912#endif 7913 7914 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7915 if_printf(pi->ifp, "transceiver unplugged.\n"); 7916 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7917 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7918 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7919 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7920 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7921 if_printf(pi->ifp, "%s transceiver inserted.\n", 7922 mod_str[pi->mod_type]); 7923 } else { 7924 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7925 pi->mod_type); 7926 } 7927} 7928 7929void 7930t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7931{ 7932 struct port_info *pi = sc->port[idx]; 7933 struct ifnet *ifp = pi->ifp; 7934 7935 if (link_stat) { 7936 pi->linkdnrc = -1; 7937 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7938 if_link_state_change(ifp, LINK_STATE_UP); 7939 } else { 7940 if (reason >= 0) 7941 pi->linkdnrc = reason; 7942 if_link_state_change(ifp, LINK_STATE_DOWN); 7943 } 7944} 7945 7946void 7947t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7948{ 7949 struct adapter *sc; 7950 7951 sx_slock(&t4_list_lock); 7952 SLIST_FOREACH(sc, &t4_list, link) { 7953 /* 7954 * func should not make any assumptions about what state sc is 7955 * in - the only guarantee is that sc->sc_lock is a valid lock. 7956 */ 7957 func(sc, arg); 7958 } 7959 sx_sunlock(&t4_list_lock); 7960} 7961 7962static int 7963t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7964{ 7965 return (0); 7966} 7967 7968static int 7969t4_close(struct cdev *dev, int flags, int type, struct thread *td) 7970{ 7971 return (0); 7972} 7973 7974static int 7975t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 7976 struct thread *td) 7977{ 7978 int rc; 7979 struct adapter *sc = dev->si_drv1; 7980 7981 rc = priv_check(td, PRIV_DRIVER); 7982 if (rc != 0) 7983 return (rc); 7984 7985 switch (cmd) { 7986 case CHELSIO_T4_GETREG: { 7987 struct t4_reg *edata = (struct t4_reg *)data; 7988 7989 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 7990 return (EFAULT); 7991 7992 if (edata->size == 4) 7993 edata->val = t4_read_reg(sc, edata->addr); 7994 else if (edata->size == 8) 7995 edata->val = t4_read_reg64(sc, edata->addr); 7996 else 7997 return (EINVAL); 7998 7999 break; 8000 } 8001 case CHELSIO_T4_SETREG: { 8002 struct t4_reg *edata = (struct t4_reg *)data; 8003 8004 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8005 return (EFAULT); 8006 8007 if (edata->size == 4) { 8008 if (edata->val & 0xffffffff00000000) 8009 return (EINVAL); 8010 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8011 } else if (edata->size == 8) 8012 t4_write_reg64(sc, edata->addr, edata->val); 8013 else 8014 return (EINVAL); 8015 break; 8016 } 8017 case CHELSIO_T4_REGDUMP: { 8018 struct t4_regdump *regs = (struct t4_regdump *)data; 8019 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8020 uint8_t *buf; 8021 8022 if (regs->len < reglen) { 8023 regs->len = reglen; /* hint to the caller */ 8024 return (ENOBUFS); 8025 } 8026 8027 regs->len = reglen; 8028 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8029 t4_get_regs(sc, regs, buf); 8030 rc = copyout(buf, regs->data, reglen); 8031 free(buf, M_CXGBE); 8032 break; 8033 } 8034 case CHELSIO_T4_GET_FILTER_MODE: 8035 rc = get_filter_mode(sc, (uint32_t *)data); 8036 break; 8037 case CHELSIO_T4_SET_FILTER_MODE: 8038 rc = set_filter_mode(sc, *(uint32_t *)data); 8039 break; 8040 case CHELSIO_T4_GET_FILTER: 8041 rc = get_filter(sc, (struct t4_filter *)data); 8042 break; 8043 case CHELSIO_T4_SET_FILTER: 8044 rc = set_filter(sc, (struct t4_filter *)data); 8045 break; 8046 case CHELSIO_T4_DEL_FILTER: 8047 rc = del_filter(sc, (struct t4_filter *)data); 8048 break; 8049 case CHELSIO_T4_GET_SGE_CONTEXT: 8050 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8051 break; 8052 case CHELSIO_T4_LOAD_FW: 8053 rc = load_fw(sc, (struct t4_data *)data); 8054 break; 8055 case CHELSIO_T4_GET_MEM: 8056 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8057 break; 8058 case CHELSIO_T4_GET_I2C: 8059 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8060 break; 8061 case CHELSIO_T4_CLEAR_STATS: { 8062 int i; 8063 u_int port_id = *(uint32_t *)data; 8064 struct port_info *pi; 8065 8066 if (port_id >= sc->params.nports) 8067 return (EINVAL); 8068 pi = sc->port[port_id]; 8069 8070 /* MAC stats */ 8071 t4_clr_port_stats(sc, pi->tx_chan); 8072 8073 if (pi->flags & PORT_INIT_DONE) { 8074 struct sge_rxq *rxq; 8075 struct sge_txq *txq; 8076 struct sge_wrq *wrq; 8077 8078 for_each_rxq(pi, i, rxq) { 8079#if defined(INET) || defined(INET6) 8080 rxq->lro.lro_queued = 0; 8081 rxq->lro.lro_flushed = 0; 8082#endif 8083 rxq->rxcsum = 0; 8084 rxq->vlan_extraction = 0; 8085 } 8086 8087 for_each_txq(pi, i, txq) { 8088 txq->txcsum = 0; 8089 txq->tso_wrs = 0; 8090 txq->vlan_insertion = 0; 8091 txq->imm_wrs = 0; 8092 txq->sgl_wrs = 0; 8093 txq->txpkt_wrs = 0; 8094 txq->txpkts_wrs = 0; 8095 txq->txpkts_pkts = 0; 8096 txq->br->br_drops = 0; 8097 txq->no_dmamap = 0; 8098 txq->no_desc = 0; 8099 } 8100 8101#ifdef TCP_OFFLOAD 8102 /* nothing to clear for each ofld_rxq */ 8103 8104 for_each_ofld_txq(pi, i, wrq) { 8105 wrq->tx_wrs = 0; 8106 wrq->no_desc = 0; 8107 } 8108#endif 8109 wrq = &sc->sge.ctrlq[pi->port_id]; 8110 wrq->tx_wrs = 0; 8111 wrq->no_desc = 0; 8112 } 8113 break; 8114 } 8115 case CHELSIO_T4_SCHED_CLASS: 8116 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8117 break; 8118 case CHELSIO_T4_SCHED_QUEUE: 8119 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8120 break; 8121 case CHELSIO_T4_GET_TRACER: 8122 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8123 break; 8124 case CHELSIO_T4_SET_TRACER: 8125 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8126 break; 8127 default: 8128 rc = EINVAL; 8129 } 8130 8131 return (rc); 8132} 8133 8134#ifdef TCP_OFFLOAD 8135void 8136t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8137 const unsigned int *pgsz_order) 8138{ 8139 struct port_info *pi = ifp->if_softc; 8140 struct adapter *sc = pi->adapter; 8141 8142 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8143 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8144 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8145 V_HPZ3(pgsz_order[3])); 8146} 8147 8148static int 8149toe_capability(struct port_info *pi, int enable) 8150{ 8151 int rc; 8152 struct adapter *sc = pi->adapter; 8153 8154 ASSERT_SYNCHRONIZED_OP(sc); 8155 8156 if (!is_offload(sc)) 8157 return (ENODEV); 8158 8159 if (enable) { 8160 if (!(sc->flags & FULL_INIT_DONE)) { 8161 rc = cxgbe_init_synchronized(pi); 8162 if (rc) 8163 return (rc); 8164 } 8165 8166 if (isset(&sc->offload_map, pi->port_id)) 8167 return (0); 8168 8169 if (!(sc->flags & TOM_INIT_DONE)) { 8170 rc = t4_activate_uld(sc, ULD_TOM); 8171 if (rc == EAGAIN) { 8172 log(LOG_WARNING, 8173 "You must kldload t4_tom.ko before trying " 8174 "to enable TOE on a cxgbe interface.\n"); 8175 } 8176 if (rc != 0) 8177 return (rc); 8178 KASSERT(sc->tom_softc != NULL, 8179 ("%s: TOM activated but softc NULL", __func__)); 8180 KASSERT(sc->flags & TOM_INIT_DONE, 8181 ("%s: TOM activated but flag not set", __func__)); 8182 } 8183 8184 setbit(&sc->offload_map, pi->port_id); 8185 } else { 8186 if (!isset(&sc->offload_map, pi->port_id)) 8187 return (0); 8188 8189 KASSERT(sc->flags & TOM_INIT_DONE, 8190 ("%s: TOM never initialized?", __func__)); 8191 clrbit(&sc->offload_map, pi->port_id); 8192 } 8193 8194 return (0); 8195} 8196 8197/* 8198 * Add an upper layer driver to the global list. 8199 */ 8200int 8201t4_register_uld(struct uld_info *ui) 8202{ 8203 int rc = 0; 8204 struct uld_info *u; 8205 8206 sx_xlock(&t4_uld_list_lock); 8207 SLIST_FOREACH(u, &t4_uld_list, link) { 8208 if (u->uld_id == ui->uld_id) { 8209 rc = EEXIST; 8210 goto done; 8211 } 8212 } 8213 8214 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8215 ui->refcount = 0; 8216done: 8217 sx_xunlock(&t4_uld_list_lock); 8218 return (rc); 8219} 8220 8221int 8222t4_unregister_uld(struct uld_info *ui) 8223{ 8224 int rc = EINVAL; 8225 struct uld_info *u; 8226 8227 sx_xlock(&t4_uld_list_lock); 8228 8229 SLIST_FOREACH(u, &t4_uld_list, link) { 8230 if (u == ui) { 8231 if (ui->refcount > 0) { 8232 rc = EBUSY; 8233 goto done; 8234 } 8235 8236 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8237 rc = 0; 8238 goto done; 8239 } 8240 } 8241done: 8242 sx_xunlock(&t4_uld_list_lock); 8243 return (rc); 8244} 8245 8246int 8247t4_activate_uld(struct adapter *sc, int id) 8248{ 8249 int rc = EAGAIN; 8250 struct uld_info *ui; 8251 8252 ASSERT_SYNCHRONIZED_OP(sc); 8253 8254 sx_slock(&t4_uld_list_lock); 8255 8256 SLIST_FOREACH(ui, &t4_uld_list, link) { 8257 if (ui->uld_id == id) { 8258 rc = ui->activate(sc); 8259 if (rc == 0) 8260 ui->refcount++; 8261 goto done; 8262 } 8263 } 8264done: 8265 sx_sunlock(&t4_uld_list_lock); 8266 8267 return (rc); 8268} 8269 8270int 8271t4_deactivate_uld(struct adapter *sc, int id) 8272{ 8273 int rc = EINVAL; 8274 struct uld_info *ui; 8275 8276 ASSERT_SYNCHRONIZED_OP(sc); 8277 8278 sx_slock(&t4_uld_list_lock); 8279 8280 SLIST_FOREACH(ui, &t4_uld_list, link) { 8281 if (ui->uld_id == id) { 8282 rc = ui->deactivate(sc); 8283 if (rc == 0) 8284 ui->refcount--; 8285 goto done; 8286 } 8287 } 8288done: 8289 sx_sunlock(&t4_uld_list_lock); 8290 8291 return (rc); 8292} 8293#endif 8294 8295/* 8296 * Come up with reasonable defaults for some of the tunables, provided they're 8297 * not set by the user (in which case we'll use the values as is). 8298 */ 8299static void 8300tweak_tunables(void) 8301{ 8302 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8303 8304 if (t4_ntxq10g < 1) 8305 t4_ntxq10g = min(nc, NTXQ_10G); 8306 8307 if (t4_ntxq1g < 1) 8308 t4_ntxq1g = min(nc, NTXQ_1G); 8309 8310 if (t4_nrxq10g < 1) 8311 t4_nrxq10g = min(nc, NRXQ_10G); 8312 8313 if (t4_nrxq1g < 1) 8314 t4_nrxq1g = min(nc, NRXQ_1G); 8315 8316#ifdef TCP_OFFLOAD 8317 if (t4_nofldtxq10g < 1) 8318 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8319 8320 if (t4_nofldtxq1g < 1) 8321 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8322 8323 if (t4_nofldrxq10g < 1) 8324 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8325 8326 if (t4_nofldrxq1g < 1) 8327 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8328 8329 if (t4_toecaps_allowed == -1) 8330 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8331#else 8332 if (t4_toecaps_allowed == -1) 8333 t4_toecaps_allowed = 0; 8334#endif 8335 8336#ifdef DEV_NETMAP 8337 if (t4_nnmtxq10g < 1) 8338 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8339 8340 if (t4_nnmtxq1g < 1) 8341 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8342 8343 if (t4_nnmrxq10g < 1) 8344 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8345 8346 if (t4_nnmrxq1g < 1) 8347 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8348#endif 8349 8350 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8351 t4_tmr_idx_10g = TMR_IDX_10G; 8352 8353 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8354 t4_pktc_idx_10g = PKTC_IDX_10G; 8355 8356 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8357 t4_tmr_idx_1g = TMR_IDX_1G; 8358 8359 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8360 t4_pktc_idx_1g = PKTC_IDX_1G; 8361 8362 if (t4_qsize_txq < 128) 8363 t4_qsize_txq = 128; 8364 8365 if (t4_qsize_rxq < 128) 8366 t4_qsize_rxq = 128; 8367 while (t4_qsize_rxq & 7) 8368 t4_qsize_rxq++; 8369 8370 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8371} 8372 8373static struct sx mlu; /* mod load unload */ 8374SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8375 8376static int 8377mod_event(module_t mod, int cmd, void *arg) 8378{ 8379 int rc = 0; 8380 static int loaded = 0; 8381 8382 switch (cmd) { 8383 case MOD_LOAD: 8384 sx_xlock(&mlu); 8385 if (loaded++ == 0) { 8386 t4_sge_modload(); 8387 sx_init(&t4_list_lock, "T4/T5 adapters"); 8388 SLIST_INIT(&t4_list); 8389#ifdef TCP_OFFLOAD 8390 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8391 SLIST_INIT(&t4_uld_list); 8392#endif 8393 t4_tracer_modload(); 8394 tweak_tunables(); 8395 } 8396 sx_xunlock(&mlu); 8397 break; 8398 8399 case MOD_UNLOAD: 8400 sx_xlock(&mlu); 8401 if (--loaded == 0) { 8402 int tries; 8403 8404 sx_slock(&t4_list_lock); 8405 if (!SLIST_EMPTY(&t4_list)) { 8406 rc = EBUSY; 8407 sx_sunlock(&t4_list_lock); 8408 goto done_unload; 8409 } 8410#ifdef TCP_OFFLOAD 8411 sx_slock(&t4_uld_list_lock); 8412 if (!SLIST_EMPTY(&t4_uld_list)) { 8413 rc = EBUSY; 8414 sx_sunlock(&t4_uld_list_lock); 8415 sx_sunlock(&t4_list_lock); 8416 goto done_unload; 8417 } 8418#endif 8419 tries = 0; 8420 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8421 uprintf("%ju clusters with custom free routine " 8422 "still is use.\n", t4_sge_extfree_refs()); 8423 pause("t4unload", 2 * hz); 8424 } 8425#ifdef TCP_OFFLOAD 8426 sx_sunlock(&t4_uld_list_lock); 8427#endif 8428 sx_sunlock(&t4_list_lock); 8429 8430 if (t4_sge_extfree_refs() == 0) { 8431 t4_tracer_modunload(); 8432#ifdef TCP_OFFLOAD 8433 sx_destroy(&t4_uld_list_lock); 8434#endif 8435 sx_destroy(&t4_list_lock); 8436 t4_sge_modunload(); 8437 loaded = 0; 8438 } else { 8439 rc = EBUSY; 8440 loaded++; /* undo earlier decrement */ 8441 } 8442 } 8443done_unload: 8444 sx_xunlock(&mlu); 8445 break; 8446 } 8447 8448 return (rc); 8449} 8450 8451static devclass_t t4_devclass, t5_devclass; 8452static devclass_t cxgbe_devclass, cxl_devclass; 8453 8454DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8455MODULE_VERSION(t4nex, 1); 8456MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8457 8458DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8459MODULE_VERSION(t5nex, 1); 8460MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8461 8462DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8463MODULE_VERSION(cxgbe, 1); 8464 8465DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8466MODULE_VERSION(cxl, 1); 8467