t4_main.c revision 291083
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 291083 2015-11-19 19:40:30Z jhb $"); 30 31#include "opt_inet.h" 32#include "opt_inet6.h" 33 34#include <sys/param.h> 35#include <sys/conf.h> 36#include <sys/priv.h> 37#include <sys/kernel.h> 38#include <sys/bus.h> 39#include <sys/systm.h> 40#include <sys/counter.h> 41#include <sys/module.h> 42#include <sys/malloc.h> 43#include <sys/queue.h> 44#include <sys/taskqueue.h> 45#include <sys/pciio.h> 46#include <dev/pci/pcireg.h> 47#include <dev/pci/pcivar.h> 48#include <dev/pci/pci_private.h> 49#include <sys/firmware.h> 50#include <sys/sbuf.h> 51#include <sys/smp.h> 52#include <sys/socket.h> 53#include <sys/sockio.h> 54#include <sys/sysctl.h> 55#include <net/ethernet.h> 56#include <net/if.h> 57#include <net/if_types.h> 58#include <net/if_dl.h> 59#include <net/if_vlan_var.h> 60#if defined(__i386__) || defined(__amd64__) 61#include <vm/vm.h> 62#include <vm/pmap.h> 63#endif 64 65#include "common/common.h" 66#include "common/t4_msg.h" 67#include "common/t4_regs.h" 68#include "common/t4_regs_values.h" 69#include "t4_ioctl.h" 70#include "t4_l2t.h" 71#include "t4_mp_ring.h" 72 73/* T4 bus driver interface */ 74static int t4_probe(device_t); 75static int t4_attach(device_t); 76static int t4_detach(device_t); 77static device_method_t t4_methods[] = { 78 DEVMETHOD(device_probe, t4_probe), 79 DEVMETHOD(device_attach, t4_attach), 80 DEVMETHOD(device_detach, t4_detach), 81 82 DEVMETHOD_END 83}; 84static driver_t t4_driver = { 85 "t4nex", 86 t4_methods, 87 sizeof(struct adapter) 88}; 89 90 91/* T4 port (cxgbe) interface */ 92static int cxgbe_probe(device_t); 93static int cxgbe_attach(device_t); 94static int cxgbe_detach(device_t); 95static device_method_t cxgbe_methods[] = { 96 DEVMETHOD(device_probe, cxgbe_probe), 97 DEVMETHOD(device_attach, cxgbe_attach), 98 DEVMETHOD(device_detach, cxgbe_detach), 99 { 0, 0 } 100}; 101static driver_t cxgbe_driver = { 102 "cxgbe", 103 cxgbe_methods, 104 sizeof(struct port_info) 105}; 106 107static d_ioctl_t t4_ioctl; 108static d_open_t t4_open; 109static d_close_t t4_close; 110 111static struct cdevsw t4_cdevsw = { 112 .d_version = D_VERSION, 113 .d_flags = 0, 114 .d_open = t4_open, 115 .d_close = t4_close, 116 .d_ioctl = t4_ioctl, 117 .d_name = "t4nex", 118}; 119 120/* T5 bus driver interface */ 121static int t5_probe(device_t); 122static device_method_t t5_methods[] = { 123 DEVMETHOD(device_probe, t5_probe), 124 DEVMETHOD(device_attach, t4_attach), 125 DEVMETHOD(device_detach, t4_detach), 126 127 DEVMETHOD_END 128}; 129static driver_t t5_driver = { 130 "t5nex", 131 t5_methods, 132 sizeof(struct adapter) 133}; 134 135 136/* T5 port (cxl) interface */ 137static driver_t cxl_driver = { 138 "cxl", 139 cxgbe_methods, 140 sizeof(struct port_info) 141}; 142 143static struct cdevsw t5_cdevsw = { 144 .d_version = D_VERSION, 145 .d_flags = 0, 146 .d_open = t4_open, 147 .d_close = t4_close, 148 .d_ioctl = t4_ioctl, 149 .d_name = "t5nex", 150}; 151 152/* ifnet + media interface */ 153static void cxgbe_init(void *); 154static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 155static int cxgbe_transmit(struct ifnet *, struct mbuf *); 156static void cxgbe_qflush(struct ifnet *); 157static int cxgbe_media_change(struct ifnet *); 158static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 159 160MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 161 162/* 163 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 164 * then ADAPTER_LOCK, then t4_uld_list_lock. 165 */ 166static struct sx t4_list_lock; 167SLIST_HEAD(, adapter) t4_list; 168#ifdef TCP_OFFLOAD 169static struct sx t4_uld_list_lock; 170SLIST_HEAD(, uld_info) t4_uld_list; 171#endif 172 173/* 174 * Tunables. See tweak_tunables() too. 175 * 176 * Each tunable is set to a default value here if it's known at compile-time. 177 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 178 * provide a reasonable default when the driver is loaded. 179 * 180 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 181 * T5 are under hw.cxl. 182 */ 183 184/* 185 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 186 */ 187#define NTXQ_10G 16 188static int t4_ntxq10g = -1; 189TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 190 191#define NRXQ_10G 8 192static int t4_nrxq10g = -1; 193TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 194 195#define NTXQ_1G 4 196static int t4_ntxq1g = -1; 197TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 198 199#define NRXQ_1G 2 200static int t4_nrxq1g = -1; 201TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 202 203static int t4_rsrv_noflowq = 0; 204TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 205 206#ifdef TCP_OFFLOAD 207#define NOFLDTXQ_10G 8 208static int t4_nofldtxq10g = -1; 209TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 210 211#define NOFLDRXQ_10G 2 212static int t4_nofldrxq10g = -1; 213TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 214 215#define NOFLDTXQ_1G 2 216static int t4_nofldtxq1g = -1; 217TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 218 219#define NOFLDRXQ_1G 1 220static int t4_nofldrxq1g = -1; 221TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 222#endif 223 224#ifdef DEV_NETMAP 225#define NNMTXQ_10G 2 226static int t4_nnmtxq10g = -1; 227TUNABLE_INT("hw.cxgbe.nnmtxq10g", &t4_nnmtxq10g); 228 229#define NNMRXQ_10G 2 230static int t4_nnmrxq10g = -1; 231TUNABLE_INT("hw.cxgbe.nnmrxq10g", &t4_nnmrxq10g); 232 233#define NNMTXQ_1G 1 234static int t4_nnmtxq1g = -1; 235TUNABLE_INT("hw.cxgbe.nnmtxq1g", &t4_nnmtxq1g); 236 237#define NNMRXQ_1G 1 238static int t4_nnmrxq1g = -1; 239TUNABLE_INT("hw.cxgbe.nnmrxq1g", &t4_nnmrxq1g); 240#endif 241 242/* 243 * Holdoff parameters for 10G and 1G ports. 244 */ 245#define TMR_IDX_10G 1 246static int t4_tmr_idx_10g = TMR_IDX_10G; 247TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 248 249#define PKTC_IDX_10G (-1) 250static int t4_pktc_idx_10g = PKTC_IDX_10G; 251TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 252 253#define TMR_IDX_1G 1 254static int t4_tmr_idx_1g = TMR_IDX_1G; 255TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 256 257#define PKTC_IDX_1G (-1) 258static int t4_pktc_idx_1g = PKTC_IDX_1G; 259TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 260 261/* 262 * Size (# of entries) of each tx and rx queue. 263 */ 264static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 265TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 266 267static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 268TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 269 270/* 271 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 272 */ 273static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 274TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 275 276/* 277 * Configuration file. 278 */ 279#define DEFAULT_CF "default" 280#define FLASH_CF "flash" 281#define UWIRE_CF "uwire" 282#define FPGA_CF "fpga" 283static char t4_cfg_file[32] = DEFAULT_CF; 284TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 285 286/* 287 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 288 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 289 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 290 * mark or when signalled to do so, 0 to never emit PAUSE. 291 */ 292static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 293TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 294 295/* 296 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 297 * encouraged respectively). 298 */ 299static unsigned int t4_fw_install = 1; 300TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 301 302/* 303 * ASIC features that will be used. Disable the ones you don't want so that the 304 * chip resources aren't wasted on features that will not be used. 305 */ 306static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 307TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 308 309static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 310TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 311 312static int t4_toecaps_allowed = -1; 313TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 314 315static int t4_rdmacaps_allowed = 0; 316TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 317 318static int t4_iscsicaps_allowed = 0; 319TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 320 321static int t4_fcoecaps_allowed = 0; 322TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 323 324static int t5_write_combine = 0; 325TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 326 327struct intrs_and_queues { 328 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 329 uint16_t nirq; /* Total # of vectors */ 330 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 331 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 332 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 333 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 334 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 335 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 336 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 337#ifdef TCP_OFFLOAD 338 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 339 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 340 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 341 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 342#endif 343#ifdef DEV_NETMAP 344 uint16_t nnmtxq10g; /* # of netmap txq's for each 10G port */ 345 uint16_t nnmrxq10g; /* # of netmap rxq's for each 10G port */ 346 uint16_t nnmtxq1g; /* # of netmap txq's for each 1G port */ 347 uint16_t nnmrxq1g; /* # of netmap rxq's for each 1G port */ 348#endif 349}; 350 351struct filter_entry { 352 uint32_t valid:1; /* filter allocated and valid */ 353 uint32_t locked:1; /* filter is administratively locked */ 354 uint32_t pending:1; /* filter action is pending firmware reply */ 355 uint32_t smtidx:8; /* Source MAC Table index for smac */ 356 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 357 358 struct t4_filter_specification fs; 359}; 360 361static int map_bars_0_and_4(struct adapter *); 362static int map_bar_2(struct adapter *); 363static void setup_memwin(struct adapter *); 364static int validate_mem_range(struct adapter *, uint32_t, int); 365static int fwmtype_to_hwmtype(int); 366static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 367 uint32_t *); 368static void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 369static uint32_t position_memwin(struct adapter *, int, uint32_t); 370static int cfg_itype_and_nqueues(struct adapter *, int, int, 371 struct intrs_and_queues *); 372static int prep_firmware(struct adapter *); 373static int partition_resources(struct adapter *, const struct firmware *, 374 const char *); 375static int get_params__pre_init(struct adapter *); 376static int get_params__post_init(struct adapter *); 377static int set_params__post_init(struct adapter *); 378static void t4_set_desc(struct adapter *); 379static void build_medialist(struct port_info *, struct ifmedia *); 380static int cxgbe_init_synchronized(struct port_info *); 381static int cxgbe_uninit_synchronized(struct port_info *); 382static int setup_intr_handlers(struct adapter *); 383static void quiesce_txq(struct adapter *, struct sge_txq *); 384static void quiesce_wrq(struct adapter *, struct sge_wrq *); 385static void quiesce_iq(struct adapter *, struct sge_iq *); 386static void quiesce_fl(struct adapter *, struct sge_fl *); 387static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 388 driver_intr_t *, void *, char *); 389static int t4_free_irq(struct adapter *, struct irq *); 390static void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 391 unsigned int); 392static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 393static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 394static void cxgbe_tick(void *); 395static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 396static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 397 struct mbuf *); 398static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 399static int fw_msg_not_handled(struct adapter *, const __be64 *); 400static int t4_sysctls(struct adapter *); 401static int cxgbe_sysctls(struct port_info *); 402static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 403static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 404static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 405static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 406static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 407static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 408static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 409static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 410static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 411static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 412static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 413#ifdef SBUF_DRAIN 414static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 415static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 416static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 417static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 418static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 419static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 420static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 421static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 422static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 423static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 424static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 425static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 426static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 427static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 428static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 429static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 430static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 431static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 432static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 433static int sysctl_tids(SYSCTL_HANDLER_ARGS); 434static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 435static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 436static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 437static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 438static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 439#endif 440static uint32_t fconf_to_mode(uint32_t); 441static uint32_t mode_to_fconf(uint32_t); 442static uint32_t fspec_to_fconf(struct t4_filter_specification *); 443static int get_filter_mode(struct adapter *, uint32_t *); 444static int set_filter_mode(struct adapter *, uint32_t); 445static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 446static int get_filter(struct adapter *, struct t4_filter *); 447static int set_filter(struct adapter *, struct t4_filter *); 448static int del_filter(struct adapter *, struct t4_filter *); 449static void clear_filter(struct filter_entry *); 450static int set_filter_wr(struct adapter *, int); 451static int del_filter_wr(struct adapter *, int); 452static int get_sge_context(struct adapter *, struct t4_sge_context *); 453static int load_fw(struct adapter *, struct t4_data *); 454static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 455static int read_i2c(struct adapter *, struct t4_i2c_data *); 456static int set_sched_class(struct adapter *, struct t4_sched_params *); 457static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 458#ifdef TCP_OFFLOAD 459static int toe_capability(struct port_info *, int); 460#endif 461static int mod_event(module_t, int, void *); 462 463struct { 464 uint16_t device; 465 char *desc; 466} t4_pciids[] = { 467 {0xa000, "Chelsio Terminator 4 FPGA"}, 468 {0x4400, "Chelsio T440-dbg"}, 469 {0x4401, "Chelsio T420-CR"}, 470 {0x4402, "Chelsio T422-CR"}, 471 {0x4403, "Chelsio T440-CR"}, 472 {0x4404, "Chelsio T420-BCH"}, 473 {0x4405, "Chelsio T440-BCH"}, 474 {0x4406, "Chelsio T440-CH"}, 475 {0x4407, "Chelsio T420-SO"}, 476 {0x4408, "Chelsio T420-CX"}, 477 {0x4409, "Chelsio T420-BT"}, 478 {0x440a, "Chelsio T404-BT"}, 479 {0x440e, "Chelsio T440-LP-CR"}, 480}, t5_pciids[] = { 481 {0xb000, "Chelsio Terminator 5 FPGA"}, 482 {0x5400, "Chelsio T580-dbg"}, 483 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 484 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 485 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 486 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 487 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 488 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 489 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 490 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 491 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 492 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 493 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 494 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 495 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 496#ifdef notyet 497 {0x5404, "Chelsio T520-BCH"}, 498 {0x5405, "Chelsio T540-BCH"}, 499 {0x5406, "Chelsio T540-CH"}, 500 {0x5408, "Chelsio T520-CX"}, 501 {0x540b, "Chelsio B520-SR"}, 502 {0x540c, "Chelsio B504-BT"}, 503 {0x540f, "Chelsio Amsterdam"}, 504 {0x5413, "Chelsio T580-CHR"}, 505#endif 506}; 507 508#ifdef TCP_OFFLOAD 509/* 510 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 511 * exactly the same for both rxq and ofld_rxq. 512 */ 513CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 514CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 515#endif 516 517/* No easy way to include t4_msg.h before adapter.h so we check this way */ 518CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 519CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 520 521CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 522 523static int 524t4_probe(device_t dev) 525{ 526 int i; 527 uint16_t v = pci_get_vendor(dev); 528 uint16_t d = pci_get_device(dev); 529 uint8_t f = pci_get_function(dev); 530 531 if (v != PCI_VENDOR_ID_CHELSIO) 532 return (ENXIO); 533 534 /* Attach only to PF0 of the FPGA */ 535 if (d == 0xa000 && f != 0) 536 return (ENXIO); 537 538 for (i = 0; i < nitems(t4_pciids); i++) { 539 if (d == t4_pciids[i].device) { 540 device_set_desc(dev, t4_pciids[i].desc); 541 return (BUS_PROBE_DEFAULT); 542 } 543 } 544 545 return (ENXIO); 546} 547 548static int 549t5_probe(device_t dev) 550{ 551 int i; 552 uint16_t v = pci_get_vendor(dev); 553 uint16_t d = pci_get_device(dev); 554 uint8_t f = pci_get_function(dev); 555 556 if (v != PCI_VENDOR_ID_CHELSIO) 557 return (ENXIO); 558 559 /* Attach only to PF0 of the FPGA */ 560 if (d == 0xb000 && f != 0) 561 return (ENXIO); 562 563 for (i = 0; i < nitems(t5_pciids); i++) { 564 if (d == t5_pciids[i].device) { 565 device_set_desc(dev, t5_pciids[i].desc); 566 return (BUS_PROBE_DEFAULT); 567 } 568 } 569 570 return (ENXIO); 571} 572 573static void 574t5_attribute_workaround(device_t dev) 575{ 576 device_t root_port; 577 uint32_t v; 578 579 /* 580 * The T5 chips do not properly echo the No Snoop and Relaxed 581 * Ordering attributes when replying to a TLP from a Root 582 * Port. As a workaround, find the parent Root Port and 583 * disable No Snoop and Relaxed Ordering. Note that this 584 * affects all devices under this root port. 585 */ 586 root_port = pci_find_pcie_root_port(dev); 587 if (root_port == NULL) { 588 device_printf(dev, "Unable to find parent root port\n"); 589 return; 590 } 591 592 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 593 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 594 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 595 0) 596 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 597 device_get_nameunit(root_port)); 598} 599 600static int 601t4_attach(device_t dev) 602{ 603 struct adapter *sc; 604 int rc = 0, i, n10g, n1g, rqidx, tqidx; 605 struct intrs_and_queues iaq; 606 struct sge *s; 607#ifdef TCP_OFFLOAD 608 int ofld_rqidx, ofld_tqidx; 609#endif 610#ifdef DEV_NETMAP 611 int nm_rqidx, nm_tqidx; 612#endif 613 614 sc = device_get_softc(dev); 615 sc->dev = dev; 616 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 617 618 if ((pci_get_device(dev) & 0xff00) == 0x5400) 619 t5_attribute_workaround(dev); 620 pci_enable_busmaster(dev); 621 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 622 uint32_t v; 623 624 pci_set_max_read_req(dev, 4096); 625 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 626 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 627 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 628 629 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 630 } 631 632 sc->traceq = -1; 633 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 634 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 635 device_get_nameunit(dev)); 636 637 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 638 device_get_nameunit(dev)); 639 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 640 sx_xlock(&t4_list_lock); 641 SLIST_INSERT_HEAD(&t4_list, sc, link); 642 sx_xunlock(&t4_list_lock); 643 644 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 645 TAILQ_INIT(&sc->sfl); 646 callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 647 648 mtx_init(&sc->regwin_lock, "register and memory window", 0, MTX_DEF); 649 650 rc = map_bars_0_and_4(sc); 651 if (rc != 0) 652 goto done; /* error message displayed already */ 653 654 /* 655 * This is the real PF# to which we're attaching. Works from within PCI 656 * passthrough environments too, where pci_get_function() could return a 657 * different PF# depending on the passthrough configuration. We need to 658 * use the real PF# in all our communication with the firmware. 659 */ 660 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 661 sc->mbox = sc->pf; 662 663 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 664 sc->an_handler = an_not_handled; 665 for (i = 0; i < nitems(sc->cpl_handler); i++) 666 sc->cpl_handler[i] = cpl_not_handled; 667 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 668 sc->fw_msg_handler[i] = fw_msg_not_handled; 669 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 670 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 671 t4_register_cpl_handler(sc, CPL_TRACE_PKT_T5, t5_trace_pkt); 672 t4_init_sge_cpl_handlers(sc); 673 674 /* Prepare the adapter for operation */ 675 rc = -t4_prep_adapter(sc); 676 if (rc != 0) { 677 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 678 goto done; 679 } 680 681 /* 682 * Do this really early, with the memory windows set up even before the 683 * character device. The userland tool's register i/o and mem read 684 * will work even in "recovery mode". 685 */ 686 setup_memwin(sc); 687 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 688 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 689 device_get_nameunit(dev)); 690 if (sc->cdev == NULL) 691 device_printf(dev, "failed to create nexus char device.\n"); 692 else 693 sc->cdev->si_drv1 = sc; 694 695 /* Go no further if recovery mode has been requested. */ 696 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 697 device_printf(dev, "recovery mode.\n"); 698 goto done; 699 } 700 701#if defined(__i386__) 702 if ((cpu_feature & CPUID_CX8) == 0) { 703 device_printf(dev, "64 bit atomics not available.\n"); 704 rc = ENOTSUP; 705 goto done; 706 } 707#endif 708 709 /* Prepare the firmware for operation */ 710 rc = prep_firmware(sc); 711 if (rc != 0) 712 goto done; /* error message displayed already */ 713 714 rc = get_params__post_init(sc); 715 if (rc != 0) 716 goto done; /* error message displayed already */ 717 718 rc = set_params__post_init(sc); 719 if (rc != 0) 720 goto done; /* error message displayed already */ 721 722 rc = map_bar_2(sc); 723 if (rc != 0) 724 goto done; /* error message displayed already */ 725 726 rc = t4_create_dma_tag(sc); 727 if (rc != 0) 728 goto done; /* error message displayed already */ 729 730 /* 731 * First pass over all the ports - allocate VIs and initialize some 732 * basic parameters like mac address, port type, etc. We also figure 733 * out whether a port is 10G or 1G and use that information when 734 * calculating how many interrupts to attempt to allocate. 735 */ 736 n10g = n1g = 0; 737 for_each_port(sc, i) { 738 struct port_info *pi; 739 740 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 741 sc->port[i] = pi; 742 743 /* These must be set before t4_port_init */ 744 pi->adapter = sc; 745 pi->port_id = i; 746 747 /* Allocate the vi and initialize parameters like mac addr */ 748 rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 749 if (rc != 0) { 750 device_printf(dev, "unable to initialize port %d: %d\n", 751 i, rc); 752 free(pi, M_CXGBE); 753 sc->port[i] = NULL; 754 goto done; 755 } 756 757 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 758 pi->link_cfg.requested_fc |= t4_pause_settings; 759 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 760 pi->link_cfg.fc |= t4_pause_settings; 761 762 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 763 if (rc != 0) { 764 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 765 free(pi, M_CXGBE); 766 sc->port[i] = NULL; 767 goto done; 768 } 769 770 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 771 device_get_nameunit(dev), i); 772 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 773 sc->chan_map[pi->tx_chan] = i; 774 775 if (is_10G_port(pi) || is_40G_port(pi)) { 776 n10g++; 777 pi->tmr_idx = t4_tmr_idx_10g; 778 pi->pktc_idx = t4_pktc_idx_10g; 779 } else { 780 n1g++; 781 pi->tmr_idx = t4_tmr_idx_1g; 782 pi->pktc_idx = t4_pktc_idx_1g; 783 } 784 785 pi->xact_addr_filt = -1; 786 pi->linkdnrc = -1; 787 788 pi->qsize_rxq = t4_qsize_rxq; 789 pi->qsize_txq = t4_qsize_txq; 790 791 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 792 if (pi->dev == NULL) { 793 device_printf(dev, 794 "failed to add device for port %d.\n", i); 795 rc = ENXIO; 796 goto done; 797 } 798 device_set_softc(pi->dev, pi); 799 } 800 801 /* 802 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 803 */ 804 rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 805 if (rc != 0) 806 goto done; /* error message displayed already */ 807 808 sc->intr_type = iaq.intr_type; 809 sc->intr_count = iaq.nirq; 810 811 s = &sc->sge; 812 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 813 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 814 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 815 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 816 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 817#ifdef TCP_OFFLOAD 818 if (is_offload(sc)) { 819 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 820 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 821 s->neq += s->nofldtxq + s->nofldrxq; 822 s->niq += s->nofldrxq; 823 824 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 825 M_CXGBE, M_ZERO | M_WAITOK); 826 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 827 M_CXGBE, M_ZERO | M_WAITOK); 828 } 829#endif 830#ifdef DEV_NETMAP 831 s->nnmrxq = n10g * iaq.nnmrxq10g + n1g * iaq.nnmrxq1g; 832 s->nnmtxq = n10g * iaq.nnmtxq10g + n1g * iaq.nnmtxq1g; 833 s->neq += s->nnmtxq + s->nnmrxq; 834 s->niq += s->nnmrxq; 835 836 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 837 M_CXGBE, M_ZERO | M_WAITOK); 838 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 839 M_CXGBE, M_ZERO | M_WAITOK); 840#endif 841 842 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 843 M_ZERO | M_WAITOK); 844 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 845 M_ZERO | M_WAITOK); 846 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 847 M_ZERO | M_WAITOK); 848 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 849 M_ZERO | M_WAITOK); 850 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 851 M_ZERO | M_WAITOK); 852 853 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 854 M_ZERO | M_WAITOK); 855 856 t4_init_l2t(sc, M_WAITOK); 857 858 /* 859 * Second pass over the ports. This time we know the number of rx and 860 * tx queues that each port should get. 861 */ 862 rqidx = tqidx = 0; 863#ifdef TCP_OFFLOAD 864 ofld_rqidx = ofld_tqidx = 0; 865#endif 866#ifdef DEV_NETMAP 867 nm_rqidx = nm_tqidx = 0; 868#endif 869 for_each_port(sc, i) { 870 struct port_info *pi = sc->port[i]; 871 872 if (pi == NULL) 873 continue; 874 875 pi->first_rxq = rqidx; 876 pi->first_txq = tqidx; 877 if (is_10G_port(pi) || is_40G_port(pi)) { 878 pi->flags |= iaq.intr_flags_10g; 879 pi->nrxq = iaq.nrxq10g; 880 pi->ntxq = iaq.ntxq10g; 881 } else { 882 pi->flags |= iaq.intr_flags_1g; 883 pi->nrxq = iaq.nrxq1g; 884 pi->ntxq = iaq.ntxq1g; 885 } 886 887 if (pi->ntxq > 1) 888 pi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 889 else 890 pi->rsrv_noflowq = 0; 891 892 rqidx += pi->nrxq; 893 tqidx += pi->ntxq; 894#ifdef TCP_OFFLOAD 895 if (is_offload(sc)) { 896 pi->first_ofld_rxq = ofld_rqidx; 897 pi->first_ofld_txq = ofld_tqidx; 898 if (is_10G_port(pi) || is_40G_port(pi)) { 899 pi->nofldrxq = iaq.nofldrxq10g; 900 pi->nofldtxq = iaq.nofldtxq10g; 901 } else { 902 pi->nofldrxq = iaq.nofldrxq1g; 903 pi->nofldtxq = iaq.nofldtxq1g; 904 } 905 ofld_rqidx += pi->nofldrxq; 906 ofld_tqidx += pi->nofldtxq; 907 } 908#endif 909#ifdef DEV_NETMAP 910 pi->first_nm_rxq = nm_rqidx; 911 pi->first_nm_txq = nm_tqidx; 912 if (is_10G_port(pi) || is_40G_port(pi)) { 913 pi->nnmrxq = iaq.nnmrxq10g; 914 pi->nnmtxq = iaq.nnmtxq10g; 915 } else { 916 pi->nnmrxq = iaq.nnmrxq1g; 917 pi->nnmtxq = iaq.nnmtxq1g; 918 } 919 nm_rqidx += pi->nnmrxq; 920 nm_tqidx += pi->nnmtxq; 921#endif 922 } 923 924 rc = setup_intr_handlers(sc); 925 if (rc != 0) { 926 device_printf(dev, 927 "failed to setup interrupt handlers: %d\n", rc); 928 goto done; 929 } 930 931 rc = bus_generic_attach(dev); 932 if (rc != 0) { 933 device_printf(dev, 934 "failed to attach all child ports: %d\n", rc); 935 goto done; 936 } 937 938 device_printf(dev, 939 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 940 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 941 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 942 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 943 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 944 945 t4_set_desc(sc); 946 947done: 948 if (rc != 0 && sc->cdev) { 949 /* cdev was created and so cxgbetool works; recover that way. */ 950 device_printf(dev, 951 "error during attach, adapter is now in recovery mode.\n"); 952 rc = 0; 953 } 954 955 if (rc != 0) 956 t4_detach(dev); 957 else 958 t4_sysctls(sc); 959 960 return (rc); 961} 962 963/* 964 * Idempotent 965 */ 966static int 967t4_detach(device_t dev) 968{ 969 struct adapter *sc; 970 struct port_info *pi; 971 int i, rc; 972 973 sc = device_get_softc(dev); 974 975 if (sc->flags & FULL_INIT_DONE) 976 t4_intr_disable(sc); 977 978 if (sc->cdev) { 979 destroy_dev(sc->cdev); 980 sc->cdev = NULL; 981 } 982 983 rc = bus_generic_detach(dev); 984 if (rc) { 985 device_printf(dev, 986 "failed to detach child devices: %d\n", rc); 987 return (rc); 988 } 989 990 for (i = 0; i < sc->intr_count; i++) 991 t4_free_irq(sc, &sc->irq[i]); 992 993 for (i = 0; i < MAX_NPORTS; i++) { 994 pi = sc->port[i]; 995 if (pi) { 996 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->viid); 997 if (pi->dev) 998 device_delete_child(dev, pi->dev); 999 1000 mtx_destroy(&pi->pi_lock); 1001 free(pi, M_CXGBE); 1002 } 1003 } 1004 1005 if (sc->flags & FULL_INIT_DONE) 1006 adapter_full_uninit(sc); 1007 1008 if (sc->flags & FW_OK) 1009 t4_fw_bye(sc, sc->mbox); 1010 1011 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1012 pci_release_msi(dev); 1013 1014 if (sc->regs_res) 1015 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1016 sc->regs_res); 1017 1018 if (sc->udbs_res) 1019 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1020 sc->udbs_res); 1021 1022 if (sc->msix_res) 1023 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1024 sc->msix_res); 1025 1026 if (sc->l2t) 1027 t4_free_l2t(sc->l2t); 1028 1029#ifdef TCP_OFFLOAD 1030 free(sc->sge.ofld_rxq, M_CXGBE); 1031 free(sc->sge.ofld_txq, M_CXGBE); 1032#endif 1033#ifdef DEV_NETMAP 1034 free(sc->sge.nm_rxq, M_CXGBE); 1035 free(sc->sge.nm_txq, M_CXGBE); 1036#endif 1037 free(sc->irq, M_CXGBE); 1038 free(sc->sge.rxq, M_CXGBE); 1039 free(sc->sge.txq, M_CXGBE); 1040 free(sc->sge.ctrlq, M_CXGBE); 1041 free(sc->sge.iqmap, M_CXGBE); 1042 free(sc->sge.eqmap, M_CXGBE); 1043 free(sc->tids.ftid_tab, M_CXGBE); 1044 t4_destroy_dma_tag(sc); 1045 if (mtx_initialized(&sc->sc_lock)) { 1046 sx_xlock(&t4_list_lock); 1047 SLIST_REMOVE(&t4_list, sc, adapter, link); 1048 sx_xunlock(&t4_list_lock); 1049 mtx_destroy(&sc->sc_lock); 1050 } 1051 1052 if (mtx_initialized(&sc->tids.ftid_lock)) 1053 mtx_destroy(&sc->tids.ftid_lock); 1054 if (mtx_initialized(&sc->sfl_lock)) 1055 mtx_destroy(&sc->sfl_lock); 1056 if (mtx_initialized(&sc->ifp_lock)) 1057 mtx_destroy(&sc->ifp_lock); 1058 if (mtx_initialized(&sc->regwin_lock)) 1059 mtx_destroy(&sc->regwin_lock); 1060 1061 bzero(sc, sizeof(*sc)); 1062 1063 return (0); 1064} 1065 1066static int 1067cxgbe_probe(device_t dev) 1068{ 1069 char buf[128]; 1070 struct port_info *pi = device_get_softc(dev); 1071 1072 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1073 device_set_desc_copy(dev, buf); 1074 1075 return (BUS_PROBE_DEFAULT); 1076} 1077 1078#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1079 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1080 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1081#define T4_CAP_ENABLE (T4_CAP) 1082 1083static int 1084cxgbe_attach(device_t dev) 1085{ 1086 struct port_info *pi = device_get_softc(dev); 1087 struct ifnet *ifp; 1088 char *s; 1089 int n, o; 1090 1091 /* Allocate an ifnet and set it up */ 1092 ifp = if_alloc(IFT_ETHER); 1093 if (ifp == NULL) { 1094 device_printf(dev, "Cannot allocate ifnet\n"); 1095 return (ENOMEM); 1096 } 1097 pi->ifp = ifp; 1098 ifp->if_softc = pi; 1099 1100 callout_init(&pi->tick, CALLOUT_MPSAFE); 1101 1102 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1103 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1104 1105 ifp->if_init = cxgbe_init; 1106 ifp->if_ioctl = cxgbe_ioctl; 1107 ifp->if_transmit = cxgbe_transmit; 1108 ifp->if_qflush = cxgbe_qflush; 1109 1110 ifp->if_capabilities = T4_CAP; 1111#ifdef TCP_OFFLOAD 1112 if (is_offload(pi->adapter)) 1113 ifp->if_capabilities |= IFCAP_TOE; 1114#endif 1115 ifp->if_capenable = T4_CAP_ENABLE; 1116 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1117 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1118 1119 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1120 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1121 ifp->if_hw_tsomaxsegsize = 65536; 1122 1123 /* Initialize ifmedia for this port */ 1124 ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 1125 cxgbe_media_status); 1126 build_medialist(pi, &pi->media); 1127 1128 pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1129 EVENTHANDLER_PRI_ANY); 1130 1131 ether_ifattach(ifp, pi->hw_addr); 1132 1133 n = 128; 1134 s = malloc(n, M_CXGBE, M_WAITOK); 1135 o = snprintf(s, n, "%d txq, %d rxq (NIC)", pi->ntxq, pi->nrxq); 1136 MPASS(n > o); 1137#ifdef TCP_OFFLOAD 1138 if (is_offload(pi->adapter)) { 1139 o += snprintf(s + o, n - o, "; %d txq, %d rxq (TOE)", 1140 pi->nofldtxq, pi->nofldrxq); 1141 MPASS(n > o); 1142 } 1143#endif 1144#ifdef DEV_NETMAP 1145 o += snprintf(s + o, n - o, "; %d txq, %d rxq (netmap)", pi->nnmtxq, 1146 pi->nnmrxq); 1147 MPASS(n > o); 1148#endif 1149 device_printf(dev, "%s\n", s); 1150 free(s, M_CXGBE); 1151 1152#ifdef DEV_NETMAP 1153 /* nm_media handled here to keep implementation private to this file */ 1154 ifmedia_init(&pi->nm_media, IFM_IMASK, cxgbe_media_change, 1155 cxgbe_media_status); 1156 build_medialist(pi, &pi->nm_media); 1157 create_netmap_ifnet(pi); /* logs errors it something fails */ 1158#endif 1159 cxgbe_sysctls(pi); 1160 1161 return (0); 1162} 1163 1164static int 1165cxgbe_detach(device_t dev) 1166{ 1167 struct port_info *pi = device_get_softc(dev); 1168 struct adapter *sc = pi->adapter; 1169 struct ifnet *ifp = pi->ifp; 1170 1171 /* Tell if_ioctl and if_init that the port is going away */ 1172 ADAPTER_LOCK(sc); 1173 SET_DOOMED(pi); 1174 wakeup(&sc->flags); 1175 while (IS_BUSY(sc)) 1176 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 1177 SET_BUSY(sc); 1178#ifdef INVARIANTS 1179 sc->last_op = "t4detach"; 1180 sc->last_op_thr = curthread; 1181 sc->last_op_flags = 0; 1182#endif 1183 ADAPTER_UNLOCK(sc); 1184 1185 if (pi->flags & HAS_TRACEQ) { 1186 sc->traceq = -1; /* cloner should not create ifnet */ 1187 t4_tracer_port_detach(sc); 1188 } 1189 1190 if (pi->vlan_c) 1191 EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 1192 1193 PORT_LOCK(pi); 1194 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1195 callout_stop(&pi->tick); 1196 PORT_UNLOCK(pi); 1197 callout_drain(&pi->tick); 1198 1199 /* Let detach proceed even if these fail. */ 1200 cxgbe_uninit_synchronized(pi); 1201 port_full_uninit(pi); 1202 1203 ifmedia_removeall(&pi->media); 1204 ether_ifdetach(pi->ifp); 1205 if_free(pi->ifp); 1206 1207#ifdef DEV_NETMAP 1208 /* XXXNM: equivalent of cxgbe_uninit_synchronized to ifdown nm_ifp */ 1209 destroy_netmap_ifnet(pi); 1210#endif 1211 1212 ADAPTER_LOCK(sc); 1213 CLR_BUSY(sc); 1214 wakeup(&sc->flags); 1215 ADAPTER_UNLOCK(sc); 1216 1217 return (0); 1218} 1219 1220static void 1221cxgbe_init(void *arg) 1222{ 1223 struct port_info *pi = arg; 1224 struct adapter *sc = pi->adapter; 1225 1226 if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1227 return; 1228 cxgbe_init_synchronized(pi); 1229 end_synchronized_op(sc, 0); 1230} 1231 1232static int 1233cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1234{ 1235 int rc = 0, mtu, flags, can_sleep; 1236 struct port_info *pi = ifp->if_softc; 1237 struct adapter *sc = pi->adapter; 1238 struct ifreq *ifr = (struct ifreq *)data; 1239 uint32_t mask; 1240 1241 switch (cmd) { 1242 case SIOCSIFMTU: 1243 mtu = ifr->ifr_mtu; 1244 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1245 return (EINVAL); 1246 1247 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1248 if (rc) 1249 return (rc); 1250 ifp->if_mtu = mtu; 1251 if (pi->flags & PORT_INIT_DONE) { 1252 t4_update_fl_bufsize(ifp); 1253 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1254 rc = update_mac_settings(ifp, XGMAC_MTU); 1255 } 1256 end_synchronized_op(sc, 0); 1257 break; 1258 1259 case SIOCSIFFLAGS: 1260 can_sleep = 0; 1261redo_sifflags: 1262 rc = begin_synchronized_op(sc, pi, 1263 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1264 if (rc) 1265 return (rc); 1266 1267 if (ifp->if_flags & IFF_UP) { 1268 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1269 flags = pi->if_flags; 1270 if ((ifp->if_flags ^ flags) & 1271 (IFF_PROMISC | IFF_ALLMULTI)) { 1272 if (can_sleep == 1) { 1273 end_synchronized_op(sc, 0); 1274 can_sleep = 0; 1275 goto redo_sifflags; 1276 } 1277 rc = update_mac_settings(ifp, 1278 XGMAC_PROMISC | XGMAC_ALLMULTI); 1279 } 1280 } else { 1281 if (can_sleep == 0) { 1282 end_synchronized_op(sc, LOCK_HELD); 1283 can_sleep = 1; 1284 goto redo_sifflags; 1285 } 1286 rc = cxgbe_init_synchronized(pi); 1287 } 1288 pi->if_flags = ifp->if_flags; 1289 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1290 if (can_sleep == 0) { 1291 end_synchronized_op(sc, LOCK_HELD); 1292 can_sleep = 1; 1293 goto redo_sifflags; 1294 } 1295 rc = cxgbe_uninit_synchronized(pi); 1296 } 1297 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1298 break; 1299 1300 case SIOCADDMULTI: 1301 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1302 rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1303 if (rc) 1304 return (rc); 1305 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1306 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1307 end_synchronized_op(sc, LOCK_HELD); 1308 break; 1309 1310 case SIOCSIFCAP: 1311 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1312 if (rc) 1313 return (rc); 1314 1315 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1316 if (mask & IFCAP_TXCSUM) { 1317 ifp->if_capenable ^= IFCAP_TXCSUM; 1318 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1319 1320 if (IFCAP_TSO4 & ifp->if_capenable && 1321 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1322 ifp->if_capenable &= ~IFCAP_TSO4; 1323 if_printf(ifp, 1324 "tso4 disabled due to -txcsum.\n"); 1325 } 1326 } 1327 if (mask & IFCAP_TXCSUM_IPV6) { 1328 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1329 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1330 1331 if (IFCAP_TSO6 & ifp->if_capenable && 1332 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1333 ifp->if_capenable &= ~IFCAP_TSO6; 1334 if_printf(ifp, 1335 "tso6 disabled due to -txcsum6.\n"); 1336 } 1337 } 1338 if (mask & IFCAP_RXCSUM) 1339 ifp->if_capenable ^= IFCAP_RXCSUM; 1340 if (mask & IFCAP_RXCSUM_IPV6) 1341 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1342 1343 /* 1344 * Note that we leave CSUM_TSO alone (it is always set). The 1345 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1346 * sending a TSO request our way, so it's sufficient to toggle 1347 * IFCAP_TSOx only. 1348 */ 1349 if (mask & IFCAP_TSO4) { 1350 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1351 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1352 if_printf(ifp, "enable txcsum first.\n"); 1353 rc = EAGAIN; 1354 goto fail; 1355 } 1356 ifp->if_capenable ^= IFCAP_TSO4; 1357 } 1358 if (mask & IFCAP_TSO6) { 1359 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1360 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1361 if_printf(ifp, "enable txcsum6 first.\n"); 1362 rc = EAGAIN; 1363 goto fail; 1364 } 1365 ifp->if_capenable ^= IFCAP_TSO6; 1366 } 1367 if (mask & IFCAP_LRO) { 1368#if defined(INET) || defined(INET6) 1369 int i; 1370 struct sge_rxq *rxq; 1371 1372 ifp->if_capenable ^= IFCAP_LRO; 1373 for_each_rxq(pi, i, rxq) { 1374 if (ifp->if_capenable & IFCAP_LRO) 1375 rxq->iq.flags |= IQ_LRO_ENABLED; 1376 else 1377 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1378 } 1379#endif 1380 } 1381#ifdef TCP_OFFLOAD 1382 if (mask & IFCAP_TOE) { 1383 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1384 1385 rc = toe_capability(pi, enable); 1386 if (rc != 0) 1387 goto fail; 1388 1389 ifp->if_capenable ^= mask; 1390 } 1391#endif 1392 if (mask & IFCAP_VLAN_HWTAGGING) { 1393 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1394 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1395 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1396 } 1397 if (mask & IFCAP_VLAN_MTU) { 1398 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1399 1400 /* Need to find out how to disable auto-mtu-inflation */ 1401 } 1402 if (mask & IFCAP_VLAN_HWTSO) 1403 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1404 if (mask & IFCAP_VLAN_HWCSUM) 1405 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1406 1407#ifdef VLAN_CAPABILITIES 1408 VLAN_CAPABILITIES(ifp); 1409#endif 1410fail: 1411 end_synchronized_op(sc, 0); 1412 break; 1413 1414 case SIOCSIFMEDIA: 1415 case SIOCGIFMEDIA: 1416 ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1417 break; 1418 1419 case SIOCGI2C: { 1420 struct ifi2creq i2c; 1421 1422 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1423 if (rc != 0) 1424 break; 1425 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1426 rc = EPERM; 1427 break; 1428 } 1429 if (i2c.len > sizeof(i2c.data)) { 1430 rc = EINVAL; 1431 break; 1432 } 1433 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4i2c"); 1434 if (rc) 1435 return (rc); 1436 rc = -t4_i2c_rd(sc, sc->mbox, pi->port_id, i2c.dev_addr, 1437 i2c.offset, i2c.len, &i2c.data[0]); 1438 end_synchronized_op(sc, 0); 1439 if (rc == 0) 1440 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1441 break; 1442 } 1443 1444 default: 1445 rc = ether_ioctl(ifp, cmd, data); 1446 } 1447 1448 return (rc); 1449} 1450 1451static int 1452cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1453{ 1454 struct port_info *pi = ifp->if_softc; 1455 struct adapter *sc = pi->adapter; 1456 struct sge_txq *txq; 1457 void *items[1]; 1458 int rc; 1459 1460 M_ASSERTPKTHDR(m); 1461 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1462 1463 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1464 m_freem(m); 1465 return (ENETDOWN); 1466 } 1467 1468 rc = parse_pkt(&m); 1469 if (__predict_false(rc != 0)) { 1470 MPASS(m == NULL); /* was freed already */ 1471 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1472 return (rc); 1473 } 1474 1475 /* Select a txq. */ 1476 txq = &sc->sge.txq[pi->first_txq]; 1477 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1478 txq += ((m->m_pkthdr.flowid % (pi->ntxq - pi->rsrv_noflowq)) + 1479 pi->rsrv_noflowq); 1480 1481 items[0] = m; 1482 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1483 if (__predict_false(rc != 0)) 1484 m_freem(m); 1485 1486 return (rc); 1487} 1488 1489static void 1490cxgbe_qflush(struct ifnet *ifp) 1491{ 1492 struct port_info *pi = ifp->if_softc; 1493 struct sge_txq *txq; 1494 int i; 1495 1496 /* queues do not exist if !PORT_INIT_DONE. */ 1497 if (pi->flags & PORT_INIT_DONE) { 1498 for_each_txq(pi, i, txq) { 1499 TXQ_LOCK(txq); 1500 txq->eq.flags &= ~EQ_ENABLED; 1501 TXQ_UNLOCK(txq); 1502 while (!mp_ring_is_idle(txq->r)) { 1503 mp_ring_check_drainage(txq->r, 0); 1504 pause("qflush", 1); 1505 } 1506 } 1507 } 1508 if_qflush(ifp); 1509} 1510 1511static int 1512cxgbe_media_change(struct ifnet *ifp) 1513{ 1514 struct port_info *pi = ifp->if_softc; 1515 1516 device_printf(pi->dev, "%s unimplemented.\n", __func__); 1517 1518 return (EOPNOTSUPP); 1519} 1520 1521static void 1522cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1523{ 1524 struct port_info *pi = ifp->if_softc; 1525 struct ifmedia *media = NULL; 1526 struct ifmedia_entry *cur; 1527 int speed = pi->link_cfg.speed; 1528#ifdef INVARIANTS 1529 int data = (pi->port_type << 8) | pi->mod_type; 1530#endif 1531 1532 if (ifp == pi->ifp) 1533 media = &pi->media; 1534#ifdef DEV_NETMAP 1535 else if (ifp == pi->nm_ifp) 1536 media = &pi->nm_media; 1537#endif 1538 MPASS(media != NULL); 1539 1540 cur = media->ifm_cur; 1541 MPASS(cur->ifm_data == data); 1542 1543 ifmr->ifm_status = IFM_AVALID; 1544 if (!pi->link_cfg.link_ok) 1545 return; 1546 1547 ifmr->ifm_status |= IFM_ACTIVE; 1548 1549 /* active and current will differ iff current media is autoselect. */ 1550 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1551 return; 1552 1553 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1554 if (speed == SPEED_10000) 1555 ifmr->ifm_active |= IFM_10G_T; 1556 else if (speed == SPEED_1000) 1557 ifmr->ifm_active |= IFM_1000_T; 1558 else if (speed == SPEED_100) 1559 ifmr->ifm_active |= IFM_100_TX; 1560 else if (speed == SPEED_10) 1561 ifmr->ifm_active |= IFM_10_T; 1562 else 1563 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1564 speed)); 1565} 1566 1567void 1568t4_fatal_err(struct adapter *sc) 1569{ 1570 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1571 t4_intr_disable(sc); 1572 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1573 device_get_nameunit(sc->dev)); 1574} 1575 1576static int 1577map_bars_0_and_4(struct adapter *sc) 1578{ 1579 sc->regs_rid = PCIR_BAR(0); 1580 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1581 &sc->regs_rid, RF_ACTIVE); 1582 if (sc->regs_res == NULL) { 1583 device_printf(sc->dev, "cannot map registers.\n"); 1584 return (ENXIO); 1585 } 1586 sc->bt = rman_get_bustag(sc->regs_res); 1587 sc->bh = rman_get_bushandle(sc->regs_res); 1588 sc->mmio_len = rman_get_size(sc->regs_res); 1589 setbit(&sc->doorbells, DOORBELL_KDB); 1590 1591 sc->msix_rid = PCIR_BAR(4); 1592 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1593 &sc->msix_rid, RF_ACTIVE); 1594 if (sc->msix_res == NULL) { 1595 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1596 return (ENXIO); 1597 } 1598 1599 return (0); 1600} 1601 1602static int 1603map_bar_2(struct adapter *sc) 1604{ 1605 1606 /* 1607 * T4: only iWARP driver uses the userspace doorbells. There is no need 1608 * to map it if RDMA is disabled. 1609 */ 1610 if (is_t4(sc) && sc->rdmacaps == 0) 1611 return (0); 1612 1613 sc->udbs_rid = PCIR_BAR(2); 1614 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1615 &sc->udbs_rid, RF_ACTIVE); 1616 if (sc->udbs_res == NULL) { 1617 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1618 return (ENXIO); 1619 } 1620 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1621 1622 if (is_t5(sc)) { 1623 setbit(&sc->doorbells, DOORBELL_UDB); 1624#if defined(__i386__) || defined(__amd64__) 1625 if (t5_write_combine) { 1626 int rc; 1627 1628 /* 1629 * Enable write combining on BAR2. This is the 1630 * userspace doorbell BAR and is split into 128B 1631 * (UDBS_SEG_SIZE) doorbell regions, each associated 1632 * with an egress queue. The first 64B has the doorbell 1633 * and the second 64B can be used to submit a tx work 1634 * request with an implicit doorbell. 1635 */ 1636 1637 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1638 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1639 if (rc == 0) { 1640 clrbit(&sc->doorbells, DOORBELL_UDB); 1641 setbit(&sc->doorbells, DOORBELL_WCWR); 1642 setbit(&sc->doorbells, DOORBELL_UDBWC); 1643 } else { 1644 device_printf(sc->dev, 1645 "couldn't enable write combining: %d\n", 1646 rc); 1647 } 1648 1649 t4_write_reg(sc, A_SGE_STAT_CFG, 1650 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1651 } 1652#endif 1653 } 1654 1655 return (0); 1656} 1657 1658static const struct memwin t4_memwin[] = { 1659 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1660 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1661 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1662}; 1663 1664static const struct memwin t5_memwin[] = { 1665 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1666 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1667 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1668}; 1669 1670static void 1671setup_memwin(struct adapter *sc) 1672{ 1673 const struct memwin *mw; 1674 int i, n; 1675 uint32_t bar0; 1676 1677 if (is_t4(sc)) { 1678 /* 1679 * Read low 32b of bar0 indirectly via the hardware backdoor 1680 * mechanism. Works from within PCI passthrough environments 1681 * too, where rman_get_start() can return a different value. We 1682 * need to program the T4 memory window decoders with the actual 1683 * addresses that will be coming across the PCIe link. 1684 */ 1685 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1686 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1687 1688 mw = &t4_memwin[0]; 1689 n = nitems(t4_memwin); 1690 } else { 1691 /* T5 uses the relative offset inside the PCIe BAR */ 1692 bar0 = 0; 1693 1694 mw = &t5_memwin[0]; 1695 n = nitems(t5_memwin); 1696 } 1697 1698 for (i = 0; i < n; i++, mw++) { 1699 t4_write_reg(sc, 1700 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1701 (mw->base + bar0) | V_BIR(0) | 1702 V_WINDOW(ilog2(mw->aperture) - 10)); 1703 } 1704 1705 /* flush */ 1706 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1707} 1708 1709/* 1710 * Verify that the memory range specified by the addr/len pair is valid and lies 1711 * entirely within a single region (EDCx or MCx). 1712 */ 1713static int 1714validate_mem_range(struct adapter *sc, uint32_t addr, int len) 1715{ 1716 uint32_t em, addr_len, maddr, mlen; 1717 1718 /* Memory can only be accessed in naturally aligned 4 byte units */ 1719 if (addr & 3 || len & 3 || len == 0) 1720 return (EINVAL); 1721 1722 /* Enabled memories */ 1723 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1724 if (em & F_EDRAM0_ENABLE) { 1725 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1726 maddr = G_EDRAM0_BASE(addr_len) << 20; 1727 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1728 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1729 addr + len <= maddr + mlen) 1730 return (0); 1731 } 1732 if (em & F_EDRAM1_ENABLE) { 1733 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1734 maddr = G_EDRAM1_BASE(addr_len) << 20; 1735 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1736 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1737 addr + len <= maddr + mlen) 1738 return (0); 1739 } 1740 if (em & F_EXT_MEM_ENABLE) { 1741 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1742 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1743 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1744 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1745 addr + len <= maddr + mlen) 1746 return (0); 1747 } 1748 if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1749 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1750 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1751 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1752 if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1753 addr + len <= maddr + mlen) 1754 return (0); 1755 } 1756 1757 return (EFAULT); 1758} 1759 1760static int 1761fwmtype_to_hwmtype(int mtype) 1762{ 1763 1764 switch (mtype) { 1765 case FW_MEMTYPE_EDC0: 1766 return (MEM_EDC0); 1767 case FW_MEMTYPE_EDC1: 1768 return (MEM_EDC1); 1769 case FW_MEMTYPE_EXTMEM: 1770 return (MEM_MC0); 1771 case FW_MEMTYPE_EXTMEM1: 1772 return (MEM_MC1); 1773 default: 1774 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 1775 } 1776} 1777 1778/* 1779 * Verify that the memory range specified by the memtype/offset/len pair is 1780 * valid and lies entirely within the memtype specified. The global address of 1781 * the start of the range is returned in addr. 1782 */ 1783static int 1784validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1785 uint32_t *addr) 1786{ 1787 uint32_t em, addr_len, maddr, mlen; 1788 1789 /* Memory can only be accessed in naturally aligned 4 byte units */ 1790 if (off & 3 || len & 3 || len == 0) 1791 return (EINVAL); 1792 1793 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1794 switch (fwmtype_to_hwmtype(mtype)) { 1795 case MEM_EDC0: 1796 if (!(em & F_EDRAM0_ENABLE)) 1797 return (EINVAL); 1798 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1799 maddr = G_EDRAM0_BASE(addr_len) << 20; 1800 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1801 break; 1802 case MEM_EDC1: 1803 if (!(em & F_EDRAM1_ENABLE)) 1804 return (EINVAL); 1805 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1806 maddr = G_EDRAM1_BASE(addr_len) << 20; 1807 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1808 break; 1809 case MEM_MC: 1810 if (!(em & F_EXT_MEM_ENABLE)) 1811 return (EINVAL); 1812 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1813 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1814 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1815 break; 1816 case MEM_MC1: 1817 if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1818 return (EINVAL); 1819 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1820 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1821 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1822 break; 1823 default: 1824 return (EINVAL); 1825 } 1826 1827 if (mlen > 0 && off < mlen && off + len <= mlen) { 1828 *addr = maddr + off; /* global address */ 1829 return (0); 1830 } 1831 1832 return (EFAULT); 1833} 1834 1835static void 1836memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1837{ 1838 const struct memwin *mw; 1839 1840 if (is_t4(sc)) { 1841 KASSERT(win >= 0 && win < nitems(t4_memwin), 1842 ("%s: incorrect memwin# (%d)", __func__, win)); 1843 mw = &t4_memwin[win]; 1844 } else { 1845 KASSERT(win >= 0 && win < nitems(t5_memwin), 1846 ("%s: incorrect memwin# (%d)", __func__, win)); 1847 mw = &t5_memwin[win]; 1848 } 1849 1850 if (base != NULL) 1851 *base = mw->base; 1852 if (aperture != NULL) 1853 *aperture = mw->aperture; 1854} 1855 1856/* 1857 * Positions the memory window such that it can be used to access the specified 1858 * address in the chip's address space. The return value is the offset of addr 1859 * from the start of the window. 1860 */ 1861static uint32_t 1862position_memwin(struct adapter *sc, int n, uint32_t addr) 1863{ 1864 uint32_t start, pf; 1865 uint32_t reg; 1866 1867 KASSERT(n >= 0 && n <= 3, 1868 ("%s: invalid window %d.", __func__, n)); 1869 KASSERT((addr & 3) == 0, 1870 ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1871 1872 if (is_t4(sc)) { 1873 pf = 0; 1874 start = addr & ~0xf; /* start must be 16B aligned */ 1875 } else { 1876 pf = V_PFNUM(sc->pf); 1877 start = addr & ~0x7f; /* start must be 128B aligned */ 1878 } 1879 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1880 1881 t4_write_reg(sc, reg, start | pf); 1882 t4_read_reg(sc, reg); 1883 1884 return (addr - start); 1885} 1886 1887static int 1888cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1889 struct intrs_and_queues *iaq) 1890{ 1891 int rc, itype, navail, nrxq10g, nrxq1g, n; 1892 int nofldrxq10g = 0, nofldrxq1g = 0; 1893 int nnmrxq10g = 0, nnmrxq1g = 0; 1894 1895 bzero(iaq, sizeof(*iaq)); 1896 1897 iaq->ntxq10g = t4_ntxq10g; 1898 iaq->ntxq1g = t4_ntxq1g; 1899 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1900 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1901 iaq->rsrv_noflowq = t4_rsrv_noflowq; 1902#ifdef TCP_OFFLOAD 1903 if (is_offload(sc)) { 1904 iaq->nofldtxq10g = t4_nofldtxq10g; 1905 iaq->nofldtxq1g = t4_nofldtxq1g; 1906 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1907 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1908 } 1909#endif 1910#ifdef DEV_NETMAP 1911 iaq->nnmtxq10g = t4_nnmtxq10g; 1912 iaq->nnmtxq1g = t4_nnmtxq1g; 1913 iaq->nnmrxq10g = nnmrxq10g = t4_nnmrxq10g; 1914 iaq->nnmrxq1g = nnmrxq1g = t4_nnmrxq1g; 1915#endif 1916 1917 for (itype = INTR_MSIX; itype; itype >>= 1) { 1918 1919 if ((itype & t4_intr_types) == 0) 1920 continue; /* not allowed */ 1921 1922 if (itype == INTR_MSIX) 1923 navail = pci_msix_count(sc->dev); 1924 else if (itype == INTR_MSI) 1925 navail = pci_msi_count(sc->dev); 1926 else 1927 navail = 1; 1928restart: 1929 if (navail == 0) 1930 continue; 1931 1932 iaq->intr_type = itype; 1933 iaq->intr_flags_10g = 0; 1934 iaq->intr_flags_1g = 0; 1935 1936 /* 1937 * Best option: an interrupt vector for errors, one for the 1938 * firmware event queue, and one for every rxq (NIC, TOE, and 1939 * netmap). 1940 */ 1941 iaq->nirq = T4_EXTRA_INTR; 1942 iaq->nirq += n10g * (nrxq10g + nofldrxq10g + nnmrxq10g); 1943 iaq->nirq += n1g * (nrxq1g + nofldrxq1g + nnmrxq1g); 1944 if (iaq->nirq <= navail && 1945 (itype != INTR_MSI || powerof2(iaq->nirq))) { 1946 iaq->intr_flags_10g = INTR_ALL; 1947 iaq->intr_flags_1g = INTR_ALL; 1948 goto allocate; 1949 } 1950 1951 /* 1952 * Second best option: a vector for errors, one for the firmware 1953 * event queue, and vectors for either all the NIC rx queues or 1954 * all the TOE rx queues. The queues that don't get vectors 1955 * will forward their interrupts to those that do. 1956 * 1957 * Note: netmap rx queues cannot be created early and so they 1958 * can't be setup to receive forwarded interrupts for others. 1959 */ 1960 iaq->nirq = T4_EXTRA_INTR; 1961 if (nrxq10g >= nofldrxq10g) { 1962 iaq->intr_flags_10g = INTR_RXQ; 1963 iaq->nirq += n10g * nrxq10g; 1964#ifdef DEV_NETMAP 1965 iaq->nnmrxq10g = min(nnmrxq10g, nrxq10g); 1966#endif 1967 } else { 1968 iaq->intr_flags_10g = INTR_OFLD_RXQ; 1969 iaq->nirq += n10g * nofldrxq10g; 1970#ifdef DEV_NETMAP 1971 iaq->nnmrxq10g = min(nnmrxq10g, nofldrxq10g); 1972#endif 1973 } 1974 if (nrxq1g >= nofldrxq1g) { 1975 iaq->intr_flags_1g = INTR_RXQ; 1976 iaq->nirq += n1g * nrxq1g; 1977#ifdef DEV_NETMAP 1978 iaq->nnmrxq1g = min(nnmrxq1g, nrxq1g); 1979#endif 1980 } else { 1981 iaq->intr_flags_1g = INTR_OFLD_RXQ; 1982 iaq->nirq += n1g * nofldrxq1g; 1983#ifdef DEV_NETMAP 1984 iaq->nnmrxq1g = min(nnmrxq1g, nofldrxq1g); 1985#endif 1986 } 1987 if (iaq->nirq <= navail && 1988 (itype != INTR_MSI || powerof2(iaq->nirq))) 1989 goto allocate; 1990 1991 /* 1992 * Next best option: an interrupt vector for errors, one for the 1993 * firmware event queue, and at least one per port. At this 1994 * point we know we'll have to downsize nrxq and/or nofldrxq 1995 * and/or nnmrxq to fit what's available to us. 1996 */ 1997 iaq->nirq = T4_EXTRA_INTR; 1998 iaq->nirq += n10g + n1g; 1999 if (iaq->nirq <= navail) { 2000 int leftover = navail - iaq->nirq; 2001 2002 if (n10g > 0) { 2003 int target = max(nrxq10g, nofldrxq10g); 2004 2005 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2006 INTR_RXQ : INTR_OFLD_RXQ; 2007 2008 n = 1; 2009 while (n < target && leftover >= n10g) { 2010 leftover -= n10g; 2011 iaq->nirq += n10g; 2012 n++; 2013 } 2014 iaq->nrxq10g = min(n, nrxq10g); 2015#ifdef TCP_OFFLOAD 2016 iaq->nofldrxq10g = min(n, nofldrxq10g); 2017#endif 2018#ifdef DEV_NETMAP 2019 iaq->nnmrxq10g = min(n, nnmrxq10g); 2020#endif 2021 } 2022 2023 if (n1g > 0) { 2024 int target = max(nrxq1g, nofldrxq1g); 2025 2026 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2027 INTR_RXQ : INTR_OFLD_RXQ; 2028 2029 n = 1; 2030 while (n < target && leftover >= n1g) { 2031 leftover -= n1g; 2032 iaq->nirq += n1g; 2033 n++; 2034 } 2035 iaq->nrxq1g = min(n, nrxq1g); 2036#ifdef TCP_OFFLOAD 2037 iaq->nofldrxq1g = min(n, nofldrxq1g); 2038#endif 2039#ifdef DEV_NETMAP 2040 iaq->nnmrxq1g = min(n, nnmrxq1g); 2041#endif 2042 } 2043 2044 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2045 goto allocate; 2046 } 2047 2048 /* 2049 * Least desirable option: one interrupt vector for everything. 2050 */ 2051 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2052 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2053#ifdef TCP_OFFLOAD 2054 if (is_offload(sc)) 2055 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2056#endif 2057#ifdef DEV_NETMAP 2058 iaq->nnmrxq10g = iaq->nnmrxq1g = 1; 2059#endif 2060 2061allocate: 2062 navail = iaq->nirq; 2063 rc = 0; 2064 if (itype == INTR_MSIX) 2065 rc = pci_alloc_msix(sc->dev, &navail); 2066 else if (itype == INTR_MSI) 2067 rc = pci_alloc_msi(sc->dev, &navail); 2068 2069 if (rc == 0) { 2070 if (navail == iaq->nirq) 2071 return (0); 2072 2073 /* 2074 * Didn't get the number requested. Use whatever number 2075 * the kernel is willing to allocate (it's in navail). 2076 */ 2077 device_printf(sc->dev, "fewer vectors than requested, " 2078 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2079 itype, iaq->nirq, navail); 2080 pci_release_msi(sc->dev); 2081 goto restart; 2082 } 2083 2084 device_printf(sc->dev, 2085 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2086 itype, rc, iaq->nirq, navail); 2087 } 2088 2089 device_printf(sc->dev, 2090 "failed to find a usable interrupt type. " 2091 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2092 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2093 2094 return (ENXIO); 2095} 2096 2097#define FW_VERSION(chip) ( \ 2098 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2099 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2100 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2101 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2102#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2103 2104struct fw_info { 2105 uint8_t chip; 2106 char *kld_name; 2107 char *fw_mod_name; 2108 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2109} fw_info[] = { 2110 { 2111 .chip = CHELSIO_T4, 2112 .kld_name = "t4fw_cfg", 2113 .fw_mod_name = "t4fw", 2114 .fw_hdr = { 2115 .chip = FW_HDR_CHIP_T4, 2116 .fw_ver = htobe32_const(FW_VERSION(T4)), 2117 .intfver_nic = FW_INTFVER(T4, NIC), 2118 .intfver_vnic = FW_INTFVER(T4, VNIC), 2119 .intfver_ofld = FW_INTFVER(T4, OFLD), 2120 .intfver_ri = FW_INTFVER(T4, RI), 2121 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2122 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2123 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2124 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2125 }, 2126 }, { 2127 .chip = CHELSIO_T5, 2128 .kld_name = "t5fw_cfg", 2129 .fw_mod_name = "t5fw", 2130 .fw_hdr = { 2131 .chip = FW_HDR_CHIP_T5, 2132 .fw_ver = htobe32_const(FW_VERSION(T5)), 2133 .intfver_nic = FW_INTFVER(T5, NIC), 2134 .intfver_vnic = FW_INTFVER(T5, VNIC), 2135 .intfver_ofld = FW_INTFVER(T5, OFLD), 2136 .intfver_ri = FW_INTFVER(T5, RI), 2137 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2138 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2139 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2140 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2141 }, 2142 } 2143}; 2144 2145static struct fw_info * 2146find_fw_info(int chip) 2147{ 2148 int i; 2149 2150 for (i = 0; i < nitems(fw_info); i++) { 2151 if (fw_info[i].chip == chip) 2152 return (&fw_info[i]); 2153 } 2154 return (NULL); 2155} 2156 2157/* 2158 * Is the given firmware API compatible with the one the driver was compiled 2159 * with? 2160 */ 2161static int 2162fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2163{ 2164 2165 /* short circuit if it's the exact same firmware version */ 2166 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2167 return (1); 2168 2169 /* 2170 * XXX: Is this too conservative? Perhaps I should limit this to the 2171 * features that are supported in the driver. 2172 */ 2173#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2174 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2175 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2176 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2177 return (1); 2178#undef SAME_INTF 2179 2180 return (0); 2181} 2182 2183/* 2184 * The firmware in the KLD is usable, but should it be installed? This routine 2185 * explains itself in detail if it indicates the KLD firmware should be 2186 * installed. 2187 */ 2188static int 2189should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2190{ 2191 const char *reason; 2192 2193 if (!card_fw_usable) { 2194 reason = "incompatible or unusable"; 2195 goto install; 2196 } 2197 2198 if (k > c) { 2199 reason = "older than the version bundled with this driver"; 2200 goto install; 2201 } 2202 2203 if (t4_fw_install == 2 && k != c) { 2204 reason = "different than the version bundled with this driver"; 2205 goto install; 2206 } 2207 2208 return (0); 2209 2210install: 2211 if (t4_fw_install == 0) { 2212 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2213 "but the driver is prohibited from installing a different " 2214 "firmware on the card.\n", 2215 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2216 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2217 2218 return (0); 2219 } 2220 2221 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2222 "installing firmware %u.%u.%u.%u on card.\n", 2223 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2224 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2225 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2226 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2227 2228 return (1); 2229} 2230/* 2231 * Establish contact with the firmware and determine if we are the master driver 2232 * or not, and whether we are responsible for chip initialization. 2233 */ 2234static int 2235prep_firmware(struct adapter *sc) 2236{ 2237 const struct firmware *fw = NULL, *default_cfg; 2238 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2239 enum dev_state state; 2240 struct fw_info *fw_info; 2241 struct fw_hdr *card_fw; /* fw on the card */ 2242 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2243 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2244 against */ 2245 2246 /* Contact firmware. */ 2247 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2248 if (rc < 0 || state == DEV_STATE_ERR) { 2249 rc = -rc; 2250 device_printf(sc->dev, 2251 "failed to connect to the firmware: %d, %d.\n", rc, state); 2252 return (rc); 2253 } 2254 pf = rc; 2255 if (pf == sc->mbox) 2256 sc->flags |= MASTER_PF; 2257 else if (state == DEV_STATE_UNINIT) { 2258 /* 2259 * We didn't get to be the master so we definitely won't be 2260 * configuring the chip. It's a bug if someone else hasn't 2261 * configured it already. 2262 */ 2263 device_printf(sc->dev, "couldn't be master(%d), " 2264 "device not already initialized either(%d).\n", rc, state); 2265 return (EDOOFUS); 2266 } 2267 2268 /* This is the firmware whose headers the driver was compiled against */ 2269 fw_info = find_fw_info(chip_id(sc)); 2270 if (fw_info == NULL) { 2271 device_printf(sc->dev, 2272 "unable to look up firmware information for chip %d.\n", 2273 chip_id(sc)); 2274 return (EINVAL); 2275 } 2276 drv_fw = &fw_info->fw_hdr; 2277 2278 /* 2279 * The firmware KLD contains many modules. The KLD name is also the 2280 * name of the module that contains the default config file. 2281 */ 2282 default_cfg = firmware_get(fw_info->kld_name); 2283 2284 /* Read the header of the firmware on the card */ 2285 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2286 rc = -t4_read_flash(sc, FLASH_FW_START, 2287 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2288 if (rc == 0) 2289 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2290 else { 2291 device_printf(sc->dev, 2292 "Unable to read card's firmware header: %d\n", rc); 2293 card_fw_usable = 0; 2294 } 2295 2296 /* This is the firmware in the KLD */ 2297 fw = firmware_get(fw_info->fw_mod_name); 2298 if (fw != NULL) { 2299 kld_fw = (const void *)fw->data; 2300 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2301 } else { 2302 kld_fw = NULL; 2303 kld_fw_usable = 0; 2304 } 2305 2306 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2307 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2308 /* 2309 * Common case: the firmware on the card is an exact match and 2310 * the KLD is an exact match too, or the KLD is 2311 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2312 * here -- use cxgbetool loadfw if you want to reinstall the 2313 * same firmware as the one on the card. 2314 */ 2315 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2316 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2317 be32toh(card_fw->fw_ver))) { 2318 2319 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2320 if (rc != 0) { 2321 device_printf(sc->dev, 2322 "failed to install firmware: %d\n", rc); 2323 goto done; 2324 } 2325 2326 /* Installed successfully, update the cached header too. */ 2327 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2328 card_fw_usable = 1; 2329 need_fw_reset = 0; /* already reset as part of load_fw */ 2330 } 2331 2332 if (!card_fw_usable) { 2333 uint32_t d, c, k; 2334 2335 d = ntohl(drv_fw->fw_ver); 2336 c = ntohl(card_fw->fw_ver); 2337 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2338 2339 device_printf(sc->dev, "Cannot find a usable firmware: " 2340 "fw_install %d, chip state %d, " 2341 "driver compiled with %d.%d.%d.%d, " 2342 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2343 t4_fw_install, state, 2344 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2345 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2346 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2347 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2348 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2349 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2350 rc = EINVAL; 2351 goto done; 2352 } 2353 2354 /* We're using whatever's on the card and it's known to be good. */ 2355 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2356 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2357 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2358 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2359 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2360 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2361 t4_get_tp_version(sc, &sc->params.tp_vers); 2362 2363 /* Reset device */ 2364 if (need_fw_reset && 2365 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2366 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2367 if (rc != ETIMEDOUT && rc != EIO) 2368 t4_fw_bye(sc, sc->mbox); 2369 goto done; 2370 } 2371 sc->flags |= FW_OK; 2372 2373 rc = get_params__pre_init(sc); 2374 if (rc != 0) 2375 goto done; /* error message displayed already */ 2376 2377 /* Partition adapter resources as specified in the config file. */ 2378 if (state == DEV_STATE_UNINIT) { 2379 2380 KASSERT(sc->flags & MASTER_PF, 2381 ("%s: trying to change chip settings when not master.", 2382 __func__)); 2383 2384 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2385 if (rc != 0) 2386 goto done; /* error message displayed already */ 2387 2388 t4_tweak_chip_settings(sc); 2389 2390 /* get basic stuff going */ 2391 rc = -t4_fw_initialize(sc, sc->mbox); 2392 if (rc != 0) { 2393 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2394 goto done; 2395 } 2396 } else { 2397 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2398 sc->cfcsum = 0; 2399 } 2400 2401done: 2402 free(card_fw, M_CXGBE); 2403 if (fw != NULL) 2404 firmware_put(fw, FIRMWARE_UNLOAD); 2405 if (default_cfg != NULL) 2406 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2407 2408 return (rc); 2409} 2410 2411#define FW_PARAM_DEV(param) \ 2412 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2413 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2414#define FW_PARAM_PFVF(param) \ 2415 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2416 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2417 2418/* 2419 * Partition chip resources for use between various PFs, VFs, etc. 2420 */ 2421static int 2422partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2423 const char *name_prefix) 2424{ 2425 const struct firmware *cfg = NULL; 2426 int rc = 0; 2427 struct fw_caps_config_cmd caps; 2428 uint32_t mtype, moff, finicsum, cfcsum; 2429 2430 /* 2431 * Figure out what configuration file to use. Pick the default config 2432 * file for the card if the user hasn't specified one explicitly. 2433 */ 2434 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2435 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2436 /* Card specific overrides go here. */ 2437 if (pci_get_device(sc->dev) == 0x440a) 2438 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2439 if (is_fpga(sc)) 2440 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2441 } 2442 2443 /* 2444 * We need to load another module if the profile is anything except 2445 * "default" or "flash". 2446 */ 2447 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2448 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2449 char s[32]; 2450 2451 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2452 cfg = firmware_get(s); 2453 if (cfg == NULL) { 2454 if (default_cfg != NULL) { 2455 device_printf(sc->dev, 2456 "unable to load module \"%s\" for " 2457 "configuration profile \"%s\", will use " 2458 "the default config file instead.\n", 2459 s, sc->cfg_file); 2460 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2461 "%s", DEFAULT_CF); 2462 } else { 2463 device_printf(sc->dev, 2464 "unable to load module \"%s\" for " 2465 "configuration profile \"%s\", will use " 2466 "the config file on the card's flash " 2467 "instead.\n", s, sc->cfg_file); 2468 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2469 "%s", FLASH_CF); 2470 } 2471 } 2472 } 2473 2474 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2475 default_cfg == NULL) { 2476 device_printf(sc->dev, 2477 "default config file not available, will use the config " 2478 "file on the card's flash instead.\n"); 2479 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2480 } 2481 2482 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2483 u_int cflen, i, n; 2484 const uint32_t *cfdata; 2485 uint32_t param, val, addr, off, mw_base, mw_aperture; 2486 2487 KASSERT(cfg != NULL || default_cfg != NULL, 2488 ("%s: no config to upload", __func__)); 2489 2490 /* 2491 * Ask the firmware where it wants us to upload the config file. 2492 */ 2493 param = FW_PARAM_DEV(CF); 2494 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2495 if (rc != 0) { 2496 /* No support for config file? Shouldn't happen. */ 2497 device_printf(sc->dev, 2498 "failed to query config file location: %d.\n", rc); 2499 goto done; 2500 } 2501 mtype = G_FW_PARAMS_PARAM_Y(val); 2502 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2503 2504 /* 2505 * XXX: sheer laziness. We deliberately added 4 bytes of 2506 * useless stuffing/comments at the end of the config file so 2507 * it's ok to simply throw away the last remaining bytes when 2508 * the config file is not an exact multiple of 4. This also 2509 * helps with the validate_mt_off_len check. 2510 */ 2511 if (cfg != NULL) { 2512 cflen = cfg->datasize & ~3; 2513 cfdata = cfg->data; 2514 } else { 2515 cflen = default_cfg->datasize & ~3; 2516 cfdata = default_cfg->data; 2517 } 2518 2519 if (cflen > FLASH_CFG_MAX_SIZE) { 2520 device_printf(sc->dev, 2521 "config file too long (%d, max allowed is %d). " 2522 "Will try to use the config on the card, if any.\n", 2523 cflen, FLASH_CFG_MAX_SIZE); 2524 goto use_config_on_flash; 2525 } 2526 2527 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2528 if (rc != 0) { 2529 device_printf(sc->dev, 2530 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2531 "Will try to use the config on the card, if any.\n", 2532 __func__, mtype, moff, cflen, rc); 2533 goto use_config_on_flash; 2534 } 2535 2536 memwin_info(sc, 2, &mw_base, &mw_aperture); 2537 while (cflen) { 2538 off = position_memwin(sc, 2, addr); 2539 n = min(cflen, mw_aperture - off); 2540 for (i = 0; i < n; i += 4) 2541 t4_write_reg(sc, mw_base + off + i, *cfdata++); 2542 cflen -= n; 2543 addr += n; 2544 } 2545 } else { 2546use_config_on_flash: 2547 mtype = FW_MEMTYPE_FLASH; 2548 moff = t4_flash_cfg_addr(sc); 2549 } 2550 2551 bzero(&caps, sizeof(caps)); 2552 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2553 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2554 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2555 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2556 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2557 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2558 if (rc != 0) { 2559 device_printf(sc->dev, 2560 "failed to pre-process config file: %d " 2561 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2562 goto done; 2563 } 2564 2565 finicsum = be32toh(caps.finicsum); 2566 cfcsum = be32toh(caps.cfcsum); 2567 if (finicsum != cfcsum) { 2568 device_printf(sc->dev, 2569 "WARNING: config file checksum mismatch: %08x %08x\n", 2570 finicsum, cfcsum); 2571 } 2572 sc->cfcsum = cfcsum; 2573 2574#define LIMIT_CAPS(x) do { \ 2575 caps.x &= htobe16(t4_##x##_allowed); \ 2576} while (0) 2577 2578 /* 2579 * Let the firmware know what features will (not) be used so it can tune 2580 * things accordingly. 2581 */ 2582 LIMIT_CAPS(linkcaps); 2583 LIMIT_CAPS(niccaps); 2584 LIMIT_CAPS(toecaps); 2585 LIMIT_CAPS(rdmacaps); 2586 LIMIT_CAPS(iscsicaps); 2587 LIMIT_CAPS(fcoecaps); 2588#undef LIMIT_CAPS 2589 2590 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2591 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2592 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2593 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2594 if (rc != 0) { 2595 device_printf(sc->dev, 2596 "failed to process config file: %d.\n", rc); 2597 } 2598done: 2599 if (cfg != NULL) 2600 firmware_put(cfg, FIRMWARE_UNLOAD); 2601 return (rc); 2602} 2603 2604/* 2605 * Retrieve parameters that are needed (or nice to have) very early. 2606 */ 2607static int 2608get_params__pre_init(struct adapter *sc) 2609{ 2610 int rc; 2611 uint32_t param[2], val[2]; 2612 struct fw_devlog_cmd cmd; 2613 struct devlog_params *dlog = &sc->params.devlog; 2614 2615 param[0] = FW_PARAM_DEV(PORTVEC); 2616 param[1] = FW_PARAM_DEV(CCLK); 2617 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2618 if (rc != 0) { 2619 device_printf(sc->dev, 2620 "failed to query parameters (pre_init): %d.\n", rc); 2621 return (rc); 2622 } 2623 2624 sc->params.portvec = val[0]; 2625 sc->params.nports = bitcount32(val[0]); 2626 sc->params.vpd.cclk = val[1]; 2627 2628 /* Read device log parameters. */ 2629 bzero(&cmd, sizeof(cmd)); 2630 cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2631 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2632 cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2633 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2634 if (rc != 0) { 2635 device_printf(sc->dev, 2636 "failed to get devlog parameters: %d.\n", rc); 2637 bzero(dlog, sizeof (*dlog)); 2638 rc = 0; /* devlog isn't critical for device operation */ 2639 } else { 2640 val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2641 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2642 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2643 dlog->size = be32toh(cmd.memsize_devlog); 2644 } 2645 2646 return (rc); 2647} 2648 2649/* 2650 * Retrieve various parameters that are of interest to the driver. The device 2651 * has been initialized by the firmware at this point. 2652 */ 2653static int 2654get_params__post_init(struct adapter *sc) 2655{ 2656 int rc; 2657 uint32_t param[7], val[7]; 2658 struct fw_caps_config_cmd caps; 2659 2660 param[0] = FW_PARAM_PFVF(IQFLINT_START); 2661 param[1] = FW_PARAM_PFVF(EQ_START); 2662 param[2] = FW_PARAM_PFVF(FILTER_START); 2663 param[3] = FW_PARAM_PFVF(FILTER_END); 2664 param[4] = FW_PARAM_PFVF(L2T_START); 2665 param[5] = FW_PARAM_PFVF(L2T_END); 2666 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2667 if (rc != 0) { 2668 device_printf(sc->dev, 2669 "failed to query parameters (post_init): %d.\n", rc); 2670 return (rc); 2671 } 2672 2673 sc->sge.iq_start = val[0]; 2674 sc->sge.eq_start = val[1]; 2675 sc->tids.ftid_base = val[2]; 2676 sc->tids.nftids = val[3] - val[2] + 1; 2677 sc->params.ftid_min = val[2]; 2678 sc->params.ftid_max = val[3]; 2679 sc->vres.l2t.start = val[4]; 2680 sc->vres.l2t.size = val[5] - val[4] + 1; 2681 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2682 ("%s: L2 table size (%u) larger than expected (%u)", 2683 __func__, sc->vres.l2t.size, L2T_SIZE)); 2684 2685 /* get capabilites */ 2686 bzero(&caps, sizeof(caps)); 2687 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2688 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2689 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2690 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2691 if (rc != 0) { 2692 device_printf(sc->dev, 2693 "failed to get card capabilities: %d.\n", rc); 2694 return (rc); 2695 } 2696 2697#define READ_CAPS(x) do { \ 2698 sc->x = htobe16(caps.x); \ 2699} while (0) 2700 READ_CAPS(linkcaps); 2701 READ_CAPS(niccaps); 2702 READ_CAPS(toecaps); 2703 READ_CAPS(rdmacaps); 2704 READ_CAPS(iscsicaps); 2705 READ_CAPS(fcoecaps); 2706 2707 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 2708 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 2709 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 2710 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2711 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 2712 if (rc != 0) { 2713 device_printf(sc->dev, 2714 "failed to query NIC parameters: %d.\n", rc); 2715 return (rc); 2716 } 2717 sc->tids.etid_base = val[0]; 2718 sc->params.etid_min = val[0]; 2719 sc->tids.netids = val[1] - val[0] + 1; 2720 sc->params.netids = sc->tids.netids; 2721 sc->params.eo_wr_cred = val[2]; 2722 sc->params.ethoffload = 1; 2723 } 2724 2725 if (sc->toecaps) { 2726 /* query offload-related parameters */ 2727 param[0] = FW_PARAM_DEV(NTID); 2728 param[1] = FW_PARAM_PFVF(SERVER_START); 2729 param[2] = FW_PARAM_PFVF(SERVER_END); 2730 param[3] = FW_PARAM_PFVF(TDDP_START); 2731 param[4] = FW_PARAM_PFVF(TDDP_END); 2732 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2733 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2734 if (rc != 0) { 2735 device_printf(sc->dev, 2736 "failed to query TOE parameters: %d.\n", rc); 2737 return (rc); 2738 } 2739 sc->tids.ntids = val[0]; 2740 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2741 sc->tids.stid_base = val[1]; 2742 sc->tids.nstids = val[2] - val[1] + 1; 2743 sc->vres.ddp.start = val[3]; 2744 sc->vres.ddp.size = val[4] - val[3] + 1; 2745 sc->params.ofldq_wr_cred = val[5]; 2746 sc->params.offload = 1; 2747 } 2748 if (sc->rdmacaps) { 2749 param[0] = FW_PARAM_PFVF(STAG_START); 2750 param[1] = FW_PARAM_PFVF(STAG_END); 2751 param[2] = FW_PARAM_PFVF(RQ_START); 2752 param[3] = FW_PARAM_PFVF(RQ_END); 2753 param[4] = FW_PARAM_PFVF(PBL_START); 2754 param[5] = FW_PARAM_PFVF(PBL_END); 2755 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2756 if (rc != 0) { 2757 device_printf(sc->dev, 2758 "failed to query RDMA parameters(1): %d.\n", rc); 2759 return (rc); 2760 } 2761 sc->vres.stag.start = val[0]; 2762 sc->vres.stag.size = val[1] - val[0] + 1; 2763 sc->vres.rq.start = val[2]; 2764 sc->vres.rq.size = val[3] - val[2] + 1; 2765 sc->vres.pbl.start = val[4]; 2766 sc->vres.pbl.size = val[5] - val[4] + 1; 2767 2768 param[0] = FW_PARAM_PFVF(SQRQ_START); 2769 param[1] = FW_PARAM_PFVF(SQRQ_END); 2770 param[2] = FW_PARAM_PFVF(CQ_START); 2771 param[3] = FW_PARAM_PFVF(CQ_END); 2772 param[4] = FW_PARAM_PFVF(OCQ_START); 2773 param[5] = FW_PARAM_PFVF(OCQ_END); 2774 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2775 if (rc != 0) { 2776 device_printf(sc->dev, 2777 "failed to query RDMA parameters(2): %d.\n", rc); 2778 return (rc); 2779 } 2780 sc->vres.qp.start = val[0]; 2781 sc->vres.qp.size = val[1] - val[0] + 1; 2782 sc->vres.cq.start = val[2]; 2783 sc->vres.cq.size = val[3] - val[2] + 1; 2784 sc->vres.ocq.start = val[4]; 2785 sc->vres.ocq.size = val[5] - val[4] + 1; 2786 } 2787 if (sc->iscsicaps) { 2788 param[0] = FW_PARAM_PFVF(ISCSI_START); 2789 param[1] = FW_PARAM_PFVF(ISCSI_END); 2790 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2791 if (rc != 0) { 2792 device_printf(sc->dev, 2793 "failed to query iSCSI parameters: %d.\n", rc); 2794 return (rc); 2795 } 2796 sc->vres.iscsi.start = val[0]; 2797 sc->vres.iscsi.size = val[1] - val[0] + 1; 2798 } 2799 2800 /* 2801 * We've got the params we wanted to query via the firmware. Now grab 2802 * some others directly from the chip. 2803 */ 2804 rc = t4_read_chip_settings(sc); 2805 2806 return (rc); 2807} 2808 2809static int 2810set_params__post_init(struct adapter *sc) 2811{ 2812 uint32_t param, val; 2813 2814 /* ask for encapsulated CPLs */ 2815 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2816 val = 1; 2817 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2818 2819 return (0); 2820} 2821 2822#undef FW_PARAM_PFVF 2823#undef FW_PARAM_DEV 2824 2825static void 2826t4_set_desc(struct adapter *sc) 2827{ 2828 char buf[128]; 2829 struct adapter_params *p = &sc->params; 2830 2831 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 2832 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 2833 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 2834 2835 device_set_desc_copy(sc->dev, buf); 2836} 2837 2838static void 2839build_medialist(struct port_info *pi, struct ifmedia *media) 2840{ 2841 int data, m; 2842 2843 PORT_LOCK(pi); 2844 2845 ifmedia_removeall(media); 2846 2847 m = IFM_ETHER | IFM_FDX; 2848 data = (pi->port_type << 8) | pi->mod_type; 2849 2850 switch(pi->port_type) { 2851 case FW_PORT_TYPE_BT_XFI: 2852 case FW_PORT_TYPE_BT_XAUI: 2853 ifmedia_add(media, m | IFM_10G_T, data, NULL); 2854 /* fall through */ 2855 2856 case FW_PORT_TYPE_BT_SGMII: 2857 ifmedia_add(media, m | IFM_1000_T, data, NULL); 2858 ifmedia_add(media, m | IFM_100_TX, data, NULL); 2859 ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2860 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2861 break; 2862 2863 case FW_PORT_TYPE_CX4: 2864 ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2865 ifmedia_set(media, m | IFM_10G_CX4); 2866 break; 2867 2868 case FW_PORT_TYPE_QSFP_10G: 2869 case FW_PORT_TYPE_SFP: 2870 case FW_PORT_TYPE_FIBER_XFI: 2871 case FW_PORT_TYPE_FIBER_XAUI: 2872 switch (pi->mod_type) { 2873 2874 case FW_PORT_MOD_TYPE_LR: 2875 ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2876 ifmedia_set(media, m | IFM_10G_LR); 2877 break; 2878 2879 case FW_PORT_MOD_TYPE_SR: 2880 ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2881 ifmedia_set(media, m | IFM_10G_SR); 2882 break; 2883 2884 case FW_PORT_MOD_TYPE_LRM: 2885 ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2886 ifmedia_set(media, m | IFM_10G_LRM); 2887 break; 2888 2889 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2890 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2891 ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2892 ifmedia_set(media, m | IFM_10G_TWINAX); 2893 break; 2894 2895 case FW_PORT_MOD_TYPE_NONE: 2896 m &= ~IFM_FDX; 2897 ifmedia_add(media, m | IFM_NONE, data, NULL); 2898 ifmedia_set(media, m | IFM_NONE); 2899 break; 2900 2901 case FW_PORT_MOD_TYPE_NA: 2902 case FW_PORT_MOD_TYPE_ER: 2903 default: 2904 device_printf(pi->dev, 2905 "unknown port_type (%d), mod_type (%d)\n", 2906 pi->port_type, pi->mod_type); 2907 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2908 ifmedia_set(media, m | IFM_UNKNOWN); 2909 break; 2910 } 2911 break; 2912 2913 case FW_PORT_TYPE_QSFP: 2914 switch (pi->mod_type) { 2915 2916 case FW_PORT_MOD_TYPE_LR: 2917 ifmedia_add(media, m | IFM_40G_LR4, data, NULL); 2918 ifmedia_set(media, m | IFM_40G_LR4); 2919 break; 2920 2921 case FW_PORT_MOD_TYPE_SR: 2922 ifmedia_add(media, m | IFM_40G_SR4, data, NULL); 2923 ifmedia_set(media, m | IFM_40G_SR4); 2924 break; 2925 2926 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2927 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2928 ifmedia_add(media, m | IFM_40G_CR4, data, NULL); 2929 ifmedia_set(media, m | IFM_40G_CR4); 2930 break; 2931 2932 case FW_PORT_MOD_TYPE_NONE: 2933 m &= ~IFM_FDX; 2934 ifmedia_add(media, m | IFM_NONE, data, NULL); 2935 ifmedia_set(media, m | IFM_NONE); 2936 break; 2937 2938 default: 2939 device_printf(pi->dev, 2940 "unknown port_type (%d), mod_type (%d)\n", 2941 pi->port_type, pi->mod_type); 2942 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2943 ifmedia_set(media, m | IFM_UNKNOWN); 2944 break; 2945 } 2946 break; 2947 2948 default: 2949 device_printf(pi->dev, 2950 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 2951 pi->mod_type); 2952 ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2953 ifmedia_set(media, m | IFM_UNKNOWN); 2954 break; 2955 } 2956 2957 PORT_UNLOCK(pi); 2958} 2959 2960#define FW_MAC_EXACT_CHUNK 7 2961 2962/* 2963 * Program the port's XGMAC based on parameters in ifnet. The caller also 2964 * indicates which parameters should be programmed (the rest are left alone). 2965 */ 2966int 2967update_mac_settings(struct ifnet *ifp, int flags) 2968{ 2969 int rc = 0; 2970 struct port_info *pi = ifp->if_softc; 2971 struct adapter *sc = pi->adapter; 2972 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2973 uint16_t viid = 0xffff; 2974 int16_t *xact_addr_filt = NULL; 2975 2976 ASSERT_SYNCHRONIZED_OP(sc); 2977 KASSERT(flags, ("%s: not told what to update.", __func__)); 2978 2979 if (ifp == pi->ifp) { 2980 viid = pi->viid; 2981 xact_addr_filt = &pi->xact_addr_filt; 2982 } 2983#ifdef DEV_NETMAP 2984 else if (ifp == pi->nm_ifp) { 2985 viid = pi->nm_viid; 2986 xact_addr_filt = &pi->nm_xact_addr_filt; 2987 } 2988#endif 2989 if (flags & XGMAC_MTU) 2990 mtu = ifp->if_mtu; 2991 2992 if (flags & XGMAC_PROMISC) 2993 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2994 2995 if (flags & XGMAC_ALLMULTI) 2996 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2997 2998 if (flags & XGMAC_VLANEX) 2999 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3000 3001 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3002 rc = -t4_set_rxmode(sc, sc->mbox, viid, mtu, promisc, allmulti, 3003 1, vlanex, false); 3004 if (rc) { 3005 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3006 rc); 3007 return (rc); 3008 } 3009 } 3010 3011 if (flags & XGMAC_UCADDR) { 3012 uint8_t ucaddr[ETHER_ADDR_LEN]; 3013 3014 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3015 rc = t4_change_mac(sc, sc->mbox, viid, *xact_addr_filt, ucaddr, 3016 true, true); 3017 if (rc < 0) { 3018 rc = -rc; 3019 if_printf(ifp, "change_mac failed: %d\n", rc); 3020 return (rc); 3021 } else { 3022 *xact_addr_filt = rc; 3023 rc = 0; 3024 } 3025 } 3026 3027 if (flags & XGMAC_MCADDRS) { 3028 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3029 int del = 1; 3030 uint64_t hash = 0; 3031 struct ifmultiaddr *ifma; 3032 int i = 0, j; 3033 3034 if_maddr_rlock(ifp); 3035 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3036 if (ifma->ifma_addr->sa_family != AF_LINK) 3037 continue; 3038 mcaddr[i] = 3039 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3040 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3041 i++; 3042 3043 if (i == FW_MAC_EXACT_CHUNK) { 3044 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, 3045 i, mcaddr, NULL, &hash, 0); 3046 if (rc < 0) { 3047 rc = -rc; 3048 for (j = 0; j < i; j++) { 3049 if_printf(ifp, 3050 "failed to add mc address" 3051 " %02x:%02x:%02x:" 3052 "%02x:%02x:%02x rc=%d\n", 3053 mcaddr[j][0], mcaddr[j][1], 3054 mcaddr[j][2], mcaddr[j][3], 3055 mcaddr[j][4], mcaddr[j][5], 3056 rc); 3057 } 3058 goto mcfail; 3059 } 3060 del = 0; 3061 i = 0; 3062 } 3063 } 3064 if (i > 0) { 3065 rc = t4_alloc_mac_filt(sc, sc->mbox, viid, del, i, 3066 mcaddr, NULL, &hash, 0); 3067 if (rc < 0) { 3068 rc = -rc; 3069 for (j = 0; j < i; j++) { 3070 if_printf(ifp, 3071 "failed to add mc address" 3072 " %02x:%02x:%02x:" 3073 "%02x:%02x:%02x rc=%d\n", 3074 mcaddr[j][0], mcaddr[j][1], 3075 mcaddr[j][2], mcaddr[j][3], 3076 mcaddr[j][4], mcaddr[j][5], 3077 rc); 3078 } 3079 goto mcfail; 3080 } 3081 } 3082 3083 rc = -t4_set_addr_hash(sc, sc->mbox, viid, 0, hash, 0); 3084 if (rc != 0) 3085 if_printf(ifp, "failed to set mc address hash: %d", rc); 3086mcfail: 3087 if_maddr_runlock(ifp); 3088 } 3089 3090 return (rc); 3091} 3092 3093/* 3094 * {begin|end}_synchronized_op must be called from the same thread. 3095 */ 3096int 3097begin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 3098 char *wmesg) 3099{ 3100 int rc, pri; 3101 3102#ifdef WITNESS 3103 /* the caller thinks it's ok to sleep, but is it really? */ 3104 if (flags & SLEEP_OK) 3105 pause("t4slptst", 1); 3106#endif 3107 3108 if (INTR_OK) 3109 pri = PCATCH; 3110 else 3111 pri = 0; 3112 3113 ADAPTER_LOCK(sc); 3114 for (;;) { 3115 3116 if (pi && IS_DOOMED(pi)) { 3117 rc = ENXIO; 3118 goto done; 3119 } 3120 3121 if (!IS_BUSY(sc)) { 3122 rc = 0; 3123 break; 3124 } 3125 3126 if (!(flags & SLEEP_OK)) { 3127 rc = EBUSY; 3128 goto done; 3129 } 3130 3131 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3132 rc = EINTR; 3133 goto done; 3134 } 3135 } 3136 3137 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3138 SET_BUSY(sc); 3139#ifdef INVARIANTS 3140 sc->last_op = wmesg; 3141 sc->last_op_thr = curthread; 3142 sc->last_op_flags = flags; 3143#endif 3144 3145done: 3146 if (!(flags & HOLD_LOCK) || rc) 3147 ADAPTER_UNLOCK(sc); 3148 3149 return (rc); 3150} 3151 3152/* 3153 * {begin|end}_synchronized_op must be called from the same thread. 3154 */ 3155void 3156end_synchronized_op(struct adapter *sc, int flags) 3157{ 3158 3159 if (flags & LOCK_HELD) 3160 ADAPTER_LOCK_ASSERT_OWNED(sc); 3161 else 3162 ADAPTER_LOCK(sc); 3163 3164 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3165 CLR_BUSY(sc); 3166 wakeup(&sc->flags); 3167 ADAPTER_UNLOCK(sc); 3168} 3169 3170static int 3171cxgbe_init_synchronized(struct port_info *pi) 3172{ 3173 struct adapter *sc = pi->adapter; 3174 struct ifnet *ifp = pi->ifp; 3175 int rc = 0, i; 3176 struct sge_txq *txq; 3177 3178 ASSERT_SYNCHRONIZED_OP(sc); 3179 3180 if (isset(&sc->open_device_map, pi->port_id)) { 3181 KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 3182 ("mismatch between open_device_map and if_drv_flags")); 3183 return (0); /* already running */ 3184 } 3185 3186 if (!(sc->flags & FULL_INIT_DONE) && 3187 ((rc = adapter_full_init(sc)) != 0)) 3188 return (rc); /* error message displayed already */ 3189 3190 if (!(pi->flags & PORT_INIT_DONE) && 3191 ((rc = port_full_init(pi)) != 0)) 3192 return (rc); /* error message displayed already */ 3193 3194 rc = update_mac_settings(ifp, XGMAC_ALL); 3195 if (rc) 3196 goto done; /* error message displayed already */ 3197 3198 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 3199 if (rc != 0) { 3200 if_printf(ifp, "enable_vi failed: %d\n", rc); 3201 goto done; 3202 } 3203 3204 /* 3205 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3206 * if this changes. 3207 */ 3208 3209 for_each_txq(pi, i, txq) { 3210 TXQ_LOCK(txq); 3211 txq->eq.flags |= EQ_ENABLED; 3212 TXQ_UNLOCK(txq); 3213 } 3214 3215 /* 3216 * The first iq of the first port to come up is used for tracing. 3217 */ 3218 if (sc->traceq < 0) { 3219 sc->traceq = sc->sge.rxq[pi->first_rxq].iq.abs_id; 3220 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3221 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3222 V_QUEUENUMBER(sc->traceq)); 3223 pi->flags |= HAS_TRACEQ; 3224 } 3225 3226 /* all ok */ 3227 setbit(&sc->open_device_map, pi->port_id); 3228 PORT_LOCK(pi); 3229 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3230 PORT_UNLOCK(pi); 3231 3232 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3233done: 3234 if (rc != 0) 3235 cxgbe_uninit_synchronized(pi); 3236 3237 return (rc); 3238} 3239 3240/* 3241 * Idempotent. 3242 */ 3243static int 3244cxgbe_uninit_synchronized(struct port_info *pi) 3245{ 3246 struct adapter *sc = pi->adapter; 3247 struct ifnet *ifp = pi->ifp; 3248 int rc, i; 3249 struct sge_txq *txq; 3250 3251 ASSERT_SYNCHRONIZED_OP(sc); 3252 3253 /* 3254 * Disable the VI so that all its data in either direction is discarded 3255 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3256 * tick) intact as the TP can deliver negative advice or data that it's 3257 * holding in its RAM (for an offloaded connection) even after the VI is 3258 * disabled. 3259 */ 3260 rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 3261 if (rc) { 3262 if_printf(ifp, "disable_vi failed: %d\n", rc); 3263 return (rc); 3264 } 3265 3266 for_each_txq(pi, i, txq) { 3267 TXQ_LOCK(txq); 3268 txq->eq.flags &= ~EQ_ENABLED; 3269 TXQ_UNLOCK(txq); 3270 } 3271 3272 clrbit(&sc->open_device_map, pi->port_id); 3273 PORT_LOCK(pi); 3274 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3275 PORT_UNLOCK(pi); 3276 3277 pi->link_cfg.link_ok = 0; 3278 pi->link_cfg.speed = 0; 3279 pi->linkdnrc = -1; 3280 t4_os_link_changed(sc, pi->port_id, 0, -1); 3281 3282 return (0); 3283} 3284 3285/* 3286 * It is ok for this function to fail midway and return right away. t4_detach 3287 * will walk the entire sc->irq list and clean up whatever is valid. 3288 */ 3289static int 3290setup_intr_handlers(struct adapter *sc) 3291{ 3292 int rc, rid, p, q; 3293 char s[8]; 3294 struct irq *irq; 3295 struct port_info *pi; 3296 struct sge_rxq *rxq; 3297#ifdef TCP_OFFLOAD 3298 struct sge_ofld_rxq *ofld_rxq; 3299#endif 3300#ifdef DEV_NETMAP 3301 struct sge_nm_rxq *nm_rxq; 3302#endif 3303 3304 /* 3305 * Setup interrupts. 3306 */ 3307 irq = &sc->irq[0]; 3308 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3309 if (sc->intr_count == 1) 3310 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3311 3312 /* Multiple interrupts. */ 3313 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3314 ("%s: too few intr.", __func__)); 3315 3316 /* The first one is always error intr */ 3317 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3318 if (rc != 0) 3319 return (rc); 3320 irq++; 3321 rid++; 3322 3323 /* The second one is always the firmware event queue */ 3324 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt"); 3325 if (rc != 0) 3326 return (rc); 3327 irq++; 3328 rid++; 3329 3330 for_each_port(sc, p) { 3331 pi = sc->port[p]; 3332 3333 if (pi->flags & INTR_RXQ) { 3334 for_each_rxq(pi, q, rxq) { 3335 snprintf(s, sizeof(s), "%d.%d", p, q); 3336 rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 3337 s); 3338 if (rc != 0) 3339 return (rc); 3340 irq++; 3341 rid++; 3342 } 3343 } 3344#ifdef TCP_OFFLOAD 3345 if (pi->flags & INTR_OFLD_RXQ) { 3346 for_each_ofld_rxq(pi, q, ofld_rxq) { 3347 snprintf(s, sizeof(s), "%d,%d", p, q); 3348 rc = t4_alloc_irq(sc, irq, rid, t4_intr, 3349 ofld_rxq, s); 3350 if (rc != 0) 3351 return (rc); 3352 irq++; 3353 rid++; 3354 } 3355 } 3356#endif 3357#ifdef DEV_NETMAP 3358 if (pi->flags & INTR_NM_RXQ) { 3359 for_each_nm_rxq(pi, q, nm_rxq) { 3360 snprintf(s, sizeof(s), "%d-%d", p, q); 3361 rc = t4_alloc_irq(sc, irq, rid, t4_nm_intr, 3362 nm_rxq, s); 3363 if (rc != 0) 3364 return (rc); 3365 irq++; 3366 rid++; 3367 } 3368 } 3369#endif 3370 } 3371 MPASS(irq == &sc->irq[sc->intr_count]); 3372 3373 return (0); 3374} 3375 3376int 3377adapter_full_init(struct adapter *sc) 3378{ 3379 int rc, i; 3380 3381 ASSERT_SYNCHRONIZED_OP(sc); 3382 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3383 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3384 ("%s: FULL_INIT_DONE already", __func__)); 3385 3386 /* 3387 * queues that belong to the adapter (not any particular port). 3388 */ 3389 rc = t4_setup_adapter_queues(sc); 3390 if (rc != 0) 3391 goto done; 3392 3393 for (i = 0; i < nitems(sc->tq); i++) { 3394 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3395 taskqueue_thread_enqueue, &sc->tq[i]); 3396 if (sc->tq[i] == NULL) { 3397 device_printf(sc->dev, 3398 "failed to allocate task queue %d\n", i); 3399 rc = ENOMEM; 3400 goto done; 3401 } 3402 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3403 device_get_nameunit(sc->dev), i); 3404 } 3405 3406 t4_intr_enable(sc); 3407 sc->flags |= FULL_INIT_DONE; 3408done: 3409 if (rc != 0) 3410 adapter_full_uninit(sc); 3411 3412 return (rc); 3413} 3414 3415int 3416adapter_full_uninit(struct adapter *sc) 3417{ 3418 int i; 3419 3420 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3421 3422 t4_teardown_adapter_queues(sc); 3423 3424 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3425 taskqueue_free(sc->tq[i]); 3426 sc->tq[i] = NULL; 3427 } 3428 3429 sc->flags &= ~FULL_INIT_DONE; 3430 3431 return (0); 3432} 3433 3434int 3435port_full_init(struct port_info *pi) 3436{ 3437 struct adapter *sc = pi->adapter; 3438 struct ifnet *ifp = pi->ifp; 3439 uint16_t *rss; 3440 struct sge_rxq *rxq; 3441 int rc, i, j; 3442 3443 ASSERT_SYNCHRONIZED_OP(sc); 3444 KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3445 ("%s: PORT_INIT_DONE already", __func__)); 3446 3447 sysctl_ctx_init(&pi->ctx); 3448 pi->flags |= PORT_SYSCTL_CTX; 3449 3450 /* 3451 * Allocate tx/rx/fl queues for this port. 3452 */ 3453 rc = t4_setup_port_queues(pi); 3454 if (rc != 0) 3455 goto done; /* error message displayed already */ 3456 3457 /* 3458 * Setup RSS for this port. Save a copy of the RSS table for later use. 3459 */ 3460 rss = malloc(pi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3461 for (i = 0; i < pi->rss_size;) { 3462 for_each_rxq(pi, j, rxq) { 3463 rss[i++] = rxq->iq.abs_id; 3464 if (i == pi->rss_size) 3465 break; 3466 } 3467 } 3468 3469 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss, 3470 pi->rss_size); 3471 if (rc != 0) { 3472 if_printf(ifp, "rss_config failed: %d\n", rc); 3473 goto done; 3474 } 3475 3476 pi->rss = rss; 3477 pi->flags |= PORT_INIT_DONE; 3478done: 3479 if (rc != 0) 3480 port_full_uninit(pi); 3481 3482 return (rc); 3483} 3484 3485/* 3486 * Idempotent. 3487 */ 3488int 3489port_full_uninit(struct port_info *pi) 3490{ 3491 struct adapter *sc = pi->adapter; 3492 int i; 3493 struct sge_rxq *rxq; 3494 struct sge_txq *txq; 3495#ifdef TCP_OFFLOAD 3496 struct sge_ofld_rxq *ofld_rxq; 3497 struct sge_wrq *ofld_txq; 3498#endif 3499 3500 if (pi->flags & PORT_INIT_DONE) { 3501 3502 /* Need to quiesce queues. */ 3503 3504 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 3505 3506 for_each_txq(pi, i, txq) { 3507 quiesce_txq(sc, txq); 3508 } 3509 3510#ifdef TCP_OFFLOAD 3511 for_each_ofld_txq(pi, i, ofld_txq) { 3512 quiesce_wrq(sc, ofld_txq); 3513 } 3514#endif 3515 3516 for_each_rxq(pi, i, rxq) { 3517 quiesce_iq(sc, &rxq->iq); 3518 quiesce_fl(sc, &rxq->fl); 3519 } 3520 3521#ifdef TCP_OFFLOAD 3522 for_each_ofld_rxq(pi, i, ofld_rxq) { 3523 quiesce_iq(sc, &ofld_rxq->iq); 3524 quiesce_fl(sc, &ofld_rxq->fl); 3525 } 3526#endif 3527 free(pi->rss, M_CXGBE); 3528 } 3529 3530 t4_teardown_port_queues(pi); 3531 pi->flags &= ~PORT_INIT_DONE; 3532 3533 return (0); 3534} 3535 3536static void 3537quiesce_txq(struct adapter *sc, struct sge_txq *txq) 3538{ 3539 struct sge_eq *eq = &txq->eq; 3540 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 3541 3542 (void) sc; /* unused */ 3543 3544#ifdef INVARIANTS 3545 TXQ_LOCK(txq); 3546 MPASS((eq->flags & EQ_ENABLED) == 0); 3547 TXQ_UNLOCK(txq); 3548#endif 3549 3550 /* Wait for the mp_ring to empty. */ 3551 while (!mp_ring_is_idle(txq->r)) { 3552 mp_ring_check_drainage(txq->r, 0); 3553 pause("rquiesce", 1); 3554 } 3555 3556 /* Then wait for the hardware to finish. */ 3557 while (spg->cidx != htobe16(eq->pidx)) 3558 pause("equiesce", 1); 3559 3560 /* Finally, wait for the driver to reclaim all descriptors. */ 3561 while (eq->cidx != eq->pidx) 3562 pause("dquiesce", 1); 3563} 3564 3565static void 3566quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 3567{ 3568 3569 /* XXXTX */ 3570} 3571 3572static void 3573quiesce_iq(struct adapter *sc, struct sge_iq *iq) 3574{ 3575 (void) sc; /* unused */ 3576 3577 /* Synchronize with the interrupt handler */ 3578 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3579 pause("iqfree", 1); 3580} 3581 3582static void 3583quiesce_fl(struct adapter *sc, struct sge_fl *fl) 3584{ 3585 mtx_lock(&sc->sfl_lock); 3586 FL_LOCK(fl); 3587 fl->flags |= FL_DOOMED; 3588 FL_UNLOCK(fl); 3589 mtx_unlock(&sc->sfl_lock); 3590 3591 callout_drain(&sc->sfl_callout); 3592 KASSERT((fl->flags & FL_STARVING) == 0, 3593 ("%s: still starving", __func__)); 3594} 3595 3596static int 3597t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3598 driver_intr_t *handler, void *arg, char *name) 3599{ 3600 int rc; 3601 3602 irq->rid = rid; 3603 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3604 RF_SHAREABLE | RF_ACTIVE); 3605 if (irq->res == NULL) { 3606 device_printf(sc->dev, 3607 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3608 return (ENOMEM); 3609 } 3610 3611 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3612 NULL, handler, arg, &irq->tag); 3613 if (rc != 0) { 3614 device_printf(sc->dev, 3615 "failed to setup interrupt for rid %d, name %s: %d\n", 3616 rid, name, rc); 3617 } else if (name) 3618 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3619 3620 return (rc); 3621} 3622 3623static int 3624t4_free_irq(struct adapter *sc, struct irq *irq) 3625{ 3626 if (irq->tag) 3627 bus_teardown_intr(sc->dev, irq->res, irq->tag); 3628 if (irq->res) 3629 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3630 3631 bzero(irq, sizeof(*irq)); 3632 3633 return (0); 3634} 3635 3636static void 3637reg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3638 unsigned int end) 3639{ 3640 uint32_t *p = (uint32_t *)(buf + start); 3641 3642 for ( ; start <= end; start += sizeof(uint32_t)) 3643 *p++ = t4_read_reg(sc, start); 3644} 3645 3646static void 3647t4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3648{ 3649 int i, n; 3650 const unsigned int *reg_ranges; 3651 static const unsigned int t4_reg_ranges[] = { 3652 0x1008, 0x1108, 3653 0x1180, 0x11b4, 3654 0x11fc, 0x123c, 3655 0x1300, 0x173c, 3656 0x1800, 0x18fc, 3657 0x3000, 0x30d8, 3658 0x30e0, 0x5924, 3659 0x5960, 0x59d4, 3660 0x5a00, 0x5af8, 3661 0x6000, 0x6098, 3662 0x6100, 0x6150, 3663 0x6200, 0x6208, 3664 0x6240, 0x6248, 3665 0x6280, 0x6338, 3666 0x6370, 0x638c, 3667 0x6400, 0x643c, 3668 0x6500, 0x6524, 3669 0x6a00, 0x6a38, 3670 0x6a60, 0x6a78, 3671 0x6b00, 0x6b84, 3672 0x6bf0, 0x6c84, 3673 0x6cf0, 0x6d84, 3674 0x6df0, 0x6e84, 3675 0x6ef0, 0x6f84, 3676 0x6ff0, 0x7084, 3677 0x70f0, 0x7184, 3678 0x71f0, 0x7284, 3679 0x72f0, 0x7384, 3680 0x73f0, 0x7450, 3681 0x7500, 0x7530, 3682 0x7600, 0x761c, 3683 0x7680, 0x76cc, 3684 0x7700, 0x7798, 3685 0x77c0, 0x77fc, 3686 0x7900, 0x79fc, 3687 0x7b00, 0x7c38, 3688 0x7d00, 0x7efc, 3689 0x8dc0, 0x8e1c, 3690 0x8e30, 0x8e78, 3691 0x8ea0, 0x8f6c, 3692 0x8fc0, 0x9074, 3693 0x90fc, 0x90fc, 3694 0x9400, 0x9458, 3695 0x9600, 0x96bc, 3696 0x9800, 0x9808, 3697 0x9820, 0x983c, 3698 0x9850, 0x9864, 3699 0x9c00, 0x9c6c, 3700 0x9c80, 0x9cec, 3701 0x9d00, 0x9d6c, 3702 0x9d80, 0x9dec, 3703 0x9e00, 0x9e6c, 3704 0x9e80, 0x9eec, 3705 0x9f00, 0x9f6c, 3706 0x9f80, 0x9fec, 3707 0xd004, 0xd03c, 3708 0xdfc0, 0xdfe0, 3709 0xe000, 0xea7c, 3710 0xf000, 0x11110, 3711 0x11118, 0x11190, 3712 0x19040, 0x1906c, 3713 0x19078, 0x19080, 3714 0x1908c, 0x19124, 3715 0x19150, 0x191b0, 3716 0x191d0, 0x191e8, 3717 0x19238, 0x1924c, 3718 0x193f8, 0x19474, 3719 0x19490, 0x194f8, 3720 0x19800, 0x19f30, 3721 0x1a000, 0x1a06c, 3722 0x1a0b0, 0x1a120, 3723 0x1a128, 0x1a138, 3724 0x1a190, 0x1a1c4, 3725 0x1a1fc, 0x1a1fc, 3726 0x1e040, 0x1e04c, 3727 0x1e284, 0x1e28c, 3728 0x1e2c0, 0x1e2c0, 3729 0x1e2e0, 0x1e2e0, 3730 0x1e300, 0x1e384, 3731 0x1e3c0, 0x1e3c8, 3732 0x1e440, 0x1e44c, 3733 0x1e684, 0x1e68c, 3734 0x1e6c0, 0x1e6c0, 3735 0x1e6e0, 0x1e6e0, 3736 0x1e700, 0x1e784, 3737 0x1e7c0, 0x1e7c8, 3738 0x1e840, 0x1e84c, 3739 0x1ea84, 0x1ea8c, 3740 0x1eac0, 0x1eac0, 3741 0x1eae0, 0x1eae0, 3742 0x1eb00, 0x1eb84, 3743 0x1ebc0, 0x1ebc8, 3744 0x1ec40, 0x1ec4c, 3745 0x1ee84, 0x1ee8c, 3746 0x1eec0, 0x1eec0, 3747 0x1eee0, 0x1eee0, 3748 0x1ef00, 0x1ef84, 3749 0x1efc0, 0x1efc8, 3750 0x1f040, 0x1f04c, 3751 0x1f284, 0x1f28c, 3752 0x1f2c0, 0x1f2c0, 3753 0x1f2e0, 0x1f2e0, 3754 0x1f300, 0x1f384, 3755 0x1f3c0, 0x1f3c8, 3756 0x1f440, 0x1f44c, 3757 0x1f684, 0x1f68c, 3758 0x1f6c0, 0x1f6c0, 3759 0x1f6e0, 0x1f6e0, 3760 0x1f700, 0x1f784, 3761 0x1f7c0, 0x1f7c8, 3762 0x1f840, 0x1f84c, 3763 0x1fa84, 0x1fa8c, 3764 0x1fac0, 0x1fac0, 3765 0x1fae0, 0x1fae0, 3766 0x1fb00, 0x1fb84, 3767 0x1fbc0, 0x1fbc8, 3768 0x1fc40, 0x1fc4c, 3769 0x1fe84, 0x1fe8c, 3770 0x1fec0, 0x1fec0, 3771 0x1fee0, 0x1fee0, 3772 0x1ff00, 0x1ff84, 3773 0x1ffc0, 0x1ffc8, 3774 0x20000, 0x2002c, 3775 0x20100, 0x2013c, 3776 0x20190, 0x201c8, 3777 0x20200, 0x20318, 3778 0x20400, 0x20528, 3779 0x20540, 0x20614, 3780 0x21000, 0x21040, 3781 0x2104c, 0x21060, 3782 0x210c0, 0x210ec, 3783 0x21200, 0x21268, 3784 0x21270, 0x21284, 3785 0x212fc, 0x21388, 3786 0x21400, 0x21404, 3787 0x21500, 0x21518, 3788 0x2152c, 0x2153c, 3789 0x21550, 0x21554, 3790 0x21600, 0x21600, 3791 0x21608, 0x21628, 3792 0x21630, 0x2163c, 3793 0x21700, 0x2171c, 3794 0x21780, 0x2178c, 3795 0x21800, 0x21c38, 3796 0x21c80, 0x21d7c, 3797 0x21e00, 0x21e04, 3798 0x22000, 0x2202c, 3799 0x22100, 0x2213c, 3800 0x22190, 0x221c8, 3801 0x22200, 0x22318, 3802 0x22400, 0x22528, 3803 0x22540, 0x22614, 3804 0x23000, 0x23040, 3805 0x2304c, 0x23060, 3806 0x230c0, 0x230ec, 3807 0x23200, 0x23268, 3808 0x23270, 0x23284, 3809 0x232fc, 0x23388, 3810 0x23400, 0x23404, 3811 0x23500, 0x23518, 3812 0x2352c, 0x2353c, 3813 0x23550, 0x23554, 3814 0x23600, 0x23600, 3815 0x23608, 0x23628, 3816 0x23630, 0x2363c, 3817 0x23700, 0x2371c, 3818 0x23780, 0x2378c, 3819 0x23800, 0x23c38, 3820 0x23c80, 0x23d7c, 3821 0x23e00, 0x23e04, 3822 0x24000, 0x2402c, 3823 0x24100, 0x2413c, 3824 0x24190, 0x241c8, 3825 0x24200, 0x24318, 3826 0x24400, 0x24528, 3827 0x24540, 0x24614, 3828 0x25000, 0x25040, 3829 0x2504c, 0x25060, 3830 0x250c0, 0x250ec, 3831 0x25200, 0x25268, 3832 0x25270, 0x25284, 3833 0x252fc, 0x25388, 3834 0x25400, 0x25404, 3835 0x25500, 0x25518, 3836 0x2552c, 0x2553c, 3837 0x25550, 0x25554, 3838 0x25600, 0x25600, 3839 0x25608, 0x25628, 3840 0x25630, 0x2563c, 3841 0x25700, 0x2571c, 3842 0x25780, 0x2578c, 3843 0x25800, 0x25c38, 3844 0x25c80, 0x25d7c, 3845 0x25e00, 0x25e04, 3846 0x26000, 0x2602c, 3847 0x26100, 0x2613c, 3848 0x26190, 0x261c8, 3849 0x26200, 0x26318, 3850 0x26400, 0x26528, 3851 0x26540, 0x26614, 3852 0x27000, 0x27040, 3853 0x2704c, 0x27060, 3854 0x270c0, 0x270ec, 3855 0x27200, 0x27268, 3856 0x27270, 0x27284, 3857 0x272fc, 0x27388, 3858 0x27400, 0x27404, 3859 0x27500, 0x27518, 3860 0x2752c, 0x2753c, 3861 0x27550, 0x27554, 3862 0x27600, 0x27600, 3863 0x27608, 0x27628, 3864 0x27630, 0x2763c, 3865 0x27700, 0x2771c, 3866 0x27780, 0x2778c, 3867 0x27800, 0x27c38, 3868 0x27c80, 0x27d7c, 3869 0x27e00, 0x27e04 3870 }; 3871 static const unsigned int t5_reg_ranges[] = { 3872 0x1008, 0x1148, 3873 0x1180, 0x11b4, 3874 0x11fc, 0x123c, 3875 0x1280, 0x173c, 3876 0x1800, 0x18fc, 3877 0x3000, 0x3028, 3878 0x3060, 0x30d8, 3879 0x30e0, 0x30fc, 3880 0x3140, 0x357c, 3881 0x35a8, 0x35cc, 3882 0x35ec, 0x35ec, 3883 0x3600, 0x5624, 3884 0x56cc, 0x575c, 3885 0x580c, 0x5814, 3886 0x5890, 0x58bc, 3887 0x5940, 0x59dc, 3888 0x59fc, 0x5a18, 3889 0x5a60, 0x5a9c, 3890 0x5b94, 0x5bfc, 3891 0x6000, 0x6040, 3892 0x6058, 0x614c, 3893 0x7700, 0x7798, 3894 0x77c0, 0x78fc, 3895 0x7b00, 0x7c54, 3896 0x7d00, 0x7efc, 3897 0x8dc0, 0x8de0, 3898 0x8df8, 0x8e84, 3899 0x8ea0, 0x8f84, 3900 0x8fc0, 0x90f8, 3901 0x9400, 0x9470, 3902 0x9600, 0x96f4, 3903 0x9800, 0x9808, 3904 0x9820, 0x983c, 3905 0x9850, 0x9864, 3906 0x9c00, 0x9c6c, 3907 0x9c80, 0x9cec, 3908 0x9d00, 0x9d6c, 3909 0x9d80, 0x9dec, 3910 0x9e00, 0x9e6c, 3911 0x9e80, 0x9eec, 3912 0x9f00, 0x9f6c, 3913 0x9f80, 0xa020, 3914 0xd004, 0xd03c, 3915 0xdfc0, 0xdfe0, 3916 0xe000, 0x11088, 3917 0x1109c, 0x11110, 3918 0x11118, 0x1117c, 3919 0x11190, 0x11204, 3920 0x19040, 0x1906c, 3921 0x19078, 0x19080, 3922 0x1908c, 0x19124, 3923 0x19150, 0x191b0, 3924 0x191d0, 0x191e8, 3925 0x19238, 0x19290, 3926 0x193f8, 0x19474, 3927 0x19490, 0x194cc, 3928 0x194f0, 0x194f8, 3929 0x19c00, 0x19c60, 3930 0x19c94, 0x19e10, 3931 0x19e50, 0x19f34, 3932 0x19f40, 0x19f50, 3933 0x19f90, 0x19fe4, 3934 0x1a000, 0x1a06c, 3935 0x1a0b0, 0x1a120, 3936 0x1a128, 0x1a138, 3937 0x1a190, 0x1a1c4, 3938 0x1a1fc, 0x1a1fc, 3939 0x1e008, 0x1e00c, 3940 0x1e040, 0x1e04c, 3941 0x1e284, 0x1e290, 3942 0x1e2c0, 0x1e2c0, 3943 0x1e2e0, 0x1e2e0, 3944 0x1e300, 0x1e384, 3945 0x1e3c0, 0x1e3c8, 3946 0x1e408, 0x1e40c, 3947 0x1e440, 0x1e44c, 3948 0x1e684, 0x1e690, 3949 0x1e6c0, 0x1e6c0, 3950 0x1e6e0, 0x1e6e0, 3951 0x1e700, 0x1e784, 3952 0x1e7c0, 0x1e7c8, 3953 0x1e808, 0x1e80c, 3954 0x1e840, 0x1e84c, 3955 0x1ea84, 0x1ea90, 3956 0x1eac0, 0x1eac0, 3957 0x1eae0, 0x1eae0, 3958 0x1eb00, 0x1eb84, 3959 0x1ebc0, 0x1ebc8, 3960 0x1ec08, 0x1ec0c, 3961 0x1ec40, 0x1ec4c, 3962 0x1ee84, 0x1ee90, 3963 0x1eec0, 0x1eec0, 3964 0x1eee0, 0x1eee0, 3965 0x1ef00, 0x1ef84, 3966 0x1efc0, 0x1efc8, 3967 0x1f008, 0x1f00c, 3968 0x1f040, 0x1f04c, 3969 0x1f284, 0x1f290, 3970 0x1f2c0, 0x1f2c0, 3971 0x1f2e0, 0x1f2e0, 3972 0x1f300, 0x1f384, 3973 0x1f3c0, 0x1f3c8, 3974 0x1f408, 0x1f40c, 3975 0x1f440, 0x1f44c, 3976 0x1f684, 0x1f690, 3977 0x1f6c0, 0x1f6c0, 3978 0x1f6e0, 0x1f6e0, 3979 0x1f700, 0x1f784, 3980 0x1f7c0, 0x1f7c8, 3981 0x1f808, 0x1f80c, 3982 0x1f840, 0x1f84c, 3983 0x1fa84, 0x1fa90, 3984 0x1fac0, 0x1fac0, 3985 0x1fae0, 0x1fae0, 3986 0x1fb00, 0x1fb84, 3987 0x1fbc0, 0x1fbc8, 3988 0x1fc08, 0x1fc0c, 3989 0x1fc40, 0x1fc4c, 3990 0x1fe84, 0x1fe90, 3991 0x1fec0, 0x1fec0, 3992 0x1fee0, 0x1fee0, 3993 0x1ff00, 0x1ff84, 3994 0x1ffc0, 0x1ffc8, 3995 0x30000, 0x30030, 3996 0x30100, 0x30144, 3997 0x30190, 0x301d0, 3998 0x30200, 0x30318, 3999 0x30400, 0x3052c, 4000 0x30540, 0x3061c, 4001 0x30800, 0x30834, 4002 0x308c0, 0x30908, 4003 0x30910, 0x309ac, 4004 0x30a00, 0x30a2c, 4005 0x30a44, 0x30a50, 4006 0x30a74, 0x30c24, 4007 0x30d00, 0x30d00, 4008 0x30d08, 0x30d14, 4009 0x30d1c, 0x30d20, 4010 0x30d3c, 0x30d50, 4011 0x31200, 0x3120c, 4012 0x31220, 0x31220, 4013 0x31240, 0x31240, 4014 0x31600, 0x3160c, 4015 0x31a00, 0x31a1c, 4016 0x31e00, 0x31e20, 4017 0x31e38, 0x31e3c, 4018 0x31e80, 0x31e80, 4019 0x31e88, 0x31ea8, 4020 0x31eb0, 0x31eb4, 4021 0x31ec8, 0x31ed4, 4022 0x31fb8, 0x32004, 4023 0x32200, 0x32200, 4024 0x32208, 0x32240, 4025 0x32248, 0x32280, 4026 0x32288, 0x322c0, 4027 0x322c8, 0x322fc, 4028 0x32600, 0x32630, 4029 0x32a00, 0x32abc, 4030 0x32b00, 0x32b70, 4031 0x33000, 0x33048, 4032 0x33060, 0x3309c, 4033 0x330f0, 0x33148, 4034 0x33160, 0x3319c, 4035 0x331f0, 0x332e4, 4036 0x332f8, 0x333e4, 4037 0x333f8, 0x33448, 4038 0x33460, 0x3349c, 4039 0x334f0, 0x33548, 4040 0x33560, 0x3359c, 4041 0x335f0, 0x336e4, 4042 0x336f8, 0x337e4, 4043 0x337f8, 0x337fc, 4044 0x33814, 0x33814, 4045 0x3382c, 0x3382c, 4046 0x33880, 0x3388c, 4047 0x338e8, 0x338ec, 4048 0x33900, 0x33948, 4049 0x33960, 0x3399c, 4050 0x339f0, 0x33ae4, 4051 0x33af8, 0x33b10, 4052 0x33b28, 0x33b28, 4053 0x33b3c, 0x33b50, 4054 0x33bf0, 0x33c10, 4055 0x33c28, 0x33c28, 4056 0x33c3c, 0x33c50, 4057 0x33cf0, 0x33cfc, 4058 0x34000, 0x34030, 4059 0x34100, 0x34144, 4060 0x34190, 0x341d0, 4061 0x34200, 0x34318, 4062 0x34400, 0x3452c, 4063 0x34540, 0x3461c, 4064 0x34800, 0x34834, 4065 0x348c0, 0x34908, 4066 0x34910, 0x349ac, 4067 0x34a00, 0x34a2c, 4068 0x34a44, 0x34a50, 4069 0x34a74, 0x34c24, 4070 0x34d00, 0x34d00, 4071 0x34d08, 0x34d14, 4072 0x34d1c, 0x34d20, 4073 0x34d3c, 0x34d50, 4074 0x35200, 0x3520c, 4075 0x35220, 0x35220, 4076 0x35240, 0x35240, 4077 0x35600, 0x3560c, 4078 0x35a00, 0x35a1c, 4079 0x35e00, 0x35e20, 4080 0x35e38, 0x35e3c, 4081 0x35e80, 0x35e80, 4082 0x35e88, 0x35ea8, 4083 0x35eb0, 0x35eb4, 4084 0x35ec8, 0x35ed4, 4085 0x35fb8, 0x36004, 4086 0x36200, 0x36200, 4087 0x36208, 0x36240, 4088 0x36248, 0x36280, 4089 0x36288, 0x362c0, 4090 0x362c8, 0x362fc, 4091 0x36600, 0x36630, 4092 0x36a00, 0x36abc, 4093 0x36b00, 0x36b70, 4094 0x37000, 0x37048, 4095 0x37060, 0x3709c, 4096 0x370f0, 0x37148, 4097 0x37160, 0x3719c, 4098 0x371f0, 0x372e4, 4099 0x372f8, 0x373e4, 4100 0x373f8, 0x37448, 4101 0x37460, 0x3749c, 4102 0x374f0, 0x37548, 4103 0x37560, 0x3759c, 4104 0x375f0, 0x376e4, 4105 0x376f8, 0x377e4, 4106 0x377f8, 0x377fc, 4107 0x37814, 0x37814, 4108 0x3782c, 0x3782c, 4109 0x37880, 0x3788c, 4110 0x378e8, 0x378ec, 4111 0x37900, 0x37948, 4112 0x37960, 0x3799c, 4113 0x379f0, 0x37ae4, 4114 0x37af8, 0x37b10, 4115 0x37b28, 0x37b28, 4116 0x37b3c, 0x37b50, 4117 0x37bf0, 0x37c10, 4118 0x37c28, 0x37c28, 4119 0x37c3c, 0x37c50, 4120 0x37cf0, 0x37cfc, 4121 0x38000, 0x38030, 4122 0x38100, 0x38144, 4123 0x38190, 0x381d0, 4124 0x38200, 0x38318, 4125 0x38400, 0x3852c, 4126 0x38540, 0x3861c, 4127 0x38800, 0x38834, 4128 0x388c0, 0x38908, 4129 0x38910, 0x389ac, 4130 0x38a00, 0x38a2c, 4131 0x38a44, 0x38a50, 4132 0x38a74, 0x38c24, 4133 0x38d00, 0x38d00, 4134 0x38d08, 0x38d14, 4135 0x38d1c, 0x38d20, 4136 0x38d3c, 0x38d50, 4137 0x39200, 0x3920c, 4138 0x39220, 0x39220, 4139 0x39240, 0x39240, 4140 0x39600, 0x3960c, 4141 0x39a00, 0x39a1c, 4142 0x39e00, 0x39e20, 4143 0x39e38, 0x39e3c, 4144 0x39e80, 0x39e80, 4145 0x39e88, 0x39ea8, 4146 0x39eb0, 0x39eb4, 4147 0x39ec8, 0x39ed4, 4148 0x39fb8, 0x3a004, 4149 0x3a200, 0x3a200, 4150 0x3a208, 0x3a240, 4151 0x3a248, 0x3a280, 4152 0x3a288, 0x3a2c0, 4153 0x3a2c8, 0x3a2fc, 4154 0x3a600, 0x3a630, 4155 0x3aa00, 0x3aabc, 4156 0x3ab00, 0x3ab70, 4157 0x3b000, 0x3b048, 4158 0x3b060, 0x3b09c, 4159 0x3b0f0, 0x3b148, 4160 0x3b160, 0x3b19c, 4161 0x3b1f0, 0x3b2e4, 4162 0x3b2f8, 0x3b3e4, 4163 0x3b3f8, 0x3b448, 4164 0x3b460, 0x3b49c, 4165 0x3b4f0, 0x3b548, 4166 0x3b560, 0x3b59c, 4167 0x3b5f0, 0x3b6e4, 4168 0x3b6f8, 0x3b7e4, 4169 0x3b7f8, 0x3b7fc, 4170 0x3b814, 0x3b814, 4171 0x3b82c, 0x3b82c, 4172 0x3b880, 0x3b88c, 4173 0x3b8e8, 0x3b8ec, 4174 0x3b900, 0x3b948, 4175 0x3b960, 0x3b99c, 4176 0x3b9f0, 0x3bae4, 4177 0x3baf8, 0x3bb10, 4178 0x3bb28, 0x3bb28, 4179 0x3bb3c, 0x3bb50, 4180 0x3bbf0, 0x3bc10, 4181 0x3bc28, 0x3bc28, 4182 0x3bc3c, 0x3bc50, 4183 0x3bcf0, 0x3bcfc, 4184 0x3c000, 0x3c030, 4185 0x3c100, 0x3c144, 4186 0x3c190, 0x3c1d0, 4187 0x3c200, 0x3c318, 4188 0x3c400, 0x3c52c, 4189 0x3c540, 0x3c61c, 4190 0x3c800, 0x3c834, 4191 0x3c8c0, 0x3c908, 4192 0x3c910, 0x3c9ac, 4193 0x3ca00, 0x3ca2c, 4194 0x3ca44, 0x3ca50, 4195 0x3ca74, 0x3cc24, 4196 0x3cd00, 0x3cd00, 4197 0x3cd08, 0x3cd14, 4198 0x3cd1c, 0x3cd20, 4199 0x3cd3c, 0x3cd50, 4200 0x3d200, 0x3d20c, 4201 0x3d220, 0x3d220, 4202 0x3d240, 0x3d240, 4203 0x3d600, 0x3d60c, 4204 0x3da00, 0x3da1c, 4205 0x3de00, 0x3de20, 4206 0x3de38, 0x3de3c, 4207 0x3de80, 0x3de80, 4208 0x3de88, 0x3dea8, 4209 0x3deb0, 0x3deb4, 4210 0x3dec8, 0x3ded4, 4211 0x3dfb8, 0x3e004, 4212 0x3e200, 0x3e200, 4213 0x3e208, 0x3e240, 4214 0x3e248, 0x3e280, 4215 0x3e288, 0x3e2c0, 4216 0x3e2c8, 0x3e2fc, 4217 0x3e600, 0x3e630, 4218 0x3ea00, 0x3eabc, 4219 0x3eb00, 0x3eb70, 4220 0x3f000, 0x3f048, 4221 0x3f060, 0x3f09c, 4222 0x3f0f0, 0x3f148, 4223 0x3f160, 0x3f19c, 4224 0x3f1f0, 0x3f2e4, 4225 0x3f2f8, 0x3f3e4, 4226 0x3f3f8, 0x3f448, 4227 0x3f460, 0x3f49c, 4228 0x3f4f0, 0x3f548, 4229 0x3f560, 0x3f59c, 4230 0x3f5f0, 0x3f6e4, 4231 0x3f6f8, 0x3f7e4, 4232 0x3f7f8, 0x3f7fc, 4233 0x3f814, 0x3f814, 4234 0x3f82c, 0x3f82c, 4235 0x3f880, 0x3f88c, 4236 0x3f8e8, 0x3f8ec, 4237 0x3f900, 0x3f948, 4238 0x3f960, 0x3f99c, 4239 0x3f9f0, 0x3fae4, 4240 0x3faf8, 0x3fb10, 4241 0x3fb28, 0x3fb28, 4242 0x3fb3c, 0x3fb50, 4243 0x3fbf0, 0x3fc10, 4244 0x3fc28, 0x3fc28, 4245 0x3fc3c, 0x3fc50, 4246 0x3fcf0, 0x3fcfc, 4247 0x40000, 0x4000c, 4248 0x40040, 0x40068, 4249 0x4007c, 0x40144, 4250 0x40180, 0x4018c, 4251 0x40200, 0x40298, 4252 0x402ac, 0x4033c, 4253 0x403f8, 0x403fc, 4254 0x41304, 0x413c4, 4255 0x41400, 0x4141c, 4256 0x41480, 0x414d0, 4257 0x44000, 0x44078, 4258 0x440c0, 0x44278, 4259 0x442c0, 0x44478, 4260 0x444c0, 0x44678, 4261 0x446c0, 0x44878, 4262 0x448c0, 0x449fc, 4263 0x45000, 0x45068, 4264 0x45080, 0x45084, 4265 0x450a0, 0x450b0, 4266 0x45200, 0x45268, 4267 0x45280, 0x45284, 4268 0x452a0, 0x452b0, 4269 0x460c0, 0x460e4, 4270 0x47000, 0x4708c, 4271 0x47200, 0x47250, 4272 0x47400, 0x47420, 4273 0x47600, 0x47618, 4274 0x47800, 0x47814, 4275 0x48000, 0x4800c, 4276 0x48040, 0x48068, 4277 0x4807c, 0x48144, 4278 0x48180, 0x4818c, 4279 0x48200, 0x48298, 4280 0x482ac, 0x4833c, 4281 0x483f8, 0x483fc, 4282 0x49304, 0x493c4, 4283 0x49400, 0x4941c, 4284 0x49480, 0x494d0, 4285 0x4c000, 0x4c078, 4286 0x4c0c0, 0x4c278, 4287 0x4c2c0, 0x4c478, 4288 0x4c4c0, 0x4c678, 4289 0x4c6c0, 0x4c878, 4290 0x4c8c0, 0x4c9fc, 4291 0x4d000, 0x4d068, 4292 0x4d080, 0x4d084, 4293 0x4d0a0, 0x4d0b0, 4294 0x4d200, 0x4d268, 4295 0x4d280, 0x4d284, 4296 0x4d2a0, 0x4d2b0, 4297 0x4e0c0, 0x4e0e4, 4298 0x4f000, 0x4f08c, 4299 0x4f200, 0x4f250, 4300 0x4f400, 0x4f420, 4301 0x4f600, 0x4f618, 4302 0x4f800, 0x4f814, 4303 0x50000, 0x500cc, 4304 0x50400, 0x50400, 4305 0x50800, 0x508cc, 4306 0x50c00, 0x50c00, 4307 0x51000, 0x5101c, 4308 0x51300, 0x51308, 4309 }; 4310 4311 if (is_t4(sc)) { 4312 reg_ranges = &t4_reg_ranges[0]; 4313 n = nitems(t4_reg_ranges); 4314 } else { 4315 reg_ranges = &t5_reg_ranges[0]; 4316 n = nitems(t5_reg_ranges); 4317 } 4318 4319 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4320 for (i = 0; i < n; i += 2) 4321 reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 4322} 4323 4324static void 4325cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4326{ 4327 struct ifnet *ifp = pi->ifp; 4328 struct sge_txq *txq; 4329 int i, drops; 4330 struct port_stats *s = &pi->stats; 4331 struct timeval tv; 4332 const struct timeval interval = {0, 250000}; /* 250ms */ 4333 4334 getmicrotime(&tv); 4335 timevalsub(&tv, &interval); 4336 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4337 return; 4338 4339 t4_get_port_stats(sc, pi->tx_chan, s); 4340 4341 ifp->if_opackets = s->tx_frames - s->tx_pause; 4342 ifp->if_ipackets = s->rx_frames - s->rx_pause; 4343 ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 4344 ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 4345 ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 4346 ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 4347 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4348 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4349 s->rx_trunc3; 4350 for (i = 0; i < NCHAN; i++) { 4351 if (pi->rx_chan_map & (1 << i)) { 4352 uint32_t v; 4353 4354 mtx_lock(&sc->regwin_lock); 4355 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4356 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4357 mtx_unlock(&sc->regwin_lock); 4358 ifp->if_iqdrops += v; 4359 } 4360 } 4361 4362 drops = s->tx_drop; 4363 for_each_txq(pi, i, txq) 4364 drops += counter_u64_fetch(txq->r->drops); 4365 ifp->if_snd.ifq_drops = drops; 4366 4367 ifp->if_oerrors = s->tx_error_frames; 4368 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4369 s->rx_fcs_err + s->rx_len_err; 4370 4371 getmicrotime(&pi->last_refreshed); 4372} 4373 4374static void 4375cxgbe_tick(void *arg) 4376{ 4377 struct port_info *pi = arg; 4378 struct adapter *sc = pi->adapter; 4379 struct ifnet *ifp = pi->ifp; 4380 4381 PORT_LOCK(pi); 4382 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4383 PORT_UNLOCK(pi); 4384 return; /* without scheduling another callout */ 4385 } 4386 4387 cxgbe_refresh_stats(sc, pi); 4388 4389 callout_schedule(&pi->tick, hz); 4390 PORT_UNLOCK(pi); 4391} 4392 4393static void 4394cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4395{ 4396 struct ifnet *vlan; 4397 4398 if (arg != ifp || ifp->if_type != IFT_ETHER) 4399 return; 4400 4401 vlan = VLAN_DEVAT(ifp, vid); 4402 VLAN_SETCOOKIE(vlan, ifp); 4403} 4404 4405static int 4406cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4407{ 4408 4409#ifdef INVARIANTS 4410 panic("%s: opcode 0x%02x on iq %p with payload %p", 4411 __func__, rss->opcode, iq, m); 4412#else 4413 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4414 __func__, rss->opcode, iq, m); 4415 m_freem(m); 4416#endif 4417 return (EDOOFUS); 4418} 4419 4420int 4421t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4422{ 4423 uintptr_t *loc, new; 4424 4425 if (opcode >= nitems(sc->cpl_handler)) 4426 return (EINVAL); 4427 4428 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4429 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4430 atomic_store_rel_ptr(loc, new); 4431 4432 return (0); 4433} 4434 4435static int 4436an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4437{ 4438 4439#ifdef INVARIANTS 4440 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4441#else 4442 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4443 __func__, iq, ctrl); 4444#endif 4445 return (EDOOFUS); 4446} 4447 4448int 4449t4_register_an_handler(struct adapter *sc, an_handler_t h) 4450{ 4451 uintptr_t *loc, new; 4452 4453 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4454 loc = (uintptr_t *) &sc->an_handler; 4455 atomic_store_rel_ptr(loc, new); 4456 4457 return (0); 4458} 4459 4460static int 4461fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4462{ 4463 const struct cpl_fw6_msg *cpl = 4464 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4465 4466#ifdef INVARIANTS 4467 panic("%s: fw_msg type %d", __func__, cpl->type); 4468#else 4469 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4470#endif 4471 return (EDOOFUS); 4472} 4473 4474int 4475t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4476{ 4477 uintptr_t *loc, new; 4478 4479 if (type >= nitems(sc->fw_msg_handler)) 4480 return (EINVAL); 4481 4482 /* 4483 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4484 * handler dispatch table. Reject any attempt to install a handler for 4485 * this subtype. 4486 */ 4487 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4488 return (EINVAL); 4489 4490 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4491 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4492 atomic_store_rel_ptr(loc, new); 4493 4494 return (0); 4495} 4496 4497static int 4498t4_sysctls(struct adapter *sc) 4499{ 4500 struct sysctl_ctx_list *ctx; 4501 struct sysctl_oid *oid; 4502 struct sysctl_oid_list *children, *c0; 4503 static char *caps[] = { 4504 "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4505 "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL" /* caps[1] niccaps */ 4506 "\6HASHFILTER\7ETHOFLD", 4507 "\20\1TOE", /* caps[2] toecaps */ 4508 "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4509 "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4510 "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4511 "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4512 "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4513 "\4PO_INITIAOR\5PO_TARGET" 4514 }; 4515 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4516 4517 ctx = device_get_sysctl_ctx(sc->dev); 4518 4519 /* 4520 * dev.t4nex.X. 4521 */ 4522 oid = device_get_sysctl_tree(sc->dev); 4523 c0 = children = SYSCTL_CHILDREN(oid); 4524 4525 sc->sc_do_rxcopy = 1; 4526 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4527 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4528 4529 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4530 sc->params.nports, "# of ports"); 4531 4532 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4533 NULL, chip_rev(sc), "chip hardware revision"); 4534 4535 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4536 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4537 4538 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4539 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4540 4541 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4542 sc->cfcsum, "config file checksum"); 4543 4544 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4545 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4546 sysctl_bitfield, "A", "available doorbells"); 4547 4548 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4549 CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4550 sysctl_bitfield, "A", "available link capabilities"); 4551 4552 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4553 CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4554 sysctl_bitfield, "A", "available NIC capabilities"); 4555 4556 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4557 CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4558 sysctl_bitfield, "A", "available TCP offload capabilities"); 4559 4560 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4561 CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4562 sysctl_bitfield, "A", "available RDMA capabilities"); 4563 4564 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4565 CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4566 sysctl_bitfield, "A", "available iSCSI capabilities"); 4567 4568 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4569 CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4570 sysctl_bitfield, "A", "available FCoE capabilities"); 4571 4572 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4573 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4574 4575 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4576 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4577 sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4578 "interrupt holdoff timer values (us)"); 4579 4580 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4581 CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4582 sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4583 "interrupt holdoff packet counter values"); 4584 4585 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4586 NULL, sc->tids.nftids, "number of filters"); 4587 4588 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4589 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4590 "chip temperature (in Celsius)"); 4591 4592 t4_sge_sysctls(sc, ctx, children); 4593 4594 sc->lro_timeout = 100; 4595 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4596 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4597 4598 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 4599 &sc->debug_flags, 0, "flags to enable runtime debugging"); 4600 4601#ifdef SBUF_DRAIN 4602 /* 4603 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4604 */ 4605 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4606 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4607 "logs and miscellaneous information"); 4608 children = SYSCTL_CHILDREN(oid); 4609 4610 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4611 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4612 sysctl_cctrl, "A", "congestion control"); 4613 4614 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4615 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4616 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4617 4618 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4619 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4620 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4621 4622 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4623 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4624 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4625 4626 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4627 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4628 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4629 4630 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4631 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4632 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4633 4634 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4635 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4636 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4637 4638 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4639 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4640 sysctl_cim_la, "A", "CIM logic analyzer"); 4641 4642 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4643 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4644 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4645 4646 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4647 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4648 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4649 4650 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4651 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4652 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4653 4654 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4655 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4656 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4657 4658 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4659 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4660 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4661 4662 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4663 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4664 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4665 4666 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4667 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4668 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4669 4670 if (is_t5(sc)) { 4671 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4672 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4673 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4674 4675 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4676 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4677 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4678 } 4679 4680 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4681 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4682 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4683 4684 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4685 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4686 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4687 4688 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4689 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4690 sysctl_cpl_stats, "A", "CPL statistics"); 4691 4692 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4693 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4694 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4695 4696 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4697 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4698 sysctl_devlog, "A", "firmware's device log"); 4699 4700 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4701 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4702 sysctl_fcoe_stats, "A", "FCoE statistics"); 4703 4704 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4705 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4706 sysctl_hw_sched, "A", "hardware scheduler "); 4707 4708 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4709 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4710 sysctl_l2t, "A", "hardware L2 table"); 4711 4712 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4713 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4714 sysctl_lb_stats, "A", "loopback statistics"); 4715 4716 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4717 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4718 sysctl_meminfo, "A", "memory regions"); 4719 4720 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4721 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4722 sysctl_mps_tcam, "A", "MPS TCAM entries"); 4723 4724 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4725 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4726 sysctl_path_mtus, "A", "path MTUs"); 4727 4728 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4729 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4730 sysctl_pm_stats, "A", "PM statistics"); 4731 4732 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4733 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4734 sysctl_rdma_stats, "A", "RDMA statistics"); 4735 4736 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4737 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4738 sysctl_tcp_stats, "A", "TCP statistics"); 4739 4740 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4741 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4742 sysctl_tids, "A", "TID information"); 4743 4744 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4745 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4746 sysctl_tp_err_stats, "A", "TP error statistics"); 4747 4748 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4749 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4750 sysctl_tp_la, "A", "TP logic analyzer"); 4751 4752 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4753 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4754 sysctl_tx_rate, "A", "Tx rate"); 4755 4756 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4757 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4758 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4759 4760 if (is_t5(sc)) { 4761 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4762 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4763 sysctl_wcwr_stats, "A", "write combined work requests"); 4764 } 4765#endif 4766 4767#ifdef TCP_OFFLOAD 4768 if (is_offload(sc)) { 4769 /* 4770 * dev.t4nex.X.toe. 4771 */ 4772 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4773 NULL, "TOE parameters"); 4774 children = SYSCTL_CHILDREN(oid); 4775 4776 sc->tt.sndbuf = 256 * 1024; 4777 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4778 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4779 4780 sc->tt.ddp = 0; 4781 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4782 &sc->tt.ddp, 0, "DDP allowed"); 4783 4784 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4785 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4786 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4787 4788 sc->tt.ddp_thres = 4789 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4790 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4791 &sc->tt.ddp_thres, 0, "DDP threshold"); 4792 4793 sc->tt.rx_coalesce = 1; 4794 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4795 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4796 4797 sc->tt.tx_align = 1; 4798 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4799 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4800 } 4801#endif 4802 4803 4804 return (0); 4805} 4806 4807static int 4808cxgbe_sysctls(struct port_info *pi) 4809{ 4810 struct sysctl_ctx_list *ctx; 4811 struct sysctl_oid *oid; 4812 struct sysctl_oid_list *children; 4813 struct adapter *sc = pi->adapter; 4814 4815 ctx = device_get_sysctl_ctx(pi->dev); 4816 4817 /* 4818 * dev.cxgbe.X. 4819 */ 4820 oid = device_get_sysctl_tree(pi->dev); 4821 children = SYSCTL_CHILDREN(oid); 4822 4823 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4824 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4825 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4826 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4827 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4828 "PHY temperature (in Celsius)"); 4829 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4830 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4831 "PHY firmware version"); 4832 } 4833 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4834 &pi->nrxq, 0, "# of rx queues"); 4835 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4836 &pi->ntxq, 0, "# of tx queues"); 4837 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4838 &pi->first_rxq, 0, "index of first rx queue"); 4839 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4840 &pi->first_txq, 0, "index of first tx queue"); 4841 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", CTLTYPE_INT | 4842 CTLFLAG_RW, pi, 0, sysctl_noflowq, "IU", 4843 "Reserve queue 0 for non-flowid packets"); 4844 4845#ifdef TCP_OFFLOAD 4846 if (is_offload(sc)) { 4847 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4848 &pi->nofldrxq, 0, 4849 "# of rx queues for offloaded TCP connections"); 4850 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4851 &pi->nofldtxq, 0, 4852 "# of tx queues for offloaded TCP connections"); 4853 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4854 CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4855 "index of first TOE rx queue"); 4856 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4857 CTLFLAG_RD, &pi->first_ofld_txq, 0, 4858 "index of first TOE tx queue"); 4859 } 4860#endif 4861#ifdef DEV_NETMAP 4862 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4863 &pi->nnmrxq, 0, "# of rx queues for netmap"); 4864 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4865 &pi->nnmtxq, 0, "# of tx queues for netmap"); 4866 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4867 CTLFLAG_RD, &pi->first_nm_rxq, 0, 4868 "index of first netmap rx queue"); 4869 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4870 CTLFLAG_RD, &pi->first_nm_txq, 0, 4871 "index of first netmap tx queue"); 4872#endif 4873 4874 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4875 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4876 "holdoff timer index"); 4877 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4878 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4879 "holdoff packet counter index"); 4880 4881 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4882 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4883 "rx queue size"); 4884 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4885 CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4886 "tx queue size"); 4887 4888 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4889 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4890 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4891 4892 /* 4893 * dev.cxgbe.X.stats. 4894 */ 4895 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4896 NULL, "port statistics"); 4897 children = SYSCTL_CHILDREN(oid); 4898 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 4899 &pi->tx_parse_error, 0, 4900 "# of tx packets with invalid length or # of segments"); 4901 4902#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4903 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4904 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 4905 sysctl_handle_t4_reg64, "QU", desc) 4906 4907 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4908 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4909 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4910 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4911 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4912 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4913 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4914 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4915 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4916 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4917 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4918 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4919 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4920 "# of tx frames in this range", 4921 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4922 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4923 "# of tx frames in this range", 4924 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4925 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4926 "# of tx frames in this range", 4927 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4928 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4929 "# of tx frames in this range", 4930 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4931 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4932 "# of tx frames in this range", 4933 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4934 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4935 "# of tx frames in this range", 4936 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4937 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4938 "# of tx frames in this range", 4939 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4940 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4941 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4942 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4943 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4944 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4945 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4946 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4947 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4948 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4949 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4950 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4951 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4952 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4953 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4954 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4955 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4956 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4957 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4958 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4959 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4960 4961 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4962 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4963 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4964 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4965 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4966 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4967 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4968 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4969 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4970 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4971 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4972 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4973 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4974 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4975 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4976 "# of frames received with bad FCS", 4977 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4978 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4979 "# of frames received with length error", 4980 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4981 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4982 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4983 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4984 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4985 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4986 "# of rx frames in this range", 4987 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4988 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4989 "# of rx frames in this range", 4990 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4991 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4992 "# of rx frames in this range", 4993 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4994 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4995 "# of rx frames in this range", 4996 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4997 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4998 "# of rx frames in this range", 4999 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5000 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5001 "# of rx frames in this range", 5002 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5003 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5004 "# of rx frames in this range", 5005 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5006 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5007 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5008 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5009 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5010 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5011 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5012 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5013 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5014 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5015 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5016 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5017 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5018 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5019 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5020 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5021 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5022 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5023 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5024 5025#undef SYSCTL_ADD_T4_REG64 5026 5027#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5028 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5029 &pi->stats.name, desc) 5030 5031 /* We get these from port_stats and they may be stale by upto 1s */ 5032 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5033 "# drops due to buffer-group 0 overflows"); 5034 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5035 "# drops due to buffer-group 1 overflows"); 5036 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5037 "# drops due to buffer-group 2 overflows"); 5038 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5039 "# drops due to buffer-group 3 overflows"); 5040 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5041 "# of buffer-group 0 truncated packets"); 5042 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5043 "# of buffer-group 1 truncated packets"); 5044 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5045 "# of buffer-group 2 truncated packets"); 5046 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5047 "# of buffer-group 3 truncated packets"); 5048 5049#undef SYSCTL_ADD_T4_PORTSTAT 5050 5051 return (0); 5052} 5053 5054static int 5055sysctl_int_array(SYSCTL_HANDLER_ARGS) 5056{ 5057 int rc, *i, space = 0; 5058 struct sbuf sb; 5059 5060 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5061 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5062 if (space) 5063 sbuf_printf(&sb, " "); 5064 sbuf_printf(&sb, "%d", *i); 5065 space = 1; 5066 } 5067 sbuf_finish(&sb); 5068 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5069 sbuf_delete(&sb); 5070 return (rc); 5071} 5072 5073static int 5074sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5075{ 5076 int rc; 5077 struct sbuf *sb; 5078 5079 rc = sysctl_wire_old_buffer(req, 0); 5080 if (rc != 0) 5081 return(rc); 5082 5083 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5084 if (sb == NULL) 5085 return (ENOMEM); 5086 5087 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5088 rc = sbuf_finish(sb); 5089 sbuf_delete(sb); 5090 5091 return (rc); 5092} 5093 5094static int 5095sysctl_btphy(SYSCTL_HANDLER_ARGS) 5096{ 5097 struct port_info *pi = arg1; 5098 int op = arg2; 5099 struct adapter *sc = pi->adapter; 5100 u_int v; 5101 int rc; 5102 5103 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4btt"); 5104 if (rc) 5105 return (rc); 5106 /* XXX: magic numbers */ 5107 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5108 &v); 5109 end_synchronized_op(sc, 0); 5110 if (rc) 5111 return (rc); 5112 if (op == 0) 5113 v /= 256; 5114 5115 rc = sysctl_handle_int(oidp, &v, 0, req); 5116 return (rc); 5117} 5118 5119static int 5120sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5121{ 5122 struct port_info *pi = arg1; 5123 int rc, val; 5124 5125 val = pi->rsrv_noflowq; 5126 rc = sysctl_handle_int(oidp, &val, 0, req); 5127 if (rc != 0 || req->newptr == NULL) 5128 return (rc); 5129 5130 if ((val >= 1) && (pi->ntxq > 1)) 5131 pi->rsrv_noflowq = 1; 5132 else 5133 pi->rsrv_noflowq = 0; 5134 5135 return (rc); 5136} 5137 5138static int 5139sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5140{ 5141 struct port_info *pi = arg1; 5142 struct adapter *sc = pi->adapter; 5143 int idx, rc, i; 5144 struct sge_rxq *rxq; 5145#ifdef TCP_OFFLOAD 5146 struct sge_ofld_rxq *ofld_rxq; 5147#endif 5148 uint8_t v; 5149 5150 idx = pi->tmr_idx; 5151 5152 rc = sysctl_handle_int(oidp, &idx, 0, req); 5153 if (rc != 0 || req->newptr == NULL) 5154 return (rc); 5155 5156 if (idx < 0 || idx >= SGE_NTIMERS) 5157 return (EINVAL); 5158 5159 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5160 "t4tmr"); 5161 if (rc) 5162 return (rc); 5163 5164 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 5165 for_each_rxq(pi, i, rxq) { 5166#ifdef atomic_store_rel_8 5167 atomic_store_rel_8(&rxq->iq.intr_params, v); 5168#else 5169 rxq->iq.intr_params = v; 5170#endif 5171 } 5172#ifdef TCP_OFFLOAD 5173 for_each_ofld_rxq(pi, i, ofld_rxq) { 5174#ifdef atomic_store_rel_8 5175 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5176#else 5177 ofld_rxq->iq.intr_params = v; 5178#endif 5179 } 5180#endif 5181 pi->tmr_idx = idx; 5182 5183 end_synchronized_op(sc, LOCK_HELD); 5184 return (0); 5185} 5186 5187static int 5188sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5189{ 5190 struct port_info *pi = arg1; 5191 struct adapter *sc = pi->adapter; 5192 int idx, rc; 5193 5194 idx = pi->pktc_idx; 5195 5196 rc = sysctl_handle_int(oidp, &idx, 0, req); 5197 if (rc != 0 || req->newptr == NULL) 5198 return (rc); 5199 5200 if (idx < -1 || idx >= SGE_NCOUNTERS) 5201 return (EINVAL); 5202 5203 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5204 "t4pktc"); 5205 if (rc) 5206 return (rc); 5207 5208 if (pi->flags & PORT_INIT_DONE) 5209 rc = EBUSY; /* cannot be changed once the queues are created */ 5210 else 5211 pi->pktc_idx = idx; 5212 5213 end_synchronized_op(sc, LOCK_HELD); 5214 return (rc); 5215} 5216 5217static int 5218sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5219{ 5220 struct port_info *pi = arg1; 5221 struct adapter *sc = pi->adapter; 5222 int qsize, rc; 5223 5224 qsize = pi->qsize_rxq; 5225 5226 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5227 if (rc != 0 || req->newptr == NULL) 5228 return (rc); 5229 5230 if (qsize < 128 || (qsize & 7)) 5231 return (EINVAL); 5232 5233 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5234 "t4rxqs"); 5235 if (rc) 5236 return (rc); 5237 5238 if (pi->flags & PORT_INIT_DONE) 5239 rc = EBUSY; /* cannot be changed once the queues are created */ 5240 else 5241 pi->qsize_rxq = qsize; 5242 5243 end_synchronized_op(sc, LOCK_HELD); 5244 return (rc); 5245} 5246 5247static int 5248sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5249{ 5250 struct port_info *pi = arg1; 5251 struct adapter *sc = pi->adapter; 5252 int qsize, rc; 5253 5254 qsize = pi->qsize_txq; 5255 5256 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5257 if (rc != 0 || req->newptr == NULL) 5258 return (rc); 5259 5260 if (qsize < 128 || qsize > 65536) 5261 return (EINVAL); 5262 5263 rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5264 "t4txqs"); 5265 if (rc) 5266 return (rc); 5267 5268 if (pi->flags & PORT_INIT_DONE) 5269 rc = EBUSY; /* cannot be changed once the queues are created */ 5270 else 5271 pi->qsize_txq = qsize; 5272 5273 end_synchronized_op(sc, LOCK_HELD); 5274 return (rc); 5275} 5276 5277static int 5278sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5279{ 5280 struct port_info *pi = arg1; 5281 struct adapter *sc = pi->adapter; 5282 struct link_config *lc = &pi->link_cfg; 5283 int rc; 5284 5285 if (req->newptr == NULL) { 5286 struct sbuf *sb; 5287 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5288 5289 rc = sysctl_wire_old_buffer(req, 0); 5290 if (rc != 0) 5291 return(rc); 5292 5293 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5294 if (sb == NULL) 5295 return (ENOMEM); 5296 5297 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5298 rc = sbuf_finish(sb); 5299 sbuf_delete(sb); 5300 } else { 5301 char s[2]; 5302 int n; 5303 5304 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5305 s[1] = 0; 5306 5307 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5308 if (rc != 0) 5309 return(rc); 5310 5311 if (s[1] != 0) 5312 return (EINVAL); 5313 if (s[0] < '0' || s[0] > '9') 5314 return (EINVAL); /* not a number */ 5315 n = s[0] - '0'; 5316 if (n & ~(PAUSE_TX | PAUSE_RX)) 5317 return (EINVAL); /* some other bit is set too */ 5318 5319 rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4PAUSE"); 5320 if (rc) 5321 return (rc); 5322 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5323 int link_ok = lc->link_ok; 5324 5325 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5326 lc->requested_fc |= n; 5327 rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, lc); 5328 lc->link_ok = link_ok; /* restore */ 5329 } 5330 end_synchronized_op(sc, 0); 5331 } 5332 5333 return (rc); 5334} 5335 5336static int 5337sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5338{ 5339 struct adapter *sc = arg1; 5340 int reg = arg2; 5341 uint64_t val; 5342 5343 val = t4_read_reg64(sc, reg); 5344 5345 return (sysctl_handle_64(oidp, &val, 0, req)); 5346} 5347 5348static int 5349sysctl_temperature(SYSCTL_HANDLER_ARGS) 5350{ 5351 struct adapter *sc = arg1; 5352 int rc, t; 5353 uint32_t param, val; 5354 5355 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5356 if (rc) 5357 return (rc); 5358 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5359 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5360 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5361 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5362 end_synchronized_op(sc, 0); 5363 if (rc) 5364 return (rc); 5365 5366 /* unknown is returned as 0 but we display -1 in that case */ 5367 t = val == 0 ? -1 : val; 5368 5369 rc = sysctl_handle_int(oidp, &t, 0, req); 5370 return (rc); 5371} 5372 5373#ifdef SBUF_DRAIN 5374static int 5375sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5376{ 5377 struct adapter *sc = arg1; 5378 struct sbuf *sb; 5379 int rc, i; 5380 uint16_t incr[NMTUS][NCCTRL_WIN]; 5381 static const char *dec_fac[] = { 5382 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5383 "0.9375" 5384 }; 5385 5386 rc = sysctl_wire_old_buffer(req, 0); 5387 if (rc != 0) 5388 return (rc); 5389 5390 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5391 if (sb == NULL) 5392 return (ENOMEM); 5393 5394 t4_read_cong_tbl(sc, incr); 5395 5396 for (i = 0; i < NCCTRL_WIN; ++i) { 5397 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5398 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5399 incr[5][i], incr[6][i], incr[7][i]); 5400 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5401 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5402 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5403 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5404 } 5405 5406 rc = sbuf_finish(sb); 5407 sbuf_delete(sb); 5408 5409 return (rc); 5410} 5411 5412static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5413 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5414 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5415 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5416}; 5417 5418static int 5419sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5420{ 5421 struct adapter *sc = arg1; 5422 struct sbuf *sb; 5423 int rc, i, n, qid = arg2; 5424 uint32_t *buf, *p; 5425 char *qtype; 5426 u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 5427 5428 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5429 ("%s: bad qid %d\n", __func__, qid)); 5430 5431 if (qid < CIM_NUM_IBQ) { 5432 /* inbound queue */ 5433 qtype = "IBQ"; 5434 n = 4 * CIM_IBQ_SIZE; 5435 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5436 rc = t4_read_cim_ibq(sc, qid, buf, n); 5437 } else { 5438 /* outbound queue */ 5439 qtype = "OBQ"; 5440 qid -= CIM_NUM_IBQ; 5441 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5442 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5443 rc = t4_read_cim_obq(sc, qid, buf, n); 5444 } 5445 5446 if (rc < 0) { 5447 rc = -rc; 5448 goto done; 5449 } 5450 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5451 5452 rc = sysctl_wire_old_buffer(req, 0); 5453 if (rc != 0) 5454 goto done; 5455 5456 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5457 if (sb == NULL) { 5458 rc = ENOMEM; 5459 goto done; 5460 } 5461 5462 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5463 for (i = 0, p = buf; i < n; i += 16, p += 4) 5464 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5465 p[2], p[3]); 5466 5467 rc = sbuf_finish(sb); 5468 sbuf_delete(sb); 5469done: 5470 free(buf, M_CXGBE); 5471 return (rc); 5472} 5473 5474static int 5475sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5476{ 5477 struct adapter *sc = arg1; 5478 u_int cfg; 5479 struct sbuf *sb; 5480 uint32_t *buf, *p; 5481 int rc; 5482 5483 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5484 if (rc != 0) 5485 return (rc); 5486 5487 rc = sysctl_wire_old_buffer(req, 0); 5488 if (rc != 0) 5489 return (rc); 5490 5491 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5492 if (sb == NULL) 5493 return (ENOMEM); 5494 5495 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5496 M_ZERO | M_WAITOK); 5497 5498 rc = -t4_cim_read_la(sc, buf, NULL); 5499 if (rc != 0) 5500 goto done; 5501 5502 sbuf_printf(sb, "Status Data PC%s", 5503 cfg & F_UPDBGLACAPTPCONLY ? "" : 5504 " LS0Stat LS0Addr LS0Data"); 5505 5506 KASSERT((sc->params.cim_la_size & 7) == 0, 5507 ("%s: p will walk off the end of buf", __func__)); 5508 5509 for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 5510 if (cfg & F_UPDBGLACAPTPCONLY) { 5511 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5512 p[6], p[7]); 5513 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5514 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5515 p[4] & 0xff, p[5] >> 8); 5516 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5517 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5518 p[1] & 0xf, p[2] >> 4); 5519 } else { 5520 sbuf_printf(sb, 5521 "\n %02x %x%07x %x%07x %08x %08x " 5522 "%08x%08x%08x%08x", 5523 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5524 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5525 p[6], p[7]); 5526 } 5527 } 5528 5529 rc = sbuf_finish(sb); 5530 sbuf_delete(sb); 5531done: 5532 free(buf, M_CXGBE); 5533 return (rc); 5534} 5535 5536static int 5537sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5538{ 5539 struct adapter *sc = arg1; 5540 u_int i; 5541 struct sbuf *sb; 5542 uint32_t *buf, *p; 5543 int rc; 5544 5545 rc = sysctl_wire_old_buffer(req, 0); 5546 if (rc != 0) 5547 return (rc); 5548 5549 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5550 if (sb == NULL) 5551 return (ENOMEM); 5552 5553 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5554 M_ZERO | M_WAITOK); 5555 5556 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5557 p = buf; 5558 5559 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5560 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5561 p[1], p[0]); 5562 } 5563 5564 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5565 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5566 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5567 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5568 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5569 (p[1] >> 2) | ((p[2] & 3) << 30), 5570 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5571 p[0] & 1); 5572 } 5573 5574 rc = sbuf_finish(sb); 5575 sbuf_delete(sb); 5576 free(buf, M_CXGBE); 5577 return (rc); 5578} 5579 5580static int 5581sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5582{ 5583 struct adapter *sc = arg1; 5584 u_int i; 5585 struct sbuf *sb; 5586 uint32_t *buf, *p; 5587 int rc; 5588 5589 rc = sysctl_wire_old_buffer(req, 0); 5590 if (rc != 0) 5591 return (rc); 5592 5593 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5594 if (sb == NULL) 5595 return (ENOMEM); 5596 5597 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5598 M_ZERO | M_WAITOK); 5599 5600 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5601 p = buf; 5602 5603 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5604 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5605 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5606 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5607 p[4], p[3], p[2], p[1], p[0]); 5608 } 5609 5610 sbuf_printf(sb, "\n\nCntl ID Data"); 5611 for (i = 0; i < CIM_MALA_SIZE; i++, p += 6) { 5612 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5613 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5614 } 5615 5616 rc = sbuf_finish(sb); 5617 sbuf_delete(sb); 5618 free(buf, M_CXGBE); 5619 return (rc); 5620} 5621 5622static int 5623sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5624{ 5625 struct adapter *sc = arg1; 5626 struct sbuf *sb; 5627 int rc, i; 5628 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5629 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5630 uint16_t thres[CIM_NUM_IBQ]; 5631 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5632 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5633 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5634 5635 if (is_t4(sc)) { 5636 cim_num_obq = CIM_NUM_OBQ; 5637 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5638 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5639 } else { 5640 cim_num_obq = CIM_NUM_OBQ_T5; 5641 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5642 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5643 } 5644 nq = CIM_NUM_IBQ + cim_num_obq; 5645 5646 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5647 if (rc == 0) 5648 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5649 if (rc != 0) 5650 return (rc); 5651 5652 t4_read_cimq_cfg(sc, base, size, thres); 5653 5654 rc = sysctl_wire_old_buffer(req, 0); 5655 if (rc != 0) 5656 return (rc); 5657 5658 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5659 if (sb == NULL) 5660 return (ENOMEM); 5661 5662 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5663 5664 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5665 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5666 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5667 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5668 G_QUEREMFLITS(p[2]) * 16); 5669 for ( ; i < nq; i++, p += 4, wr += 2) 5670 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5671 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5672 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5673 G_QUEREMFLITS(p[2]) * 16); 5674 5675 rc = sbuf_finish(sb); 5676 sbuf_delete(sb); 5677 5678 return (rc); 5679} 5680 5681static int 5682sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5683{ 5684 struct adapter *sc = arg1; 5685 struct sbuf *sb; 5686 int rc; 5687 struct tp_cpl_stats stats; 5688 5689 rc = sysctl_wire_old_buffer(req, 0); 5690 if (rc != 0) 5691 return (rc); 5692 5693 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5694 if (sb == NULL) 5695 return (ENOMEM); 5696 5697 t4_tp_get_cpl_stats(sc, &stats); 5698 5699 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5700 "channel 3\n"); 5701 sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 5702 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5703 sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 5704 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5705 5706 rc = sbuf_finish(sb); 5707 sbuf_delete(sb); 5708 5709 return (rc); 5710} 5711 5712static int 5713sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5714{ 5715 struct adapter *sc = arg1; 5716 struct sbuf *sb; 5717 int rc; 5718 struct tp_usm_stats stats; 5719 5720 rc = sysctl_wire_old_buffer(req, 0); 5721 if (rc != 0) 5722 return(rc); 5723 5724 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5725 if (sb == NULL) 5726 return (ENOMEM); 5727 5728 t4_get_usm_stats(sc, &stats); 5729 5730 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5731 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5732 sbuf_printf(sb, "Drops: %u", stats.drops); 5733 5734 rc = sbuf_finish(sb); 5735 sbuf_delete(sb); 5736 5737 return (rc); 5738} 5739 5740const char *devlog_level_strings[] = { 5741 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5742 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5743 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5744 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5745 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5746 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5747}; 5748 5749const char *devlog_facility_strings[] = { 5750 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5751 [FW_DEVLOG_FACILITY_CF] = "CF", 5752 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5753 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5754 [FW_DEVLOG_FACILITY_RES] = "RES", 5755 [FW_DEVLOG_FACILITY_HW] = "HW", 5756 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5757 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5758 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5759 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5760 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5761 [FW_DEVLOG_FACILITY_VI] = "VI", 5762 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5763 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5764 [FW_DEVLOG_FACILITY_TM] = "TM", 5765 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5766 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5767 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5768 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5769 [FW_DEVLOG_FACILITY_RI] = "RI", 5770 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5771 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5772 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5773 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5774}; 5775 5776static int 5777sysctl_devlog(SYSCTL_HANDLER_ARGS) 5778{ 5779 struct adapter *sc = arg1; 5780 struct devlog_params *dparams = &sc->params.devlog; 5781 struct fw_devlog_e *buf, *e; 5782 int i, j, rc, nentries, first = 0, m; 5783 struct sbuf *sb; 5784 uint64_t ftstamp = UINT64_MAX; 5785 5786 if (dparams->start == 0) { 5787 dparams->memtype = FW_MEMTYPE_EDC0; 5788 dparams->start = 0x84000; 5789 dparams->size = 32768; 5790 } 5791 5792 nentries = dparams->size / sizeof(struct fw_devlog_e); 5793 5794 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5795 if (buf == NULL) 5796 return (ENOMEM); 5797 5798 m = fwmtype_to_hwmtype(dparams->memtype); 5799 rc = -t4_mem_read(sc, m, dparams->start, dparams->size, (void *)buf); 5800 if (rc != 0) 5801 goto done; 5802 5803 for (i = 0; i < nentries; i++) { 5804 e = &buf[i]; 5805 5806 if (e->timestamp == 0) 5807 break; /* end */ 5808 5809 e->timestamp = be64toh(e->timestamp); 5810 e->seqno = be32toh(e->seqno); 5811 for (j = 0; j < 8; j++) 5812 e->params[j] = be32toh(e->params[j]); 5813 5814 if (e->timestamp < ftstamp) { 5815 ftstamp = e->timestamp; 5816 first = i; 5817 } 5818 } 5819 5820 if (buf[first].timestamp == 0) 5821 goto done; /* nothing in the log */ 5822 5823 rc = sysctl_wire_old_buffer(req, 0); 5824 if (rc != 0) 5825 goto done; 5826 5827 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5828 if (sb == NULL) { 5829 rc = ENOMEM; 5830 goto done; 5831 } 5832 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5833 "Seq#", "Tstamp", "Level", "Facility", "Message"); 5834 5835 i = first; 5836 do { 5837 e = &buf[i]; 5838 if (e->timestamp == 0) 5839 break; /* end */ 5840 5841 sbuf_printf(sb, "%10d %15ju %8s %8s ", 5842 e->seqno, e->timestamp, 5843 (e->level < nitems(devlog_level_strings) ? 5844 devlog_level_strings[e->level] : "UNKNOWN"), 5845 (e->facility < nitems(devlog_facility_strings) ? 5846 devlog_facility_strings[e->facility] : "UNKNOWN")); 5847 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5848 e->params[2], e->params[3], e->params[4], 5849 e->params[5], e->params[6], e->params[7]); 5850 5851 if (++i == nentries) 5852 i = 0; 5853 } while (i != first); 5854 5855 rc = sbuf_finish(sb); 5856 sbuf_delete(sb); 5857done: 5858 free(buf, M_CXGBE); 5859 return (rc); 5860} 5861 5862static int 5863sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5864{ 5865 struct adapter *sc = arg1; 5866 struct sbuf *sb; 5867 int rc; 5868 struct tp_fcoe_stats stats[4]; 5869 5870 rc = sysctl_wire_old_buffer(req, 0); 5871 if (rc != 0) 5872 return (rc); 5873 5874 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5875 if (sb == NULL) 5876 return (ENOMEM); 5877 5878 t4_get_fcoe_stats(sc, 0, &stats[0]); 5879 t4_get_fcoe_stats(sc, 1, &stats[1]); 5880 t4_get_fcoe_stats(sc, 2, &stats[2]); 5881 t4_get_fcoe_stats(sc, 3, &stats[3]); 5882 5883 sbuf_printf(sb, " channel 0 channel 1 " 5884 "channel 2 channel 3\n"); 5885 sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5886 stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5887 stats[3].octetsDDP); 5888 sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5889 stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5890 sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5891 stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5892 stats[3].framesDrop); 5893 5894 rc = sbuf_finish(sb); 5895 sbuf_delete(sb); 5896 5897 return (rc); 5898} 5899 5900static int 5901sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5902{ 5903 struct adapter *sc = arg1; 5904 struct sbuf *sb; 5905 int rc, i; 5906 unsigned int map, kbps, ipg, mode; 5907 unsigned int pace_tab[NTX_SCHED]; 5908 5909 rc = sysctl_wire_old_buffer(req, 0); 5910 if (rc != 0) 5911 return (rc); 5912 5913 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5914 if (sb == NULL) 5915 return (ENOMEM); 5916 5917 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5918 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5919 t4_read_pace_tbl(sc, pace_tab); 5920 5921 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5922 "Class IPG (0.1 ns) Flow IPG (us)"); 5923 5924 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5925 t4_get_tx_sched(sc, i, &kbps, &ipg); 5926 sbuf_printf(sb, "\n %u %-5s %u ", i, 5927 (mode & (1 << i)) ? "flow" : "class", map & 3); 5928 if (kbps) 5929 sbuf_printf(sb, "%9u ", kbps); 5930 else 5931 sbuf_printf(sb, " disabled "); 5932 5933 if (ipg) 5934 sbuf_printf(sb, "%13u ", ipg); 5935 else 5936 sbuf_printf(sb, " disabled "); 5937 5938 if (pace_tab[i]) 5939 sbuf_printf(sb, "%10u", pace_tab[i]); 5940 else 5941 sbuf_printf(sb, " disabled"); 5942 } 5943 5944 rc = sbuf_finish(sb); 5945 sbuf_delete(sb); 5946 5947 return (rc); 5948} 5949 5950static int 5951sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5952{ 5953 struct adapter *sc = arg1; 5954 struct sbuf *sb; 5955 int rc, i, j; 5956 uint64_t *p0, *p1; 5957 struct lb_port_stats s[2]; 5958 static const char *stat_name[] = { 5959 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5960 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5961 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5962 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5963 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5964 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5965 "BG2FramesTrunc:", "BG3FramesTrunc:" 5966 }; 5967 5968 rc = sysctl_wire_old_buffer(req, 0); 5969 if (rc != 0) 5970 return (rc); 5971 5972 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5973 if (sb == NULL) 5974 return (ENOMEM); 5975 5976 memset(s, 0, sizeof(s)); 5977 5978 for (i = 0; i < 4; i += 2) { 5979 t4_get_lb_stats(sc, i, &s[0]); 5980 t4_get_lb_stats(sc, i + 1, &s[1]); 5981 5982 p0 = &s[0].octets; 5983 p1 = &s[1].octets; 5984 sbuf_printf(sb, "%s Loopback %u" 5985 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5986 5987 for (j = 0; j < nitems(stat_name); j++) 5988 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5989 *p0++, *p1++); 5990 } 5991 5992 rc = sbuf_finish(sb); 5993 sbuf_delete(sb); 5994 5995 return (rc); 5996} 5997 5998static int 5999sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6000{ 6001 int rc = 0; 6002 struct port_info *pi = arg1; 6003 struct sbuf *sb; 6004 static const char *linkdnreasons[] = { 6005 "non-specific", "remote fault", "autoneg failed", "reserved3", 6006 "PHY overheated", "unknown", "rx los", "reserved7" 6007 }; 6008 6009 rc = sysctl_wire_old_buffer(req, 0); 6010 if (rc != 0) 6011 return(rc); 6012 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6013 if (sb == NULL) 6014 return (ENOMEM); 6015 6016 if (pi->linkdnrc < 0) 6017 sbuf_printf(sb, "n/a"); 6018 else if (pi->linkdnrc < nitems(linkdnreasons)) 6019 sbuf_printf(sb, "%s", linkdnreasons[pi->linkdnrc]); 6020 else 6021 sbuf_printf(sb, "%d", pi->linkdnrc); 6022 6023 rc = sbuf_finish(sb); 6024 sbuf_delete(sb); 6025 6026 return (rc); 6027} 6028 6029struct mem_desc { 6030 unsigned int base; 6031 unsigned int limit; 6032 unsigned int idx; 6033}; 6034 6035static int 6036mem_desc_cmp(const void *a, const void *b) 6037{ 6038 return ((const struct mem_desc *)a)->base - 6039 ((const struct mem_desc *)b)->base; 6040} 6041 6042static void 6043mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6044 unsigned int to) 6045{ 6046 unsigned int size; 6047 6048 size = to - from + 1; 6049 if (size == 0) 6050 return; 6051 6052 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6053 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6054} 6055 6056static int 6057sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6058{ 6059 struct adapter *sc = arg1; 6060 struct sbuf *sb; 6061 int rc, i, n; 6062 uint32_t lo, hi, used, alloc; 6063 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6064 static const char *region[] = { 6065 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6066 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6067 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6068 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6069 "RQUDP region:", "PBL region:", "TXPBL region:", 6070 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6071 "On-chip queues:" 6072 }; 6073 struct mem_desc avail[4]; 6074 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6075 struct mem_desc *md = mem; 6076 6077 rc = sysctl_wire_old_buffer(req, 0); 6078 if (rc != 0) 6079 return (rc); 6080 6081 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6082 if (sb == NULL) 6083 return (ENOMEM); 6084 6085 for (i = 0; i < nitems(mem); i++) { 6086 mem[i].limit = 0; 6087 mem[i].idx = i; 6088 } 6089 6090 /* Find and sort the populated memory ranges */ 6091 i = 0; 6092 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6093 if (lo & F_EDRAM0_ENABLE) { 6094 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6095 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6096 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6097 avail[i].idx = 0; 6098 i++; 6099 } 6100 if (lo & F_EDRAM1_ENABLE) { 6101 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6102 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6103 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6104 avail[i].idx = 1; 6105 i++; 6106 } 6107 if (lo & F_EXT_MEM_ENABLE) { 6108 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6109 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6110 avail[i].limit = avail[i].base + 6111 (G_EXT_MEM_SIZE(hi) << 20); 6112 avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 6113 i++; 6114 } 6115 if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 6116 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6117 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6118 avail[i].limit = avail[i].base + 6119 (G_EXT_MEM1_SIZE(hi) << 20); 6120 avail[i].idx = 4; 6121 i++; 6122 } 6123 if (!i) /* no memory available */ 6124 return 0; 6125 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6126 6127 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6128 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6129 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6130 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6131 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6132 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6133 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6134 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6135 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6136 6137 /* the next few have explicit upper bounds */ 6138 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6139 md->limit = md->base - 1 + 6140 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6141 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6142 md++; 6143 6144 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6145 md->limit = md->base - 1 + 6146 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6147 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6148 md++; 6149 6150 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6151 hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 6152 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6153 md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 6154 } else { 6155 md->base = 0; 6156 md->idx = nitems(region); /* hide it */ 6157 } 6158 md++; 6159 6160#define ulp_region(reg) \ 6161 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6162 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6163 6164 ulp_region(RX_ISCSI); 6165 ulp_region(RX_TDDP); 6166 ulp_region(TX_TPT); 6167 ulp_region(RX_STAG); 6168 ulp_region(RX_RQ); 6169 ulp_region(RX_RQUDP); 6170 ulp_region(RX_PBL); 6171 ulp_region(TX_PBL); 6172#undef ulp_region 6173 6174 md->base = 0; 6175 md->idx = nitems(region); 6176 if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 6177 md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 6178 md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 6179 A_SGE_DBVFIFO_SIZE))) << 2) - 1; 6180 } 6181 md++; 6182 6183 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6184 md->limit = md->base + sc->tids.ntids - 1; 6185 md++; 6186 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6187 md->limit = md->base + sc->tids.ntids - 1; 6188 md++; 6189 6190 md->base = sc->vres.ocq.start; 6191 if (sc->vres.ocq.size) 6192 md->limit = md->base + sc->vres.ocq.size - 1; 6193 else 6194 md->idx = nitems(region); /* hide it */ 6195 md++; 6196 6197 /* add any address-space holes, there can be up to 3 */ 6198 for (n = 0; n < i - 1; n++) 6199 if (avail[n].limit < avail[n + 1].base) 6200 (md++)->base = avail[n].limit; 6201 if (avail[n].limit) 6202 (md++)->base = avail[n].limit; 6203 6204 n = md - mem; 6205 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6206 6207 for (lo = 0; lo < i; lo++) 6208 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6209 avail[lo].limit - 1); 6210 6211 sbuf_printf(sb, "\n"); 6212 for (i = 0; i < n; i++) { 6213 if (mem[i].idx >= nitems(region)) 6214 continue; /* skip holes */ 6215 if (!mem[i].limit) 6216 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6217 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6218 mem[i].limit); 6219 } 6220 6221 sbuf_printf(sb, "\n"); 6222 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6223 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6224 mem_region_show(sb, "uP RAM:", lo, hi); 6225 6226 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6227 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6228 mem_region_show(sb, "uP Extmem2:", lo, hi); 6229 6230 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6231 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6232 G_PMRXMAXPAGE(lo), 6233 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6234 (lo & F_PMRXNUMCHN) ? 2 : 1); 6235 6236 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6237 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6238 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6239 G_PMTXMAXPAGE(lo), 6240 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6241 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6242 sbuf_printf(sb, "%u p-structs\n", 6243 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6244 6245 for (i = 0; i < 4; i++) { 6246 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6247 if (is_t4(sc)) { 6248 used = G_USED(lo); 6249 alloc = G_ALLOC(lo); 6250 } else { 6251 used = G_T5_USED(lo); 6252 alloc = G_T5_ALLOC(lo); 6253 } 6254 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6255 i, used, alloc); 6256 } 6257 for (i = 0; i < 4; i++) { 6258 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6259 if (is_t4(sc)) { 6260 used = G_USED(lo); 6261 alloc = G_ALLOC(lo); 6262 } else { 6263 used = G_T5_USED(lo); 6264 alloc = G_T5_ALLOC(lo); 6265 } 6266 sbuf_printf(sb, 6267 "\nLoopback %d using %u pages out of %u allocated", 6268 i, used, alloc); 6269 } 6270 6271 rc = sbuf_finish(sb); 6272 sbuf_delete(sb); 6273 6274 return (rc); 6275} 6276 6277static inline void 6278tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6279{ 6280 *mask = x | y; 6281 y = htobe64(y); 6282 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6283} 6284 6285static int 6286sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6287{ 6288 struct adapter *sc = arg1; 6289 struct sbuf *sb; 6290 int rc, i, n; 6291 6292 rc = sysctl_wire_old_buffer(req, 0); 6293 if (rc != 0) 6294 return (rc); 6295 6296 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6297 if (sb == NULL) 6298 return (ENOMEM); 6299 6300 sbuf_printf(sb, 6301 "Idx Ethernet address Mask Vld Ports PF" 6302 " VF Replication P0 P1 P2 P3 ML"); 6303 n = is_t4(sc) ? NUM_MPS_CLS_SRAM_L_INSTANCES : 6304 NUM_MPS_T5_CLS_SRAM_L_INSTANCES; 6305 for (i = 0; i < n; i++) { 6306 uint64_t tcamx, tcamy, mask; 6307 uint32_t cls_lo, cls_hi; 6308 uint8_t addr[ETHER_ADDR_LEN]; 6309 6310 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6311 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6312 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6313 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6314 6315 if (tcamx & tcamy) 6316 continue; 6317 6318 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6319 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6320 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6321 addr[3], addr[4], addr[5], (uintmax_t)mask, 6322 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6323 G_PORTMAP(cls_hi), G_PF(cls_lo), 6324 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6325 6326 if (cls_lo & F_REPLICATE) { 6327 struct fw_ldst_cmd ldst_cmd; 6328 6329 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6330 ldst_cmd.op_to_addrspace = 6331 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6332 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6333 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6334 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6335 ldst_cmd.u.mps.rplc.fid_idx = 6336 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6337 V_FW_LDST_CMD_IDX(i)); 6338 6339 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6340 "t4mps"); 6341 if (rc) 6342 break; 6343 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6344 sizeof(ldst_cmd), &ldst_cmd); 6345 end_synchronized_op(sc, 0); 6346 6347 if (rc != 0) { 6348 sbuf_printf(sb, 6349 " ------------ error %3u ------------", rc); 6350 rc = 0; 6351 } else { 6352 sbuf_printf(sb, " %08x %08x %08x %08x", 6353 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6354 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6355 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6356 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6357 } 6358 } else 6359 sbuf_printf(sb, "%36s", ""); 6360 6361 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6362 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6363 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6364 } 6365 6366 if (rc) 6367 (void) sbuf_finish(sb); 6368 else 6369 rc = sbuf_finish(sb); 6370 sbuf_delete(sb); 6371 6372 return (rc); 6373} 6374 6375static int 6376sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6377{ 6378 struct adapter *sc = arg1; 6379 struct sbuf *sb; 6380 int rc; 6381 uint16_t mtus[NMTUS]; 6382 6383 rc = sysctl_wire_old_buffer(req, 0); 6384 if (rc != 0) 6385 return (rc); 6386 6387 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6388 if (sb == NULL) 6389 return (ENOMEM); 6390 6391 t4_read_mtu_tbl(sc, mtus, NULL); 6392 6393 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6394 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6395 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6396 mtus[14], mtus[15]); 6397 6398 rc = sbuf_finish(sb); 6399 sbuf_delete(sb); 6400 6401 return (rc); 6402} 6403 6404static int 6405sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6406{ 6407 struct adapter *sc = arg1; 6408 struct sbuf *sb; 6409 int rc, i; 6410 uint32_t cnt[PM_NSTATS]; 6411 uint64_t cyc[PM_NSTATS]; 6412 static const char *rx_stats[] = { 6413 "Read:", "Write bypass:", "Write mem:", "Flush:" 6414 }; 6415 static const char *tx_stats[] = { 6416 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:" 6417 }; 6418 6419 rc = sysctl_wire_old_buffer(req, 0); 6420 if (rc != 0) 6421 return (rc); 6422 6423 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6424 if (sb == NULL) 6425 return (ENOMEM); 6426 6427 t4_pmtx_get_stats(sc, cnt, cyc); 6428 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6429 for (i = 0; i < ARRAY_SIZE(tx_stats); i++) 6430 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], cnt[i], 6431 cyc[i]); 6432 6433 t4_pmrx_get_stats(sc, cnt, cyc); 6434 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6435 for (i = 0; i < ARRAY_SIZE(rx_stats); i++) 6436 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], cnt[i], 6437 cyc[i]); 6438 6439 rc = sbuf_finish(sb); 6440 sbuf_delete(sb); 6441 6442 return (rc); 6443} 6444 6445static int 6446sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6447{ 6448 struct adapter *sc = arg1; 6449 struct sbuf *sb; 6450 int rc; 6451 struct tp_rdma_stats stats; 6452 6453 rc = sysctl_wire_old_buffer(req, 0); 6454 if (rc != 0) 6455 return (rc); 6456 6457 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6458 if (sb == NULL) 6459 return (ENOMEM); 6460 6461 t4_tp_get_rdma_stats(sc, &stats); 6462 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6463 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6464 6465 rc = sbuf_finish(sb); 6466 sbuf_delete(sb); 6467 6468 return (rc); 6469} 6470 6471static int 6472sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6473{ 6474 struct adapter *sc = arg1; 6475 struct sbuf *sb; 6476 int rc; 6477 struct tp_tcp_stats v4, v6; 6478 6479 rc = sysctl_wire_old_buffer(req, 0); 6480 if (rc != 0) 6481 return (rc); 6482 6483 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6484 if (sb == NULL) 6485 return (ENOMEM); 6486 6487 t4_tp_get_tcp_stats(sc, &v4, &v6); 6488 sbuf_printf(sb, 6489 " IP IPv6\n"); 6490 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6491 v4.tcpOutRsts, v6.tcpOutRsts); 6492 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6493 v4.tcpInSegs, v6.tcpInSegs); 6494 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6495 v4.tcpOutSegs, v6.tcpOutSegs); 6496 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6497 v4.tcpRetransSegs, v6.tcpRetransSegs); 6498 6499 rc = sbuf_finish(sb); 6500 sbuf_delete(sb); 6501 6502 return (rc); 6503} 6504 6505static int 6506sysctl_tids(SYSCTL_HANDLER_ARGS) 6507{ 6508 struct adapter *sc = arg1; 6509 struct sbuf *sb; 6510 int rc; 6511 struct tid_info *t = &sc->tids; 6512 6513 rc = sysctl_wire_old_buffer(req, 0); 6514 if (rc != 0) 6515 return (rc); 6516 6517 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6518 if (sb == NULL) 6519 return (ENOMEM); 6520 6521 if (t->natids) { 6522 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6523 t->atids_in_use); 6524 } 6525 6526 if (t->ntids) { 6527 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6528 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6529 6530 if (b) { 6531 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6532 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6533 t->ntids - 1); 6534 } else { 6535 sbuf_printf(sb, "TID range: %u-%u", 6536 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6537 t->ntids - 1); 6538 } 6539 } else 6540 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6541 sbuf_printf(sb, ", in use: %u\n", 6542 atomic_load_acq_int(&t->tids_in_use)); 6543 } 6544 6545 if (t->nstids) { 6546 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6547 t->stid_base + t->nstids - 1, t->stids_in_use); 6548 } 6549 6550 if (t->nftids) { 6551 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6552 t->ftid_base + t->nftids - 1); 6553 } 6554 6555 if (t->netids) { 6556 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6557 t->etid_base + t->netids - 1); 6558 } 6559 6560 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6561 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6562 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6563 6564 rc = sbuf_finish(sb); 6565 sbuf_delete(sb); 6566 6567 return (rc); 6568} 6569 6570static int 6571sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6572{ 6573 struct adapter *sc = arg1; 6574 struct sbuf *sb; 6575 int rc; 6576 struct tp_err_stats stats; 6577 6578 rc = sysctl_wire_old_buffer(req, 0); 6579 if (rc != 0) 6580 return (rc); 6581 6582 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6583 if (sb == NULL) 6584 return (ENOMEM); 6585 6586 t4_tp_get_err_stats(sc, &stats); 6587 6588 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6589 "channel 3\n"); 6590 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 6591 stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 6592 stats.macInErrs[3]); 6593 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 6594 stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 6595 stats.hdrInErrs[3]); 6596 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 6597 stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 6598 stats.tcpInErrs[3]); 6599 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 6600 stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 6601 stats.tcp6InErrs[3]); 6602 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 6603 stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 6604 stats.tnlCongDrops[3]); 6605 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 6606 stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 6607 stats.tnlTxDrops[3]); 6608 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 6609 stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 6610 stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 6611 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 6612 stats.ofldChanDrops[0], stats.ofldChanDrops[1], 6613 stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 6614 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 6615 stats.ofldNoNeigh, stats.ofldCongDefer); 6616 6617 rc = sbuf_finish(sb); 6618 sbuf_delete(sb); 6619 6620 return (rc); 6621} 6622 6623struct field_desc { 6624 const char *name; 6625 u_int start; 6626 u_int width; 6627}; 6628 6629static void 6630field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 6631{ 6632 char buf[32]; 6633 int line_size = 0; 6634 6635 while (f->name) { 6636 uint64_t mask = (1ULL << f->width) - 1; 6637 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 6638 ((uintmax_t)v >> f->start) & mask); 6639 6640 if (line_size + len >= 79) { 6641 line_size = 8; 6642 sbuf_printf(sb, "\n "); 6643 } 6644 sbuf_printf(sb, "%s ", buf); 6645 line_size += len + 1; 6646 f++; 6647 } 6648 sbuf_printf(sb, "\n"); 6649} 6650 6651static struct field_desc tp_la0[] = { 6652 { "RcfOpCodeOut", 60, 4 }, 6653 { "State", 56, 4 }, 6654 { "WcfState", 52, 4 }, 6655 { "RcfOpcSrcOut", 50, 2 }, 6656 { "CRxError", 49, 1 }, 6657 { "ERxError", 48, 1 }, 6658 { "SanityFailed", 47, 1 }, 6659 { "SpuriousMsg", 46, 1 }, 6660 { "FlushInputMsg", 45, 1 }, 6661 { "FlushInputCpl", 44, 1 }, 6662 { "RssUpBit", 43, 1 }, 6663 { "RssFilterHit", 42, 1 }, 6664 { "Tid", 32, 10 }, 6665 { "InitTcb", 31, 1 }, 6666 { "LineNumber", 24, 7 }, 6667 { "Emsg", 23, 1 }, 6668 { "EdataOut", 22, 1 }, 6669 { "Cmsg", 21, 1 }, 6670 { "CdataOut", 20, 1 }, 6671 { "EreadPdu", 19, 1 }, 6672 { "CreadPdu", 18, 1 }, 6673 { "TunnelPkt", 17, 1 }, 6674 { "RcfPeerFin", 16, 1 }, 6675 { "RcfReasonOut", 12, 4 }, 6676 { "TxCchannel", 10, 2 }, 6677 { "RcfTxChannel", 8, 2 }, 6678 { "RxEchannel", 6, 2 }, 6679 { "RcfRxChannel", 5, 1 }, 6680 { "RcfDataOutSrdy", 4, 1 }, 6681 { "RxDvld", 3, 1 }, 6682 { "RxOoDvld", 2, 1 }, 6683 { "RxCongestion", 1, 1 }, 6684 { "TxCongestion", 0, 1 }, 6685 { NULL } 6686}; 6687 6688static struct field_desc tp_la1[] = { 6689 { "CplCmdIn", 56, 8 }, 6690 { "CplCmdOut", 48, 8 }, 6691 { "ESynOut", 47, 1 }, 6692 { "EAckOut", 46, 1 }, 6693 { "EFinOut", 45, 1 }, 6694 { "ERstOut", 44, 1 }, 6695 { "SynIn", 43, 1 }, 6696 { "AckIn", 42, 1 }, 6697 { "FinIn", 41, 1 }, 6698 { "RstIn", 40, 1 }, 6699 { "DataIn", 39, 1 }, 6700 { "DataInVld", 38, 1 }, 6701 { "PadIn", 37, 1 }, 6702 { "RxBufEmpty", 36, 1 }, 6703 { "RxDdp", 35, 1 }, 6704 { "RxFbCongestion", 34, 1 }, 6705 { "TxFbCongestion", 33, 1 }, 6706 { "TxPktSumSrdy", 32, 1 }, 6707 { "RcfUlpType", 28, 4 }, 6708 { "Eread", 27, 1 }, 6709 { "Ebypass", 26, 1 }, 6710 { "Esave", 25, 1 }, 6711 { "Static0", 24, 1 }, 6712 { "Cread", 23, 1 }, 6713 { "Cbypass", 22, 1 }, 6714 { "Csave", 21, 1 }, 6715 { "CPktOut", 20, 1 }, 6716 { "RxPagePoolFull", 18, 2 }, 6717 { "RxLpbkPkt", 17, 1 }, 6718 { "TxLpbkPkt", 16, 1 }, 6719 { "RxVfValid", 15, 1 }, 6720 { "SynLearned", 14, 1 }, 6721 { "SetDelEntry", 13, 1 }, 6722 { "SetInvEntry", 12, 1 }, 6723 { "CpcmdDvld", 11, 1 }, 6724 { "CpcmdSave", 10, 1 }, 6725 { "RxPstructsFull", 8, 2 }, 6726 { "EpcmdDvld", 7, 1 }, 6727 { "EpcmdFlush", 6, 1 }, 6728 { "EpcmdTrimPrefix", 5, 1 }, 6729 { "EpcmdTrimPostfix", 4, 1 }, 6730 { "ERssIp4Pkt", 3, 1 }, 6731 { "ERssIp6Pkt", 2, 1 }, 6732 { "ERssTcpUdpPkt", 1, 1 }, 6733 { "ERssFceFipPkt", 0, 1 }, 6734 { NULL } 6735}; 6736 6737static struct field_desc tp_la2[] = { 6738 { "CplCmdIn", 56, 8 }, 6739 { "MpsVfVld", 55, 1 }, 6740 { "MpsPf", 52, 3 }, 6741 { "MpsVf", 44, 8 }, 6742 { "SynIn", 43, 1 }, 6743 { "AckIn", 42, 1 }, 6744 { "FinIn", 41, 1 }, 6745 { "RstIn", 40, 1 }, 6746 { "DataIn", 39, 1 }, 6747 { "DataInVld", 38, 1 }, 6748 { "PadIn", 37, 1 }, 6749 { "RxBufEmpty", 36, 1 }, 6750 { "RxDdp", 35, 1 }, 6751 { "RxFbCongestion", 34, 1 }, 6752 { "TxFbCongestion", 33, 1 }, 6753 { "TxPktSumSrdy", 32, 1 }, 6754 { "RcfUlpType", 28, 4 }, 6755 { "Eread", 27, 1 }, 6756 { "Ebypass", 26, 1 }, 6757 { "Esave", 25, 1 }, 6758 { "Static0", 24, 1 }, 6759 { "Cread", 23, 1 }, 6760 { "Cbypass", 22, 1 }, 6761 { "Csave", 21, 1 }, 6762 { "CPktOut", 20, 1 }, 6763 { "RxPagePoolFull", 18, 2 }, 6764 { "RxLpbkPkt", 17, 1 }, 6765 { "TxLpbkPkt", 16, 1 }, 6766 { "RxVfValid", 15, 1 }, 6767 { "SynLearned", 14, 1 }, 6768 { "SetDelEntry", 13, 1 }, 6769 { "SetInvEntry", 12, 1 }, 6770 { "CpcmdDvld", 11, 1 }, 6771 { "CpcmdSave", 10, 1 }, 6772 { "RxPstructsFull", 8, 2 }, 6773 { "EpcmdDvld", 7, 1 }, 6774 { "EpcmdFlush", 6, 1 }, 6775 { "EpcmdTrimPrefix", 5, 1 }, 6776 { "EpcmdTrimPostfix", 4, 1 }, 6777 { "ERssIp4Pkt", 3, 1 }, 6778 { "ERssIp6Pkt", 2, 1 }, 6779 { "ERssTcpUdpPkt", 1, 1 }, 6780 { "ERssFceFipPkt", 0, 1 }, 6781 { NULL } 6782}; 6783 6784static void 6785tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 6786{ 6787 6788 field_desc_show(sb, *p, tp_la0); 6789} 6790 6791static void 6792tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 6793{ 6794 6795 if (idx) 6796 sbuf_printf(sb, "\n"); 6797 field_desc_show(sb, p[0], tp_la0); 6798 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6799 field_desc_show(sb, p[1], tp_la0); 6800} 6801 6802static void 6803tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 6804{ 6805 6806 if (idx) 6807 sbuf_printf(sb, "\n"); 6808 field_desc_show(sb, p[0], tp_la0); 6809 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 6810 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 6811} 6812 6813static int 6814sysctl_tp_la(SYSCTL_HANDLER_ARGS) 6815{ 6816 struct adapter *sc = arg1; 6817 struct sbuf *sb; 6818 uint64_t *buf, *p; 6819 int rc; 6820 u_int i, inc; 6821 void (*show_func)(struct sbuf *, uint64_t *, int); 6822 6823 rc = sysctl_wire_old_buffer(req, 0); 6824 if (rc != 0) 6825 return (rc); 6826 6827 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6828 if (sb == NULL) 6829 return (ENOMEM); 6830 6831 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 6832 6833 t4_tp_read_la(sc, buf, NULL); 6834 p = buf; 6835 6836 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 6837 case 2: 6838 inc = 2; 6839 show_func = tp_la_show2; 6840 break; 6841 case 3: 6842 inc = 2; 6843 show_func = tp_la_show3; 6844 break; 6845 default: 6846 inc = 1; 6847 show_func = tp_la_show; 6848 } 6849 6850 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 6851 (*show_func)(sb, p, i); 6852 6853 rc = sbuf_finish(sb); 6854 sbuf_delete(sb); 6855 free(buf, M_CXGBE); 6856 return (rc); 6857} 6858 6859static int 6860sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 6861{ 6862 struct adapter *sc = arg1; 6863 struct sbuf *sb; 6864 int rc; 6865 u64 nrate[NCHAN], orate[NCHAN]; 6866 6867 rc = sysctl_wire_old_buffer(req, 0); 6868 if (rc != 0) 6869 return (rc); 6870 6871 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6872 if (sb == NULL) 6873 return (ENOMEM); 6874 6875 t4_get_chan_txrate(sc, nrate, orate); 6876 sbuf_printf(sb, " channel 0 channel 1 channel 2 " 6877 "channel 3\n"); 6878 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 6879 nrate[0], nrate[1], nrate[2], nrate[3]); 6880 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 6881 orate[0], orate[1], orate[2], orate[3]); 6882 6883 rc = sbuf_finish(sb); 6884 sbuf_delete(sb); 6885 6886 return (rc); 6887} 6888 6889static int 6890sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 6891{ 6892 struct adapter *sc = arg1; 6893 struct sbuf *sb; 6894 uint32_t *buf, *p; 6895 int rc, i; 6896 6897 rc = sysctl_wire_old_buffer(req, 0); 6898 if (rc != 0) 6899 return (rc); 6900 6901 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6902 if (sb == NULL) 6903 return (ENOMEM); 6904 6905 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 6906 M_ZERO | M_WAITOK); 6907 6908 t4_ulprx_read_la(sc, buf); 6909 p = buf; 6910 6911 sbuf_printf(sb, " Pcmd Type Message" 6912 " Data"); 6913 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 6914 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 6915 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 6916 } 6917 6918 rc = sbuf_finish(sb); 6919 sbuf_delete(sb); 6920 free(buf, M_CXGBE); 6921 return (rc); 6922} 6923 6924static int 6925sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 6926{ 6927 struct adapter *sc = arg1; 6928 struct sbuf *sb; 6929 int rc, v; 6930 6931 rc = sysctl_wire_old_buffer(req, 0); 6932 if (rc != 0) 6933 return (rc); 6934 6935 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6936 if (sb == NULL) 6937 return (ENOMEM); 6938 6939 v = t4_read_reg(sc, A_SGE_STAT_CFG); 6940 if (G_STATSOURCE_T5(v) == 7) { 6941 if (G_STATMODE(v) == 0) { 6942 sbuf_printf(sb, "total %d, incomplete %d", 6943 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6944 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6945 } else if (G_STATMODE(v) == 1) { 6946 sbuf_printf(sb, "total %d, data overflow %d", 6947 t4_read_reg(sc, A_SGE_STAT_TOTAL), 6948 t4_read_reg(sc, A_SGE_STAT_MATCH)); 6949 } 6950 } 6951 rc = sbuf_finish(sb); 6952 sbuf_delete(sb); 6953 6954 return (rc); 6955} 6956#endif 6957 6958static uint32_t 6959fconf_to_mode(uint32_t fconf) 6960{ 6961 uint32_t mode; 6962 6963 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 6964 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 6965 6966 if (fconf & F_FRAGMENTATION) 6967 mode |= T4_FILTER_IP_FRAGMENT; 6968 6969 if (fconf & F_MPSHITTYPE) 6970 mode |= T4_FILTER_MPS_HIT_TYPE; 6971 6972 if (fconf & F_MACMATCH) 6973 mode |= T4_FILTER_MAC_IDX; 6974 6975 if (fconf & F_ETHERTYPE) 6976 mode |= T4_FILTER_ETH_TYPE; 6977 6978 if (fconf & F_PROTOCOL) 6979 mode |= T4_FILTER_IP_PROTO; 6980 6981 if (fconf & F_TOS) 6982 mode |= T4_FILTER_IP_TOS; 6983 6984 if (fconf & F_VLAN) 6985 mode |= T4_FILTER_VLAN; 6986 6987 if (fconf & F_VNIC_ID) 6988 mode |= T4_FILTER_VNIC; 6989 6990 if (fconf & F_PORT) 6991 mode |= T4_FILTER_PORT; 6992 6993 if (fconf & F_FCOE) 6994 mode |= T4_FILTER_FCoE; 6995 6996 return (mode); 6997} 6998 6999static uint32_t 7000mode_to_fconf(uint32_t mode) 7001{ 7002 uint32_t fconf = 0; 7003 7004 if (mode & T4_FILTER_IP_FRAGMENT) 7005 fconf |= F_FRAGMENTATION; 7006 7007 if (mode & T4_FILTER_MPS_HIT_TYPE) 7008 fconf |= F_MPSHITTYPE; 7009 7010 if (mode & T4_FILTER_MAC_IDX) 7011 fconf |= F_MACMATCH; 7012 7013 if (mode & T4_FILTER_ETH_TYPE) 7014 fconf |= F_ETHERTYPE; 7015 7016 if (mode & T4_FILTER_IP_PROTO) 7017 fconf |= F_PROTOCOL; 7018 7019 if (mode & T4_FILTER_IP_TOS) 7020 fconf |= F_TOS; 7021 7022 if (mode & T4_FILTER_VLAN) 7023 fconf |= F_VLAN; 7024 7025 if (mode & T4_FILTER_VNIC) 7026 fconf |= F_VNIC_ID; 7027 7028 if (mode & T4_FILTER_PORT) 7029 fconf |= F_PORT; 7030 7031 if (mode & T4_FILTER_FCoE) 7032 fconf |= F_FCOE; 7033 7034 return (fconf); 7035} 7036 7037static uint32_t 7038fspec_to_fconf(struct t4_filter_specification *fs) 7039{ 7040 uint32_t fconf = 0; 7041 7042 if (fs->val.frag || fs->mask.frag) 7043 fconf |= F_FRAGMENTATION; 7044 7045 if (fs->val.matchtype || fs->mask.matchtype) 7046 fconf |= F_MPSHITTYPE; 7047 7048 if (fs->val.macidx || fs->mask.macidx) 7049 fconf |= F_MACMATCH; 7050 7051 if (fs->val.ethtype || fs->mask.ethtype) 7052 fconf |= F_ETHERTYPE; 7053 7054 if (fs->val.proto || fs->mask.proto) 7055 fconf |= F_PROTOCOL; 7056 7057 if (fs->val.tos || fs->mask.tos) 7058 fconf |= F_TOS; 7059 7060 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7061 fconf |= F_VLAN; 7062 7063 if (fs->val.vnic_vld || fs->mask.vnic_vld) 7064 fconf |= F_VNIC_ID; 7065 7066 if (fs->val.iport || fs->mask.iport) 7067 fconf |= F_PORT; 7068 7069 if (fs->val.fcoe || fs->mask.fcoe) 7070 fconf |= F_FCOE; 7071 7072 return (fconf); 7073} 7074 7075static int 7076get_filter_mode(struct adapter *sc, uint32_t *mode) 7077{ 7078 int rc; 7079 uint32_t fconf; 7080 7081 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7082 "t4getfm"); 7083 if (rc) 7084 return (rc); 7085 7086 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 7087 A_TP_VLAN_PRI_MAP); 7088 7089 if (sc->params.tp.vlan_pri_map != fconf) { 7090 log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 7091 device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map, 7092 fconf); 7093 } 7094 7095 *mode = fconf_to_mode(fconf); 7096 7097 end_synchronized_op(sc, LOCK_HELD); 7098 return (0); 7099} 7100 7101static int 7102set_filter_mode(struct adapter *sc, uint32_t mode) 7103{ 7104 uint32_t fconf; 7105 int rc; 7106 7107 fconf = mode_to_fconf(mode); 7108 7109 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7110 "t4setfm"); 7111 if (rc) 7112 return (rc); 7113 7114 if (sc->tids.ftids_in_use > 0) { 7115 rc = EBUSY; 7116 goto done; 7117 } 7118 7119#ifdef TCP_OFFLOAD 7120 if (uld_active(sc, ULD_TOM)) { 7121 rc = EBUSY; 7122 goto done; 7123 } 7124#endif 7125 7126 rc = -t4_set_filter_mode(sc, fconf); 7127done: 7128 end_synchronized_op(sc, LOCK_HELD); 7129 return (rc); 7130} 7131 7132static inline uint64_t 7133get_filter_hits(struct adapter *sc, uint32_t fid) 7134{ 7135 uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 7136 uint64_t hits; 7137 7138 memwin_info(sc, 0, &mw_base, NULL); 7139 off = position_memwin(sc, 0, 7140 tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 7141 if (is_t4(sc)) { 7142 hits = t4_read_reg64(sc, mw_base + off + 16); 7143 hits = be64toh(hits); 7144 } else { 7145 hits = t4_read_reg(sc, mw_base + off + 24); 7146 hits = be32toh(hits); 7147 } 7148 7149 return (hits); 7150} 7151 7152static int 7153get_filter(struct adapter *sc, struct t4_filter *t) 7154{ 7155 int i, rc, nfilters = sc->tids.nftids; 7156 struct filter_entry *f; 7157 7158 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7159 "t4getf"); 7160 if (rc) 7161 return (rc); 7162 7163 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7164 t->idx >= nfilters) { 7165 t->idx = 0xffffffff; 7166 goto done; 7167 } 7168 7169 f = &sc->tids.ftid_tab[t->idx]; 7170 for (i = t->idx; i < nfilters; i++, f++) { 7171 if (f->valid) { 7172 t->idx = i; 7173 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7174 t->smtidx = f->smtidx; 7175 if (f->fs.hitcnts) 7176 t->hits = get_filter_hits(sc, t->idx); 7177 else 7178 t->hits = UINT64_MAX; 7179 t->fs = f->fs; 7180 7181 goto done; 7182 } 7183 } 7184 7185 t->idx = 0xffffffff; 7186done: 7187 end_synchronized_op(sc, LOCK_HELD); 7188 return (0); 7189} 7190 7191static int 7192set_filter(struct adapter *sc, struct t4_filter *t) 7193{ 7194 unsigned int nfilters, nports; 7195 struct filter_entry *f; 7196 int i, rc; 7197 7198 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7199 if (rc) 7200 return (rc); 7201 7202 nfilters = sc->tids.nftids; 7203 nports = sc->params.nports; 7204 7205 if (nfilters == 0) { 7206 rc = ENOTSUP; 7207 goto done; 7208 } 7209 7210 if (!(sc->flags & FULL_INIT_DONE)) { 7211 rc = EAGAIN; 7212 goto done; 7213 } 7214 7215 if (t->idx >= nfilters) { 7216 rc = EINVAL; 7217 goto done; 7218 } 7219 7220 /* Validate against the global filter mode */ 7221 if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) != 7222 sc->params.tp.vlan_pri_map) { 7223 rc = E2BIG; 7224 goto done; 7225 } 7226 7227 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7228 rc = EINVAL; 7229 goto done; 7230 } 7231 7232 if (t->fs.val.iport >= nports) { 7233 rc = EINVAL; 7234 goto done; 7235 } 7236 7237 /* Can't specify an iq if not steering to it */ 7238 if (!t->fs.dirsteer && t->fs.iq) { 7239 rc = EINVAL; 7240 goto done; 7241 } 7242 7243 /* IPv6 filter idx must be 4 aligned */ 7244 if (t->fs.type == 1 && 7245 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7246 rc = EINVAL; 7247 goto done; 7248 } 7249 7250 if (sc->tids.ftid_tab == NULL) { 7251 KASSERT(sc->tids.ftids_in_use == 0, 7252 ("%s: no memory allocated but filters_in_use > 0", 7253 __func__)); 7254 7255 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7256 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7257 if (sc->tids.ftid_tab == NULL) { 7258 rc = ENOMEM; 7259 goto done; 7260 } 7261 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7262 } 7263 7264 for (i = 0; i < 4; i++) { 7265 f = &sc->tids.ftid_tab[t->idx + i]; 7266 7267 if (f->pending || f->valid) { 7268 rc = EBUSY; 7269 goto done; 7270 } 7271 if (f->locked) { 7272 rc = EPERM; 7273 goto done; 7274 } 7275 7276 if (t->fs.type == 0) 7277 break; 7278 } 7279 7280 f = &sc->tids.ftid_tab[t->idx]; 7281 f->fs = t->fs; 7282 7283 rc = set_filter_wr(sc, t->idx); 7284done: 7285 end_synchronized_op(sc, 0); 7286 7287 if (rc == 0) { 7288 mtx_lock(&sc->tids.ftid_lock); 7289 for (;;) { 7290 if (f->pending == 0) { 7291 rc = f->valid ? 0 : EIO; 7292 break; 7293 } 7294 7295 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7296 PCATCH, "t4setfw", 0)) { 7297 rc = EINPROGRESS; 7298 break; 7299 } 7300 } 7301 mtx_unlock(&sc->tids.ftid_lock); 7302 } 7303 return (rc); 7304} 7305 7306static int 7307del_filter(struct adapter *sc, struct t4_filter *t) 7308{ 7309 unsigned int nfilters; 7310 struct filter_entry *f; 7311 int rc; 7312 7313 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7314 if (rc) 7315 return (rc); 7316 7317 nfilters = sc->tids.nftids; 7318 7319 if (nfilters == 0) { 7320 rc = ENOTSUP; 7321 goto done; 7322 } 7323 7324 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7325 t->idx >= nfilters) { 7326 rc = EINVAL; 7327 goto done; 7328 } 7329 7330 if (!(sc->flags & FULL_INIT_DONE)) { 7331 rc = EAGAIN; 7332 goto done; 7333 } 7334 7335 f = &sc->tids.ftid_tab[t->idx]; 7336 7337 if (f->pending) { 7338 rc = EBUSY; 7339 goto done; 7340 } 7341 if (f->locked) { 7342 rc = EPERM; 7343 goto done; 7344 } 7345 7346 if (f->valid) { 7347 t->fs = f->fs; /* extra info for the caller */ 7348 rc = del_filter_wr(sc, t->idx); 7349 } 7350 7351done: 7352 end_synchronized_op(sc, 0); 7353 7354 if (rc == 0) { 7355 mtx_lock(&sc->tids.ftid_lock); 7356 for (;;) { 7357 if (f->pending == 0) { 7358 rc = f->valid ? EIO : 0; 7359 break; 7360 } 7361 7362 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7363 PCATCH, "t4delfw", 0)) { 7364 rc = EINPROGRESS; 7365 break; 7366 } 7367 } 7368 mtx_unlock(&sc->tids.ftid_lock); 7369 } 7370 7371 return (rc); 7372} 7373 7374static void 7375clear_filter(struct filter_entry *f) 7376{ 7377 if (f->l2t) 7378 t4_l2t_release(f->l2t); 7379 7380 bzero(f, sizeof (*f)); 7381} 7382 7383static int 7384set_filter_wr(struct adapter *sc, int fidx) 7385{ 7386 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7387 struct fw_filter_wr *fwr; 7388 unsigned int ftid; 7389 struct wrq_cookie cookie; 7390 7391 ASSERT_SYNCHRONIZED_OP(sc); 7392 7393 if (f->fs.newdmac || f->fs.newvlan) { 7394 /* This filter needs an L2T entry; allocate one. */ 7395 f->l2t = t4_l2t_alloc_switching(sc->l2t); 7396 if (f->l2t == NULL) 7397 return (EAGAIN); 7398 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 7399 f->fs.dmac)) { 7400 t4_l2t_release(f->l2t); 7401 f->l2t = NULL; 7402 return (ENOMEM); 7403 } 7404 } 7405 7406 ftid = sc->tids.ftid_base + fidx; 7407 7408 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7409 if (fwr == NULL) 7410 return (ENOMEM); 7411 bzero(fwr, sizeof(*fwr)); 7412 7413 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 7414 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 7415 fwr->tid_to_iq = 7416 htobe32(V_FW_FILTER_WR_TID(ftid) | 7417 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 7418 V_FW_FILTER_WR_NOREPLY(0) | 7419 V_FW_FILTER_WR_IQ(f->fs.iq)); 7420 fwr->del_filter_to_l2tix = 7421 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 7422 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 7423 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 7424 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 7425 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 7426 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 7427 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 7428 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 7429 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 7430 f->fs.newvlan == VLAN_REWRITE) | 7431 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 7432 f->fs.newvlan == VLAN_REWRITE) | 7433 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 7434 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 7435 V_FW_FILTER_WR_PRIO(f->fs.prio) | 7436 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 7437 fwr->ethtype = htobe16(f->fs.val.ethtype); 7438 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 7439 fwr->frag_to_ovlan_vldm = 7440 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 7441 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 7442 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 7443 V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 7444 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 7445 V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 7446 fwr->smac_sel = 0; 7447 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 7448 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 7449 fwr->maci_to_matchtypem = 7450 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 7451 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 7452 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 7453 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 7454 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 7455 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 7456 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 7457 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 7458 fwr->ptcl = f->fs.val.proto; 7459 fwr->ptclm = f->fs.mask.proto; 7460 fwr->ttyp = f->fs.val.tos; 7461 fwr->ttypm = f->fs.mask.tos; 7462 fwr->ivlan = htobe16(f->fs.val.vlan); 7463 fwr->ivlanm = htobe16(f->fs.mask.vlan); 7464 fwr->ovlan = htobe16(f->fs.val.vnic); 7465 fwr->ovlanm = htobe16(f->fs.mask.vnic); 7466 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 7467 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 7468 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 7469 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 7470 fwr->lp = htobe16(f->fs.val.dport); 7471 fwr->lpm = htobe16(f->fs.mask.dport); 7472 fwr->fp = htobe16(f->fs.val.sport); 7473 fwr->fpm = htobe16(f->fs.mask.sport); 7474 if (f->fs.newsmac) 7475 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 7476 7477 f->pending = 1; 7478 sc->tids.ftids_in_use++; 7479 7480 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7481 return (0); 7482} 7483 7484static int 7485del_filter_wr(struct adapter *sc, int fidx) 7486{ 7487 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 7488 struct fw_filter_wr *fwr; 7489 unsigned int ftid; 7490 struct wrq_cookie cookie; 7491 7492 ftid = sc->tids.ftid_base + fidx; 7493 7494 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 7495 if (fwr == NULL) 7496 return (ENOMEM); 7497 bzero(fwr, sizeof (*fwr)); 7498 7499 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 7500 7501 f->pending = 1; 7502 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 7503 return (0); 7504} 7505 7506int 7507t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 7508{ 7509 struct adapter *sc = iq->adapter; 7510 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 7511 unsigned int idx = GET_TID(rpl); 7512 unsigned int rc; 7513 struct filter_entry *f; 7514 7515 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 7516 rss->opcode)); 7517 7518 if (is_ftid(sc, idx)) { 7519 7520 idx -= sc->tids.ftid_base; 7521 f = &sc->tids.ftid_tab[idx]; 7522 rc = G_COOKIE(rpl->cookie); 7523 7524 mtx_lock(&sc->tids.ftid_lock); 7525 if (rc == FW_FILTER_WR_FLT_ADDED) { 7526 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 7527 __func__, idx)); 7528 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 7529 f->pending = 0; /* asynchronous setup completed */ 7530 f->valid = 1; 7531 } else { 7532 if (rc != FW_FILTER_WR_FLT_DELETED) { 7533 /* Add or delete failed, display an error */ 7534 log(LOG_ERR, 7535 "filter %u setup failed with error %u\n", 7536 idx, rc); 7537 } 7538 7539 clear_filter(f); 7540 sc->tids.ftids_in_use--; 7541 } 7542 wakeup(&sc->tids.ftid_tab); 7543 mtx_unlock(&sc->tids.ftid_lock); 7544 } 7545 7546 return (0); 7547} 7548 7549static int 7550get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 7551{ 7552 int rc; 7553 7554 if (cntxt->cid > M_CTXTQID) 7555 return (EINVAL); 7556 7557 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 7558 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 7559 return (EINVAL); 7560 7561 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 7562 if (rc) 7563 return (rc); 7564 7565 if (sc->flags & FW_OK) { 7566 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 7567 &cntxt->data[0]); 7568 if (rc == 0) 7569 goto done; 7570 } 7571 7572 /* 7573 * Read via firmware failed or wasn't even attempted. Read directly via 7574 * the backdoor. 7575 */ 7576 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 7577done: 7578 end_synchronized_op(sc, 0); 7579 return (rc); 7580} 7581 7582static int 7583load_fw(struct adapter *sc, struct t4_data *fw) 7584{ 7585 int rc; 7586 uint8_t *fw_data; 7587 7588 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 7589 if (rc) 7590 return (rc); 7591 7592 if (sc->flags & FULL_INIT_DONE) { 7593 rc = EBUSY; 7594 goto done; 7595 } 7596 7597 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 7598 if (fw_data == NULL) { 7599 rc = ENOMEM; 7600 goto done; 7601 } 7602 7603 rc = copyin(fw->data, fw_data, fw->len); 7604 if (rc == 0) 7605 rc = -t4_load_fw(sc, fw_data, fw->len); 7606 7607 free(fw_data, M_CXGBE); 7608done: 7609 end_synchronized_op(sc, 0); 7610 return (rc); 7611} 7612 7613static int 7614read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 7615{ 7616 uint32_t addr, off, remaining, i, n; 7617 uint32_t *buf, *b; 7618 uint32_t mw_base, mw_aperture; 7619 int rc; 7620 uint8_t *dst; 7621 7622 rc = validate_mem_range(sc, mr->addr, mr->len); 7623 if (rc != 0) 7624 return (rc); 7625 7626 memwin_info(sc, win, &mw_base, &mw_aperture); 7627 buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 7628 addr = mr->addr; 7629 remaining = mr->len; 7630 dst = (void *)mr->data; 7631 7632 while (remaining) { 7633 off = position_memwin(sc, win, addr); 7634 7635 /* number of bytes that we'll copy in the inner loop */ 7636 n = min(remaining, mw_aperture - off); 7637 for (i = 0; i < n; i += 4) 7638 *b++ = t4_read_reg(sc, mw_base + off + i); 7639 7640 rc = copyout(buf, dst, n); 7641 if (rc != 0) 7642 break; 7643 7644 b = buf; 7645 dst += n; 7646 remaining -= n; 7647 addr += n; 7648 } 7649 7650 free(buf, M_CXGBE); 7651 return (rc); 7652} 7653 7654static int 7655read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 7656{ 7657 int rc; 7658 7659 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 7660 return (EINVAL); 7661 7662 if (i2cd->len > sizeof(i2cd->data)) 7663 return (EFBIG); 7664 7665 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 7666 if (rc) 7667 return (rc); 7668 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 7669 i2cd->offset, i2cd->len, &i2cd->data[0]); 7670 end_synchronized_op(sc, 0); 7671 7672 return (rc); 7673} 7674 7675static int 7676in_range(int val, int lo, int hi) 7677{ 7678 7679 return (val < 0 || (val <= hi && val >= lo)); 7680} 7681 7682static int 7683set_sched_class(struct adapter *sc, struct t4_sched_params *p) 7684{ 7685 int fw_subcmd, fw_type, rc; 7686 7687 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsc"); 7688 if (rc) 7689 return (rc); 7690 7691 if (!(sc->flags & FULL_INIT_DONE)) { 7692 rc = EAGAIN; 7693 goto done; 7694 } 7695 7696 /* 7697 * Translate the cxgbetool parameters into T4 firmware parameters. (The 7698 * sub-command and type are in common locations.) 7699 */ 7700 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 7701 fw_subcmd = FW_SCHED_SC_CONFIG; 7702 else if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 7703 fw_subcmd = FW_SCHED_SC_PARAMS; 7704 else { 7705 rc = EINVAL; 7706 goto done; 7707 } 7708 if (p->type == SCHED_CLASS_TYPE_PACKET) 7709 fw_type = FW_SCHED_TYPE_PKTSCHED; 7710 else { 7711 rc = EINVAL; 7712 goto done; 7713 } 7714 7715 if (fw_subcmd == FW_SCHED_SC_CONFIG) { 7716 /* Vet our parameters ..*/ 7717 if (p->u.config.minmax < 0) { 7718 rc = EINVAL; 7719 goto done; 7720 } 7721 7722 /* And pass the request to the firmware ...*/ 7723 rc = -t4_sched_config(sc, fw_type, p->u.config.minmax, 1); 7724 goto done; 7725 } 7726 7727 if (fw_subcmd == FW_SCHED_SC_PARAMS) { 7728 int fw_level; 7729 int fw_mode; 7730 int fw_rateunit; 7731 int fw_ratemode; 7732 7733 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL) 7734 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 7735 else if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) 7736 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 7737 else if (p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) 7738 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 7739 else { 7740 rc = EINVAL; 7741 goto done; 7742 } 7743 7744 if (p->u.params.mode == SCHED_CLASS_MODE_CLASS) 7745 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 7746 else if (p->u.params.mode == SCHED_CLASS_MODE_FLOW) 7747 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 7748 else { 7749 rc = EINVAL; 7750 goto done; 7751 } 7752 7753 if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_BITS) 7754 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 7755 else if (p->u.params.rateunit == SCHED_CLASS_RATEUNIT_PKTS) 7756 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 7757 else { 7758 rc = EINVAL; 7759 goto done; 7760 } 7761 7762 if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_REL) 7763 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 7764 else if (p->u.params.ratemode == SCHED_CLASS_RATEMODE_ABS) 7765 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 7766 else { 7767 rc = EINVAL; 7768 goto done; 7769 } 7770 7771 /* Vet our parameters ... */ 7772 if (!in_range(p->u.params.channel, 0, 3) || 7773 !in_range(p->u.params.cl, 0, is_t4(sc) ? 15 : 16) || 7774 !in_range(p->u.params.minrate, 0, 10000000) || 7775 !in_range(p->u.params.maxrate, 0, 10000000) || 7776 !in_range(p->u.params.weight, 0, 100)) { 7777 rc = ERANGE; 7778 goto done; 7779 } 7780 7781 /* 7782 * Translate any unset parameters into the firmware's 7783 * nomenclature and/or fail the call if the parameters 7784 * are required ... 7785 */ 7786 if (p->u.params.rateunit < 0 || p->u.params.ratemode < 0 || 7787 p->u.params.channel < 0 || p->u.params.cl < 0) { 7788 rc = EINVAL; 7789 goto done; 7790 } 7791 if (p->u.params.minrate < 0) 7792 p->u.params.minrate = 0; 7793 if (p->u.params.maxrate < 0) { 7794 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7795 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7796 rc = EINVAL; 7797 goto done; 7798 } else 7799 p->u.params.maxrate = 0; 7800 } 7801 if (p->u.params.weight < 0) { 7802 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_WRR) { 7803 rc = EINVAL; 7804 goto done; 7805 } else 7806 p->u.params.weight = 0; 7807 } 7808 if (p->u.params.pktsize < 0) { 7809 if (p->u.params.level == SCHED_CLASS_LEVEL_CL_RL || 7810 p->u.params.level == SCHED_CLASS_LEVEL_CH_RL) { 7811 rc = EINVAL; 7812 goto done; 7813 } else 7814 p->u.params.pktsize = 0; 7815 } 7816 7817 /* See what the firmware thinks of the request ... */ 7818 rc = -t4_sched_params(sc, fw_type, fw_level, fw_mode, 7819 fw_rateunit, fw_ratemode, p->u.params.channel, 7820 p->u.params.cl, p->u.params.minrate, p->u.params.maxrate, 7821 p->u.params.weight, p->u.params.pktsize, 1); 7822 goto done; 7823 } 7824 7825 rc = EINVAL; 7826done: 7827 end_synchronized_op(sc, 0); 7828 return (rc); 7829} 7830 7831static int 7832set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 7833{ 7834 struct port_info *pi = NULL; 7835 struct sge_txq *txq; 7836 uint32_t fw_mnem, fw_queue, fw_class; 7837 int i, rc; 7838 7839 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 7840 if (rc) 7841 return (rc); 7842 7843 if (!(sc->flags & FULL_INIT_DONE)) { 7844 rc = EAGAIN; 7845 goto done; 7846 } 7847 7848 if (p->port >= sc->params.nports) { 7849 rc = EINVAL; 7850 goto done; 7851 } 7852 7853 pi = sc->port[p->port]; 7854 if (!in_range(p->queue, 0, pi->ntxq - 1) || !in_range(p->cl, 0, 7)) { 7855 rc = EINVAL; 7856 goto done; 7857 } 7858 7859 /* 7860 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 7861 * Scheduling Class in this case). 7862 */ 7863 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 7864 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 7865 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 7866 7867 /* 7868 * If op.queue is non-negative, then we're only changing the scheduling 7869 * on a single specified TX queue. 7870 */ 7871 if (p->queue >= 0) { 7872 txq = &sc->sge.txq[pi->first_txq + p->queue]; 7873 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7874 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7875 &fw_class); 7876 goto done; 7877 } 7878 7879 /* 7880 * Change the scheduling on all the TX queues for the 7881 * interface. 7882 */ 7883 for_each_txq(pi, i, txq) { 7884 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 7885 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 7886 &fw_class); 7887 if (rc) 7888 goto done; 7889 } 7890 7891 rc = 0; 7892done: 7893 end_synchronized_op(sc, 0); 7894 return (rc); 7895} 7896 7897int 7898t4_os_find_pci_capability(struct adapter *sc, int cap) 7899{ 7900 int i; 7901 7902 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 7903} 7904 7905int 7906t4_os_pci_save_state(struct adapter *sc) 7907{ 7908 device_t dev; 7909 struct pci_devinfo *dinfo; 7910 7911 dev = sc->dev; 7912 dinfo = device_get_ivars(dev); 7913 7914 pci_cfg_save(dev, dinfo, 0); 7915 return (0); 7916} 7917 7918int 7919t4_os_pci_restore_state(struct adapter *sc) 7920{ 7921 device_t dev; 7922 struct pci_devinfo *dinfo; 7923 7924 dev = sc->dev; 7925 dinfo = device_get_ivars(dev); 7926 7927 pci_cfg_restore(dev, dinfo); 7928 return (0); 7929} 7930 7931void 7932t4_os_portmod_changed(const struct adapter *sc, int idx) 7933{ 7934 struct port_info *pi = sc->port[idx]; 7935 static const char *mod_str[] = { 7936 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 7937 }; 7938 7939 build_medialist(pi, &pi->media); 7940#ifdef DEV_NETMAP 7941 build_medialist(pi, &pi->nm_media); 7942#endif 7943 7944 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 7945 if_printf(pi->ifp, "transceiver unplugged.\n"); 7946 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 7947 if_printf(pi->ifp, "unknown transceiver inserted.\n"); 7948 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 7949 if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 7950 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 7951 if_printf(pi->ifp, "%s transceiver inserted.\n", 7952 mod_str[pi->mod_type]); 7953 } else { 7954 if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 7955 pi->mod_type); 7956 } 7957} 7958 7959void 7960t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 7961{ 7962 struct port_info *pi = sc->port[idx]; 7963 struct ifnet *ifp = pi->ifp; 7964 7965 if (link_stat) { 7966 pi->linkdnrc = -1; 7967 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 7968 if_link_state_change(ifp, LINK_STATE_UP); 7969 } else { 7970 if (reason >= 0) 7971 pi->linkdnrc = reason; 7972 if_link_state_change(ifp, LINK_STATE_DOWN); 7973 } 7974} 7975 7976void 7977t4_iterate(void (*func)(struct adapter *, void *), void *arg) 7978{ 7979 struct adapter *sc; 7980 7981 sx_slock(&t4_list_lock); 7982 SLIST_FOREACH(sc, &t4_list, link) { 7983 /* 7984 * func should not make any assumptions about what state sc is 7985 * in - the only guarantee is that sc->sc_lock is a valid lock. 7986 */ 7987 func(sc, arg); 7988 } 7989 sx_sunlock(&t4_list_lock); 7990} 7991 7992static int 7993t4_open(struct cdev *dev, int flags, int type, struct thread *td) 7994{ 7995 return (0); 7996} 7997 7998static int 7999t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8000{ 8001 return (0); 8002} 8003 8004static int 8005t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8006 struct thread *td) 8007{ 8008 int rc; 8009 struct adapter *sc = dev->si_drv1; 8010 8011 rc = priv_check(td, PRIV_DRIVER); 8012 if (rc != 0) 8013 return (rc); 8014 8015 switch (cmd) { 8016 case CHELSIO_T4_GETREG: { 8017 struct t4_reg *edata = (struct t4_reg *)data; 8018 8019 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8020 return (EFAULT); 8021 8022 if (edata->size == 4) 8023 edata->val = t4_read_reg(sc, edata->addr); 8024 else if (edata->size == 8) 8025 edata->val = t4_read_reg64(sc, edata->addr); 8026 else 8027 return (EINVAL); 8028 8029 break; 8030 } 8031 case CHELSIO_T4_SETREG: { 8032 struct t4_reg *edata = (struct t4_reg *)data; 8033 8034 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8035 return (EFAULT); 8036 8037 if (edata->size == 4) { 8038 if (edata->val & 0xffffffff00000000) 8039 return (EINVAL); 8040 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8041 } else if (edata->size == 8) 8042 t4_write_reg64(sc, edata->addr, edata->val); 8043 else 8044 return (EINVAL); 8045 break; 8046 } 8047 case CHELSIO_T4_REGDUMP: { 8048 struct t4_regdump *regs = (struct t4_regdump *)data; 8049 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8050 uint8_t *buf; 8051 8052 if (regs->len < reglen) { 8053 regs->len = reglen; /* hint to the caller */ 8054 return (ENOBUFS); 8055 } 8056 8057 regs->len = reglen; 8058 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8059 t4_get_regs(sc, regs, buf); 8060 rc = copyout(buf, regs->data, reglen); 8061 free(buf, M_CXGBE); 8062 break; 8063 } 8064 case CHELSIO_T4_GET_FILTER_MODE: 8065 rc = get_filter_mode(sc, (uint32_t *)data); 8066 break; 8067 case CHELSIO_T4_SET_FILTER_MODE: 8068 rc = set_filter_mode(sc, *(uint32_t *)data); 8069 break; 8070 case CHELSIO_T4_GET_FILTER: 8071 rc = get_filter(sc, (struct t4_filter *)data); 8072 break; 8073 case CHELSIO_T4_SET_FILTER: 8074 rc = set_filter(sc, (struct t4_filter *)data); 8075 break; 8076 case CHELSIO_T4_DEL_FILTER: 8077 rc = del_filter(sc, (struct t4_filter *)data); 8078 break; 8079 case CHELSIO_T4_GET_SGE_CONTEXT: 8080 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8081 break; 8082 case CHELSIO_T4_LOAD_FW: 8083 rc = load_fw(sc, (struct t4_data *)data); 8084 break; 8085 case CHELSIO_T4_GET_MEM: 8086 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8087 break; 8088 case CHELSIO_T4_GET_I2C: 8089 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8090 break; 8091 case CHELSIO_T4_CLEAR_STATS: { 8092 int i; 8093 u_int port_id = *(uint32_t *)data; 8094 struct port_info *pi; 8095 8096 if (port_id >= sc->params.nports) 8097 return (EINVAL); 8098 pi = sc->port[port_id]; 8099 8100 /* MAC stats */ 8101 t4_clr_port_stats(sc, pi->tx_chan); 8102 pi->tx_parse_error = 0; 8103 8104 if (pi->flags & PORT_INIT_DONE) { 8105 struct sge_rxq *rxq; 8106 struct sge_txq *txq; 8107 struct sge_wrq *wrq; 8108 8109 for_each_rxq(pi, i, rxq) { 8110#if defined(INET) || defined(INET6) 8111 rxq->lro.lro_queued = 0; 8112 rxq->lro.lro_flushed = 0; 8113#endif 8114 rxq->rxcsum = 0; 8115 rxq->vlan_extraction = 0; 8116 } 8117 8118 for_each_txq(pi, i, txq) { 8119 txq->txcsum = 0; 8120 txq->tso_wrs = 0; 8121 txq->vlan_insertion = 0; 8122 txq->imm_wrs = 0; 8123 txq->sgl_wrs = 0; 8124 txq->txpkt_wrs = 0; 8125 txq->txpkts0_wrs = 0; 8126 txq->txpkts1_wrs = 0; 8127 txq->txpkts0_pkts = 0; 8128 txq->txpkts1_pkts = 0; 8129 mp_ring_reset_stats(txq->r); 8130 } 8131 8132#ifdef TCP_OFFLOAD 8133 /* nothing to clear for each ofld_rxq */ 8134 8135 for_each_ofld_txq(pi, i, wrq) { 8136 wrq->tx_wrs_direct = 0; 8137 wrq->tx_wrs_copied = 0; 8138 } 8139#endif 8140 wrq = &sc->sge.ctrlq[pi->port_id]; 8141 wrq->tx_wrs_direct = 0; 8142 wrq->tx_wrs_copied = 0; 8143 } 8144 break; 8145 } 8146 case CHELSIO_T4_SCHED_CLASS: 8147 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8148 break; 8149 case CHELSIO_T4_SCHED_QUEUE: 8150 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8151 break; 8152 case CHELSIO_T4_GET_TRACER: 8153 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8154 break; 8155 case CHELSIO_T4_SET_TRACER: 8156 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8157 break; 8158 default: 8159 rc = EINVAL; 8160 } 8161 8162 return (rc); 8163} 8164 8165#ifdef TCP_OFFLOAD 8166void 8167t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8168 const unsigned int *pgsz_order) 8169{ 8170 struct port_info *pi = ifp->if_softc; 8171 struct adapter *sc = pi->adapter; 8172 8173 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8174 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8175 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8176 V_HPZ3(pgsz_order[3])); 8177} 8178 8179static int 8180toe_capability(struct port_info *pi, int enable) 8181{ 8182 int rc; 8183 struct adapter *sc = pi->adapter; 8184 8185 ASSERT_SYNCHRONIZED_OP(sc); 8186 8187 if (!is_offload(sc)) 8188 return (ENODEV); 8189 8190 if (enable) { 8191 /* 8192 * We need the port's queues around so that we're able to send 8193 * and receive CPLs to/from the TOE even if the ifnet for this 8194 * port has never been UP'd administratively. 8195 */ 8196 if (!(pi->flags & PORT_INIT_DONE)) { 8197 rc = cxgbe_init_synchronized(pi); 8198 if (rc) 8199 return (rc); 8200 } 8201 8202 if (isset(&sc->offload_map, pi->port_id)) 8203 return (0); 8204 8205 if (!uld_active(sc, ULD_TOM)) { 8206 rc = t4_activate_uld(sc, ULD_TOM); 8207 if (rc == EAGAIN) { 8208 log(LOG_WARNING, 8209 "You must kldload t4_tom.ko before trying " 8210 "to enable TOE on a cxgbe interface.\n"); 8211 } 8212 if (rc != 0) 8213 return (rc); 8214 KASSERT(sc->tom_softc != NULL, 8215 ("%s: TOM activated but softc NULL", __func__)); 8216 KASSERT(uld_active(sc, ULD_TOM), 8217 ("%s: TOM activated but flag not set", __func__)); 8218 } 8219 8220 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8221 if (!uld_active(sc, ULD_IWARP)) 8222 (void) t4_activate_uld(sc, ULD_IWARP); 8223 if (!uld_active(sc, ULD_ISCSI)) 8224 (void) t4_activate_uld(sc, ULD_ISCSI); 8225 8226 setbit(&sc->offload_map, pi->port_id); 8227 } else { 8228 if (!isset(&sc->offload_map, pi->port_id)) 8229 return (0); 8230 8231 KASSERT(uld_active(sc, ULD_TOM), 8232 ("%s: TOM never initialized?", __func__)); 8233 clrbit(&sc->offload_map, pi->port_id); 8234 } 8235 8236 return (0); 8237} 8238 8239/* 8240 * Add an upper layer driver to the global list. 8241 */ 8242int 8243t4_register_uld(struct uld_info *ui) 8244{ 8245 int rc = 0; 8246 struct uld_info *u; 8247 8248 sx_xlock(&t4_uld_list_lock); 8249 SLIST_FOREACH(u, &t4_uld_list, link) { 8250 if (u->uld_id == ui->uld_id) { 8251 rc = EEXIST; 8252 goto done; 8253 } 8254 } 8255 8256 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8257 ui->refcount = 0; 8258done: 8259 sx_xunlock(&t4_uld_list_lock); 8260 return (rc); 8261} 8262 8263int 8264t4_unregister_uld(struct uld_info *ui) 8265{ 8266 int rc = EINVAL; 8267 struct uld_info *u; 8268 8269 sx_xlock(&t4_uld_list_lock); 8270 8271 SLIST_FOREACH(u, &t4_uld_list, link) { 8272 if (u == ui) { 8273 if (ui->refcount > 0) { 8274 rc = EBUSY; 8275 goto done; 8276 } 8277 8278 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 8279 rc = 0; 8280 goto done; 8281 } 8282 } 8283done: 8284 sx_xunlock(&t4_uld_list_lock); 8285 return (rc); 8286} 8287 8288int 8289t4_activate_uld(struct adapter *sc, int id) 8290{ 8291 int rc; 8292 struct uld_info *ui; 8293 8294 ASSERT_SYNCHRONIZED_OP(sc); 8295 8296 if (id < 0 || id > ULD_MAX) 8297 return (EINVAL); 8298 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 8299 8300 sx_slock(&t4_uld_list_lock); 8301 8302 SLIST_FOREACH(ui, &t4_uld_list, link) { 8303 if (ui->uld_id == id) { 8304 if (!(sc->flags & FULL_INIT_DONE)) { 8305 rc = adapter_full_init(sc); 8306 if (rc != 0) 8307 break; 8308 } 8309 8310 rc = ui->activate(sc); 8311 if (rc == 0) { 8312 setbit(&sc->active_ulds, id); 8313 ui->refcount++; 8314 } 8315 break; 8316 } 8317 } 8318 8319 sx_sunlock(&t4_uld_list_lock); 8320 8321 return (rc); 8322} 8323 8324int 8325t4_deactivate_uld(struct adapter *sc, int id) 8326{ 8327 int rc; 8328 struct uld_info *ui; 8329 8330 ASSERT_SYNCHRONIZED_OP(sc); 8331 8332 if (id < 0 || id > ULD_MAX) 8333 return (EINVAL); 8334 rc = ENXIO; 8335 8336 sx_slock(&t4_uld_list_lock); 8337 8338 SLIST_FOREACH(ui, &t4_uld_list, link) { 8339 if (ui->uld_id == id) { 8340 rc = ui->deactivate(sc); 8341 if (rc == 0) { 8342 clrbit(&sc->active_ulds, id); 8343 ui->refcount--; 8344 } 8345 break; 8346 } 8347 } 8348 8349 sx_sunlock(&t4_uld_list_lock); 8350 8351 return (rc); 8352} 8353 8354int 8355uld_active(struct adapter *sc, int uld_id) 8356{ 8357 8358 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 8359 8360 return (isset(&sc->active_ulds, uld_id)); 8361} 8362#endif 8363 8364/* 8365 * Come up with reasonable defaults for some of the tunables, provided they're 8366 * not set by the user (in which case we'll use the values as is). 8367 */ 8368static void 8369tweak_tunables(void) 8370{ 8371 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 8372 8373 if (t4_ntxq10g < 1) 8374 t4_ntxq10g = min(nc, NTXQ_10G); 8375 8376 if (t4_ntxq1g < 1) 8377 t4_ntxq1g = min(nc, NTXQ_1G); 8378 8379 if (t4_nrxq10g < 1) 8380 t4_nrxq10g = min(nc, NRXQ_10G); 8381 8382 if (t4_nrxq1g < 1) 8383 t4_nrxq1g = min(nc, NRXQ_1G); 8384 8385#ifdef TCP_OFFLOAD 8386 if (t4_nofldtxq10g < 1) 8387 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 8388 8389 if (t4_nofldtxq1g < 1) 8390 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 8391 8392 if (t4_nofldrxq10g < 1) 8393 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 8394 8395 if (t4_nofldrxq1g < 1) 8396 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 8397 8398 if (t4_toecaps_allowed == -1) 8399 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 8400#else 8401 if (t4_toecaps_allowed == -1) 8402 t4_toecaps_allowed = 0; 8403#endif 8404 8405#ifdef DEV_NETMAP 8406 if (t4_nnmtxq10g < 1) 8407 t4_nnmtxq10g = min(nc, NNMTXQ_10G); 8408 8409 if (t4_nnmtxq1g < 1) 8410 t4_nnmtxq1g = min(nc, NNMTXQ_1G); 8411 8412 if (t4_nnmrxq10g < 1) 8413 t4_nnmrxq10g = min(nc, NNMRXQ_10G); 8414 8415 if (t4_nnmrxq1g < 1) 8416 t4_nnmrxq1g = min(nc, NNMRXQ_1G); 8417#endif 8418 8419 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 8420 t4_tmr_idx_10g = TMR_IDX_10G; 8421 8422 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 8423 t4_pktc_idx_10g = PKTC_IDX_10G; 8424 8425 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 8426 t4_tmr_idx_1g = TMR_IDX_1G; 8427 8428 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 8429 t4_pktc_idx_1g = PKTC_IDX_1G; 8430 8431 if (t4_qsize_txq < 128) 8432 t4_qsize_txq = 128; 8433 8434 if (t4_qsize_rxq < 128) 8435 t4_qsize_rxq = 128; 8436 while (t4_qsize_rxq & 7) 8437 t4_qsize_rxq++; 8438 8439 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 8440} 8441 8442static struct sx mlu; /* mod load unload */ 8443SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 8444 8445static int 8446mod_event(module_t mod, int cmd, void *arg) 8447{ 8448 int rc = 0; 8449 static int loaded = 0; 8450 8451 switch (cmd) { 8452 case MOD_LOAD: 8453 sx_xlock(&mlu); 8454 if (loaded++ == 0) { 8455 t4_sge_modload(); 8456 sx_init(&t4_list_lock, "T4/T5 adapters"); 8457 SLIST_INIT(&t4_list); 8458#ifdef TCP_OFFLOAD 8459 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 8460 SLIST_INIT(&t4_uld_list); 8461#endif 8462 t4_tracer_modload(); 8463 tweak_tunables(); 8464 } 8465 sx_xunlock(&mlu); 8466 break; 8467 8468 case MOD_UNLOAD: 8469 sx_xlock(&mlu); 8470 if (--loaded == 0) { 8471 int tries; 8472 8473 sx_slock(&t4_list_lock); 8474 if (!SLIST_EMPTY(&t4_list)) { 8475 rc = EBUSY; 8476 sx_sunlock(&t4_list_lock); 8477 goto done_unload; 8478 } 8479#ifdef TCP_OFFLOAD 8480 sx_slock(&t4_uld_list_lock); 8481 if (!SLIST_EMPTY(&t4_uld_list)) { 8482 rc = EBUSY; 8483 sx_sunlock(&t4_uld_list_lock); 8484 sx_sunlock(&t4_list_lock); 8485 goto done_unload; 8486 } 8487#endif 8488 tries = 0; 8489 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 8490 uprintf("%ju clusters with custom free routine " 8491 "still is use.\n", t4_sge_extfree_refs()); 8492 pause("t4unload", 2 * hz); 8493 } 8494#ifdef TCP_OFFLOAD 8495 sx_sunlock(&t4_uld_list_lock); 8496#endif 8497 sx_sunlock(&t4_list_lock); 8498 8499 if (t4_sge_extfree_refs() == 0) { 8500 t4_tracer_modunload(); 8501#ifdef TCP_OFFLOAD 8502 sx_destroy(&t4_uld_list_lock); 8503#endif 8504 sx_destroy(&t4_list_lock); 8505 t4_sge_modunload(); 8506 loaded = 0; 8507 } else { 8508 rc = EBUSY; 8509 loaded++; /* undo earlier decrement */ 8510 } 8511 } 8512done_unload: 8513 sx_xunlock(&mlu); 8514 break; 8515 } 8516 8517 return (rc); 8518} 8519 8520static devclass_t t4_devclass, t5_devclass; 8521static devclass_t cxgbe_devclass, cxl_devclass; 8522 8523DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 8524MODULE_VERSION(t4nex, 1); 8525MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 8526 8527DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 8528MODULE_VERSION(t5nex, 1); 8529MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 8530 8531DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 8532MODULE_VERSION(cxgbe, 1); 8533 8534DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 8535MODULE_VERSION(cxl, 1); 8536