t4_main.c revision 308321
1/*- 2 * Copyright (c) 2011 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/t4_main.c 308321 2016-11-04 21:59:27Z jhb $"); 30 31#include "opt_ddb.h" 32#include "opt_inet.h" 33#include "opt_inet6.h" 34 35#include <sys/param.h> 36#include <sys/conf.h> 37#include <sys/priv.h> 38#include <sys/kernel.h> 39#include <sys/bus.h> 40#include <sys/systm.h> 41#include <sys/counter.h> 42#include <sys/module.h> 43#include <sys/malloc.h> 44#include <sys/queue.h> 45#include <sys/taskqueue.h> 46#include <sys/pciio.h> 47#include <dev/pci/pcireg.h> 48#include <dev/pci/pcivar.h> 49#include <dev/pci/pci_private.h> 50#include <sys/firmware.h> 51#include <sys/sbuf.h> 52#include <sys/smp.h> 53#include <sys/socket.h> 54#include <sys/sockio.h> 55#include <sys/sysctl.h> 56#include <net/ethernet.h> 57#include <net/if.h> 58#include <net/if_types.h> 59#include <net/if_dl.h> 60#include <net/if_vlan_var.h> 61#ifdef RSS 62#include <net/rss_config.h> 63#endif 64#if defined(__i386__) || defined(__amd64__) 65#include <vm/vm.h> 66#include <vm/pmap.h> 67#endif 68#ifdef DDB 69#include <ddb/ddb.h> 70#include <ddb/db_lex.h> 71#endif 72 73#include "common/common.h" 74#include "common/t4_msg.h" 75#include "common/t4_regs.h" 76#include "common/t4_regs_values.h" 77#include "t4_ioctl.h" 78#include "t4_l2t.h" 79#include "t4_mp_ring.h" 80 81/* T4 bus driver interface */ 82static int t4_probe(device_t); 83static int t4_attach(device_t); 84static int t4_detach(device_t); 85static device_method_t t4_methods[] = { 86 DEVMETHOD(device_probe, t4_probe), 87 DEVMETHOD(device_attach, t4_attach), 88 DEVMETHOD(device_detach, t4_detach), 89 90 DEVMETHOD_END 91}; 92static driver_t t4_driver = { 93 "t4nex", 94 t4_methods, 95 sizeof(struct adapter) 96}; 97 98 99/* T4 port (cxgbe) interface */ 100static int cxgbe_probe(device_t); 101static int cxgbe_attach(device_t); 102static int cxgbe_detach(device_t); 103static device_method_t cxgbe_methods[] = { 104 DEVMETHOD(device_probe, cxgbe_probe), 105 DEVMETHOD(device_attach, cxgbe_attach), 106 DEVMETHOD(device_detach, cxgbe_detach), 107 { 0, 0 } 108}; 109static driver_t cxgbe_driver = { 110 "cxgbe", 111 cxgbe_methods, 112 sizeof(struct port_info) 113}; 114 115/* T4 VI (vcxgbe) interface */ 116static int vcxgbe_probe(device_t); 117static int vcxgbe_attach(device_t); 118static int vcxgbe_detach(device_t); 119static device_method_t vcxgbe_methods[] = { 120 DEVMETHOD(device_probe, vcxgbe_probe), 121 DEVMETHOD(device_attach, vcxgbe_attach), 122 DEVMETHOD(device_detach, vcxgbe_detach), 123 { 0, 0 } 124}; 125static driver_t vcxgbe_driver = { 126 "vcxgbe", 127 vcxgbe_methods, 128 sizeof(struct vi_info) 129}; 130 131static d_ioctl_t t4_ioctl; 132static d_open_t t4_open; 133static d_close_t t4_close; 134 135static struct cdevsw t4_cdevsw = { 136 .d_version = D_VERSION, 137 .d_flags = 0, 138 .d_open = t4_open, 139 .d_close = t4_close, 140 .d_ioctl = t4_ioctl, 141 .d_name = "t4nex", 142}; 143 144/* T5 bus driver interface */ 145static int t5_probe(device_t); 146static device_method_t t5_methods[] = { 147 DEVMETHOD(device_probe, t5_probe), 148 DEVMETHOD(device_attach, t4_attach), 149 DEVMETHOD(device_detach, t4_detach), 150 151 DEVMETHOD_END 152}; 153static driver_t t5_driver = { 154 "t5nex", 155 t5_methods, 156 sizeof(struct adapter) 157}; 158 159 160/* T5 port (cxl) interface */ 161static driver_t cxl_driver = { 162 "cxl", 163 cxgbe_methods, 164 sizeof(struct port_info) 165}; 166 167/* T5 VI (vcxl) interface */ 168static driver_t vcxl_driver = { 169 "vcxl", 170 vcxgbe_methods, 171 sizeof(struct vi_info) 172}; 173 174static struct cdevsw t5_cdevsw = { 175 .d_version = D_VERSION, 176 .d_flags = 0, 177 .d_open = t4_open, 178 .d_close = t4_close, 179 .d_ioctl = t4_ioctl, 180 .d_name = "t5nex", 181}; 182 183/* ifnet + media interface */ 184static void cxgbe_init(void *); 185static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 186static int cxgbe_transmit(struct ifnet *, struct mbuf *); 187static void cxgbe_qflush(struct ifnet *); 188static int cxgbe_media_change(struct ifnet *); 189static void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 190 191MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 192 193/* 194 * Correct lock order when you need to acquire multiple locks is t4_list_lock, 195 * then ADAPTER_LOCK, then t4_uld_list_lock. 196 */ 197static struct sx t4_list_lock; 198SLIST_HEAD(, adapter) t4_list; 199#ifdef TCP_OFFLOAD 200static struct sx t4_uld_list_lock; 201SLIST_HEAD(, uld_info) t4_uld_list; 202#endif 203 204/* 205 * Tunables. See tweak_tunables() too. 206 * 207 * Each tunable is set to a default value here if it's known at compile-time. 208 * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 209 * provide a reasonable default when the driver is loaded. 210 * 211 * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 212 * T5 are under hw.cxl. 213 */ 214 215/* 216 * Number of queues for tx and rx, 10G and 1G, NIC and offload. 217 */ 218#define NTXQ_10G 16 219static int t4_ntxq10g = -1; 220TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 221 222#define NRXQ_10G 8 223static int t4_nrxq10g = -1; 224TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 225 226#define NTXQ_1G 4 227static int t4_ntxq1g = -1; 228TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 229 230#define NRXQ_1G 2 231static int t4_nrxq1g = -1; 232TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 233 234#define NTXQ_VI 1 235static int t4_ntxq_vi = -1; 236TUNABLE_INT("hw.cxgbe.ntxq_vi", &t4_ntxq_vi); 237 238#define NRXQ_VI 1 239static int t4_nrxq_vi = -1; 240TUNABLE_INT("hw.cxgbe.nrxq_vi", &t4_nrxq_vi); 241 242static int t4_rsrv_noflowq = 0; 243TUNABLE_INT("hw.cxgbe.rsrv_noflowq", &t4_rsrv_noflowq); 244 245#ifdef TCP_OFFLOAD 246#define NOFLDTXQ_10G 8 247static int t4_nofldtxq10g = -1; 248TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 249 250#define NOFLDRXQ_10G 2 251static int t4_nofldrxq10g = -1; 252TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 253 254#define NOFLDTXQ_1G 2 255static int t4_nofldtxq1g = -1; 256TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 257 258#define NOFLDRXQ_1G 1 259static int t4_nofldrxq1g = -1; 260TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 261 262#define NOFLDTXQ_VI 1 263static int t4_nofldtxq_vi = -1; 264TUNABLE_INT("hw.cxgbe.nofldtxq_vi", &t4_nofldtxq_vi); 265 266#define NOFLDRXQ_VI 1 267static int t4_nofldrxq_vi = -1; 268TUNABLE_INT("hw.cxgbe.nofldrxq_vi", &t4_nofldrxq_vi); 269#endif 270 271#ifdef DEV_NETMAP 272#define NNMTXQ_VI 2 273static int t4_nnmtxq_vi = -1; 274TUNABLE_INT("hw.cxgbe.nnmtxq_vi", &t4_nnmtxq_vi); 275 276#define NNMRXQ_VI 2 277static int t4_nnmrxq_vi = -1; 278TUNABLE_INT("hw.cxgbe.nnmrxq_vi", &t4_nnmrxq_vi); 279#endif 280 281/* 282 * Holdoff parameters for 10G and 1G ports. 283 */ 284#define TMR_IDX_10G 1 285static int t4_tmr_idx_10g = TMR_IDX_10G; 286TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 287 288#define PKTC_IDX_10G (-1) 289static int t4_pktc_idx_10g = PKTC_IDX_10G; 290TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 291 292#define TMR_IDX_1G 1 293static int t4_tmr_idx_1g = TMR_IDX_1G; 294TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 295 296#define PKTC_IDX_1G (-1) 297static int t4_pktc_idx_1g = PKTC_IDX_1G; 298TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 299 300/* 301 * Size (# of entries) of each tx and rx queue. 302 */ 303static unsigned int t4_qsize_txq = TX_EQ_QSIZE; 304TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 305 306static unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 307TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 308 309/* 310 * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 311 */ 312static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 313TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 314 315/* 316 * Configuration file. 317 */ 318#define DEFAULT_CF "default" 319#define FLASH_CF "flash" 320#define UWIRE_CF "uwire" 321#define FPGA_CF "fpga" 322static char t4_cfg_file[32] = DEFAULT_CF; 323TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 324 325/* 326 * PAUSE settings (bit 0, 1 = rx_pause, tx_pause respectively). 327 * rx_pause = 1 to heed incoming PAUSE frames, 0 to ignore them. 328 * tx_pause = 1 to emit PAUSE frames when the rx FIFO reaches its high water 329 * mark or when signalled to do so, 0 to never emit PAUSE. 330 */ 331static int t4_pause_settings = PAUSE_TX | PAUSE_RX; 332TUNABLE_INT("hw.cxgbe.pause_settings", &t4_pause_settings); 333 334/* 335 * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 336 * encouraged respectively). 337 */ 338static unsigned int t4_fw_install = 1; 339TUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 340 341/* 342 * ASIC features that will be used. Disable the ones you don't want so that the 343 * chip resources aren't wasted on features that will not be used. 344 */ 345static int t4_nbmcaps_allowed = 0; 346TUNABLE_INT("hw.cxgbe.nbmcaps_allowed", &t4_nbmcaps_allowed); 347 348static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 349TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 350 351static int t4_switchcaps_allowed = FW_CAPS_CONFIG_SWITCH_INGRESS | 352 FW_CAPS_CONFIG_SWITCH_EGRESS; 353TUNABLE_INT("hw.cxgbe.switchcaps_allowed", &t4_switchcaps_allowed); 354 355static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 356TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 357 358static int t4_toecaps_allowed = -1; 359TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 360 361static int t4_rdmacaps_allowed = -1; 362TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 363 364static int t4_tlscaps_allowed = 0; 365TUNABLE_INT("hw.cxgbe.tlscaps_allowed", &t4_tlscaps_allowed); 366 367static int t4_iscsicaps_allowed = -1; 368TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 369 370static int t4_fcoecaps_allowed = 0; 371TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 372 373static int t5_write_combine = 0; 374TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 375 376static int t4_num_vis = 1; 377TUNABLE_INT("hw.cxgbe.num_vis", &t4_num_vis); 378 379/* Functions used by extra VIs to obtain unique MAC addresses for each VI. */ 380static int vi_mac_funcs[] = { 381 FW_VI_FUNC_OFLD, 382 FW_VI_FUNC_IWARP, 383 FW_VI_FUNC_OPENISCSI, 384 FW_VI_FUNC_OPENFCOE, 385 FW_VI_FUNC_FOISCSI, 386 FW_VI_FUNC_FOFCOE, 387}; 388 389struct intrs_and_queues { 390 uint16_t intr_type; /* INTx, MSI, or MSI-X */ 391 uint16_t nirq; /* Total # of vectors */ 392 uint16_t intr_flags_10g;/* Interrupt flags for each 10G port */ 393 uint16_t intr_flags_1g; /* Interrupt flags for each 1G port */ 394 uint16_t ntxq10g; /* # of NIC txq's for each 10G port */ 395 uint16_t nrxq10g; /* # of NIC rxq's for each 10G port */ 396 uint16_t ntxq1g; /* # of NIC txq's for each 1G port */ 397 uint16_t nrxq1g; /* # of NIC rxq's for each 1G port */ 398 uint16_t rsrv_noflowq; /* Flag whether to reserve queue 0 */ 399 uint16_t nofldtxq10g; /* # of TOE txq's for each 10G port */ 400 uint16_t nofldrxq10g; /* # of TOE rxq's for each 10G port */ 401 uint16_t nofldtxq1g; /* # of TOE txq's for each 1G port */ 402 uint16_t nofldrxq1g; /* # of TOE rxq's for each 1G port */ 403 404 /* The vcxgbe/vcxl interfaces use these and not the ones above. */ 405 uint16_t ntxq_vi; /* # of NIC txq's */ 406 uint16_t nrxq_vi; /* # of NIC rxq's */ 407 uint16_t nofldtxq_vi; /* # of TOE txq's */ 408 uint16_t nofldrxq_vi; /* # of TOE rxq's */ 409 uint16_t nnmtxq_vi; /* # of netmap txq's */ 410 uint16_t nnmrxq_vi; /* # of netmap rxq's */ 411}; 412 413struct filter_entry { 414 uint32_t valid:1; /* filter allocated and valid */ 415 uint32_t locked:1; /* filter is administratively locked */ 416 uint32_t pending:1; /* filter action is pending firmware reply */ 417 uint32_t smtidx:8; /* Source MAC Table index for smac */ 418 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 419 420 struct t4_filter_specification fs; 421}; 422 423static int map_bars_0_and_4(struct adapter *); 424static int map_bar_2(struct adapter *); 425static void setup_memwin(struct adapter *); 426static void position_memwin(struct adapter *, int, uint32_t); 427static int rw_via_memwin(struct adapter *, int, uint32_t, uint32_t *, int, int); 428static inline int read_via_memwin(struct adapter *, int, uint32_t, uint32_t *, 429 int); 430static inline int write_via_memwin(struct adapter *, int, uint32_t, 431 const uint32_t *, int); 432static int validate_mem_range(struct adapter *, uint32_t, int); 433static int fwmtype_to_hwmtype(int); 434static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 435 uint32_t *); 436static int fixup_devlog_params(struct adapter *); 437static int cfg_itype_and_nqueues(struct adapter *, int, int, int, 438 struct intrs_and_queues *); 439static int prep_firmware(struct adapter *); 440static int partition_resources(struct adapter *, const struct firmware *, 441 const char *); 442static int get_params__pre_init(struct adapter *); 443static int get_params__post_init(struct adapter *); 444static int set_params__post_init(struct adapter *); 445static void t4_set_desc(struct adapter *); 446static void build_medialist(struct port_info *, struct ifmedia *); 447static int cxgbe_init_synchronized(struct vi_info *); 448static int cxgbe_uninit_synchronized(struct vi_info *); 449static int setup_intr_handlers(struct adapter *); 450static void quiesce_txq(struct adapter *, struct sge_txq *); 451static void quiesce_wrq(struct adapter *, struct sge_wrq *); 452static void quiesce_iq(struct adapter *, struct sge_iq *); 453static void quiesce_fl(struct adapter *, struct sge_fl *); 454static int t4_alloc_irq(struct adapter *, struct irq *, int rid, 455 driver_intr_t *, void *, char *); 456static int t4_free_irq(struct adapter *, struct irq *); 457static void get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 458static void vi_refresh_stats(struct adapter *, struct vi_info *); 459static void cxgbe_refresh_stats(struct adapter *, struct port_info *); 460static void cxgbe_tick(void *); 461static void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 462static int cpl_not_handled(struct sge_iq *, const struct rss_header *, 463 struct mbuf *); 464static int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 465static int fw_msg_not_handled(struct adapter *, const __be64 *); 466static void t4_sysctls(struct adapter *); 467static void cxgbe_sysctls(struct port_info *); 468static int sysctl_int_array(SYSCTL_HANDLER_ARGS); 469static int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 470static int sysctl_btphy(SYSCTL_HANDLER_ARGS); 471static int sysctl_noflowq(SYSCTL_HANDLER_ARGS); 472static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 473static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 474static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 475static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 476static int sysctl_pause_settings(SYSCTL_HANDLER_ARGS); 477static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 478static int sysctl_temperature(SYSCTL_HANDLER_ARGS); 479#ifdef SBUF_DRAIN 480static int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 481static int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 482static int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 483static int sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS); 484static int sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS); 485static int sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS); 486static int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 487static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 488static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 489static int sysctl_devlog(SYSCTL_HANDLER_ARGS); 490static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 491static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 492static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 493static int sysctl_linkdnrc(SYSCTL_HANDLER_ARGS); 494static int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 495static int sysctl_mps_tcam(SYSCTL_HANDLER_ARGS); 496static int sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS); 497static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 498static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 499static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 500static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 501static int sysctl_tids(SYSCTL_HANDLER_ARGS); 502static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 503static int sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS); 504static int sysctl_tp_la(SYSCTL_HANDLER_ARGS); 505static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 506static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS); 507static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS); 508static int sysctl_tc_params(SYSCTL_HANDLER_ARGS); 509#endif 510#ifdef TCP_OFFLOAD 511static int sysctl_tp_tick(SYSCTL_HANDLER_ARGS); 512static int sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS); 513static int sysctl_tp_timer(SYSCTL_HANDLER_ARGS); 514#endif 515static uint32_t fconf_iconf_to_mode(uint32_t, uint32_t); 516static uint32_t mode_to_fconf(uint32_t); 517static uint32_t mode_to_iconf(uint32_t); 518static int check_fspec_against_fconf_iconf(struct adapter *, 519 struct t4_filter_specification *); 520static int get_filter_mode(struct adapter *, uint32_t *); 521static int set_filter_mode(struct adapter *, uint32_t); 522static inline uint64_t get_filter_hits(struct adapter *, uint32_t); 523static int get_filter(struct adapter *, struct t4_filter *); 524static int set_filter(struct adapter *, struct t4_filter *); 525static int del_filter(struct adapter *, struct t4_filter *); 526static void clear_filter(struct filter_entry *); 527static int set_filter_wr(struct adapter *, int); 528static int del_filter_wr(struct adapter *, int); 529static int get_sge_context(struct adapter *, struct t4_sge_context *); 530static int load_fw(struct adapter *, struct t4_data *); 531static int read_card_mem(struct adapter *, int, struct t4_mem_range *); 532static int read_i2c(struct adapter *, struct t4_i2c_data *); 533static int set_sched_class(struct adapter *, struct t4_sched_params *); 534static int set_sched_queue(struct adapter *, struct t4_sched_queue *); 535#ifdef TCP_OFFLOAD 536static int toe_capability(struct vi_info *, int); 537#endif 538static int mod_event(module_t, int, void *); 539 540struct { 541 uint16_t device; 542 char *desc; 543} t4_pciids[] = { 544 {0xa000, "Chelsio Terminator 4 FPGA"}, 545 {0x4400, "Chelsio T440-dbg"}, 546 {0x4401, "Chelsio T420-CR"}, 547 {0x4402, "Chelsio T422-CR"}, 548 {0x4403, "Chelsio T440-CR"}, 549 {0x4404, "Chelsio T420-BCH"}, 550 {0x4405, "Chelsio T440-BCH"}, 551 {0x4406, "Chelsio T440-CH"}, 552 {0x4407, "Chelsio T420-SO"}, 553 {0x4408, "Chelsio T420-CX"}, 554 {0x4409, "Chelsio T420-BT"}, 555 {0x440a, "Chelsio T404-BT"}, 556 {0x440e, "Chelsio T440-LP-CR"}, 557}, t5_pciids[] = { 558 {0xb000, "Chelsio Terminator 5 FPGA"}, 559 {0x5400, "Chelsio T580-dbg"}, 560 {0x5401, "Chelsio T520-CR"}, /* 2 x 10G */ 561 {0x5402, "Chelsio T522-CR"}, /* 2 x 10G, 2 X 1G */ 562 {0x5403, "Chelsio T540-CR"}, /* 4 x 10G */ 563 {0x5407, "Chelsio T520-SO"}, /* 2 x 10G, nomem */ 564 {0x5409, "Chelsio T520-BT"}, /* 2 x 10GBaseT */ 565 {0x540a, "Chelsio T504-BT"}, /* 4 x 1G */ 566 {0x540d, "Chelsio T580-CR"}, /* 2 x 40G */ 567 {0x540e, "Chelsio T540-LP-CR"}, /* 4 x 10G */ 568 {0x5410, "Chelsio T580-LP-CR"}, /* 2 x 40G */ 569 {0x5411, "Chelsio T520-LL-CR"}, /* 2 x 10G */ 570 {0x5412, "Chelsio T560-CR"}, /* 1 x 40G, 2 x 10G */ 571 {0x5414, "Chelsio T580-LP-SO-CR"}, /* 2 x 40G, nomem */ 572 {0x5415, "Chelsio T502-BT"}, /* 2 x 1G */ 573#ifdef notyet 574 {0x5404, "Chelsio T520-BCH"}, 575 {0x5405, "Chelsio T540-BCH"}, 576 {0x5406, "Chelsio T540-CH"}, 577 {0x5408, "Chelsio T520-CX"}, 578 {0x540b, "Chelsio B520-SR"}, 579 {0x540c, "Chelsio B504-BT"}, 580 {0x540f, "Chelsio Amsterdam"}, 581 {0x5413, "Chelsio T580-CHR"}, 582#endif 583}; 584 585#ifdef TCP_OFFLOAD 586/* 587 * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 588 * exactly the same for both rxq and ofld_rxq. 589 */ 590CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 591CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 592#endif 593 594/* No easy way to include t4_msg.h before adapter.h so we check this way */ 595CTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 596CTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 597 598CTASSERT(sizeof(struct cluster_metadata) <= CL_METADATA_SIZE); 599 600static int 601t4_probe(device_t dev) 602{ 603 int i; 604 uint16_t v = pci_get_vendor(dev); 605 uint16_t d = pci_get_device(dev); 606 uint8_t f = pci_get_function(dev); 607 608 if (v != PCI_VENDOR_ID_CHELSIO) 609 return (ENXIO); 610 611 /* Attach only to PF0 of the FPGA */ 612 if (d == 0xa000 && f != 0) 613 return (ENXIO); 614 615 for (i = 0; i < nitems(t4_pciids); i++) { 616 if (d == t4_pciids[i].device) { 617 device_set_desc(dev, t4_pciids[i].desc); 618 return (BUS_PROBE_DEFAULT); 619 } 620 } 621 622 return (ENXIO); 623} 624 625static int 626t5_probe(device_t dev) 627{ 628 int i; 629 uint16_t v = pci_get_vendor(dev); 630 uint16_t d = pci_get_device(dev); 631 uint8_t f = pci_get_function(dev); 632 633 if (v != PCI_VENDOR_ID_CHELSIO) 634 return (ENXIO); 635 636 /* Attach only to PF0 of the FPGA */ 637 if (d == 0xb000 && f != 0) 638 return (ENXIO); 639 640 for (i = 0; i < nitems(t5_pciids); i++) { 641 if (d == t5_pciids[i].device) { 642 device_set_desc(dev, t5_pciids[i].desc); 643 return (BUS_PROBE_DEFAULT); 644 } 645 } 646 647 return (ENXIO); 648} 649 650static void 651t5_attribute_workaround(device_t dev) 652{ 653 device_t root_port; 654 uint32_t v; 655 656 /* 657 * The T5 chips do not properly echo the No Snoop and Relaxed 658 * Ordering attributes when replying to a TLP from a Root 659 * Port. As a workaround, find the parent Root Port and 660 * disable No Snoop and Relaxed Ordering. Note that this 661 * affects all devices under this root port. 662 */ 663 root_port = pci_find_pcie_root_port(dev); 664 if (root_port == NULL) { 665 device_printf(dev, "Unable to find parent root port\n"); 666 return; 667 } 668 669 v = pcie_adjust_config(root_port, PCIER_DEVICE_CTL, 670 PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE, 0, 2); 671 if ((v & (PCIEM_CTL_RELAXED_ORD_ENABLE | PCIEM_CTL_NOSNOOP_ENABLE)) != 672 0) 673 device_printf(dev, "Disabled No Snoop/Relaxed Ordering on %s\n", 674 device_get_nameunit(root_port)); 675} 676 677static int 678t4_attach(device_t dev) 679{ 680 struct adapter *sc; 681 int rc = 0, i, j, n10g, n1g, rqidx, tqidx; 682 struct intrs_and_queues iaq; 683 struct sge *s; 684 uint8_t *buf; 685#ifdef TCP_OFFLOAD 686 int ofld_rqidx, ofld_tqidx; 687#endif 688#ifdef DEV_NETMAP 689 int nm_rqidx, nm_tqidx; 690#endif 691 int num_vis; 692 693 sc = device_get_softc(dev); 694 sc->dev = dev; 695 TUNABLE_INT_FETCH("hw.cxgbe.debug_flags", &sc->debug_flags); 696 697 if ((pci_get_device(dev) & 0xff00) == 0x5400) 698 t5_attribute_workaround(dev); 699 pci_enable_busmaster(dev); 700 if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 701 uint32_t v; 702 703 pci_set_max_read_req(dev, 4096); 704 v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 705 v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 706 pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 707 708 sc->params.pci.mps = 128 << ((v & PCIEM_CTL_MAX_PAYLOAD) >> 5); 709 } 710 711 sc->traceq = -1; 712 mtx_init(&sc->ifp_lock, sc->ifp_lockname, 0, MTX_DEF); 713 snprintf(sc->ifp_lockname, sizeof(sc->ifp_lockname), "%s tracer", 714 device_get_nameunit(dev)); 715 716 snprintf(sc->lockname, sizeof(sc->lockname), "%s", 717 device_get_nameunit(dev)); 718 mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 719 sx_xlock(&t4_list_lock); 720 SLIST_INSERT_HEAD(&t4_list, sc, link); 721 sx_xunlock(&t4_list_lock); 722 723 mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 724 TAILQ_INIT(&sc->sfl); 725 callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); 726 727 mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); 728 729 rc = map_bars_0_and_4(sc); 730 if (rc != 0) 731 goto done; /* error message displayed already */ 732 733 /* 734 * This is the real PF# to which we're attaching. Works from within PCI 735 * passthrough environments too, where pci_get_function() could return a 736 * different PF# depending on the passthrough configuration. We need to 737 * use the real PF# in all our communication with the firmware. 738 */ 739 sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 740 sc->mbox = sc->pf; 741 742 memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 743 sc->an_handler = an_not_handled; 744 for (i = 0; i < nitems(sc->cpl_handler); i++) 745 sc->cpl_handler[i] = cpl_not_handled; 746 for (i = 0; i < nitems(sc->fw_msg_handler); i++) 747 sc->fw_msg_handler[i] = fw_msg_not_handled; 748 t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 749 t4_register_cpl_handler(sc, CPL_TRACE_PKT, t4_trace_pkt); 750 t4_register_cpl_handler(sc, CPL_T5_TRACE_PKT, t5_trace_pkt); 751 t4_init_sge_cpl_handlers(sc); 752 753 /* Prepare the adapter for operation. */ 754 buf = malloc(PAGE_SIZE, M_CXGBE, M_ZERO | M_WAITOK); 755 rc = -t4_prep_adapter(sc, buf); 756 free(buf, M_CXGBE); 757 if (rc != 0) { 758 device_printf(dev, "failed to prepare adapter: %d.\n", rc); 759 goto done; 760 } 761 762 /* 763 * Do this really early, with the memory windows set up even before the 764 * character device. The userland tool's register i/o and mem read 765 * will work even in "recovery mode". 766 */ 767 setup_memwin(sc); 768 if (t4_init_devlog_params(sc, 0) == 0) 769 fixup_devlog_params(sc); 770 sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 771 device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 772 device_get_nameunit(dev)); 773 if (sc->cdev == NULL) 774 device_printf(dev, "failed to create nexus char device.\n"); 775 else 776 sc->cdev->si_drv1 = sc; 777 778 /* Go no further if recovery mode has been requested. */ 779 if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 780 device_printf(dev, "recovery mode.\n"); 781 goto done; 782 } 783 784#if defined(__i386__) 785 if ((cpu_feature & CPUID_CX8) == 0) { 786 device_printf(dev, "64 bit atomics not available.\n"); 787 rc = ENOTSUP; 788 goto done; 789 } 790#endif 791 792 /* Prepare the firmware for operation */ 793 rc = prep_firmware(sc); 794 if (rc != 0) 795 goto done; /* error message displayed already */ 796 797 rc = get_params__post_init(sc); 798 if (rc != 0) 799 goto done; /* error message displayed already */ 800 801 rc = set_params__post_init(sc); 802 if (rc != 0) 803 goto done; /* error message displayed already */ 804 805 rc = map_bar_2(sc); 806 if (rc != 0) 807 goto done; /* error message displayed already */ 808 809 rc = t4_create_dma_tag(sc); 810 if (rc != 0) 811 goto done; /* error message displayed already */ 812 813 /* 814 * Number of VIs to create per-port. The first VI is the "main" regular 815 * VI for the port. The rest are additional virtual interfaces on the 816 * same physical port. Note that the main VI does not have native 817 * netmap support but the extra VIs do. 818 * 819 * Limit the number of VIs per port to the number of available 820 * MAC addresses per port. 821 */ 822 if (t4_num_vis >= 1) 823 num_vis = t4_num_vis; 824 else 825 num_vis = 1; 826 if (num_vis > nitems(vi_mac_funcs)) { 827 num_vis = nitems(vi_mac_funcs); 828 device_printf(dev, "Number of VIs limited to %d\n", num_vis); 829 } 830 831 /* 832 * First pass over all the ports - allocate VIs and initialize some 833 * basic parameters like mac address, port type, etc. We also figure 834 * out whether a port is 10G or 1G and use that information when 835 * calculating how many interrupts to attempt to allocate. 836 */ 837 n10g = n1g = 0; 838 for_each_port(sc, i) { 839 struct port_info *pi; 840 841 pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 842 sc->port[i] = pi; 843 844 /* These must be set before t4_port_init */ 845 pi->adapter = sc; 846 pi->port_id = i; 847 /* 848 * XXX: vi[0] is special so we can't delay this allocation until 849 * pi->nvi's final value is known. 850 */ 851 pi->vi = malloc(sizeof(struct vi_info) * num_vis, M_CXGBE, 852 M_ZERO | M_WAITOK); 853 854 /* 855 * Allocate the "main" VI and initialize parameters 856 * like mac addr. 857 */ 858 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); 859 if (rc != 0) { 860 device_printf(dev, "unable to initialize port %d: %d\n", 861 i, rc); 862 free(pi->vi, M_CXGBE); 863 free(pi, M_CXGBE); 864 sc->port[i] = NULL; 865 goto done; 866 } 867 868 pi->link_cfg.requested_fc &= ~(PAUSE_TX | PAUSE_RX); 869 pi->link_cfg.requested_fc |= t4_pause_settings; 870 pi->link_cfg.fc &= ~(PAUSE_TX | PAUSE_RX); 871 pi->link_cfg.fc |= t4_pause_settings; 872 873 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 874 if (rc != 0) { 875 device_printf(dev, "port %d l1cfg failed: %d\n", i, rc); 876 free(pi->vi, M_CXGBE); 877 free(pi, M_CXGBE); 878 sc->port[i] = NULL; 879 goto done; 880 } 881 882 snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 883 device_get_nameunit(dev), i); 884 mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 885 sc->chan_map[pi->tx_chan] = i; 886 887 pi->tc = malloc(sizeof(struct tx_sched_class) * 888 sc->chip_params->nsched_cls, M_CXGBE, M_ZERO | M_WAITOK); 889 890 if (is_10G_port(pi) || is_40G_port(pi)) { 891 n10g++; 892 } else { 893 n1g++; 894 } 895 896 pi->linkdnrc = -1; 897 898 pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 899 if (pi->dev == NULL) { 900 device_printf(dev, 901 "failed to add device for port %d.\n", i); 902 rc = ENXIO; 903 goto done; 904 } 905 pi->vi[0].dev = pi->dev; 906 device_set_softc(pi->dev, pi); 907 } 908 909 /* 910 * Interrupt type, # of interrupts, # of rx/tx queues, etc. 911 */ 912 rc = cfg_itype_and_nqueues(sc, n10g, n1g, num_vis, &iaq); 913 if (rc != 0) 914 goto done; /* error message displayed already */ 915 if (iaq.nrxq_vi + iaq.nofldrxq_vi + iaq.nnmrxq_vi == 0) 916 num_vis = 1; 917 918 sc->intr_type = iaq.intr_type; 919 sc->intr_count = iaq.nirq; 920 921 s = &sc->sge; 922 s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 923 s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 924 if (num_vis > 1) { 925 s->nrxq += (n10g + n1g) * (num_vis - 1) * iaq.nrxq_vi; 926 s->ntxq += (n10g + n1g) * (num_vis - 1) * iaq.ntxq_vi; 927 } 928 s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 929 s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 930 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 931#ifdef TCP_OFFLOAD 932 if (is_offload(sc)) { 933 s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 934 s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 935 if (num_vis > 1) { 936 s->nofldrxq += (n10g + n1g) * (num_vis - 1) * 937 iaq.nofldrxq_vi; 938 s->nofldtxq += (n10g + n1g) * (num_vis - 1) * 939 iaq.nofldtxq_vi; 940 } 941 s->neq += s->nofldtxq + s->nofldrxq; 942 s->niq += s->nofldrxq; 943 944 s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 945 M_CXGBE, M_ZERO | M_WAITOK); 946 s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 947 M_CXGBE, M_ZERO | M_WAITOK); 948 } 949#endif 950#ifdef DEV_NETMAP 951 if (num_vis > 1) { 952 s->nnmrxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmrxq_vi; 953 s->nnmtxq = (n10g + n1g) * (num_vis - 1) * iaq.nnmtxq_vi; 954 } 955 s->neq += s->nnmtxq + s->nnmrxq; 956 s->niq += s->nnmrxq; 957 958 s->nm_rxq = malloc(s->nnmrxq * sizeof(struct sge_nm_rxq), 959 M_CXGBE, M_ZERO | M_WAITOK); 960 s->nm_txq = malloc(s->nnmtxq * sizeof(struct sge_nm_txq), 961 M_CXGBE, M_ZERO | M_WAITOK); 962#endif 963 964 s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 965 M_ZERO | M_WAITOK); 966 s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 967 M_ZERO | M_WAITOK); 968 s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 969 M_ZERO | M_WAITOK); 970 s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 971 M_ZERO | M_WAITOK); 972 s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 973 M_ZERO | M_WAITOK); 974 975 sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 976 M_ZERO | M_WAITOK); 977 978 t4_init_l2t(sc, M_WAITOK); 979 980 /* 981 * Second pass over the ports. This time we know the number of rx and 982 * tx queues that each port should get. 983 */ 984 rqidx = tqidx = 0; 985#ifdef TCP_OFFLOAD 986 ofld_rqidx = ofld_tqidx = 0; 987#endif 988#ifdef DEV_NETMAP 989 nm_rqidx = nm_tqidx = 0; 990#endif 991 for_each_port(sc, i) { 992 struct port_info *pi = sc->port[i]; 993 struct vi_info *vi; 994 995 if (pi == NULL) 996 continue; 997 998 pi->nvi = num_vis; 999 for_each_vi(pi, j, vi) { 1000 vi->pi = pi; 1001 vi->qsize_rxq = t4_qsize_rxq; 1002 vi->qsize_txq = t4_qsize_txq; 1003 1004 vi->first_rxq = rqidx; 1005 vi->first_txq = tqidx; 1006 if (is_10G_port(pi) || is_40G_port(pi)) { 1007 vi->tmr_idx = t4_tmr_idx_10g; 1008 vi->pktc_idx = t4_pktc_idx_10g; 1009 vi->flags |= iaq.intr_flags_10g & INTR_RXQ; 1010 vi->nrxq = j == 0 ? iaq.nrxq10g : iaq.nrxq_vi; 1011 vi->ntxq = j == 0 ? iaq.ntxq10g : iaq.ntxq_vi; 1012 } else { 1013 vi->tmr_idx = t4_tmr_idx_1g; 1014 vi->pktc_idx = t4_pktc_idx_1g; 1015 vi->flags |= iaq.intr_flags_1g & INTR_RXQ; 1016 vi->nrxq = j == 0 ? iaq.nrxq1g : iaq.nrxq_vi; 1017 vi->ntxq = j == 0 ? iaq.ntxq1g : iaq.ntxq_vi; 1018 } 1019 rqidx += vi->nrxq; 1020 tqidx += vi->ntxq; 1021 1022 if (j == 0 && vi->ntxq > 1) 1023 vi->rsrv_noflowq = iaq.rsrv_noflowq ? 1 : 0; 1024 else 1025 vi->rsrv_noflowq = 0; 1026 1027#ifdef TCP_OFFLOAD 1028 vi->first_ofld_rxq = ofld_rqidx; 1029 vi->first_ofld_txq = ofld_tqidx; 1030 if (is_10G_port(pi) || is_40G_port(pi)) { 1031 vi->flags |= iaq.intr_flags_10g & INTR_OFLD_RXQ; 1032 vi->nofldrxq = j == 0 ? iaq.nofldrxq10g : 1033 iaq.nofldrxq_vi; 1034 vi->nofldtxq = j == 0 ? iaq.nofldtxq10g : 1035 iaq.nofldtxq_vi; 1036 } else { 1037 vi->flags |= iaq.intr_flags_1g & INTR_OFLD_RXQ; 1038 vi->nofldrxq = j == 0 ? iaq.nofldrxq1g : 1039 iaq.nofldrxq_vi; 1040 vi->nofldtxq = j == 0 ? iaq.nofldtxq1g : 1041 iaq.nofldtxq_vi; 1042 } 1043 ofld_rqidx += vi->nofldrxq; 1044 ofld_tqidx += vi->nofldtxq; 1045#endif 1046#ifdef DEV_NETMAP 1047 if (j > 0) { 1048 vi->first_nm_rxq = nm_rqidx; 1049 vi->first_nm_txq = nm_tqidx; 1050 vi->nnmrxq = iaq.nnmrxq_vi; 1051 vi->nnmtxq = iaq.nnmtxq_vi; 1052 nm_rqidx += vi->nnmrxq; 1053 nm_tqidx += vi->nnmtxq; 1054 } 1055#endif 1056 } 1057 } 1058 1059 rc = setup_intr_handlers(sc); 1060 if (rc != 0) { 1061 device_printf(dev, 1062 "failed to setup interrupt handlers: %d\n", rc); 1063 goto done; 1064 } 1065 1066 rc = bus_generic_attach(dev); 1067 if (rc != 0) { 1068 device_printf(dev, 1069 "failed to attach all child ports: %d\n", rc); 1070 goto done; 1071 } 1072 1073 device_printf(dev, 1074 "PCIe gen%d x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 1075 sc->params.pci.speed, sc->params.pci.width, sc->params.nports, 1076 sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : 1077 (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 1078 sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 1079 1080 t4_set_desc(sc); 1081 1082done: 1083 if (rc != 0 && sc->cdev) { 1084 /* cdev was created and so cxgbetool works; recover that way. */ 1085 device_printf(dev, 1086 "error during attach, adapter is now in recovery mode.\n"); 1087 rc = 0; 1088 } 1089 1090 if (rc != 0) 1091 t4_detach(dev); 1092 else 1093 t4_sysctls(sc); 1094 1095 return (rc); 1096} 1097 1098/* 1099 * Idempotent 1100 */ 1101static int 1102t4_detach(device_t dev) 1103{ 1104 struct adapter *sc; 1105 struct port_info *pi; 1106 int i, rc; 1107 1108 sc = device_get_softc(dev); 1109 1110 if (sc->flags & FULL_INIT_DONE) 1111 t4_intr_disable(sc); 1112 1113 if (sc->cdev) { 1114 destroy_dev(sc->cdev); 1115 sc->cdev = NULL; 1116 } 1117 1118 rc = bus_generic_detach(dev); 1119 if (rc) { 1120 device_printf(dev, 1121 "failed to detach child devices: %d\n", rc); 1122 return (rc); 1123 } 1124 1125 for (i = 0; i < sc->intr_count; i++) 1126 t4_free_irq(sc, &sc->irq[i]); 1127 1128 for (i = 0; i < MAX_NPORTS; i++) { 1129 pi = sc->port[i]; 1130 if (pi) { 1131 t4_free_vi(sc, sc->mbox, sc->pf, 0, pi->vi[0].viid); 1132 if (pi->dev) 1133 device_delete_child(dev, pi->dev); 1134 1135 mtx_destroy(&pi->pi_lock); 1136 free(pi->vi, M_CXGBE); 1137 free(pi->tc, M_CXGBE); 1138 free(pi, M_CXGBE); 1139 } 1140 } 1141 1142 if (sc->flags & FULL_INIT_DONE) 1143 adapter_full_uninit(sc); 1144 1145 if (sc->flags & FW_OK) 1146 t4_fw_bye(sc, sc->mbox); 1147 1148 if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 1149 pci_release_msi(dev); 1150 1151 if (sc->regs_res) 1152 bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 1153 sc->regs_res); 1154 1155 if (sc->udbs_res) 1156 bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 1157 sc->udbs_res); 1158 1159 if (sc->msix_res) 1160 bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 1161 sc->msix_res); 1162 1163 if (sc->l2t) 1164 t4_free_l2t(sc->l2t); 1165 1166#ifdef TCP_OFFLOAD 1167 free(sc->sge.ofld_rxq, M_CXGBE); 1168 free(sc->sge.ofld_txq, M_CXGBE); 1169#endif 1170#ifdef DEV_NETMAP 1171 free(sc->sge.nm_rxq, M_CXGBE); 1172 free(sc->sge.nm_txq, M_CXGBE); 1173#endif 1174 free(sc->irq, M_CXGBE); 1175 free(sc->sge.rxq, M_CXGBE); 1176 free(sc->sge.txq, M_CXGBE); 1177 free(sc->sge.ctrlq, M_CXGBE); 1178 free(sc->sge.iqmap, M_CXGBE); 1179 free(sc->sge.eqmap, M_CXGBE); 1180 free(sc->tids.ftid_tab, M_CXGBE); 1181 t4_destroy_dma_tag(sc); 1182 if (mtx_initialized(&sc->sc_lock)) { 1183 sx_xlock(&t4_list_lock); 1184 SLIST_REMOVE(&t4_list, sc, adapter, link); 1185 sx_xunlock(&t4_list_lock); 1186 mtx_destroy(&sc->sc_lock); 1187 } 1188 1189 callout_drain(&sc->sfl_callout); 1190 if (mtx_initialized(&sc->tids.ftid_lock)) 1191 mtx_destroy(&sc->tids.ftid_lock); 1192 if (mtx_initialized(&sc->sfl_lock)) 1193 mtx_destroy(&sc->sfl_lock); 1194 if (mtx_initialized(&sc->ifp_lock)) 1195 mtx_destroy(&sc->ifp_lock); 1196 if (mtx_initialized(&sc->reg_lock)) 1197 mtx_destroy(&sc->reg_lock); 1198 1199 for (i = 0; i < NUM_MEMWIN; i++) { 1200 struct memwin *mw = &sc->memwin[i]; 1201 1202 if (rw_initialized(&mw->mw_lock)) 1203 rw_destroy(&mw->mw_lock); 1204 } 1205 1206 bzero(sc, sizeof(*sc)); 1207 1208 return (0); 1209} 1210 1211static int 1212cxgbe_probe(device_t dev) 1213{ 1214 char buf[128]; 1215 struct port_info *pi = device_get_softc(dev); 1216 1217 snprintf(buf, sizeof(buf), "port %d", pi->port_id); 1218 device_set_desc_copy(dev, buf); 1219 1220 return (BUS_PROBE_DEFAULT); 1221} 1222 1223#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 1224 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 1225 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6 | IFCAP_HWSTATS) 1226#define T4_CAP_ENABLE (T4_CAP) 1227 1228static int 1229cxgbe_vi_attach(device_t dev, struct vi_info *vi) 1230{ 1231 struct ifnet *ifp; 1232 struct sbuf *sb; 1233 1234 vi->xact_addr_filt = -1; 1235 callout_init(&vi->tick, 1); 1236 1237 /* Allocate an ifnet and set it up */ 1238 ifp = if_alloc(IFT_ETHER); 1239 if (ifp == NULL) { 1240 device_printf(dev, "Cannot allocate ifnet\n"); 1241 return (ENOMEM); 1242 } 1243 vi->ifp = ifp; 1244 ifp->if_softc = vi; 1245 1246 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1247 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1248 1249 ifp->if_init = cxgbe_init; 1250 ifp->if_ioctl = cxgbe_ioctl; 1251 ifp->if_transmit = cxgbe_transmit; 1252 ifp->if_qflush = cxgbe_qflush; 1253 1254 ifp->if_capabilities = T4_CAP; 1255#ifdef TCP_OFFLOAD 1256 if (vi->nofldrxq != 0) 1257 ifp->if_capabilities |= IFCAP_TOE; 1258#endif 1259 ifp->if_capenable = T4_CAP_ENABLE; 1260 ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 1261 CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 1262 1263 ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 1264 ifp->if_hw_tsomaxsegcount = TX_SGL_SEGS; 1265 ifp->if_hw_tsomaxsegsize = 65536; 1266 1267 /* Initialize ifmedia for this VI */ 1268 ifmedia_init(&vi->media, IFM_IMASK, cxgbe_media_change, 1269 cxgbe_media_status); 1270 build_medialist(vi->pi, &vi->media); 1271 1272 vi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 1273 EVENTHANDLER_PRI_ANY); 1274 1275 ether_ifattach(ifp, vi->hw_addr); 1276#ifdef DEV_NETMAP 1277 if (vi->nnmrxq != 0) 1278 cxgbe_nm_attach(vi); 1279#endif 1280 sb = sbuf_new_auto(); 1281 sbuf_printf(sb, "%d txq, %d rxq (NIC)", vi->ntxq, vi->nrxq); 1282#ifdef TCP_OFFLOAD 1283 if (ifp->if_capabilities & IFCAP_TOE) 1284 sbuf_printf(sb, "; %d txq, %d rxq (TOE)", 1285 vi->nofldtxq, vi->nofldrxq); 1286#endif 1287#ifdef DEV_NETMAP 1288 if (ifp->if_capabilities & IFCAP_NETMAP) 1289 sbuf_printf(sb, "; %d txq, %d rxq (netmap)", 1290 vi->nnmtxq, vi->nnmrxq); 1291#endif 1292 sbuf_finish(sb); 1293 device_printf(dev, "%s\n", sbuf_data(sb)); 1294 sbuf_delete(sb); 1295 1296 vi_sysctls(vi); 1297 1298 return (0); 1299} 1300 1301static int 1302cxgbe_attach(device_t dev) 1303{ 1304 struct port_info *pi = device_get_softc(dev); 1305 struct vi_info *vi; 1306 int i, rc; 1307 1308 callout_init_mtx(&pi->tick, &pi->pi_lock, 0); 1309 1310 rc = cxgbe_vi_attach(dev, &pi->vi[0]); 1311 if (rc) 1312 return (rc); 1313 1314 for_each_vi(pi, i, vi) { 1315 if (i == 0) 1316 continue; 1317 vi->dev = device_add_child(dev, is_t4(pi->adapter) ? 1318 "vcxgbe" : "vcxl", -1); 1319 if (vi->dev == NULL) { 1320 device_printf(dev, "failed to add VI %d\n", i); 1321 continue; 1322 } 1323 device_set_softc(vi->dev, vi); 1324 } 1325 1326 cxgbe_sysctls(pi); 1327 1328 bus_generic_attach(dev); 1329 1330 return (0); 1331} 1332 1333static void 1334cxgbe_vi_detach(struct vi_info *vi) 1335{ 1336 struct ifnet *ifp = vi->ifp; 1337 1338 ether_ifdetach(ifp); 1339 1340 if (vi->vlan_c) 1341 EVENTHANDLER_DEREGISTER(vlan_config, vi->vlan_c); 1342 1343 /* Let detach proceed even if these fail. */ 1344#ifdef DEV_NETMAP 1345 if (ifp->if_capabilities & IFCAP_NETMAP) 1346 cxgbe_nm_detach(vi); 1347#endif 1348 cxgbe_uninit_synchronized(vi); 1349 callout_drain(&vi->tick); 1350 vi_full_uninit(vi); 1351 1352 ifmedia_removeall(&vi->media); 1353 if_free(vi->ifp); 1354 vi->ifp = NULL; 1355} 1356 1357static int 1358cxgbe_detach(device_t dev) 1359{ 1360 struct port_info *pi = device_get_softc(dev); 1361 struct adapter *sc = pi->adapter; 1362 int rc; 1363 1364 /* Detach the extra VIs first. */ 1365 rc = bus_generic_detach(dev); 1366 if (rc) 1367 return (rc); 1368 device_delete_children(dev); 1369 1370 doom_vi(sc, &pi->vi[0]); 1371 1372 if (pi->flags & HAS_TRACEQ) { 1373 sc->traceq = -1; /* cloner should not create ifnet */ 1374 t4_tracer_port_detach(sc); 1375 } 1376 1377 cxgbe_vi_detach(&pi->vi[0]); 1378 callout_drain(&pi->tick); 1379 1380 end_synchronized_op(sc, 0); 1381 1382 return (0); 1383} 1384 1385static void 1386cxgbe_init(void *arg) 1387{ 1388 struct vi_info *vi = arg; 1389 struct adapter *sc = vi->pi->adapter; 1390 1391 if (begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4init") != 0) 1392 return; 1393 cxgbe_init_synchronized(vi); 1394 end_synchronized_op(sc, 0); 1395} 1396 1397static int 1398cxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1399{ 1400 int rc = 0, mtu, flags, can_sleep; 1401 struct vi_info *vi = ifp->if_softc; 1402 struct adapter *sc = vi->pi->adapter; 1403 struct ifreq *ifr = (struct ifreq *)data; 1404 uint32_t mask; 1405 1406 switch (cmd) { 1407 case SIOCSIFMTU: 1408 mtu = ifr->ifr_mtu; 1409 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1410 return (EINVAL); 1411 1412 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4mtu"); 1413 if (rc) 1414 return (rc); 1415 ifp->if_mtu = mtu; 1416 if (vi->flags & VI_INIT_DONE) { 1417 t4_update_fl_bufsize(ifp); 1418 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1419 rc = update_mac_settings(ifp, XGMAC_MTU); 1420 } 1421 end_synchronized_op(sc, 0); 1422 break; 1423 1424 case SIOCSIFFLAGS: 1425 can_sleep = 0; 1426redo_sifflags: 1427 rc = begin_synchronized_op(sc, vi, 1428 can_sleep ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4flg"); 1429 if (rc) 1430 return (rc); 1431 1432 if (ifp->if_flags & IFF_UP) { 1433 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1434 flags = vi->if_flags; 1435 if ((ifp->if_flags ^ flags) & 1436 (IFF_PROMISC | IFF_ALLMULTI)) { 1437 if (can_sleep == 1) { 1438 end_synchronized_op(sc, 0); 1439 can_sleep = 0; 1440 goto redo_sifflags; 1441 } 1442 rc = update_mac_settings(ifp, 1443 XGMAC_PROMISC | XGMAC_ALLMULTI); 1444 } 1445 } else { 1446 if (can_sleep == 0) { 1447 end_synchronized_op(sc, LOCK_HELD); 1448 can_sleep = 1; 1449 goto redo_sifflags; 1450 } 1451 rc = cxgbe_init_synchronized(vi); 1452 } 1453 vi->if_flags = ifp->if_flags; 1454 } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1455 if (can_sleep == 0) { 1456 end_synchronized_op(sc, LOCK_HELD); 1457 can_sleep = 1; 1458 goto redo_sifflags; 1459 } 1460 rc = cxgbe_uninit_synchronized(vi); 1461 } 1462 end_synchronized_op(sc, can_sleep ? 0 : LOCK_HELD); 1463 break; 1464 1465 case SIOCADDMULTI: 1466 case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1467 rc = begin_synchronized_op(sc, vi, HOLD_LOCK, "t4multi"); 1468 if (rc) 1469 return (rc); 1470 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1471 rc = update_mac_settings(ifp, XGMAC_MCADDRS); 1472 end_synchronized_op(sc, LOCK_HELD); 1473 break; 1474 1475 case SIOCSIFCAP: 1476 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4cap"); 1477 if (rc) 1478 return (rc); 1479 1480 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1481 if (mask & IFCAP_TXCSUM) { 1482 ifp->if_capenable ^= IFCAP_TXCSUM; 1483 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1484 1485 if (IFCAP_TSO4 & ifp->if_capenable && 1486 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1487 ifp->if_capenable &= ~IFCAP_TSO4; 1488 if_printf(ifp, 1489 "tso4 disabled due to -txcsum.\n"); 1490 } 1491 } 1492 if (mask & IFCAP_TXCSUM_IPV6) { 1493 ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1494 ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1495 1496 if (IFCAP_TSO6 & ifp->if_capenable && 1497 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1498 ifp->if_capenable &= ~IFCAP_TSO6; 1499 if_printf(ifp, 1500 "tso6 disabled due to -txcsum6.\n"); 1501 } 1502 } 1503 if (mask & IFCAP_RXCSUM) 1504 ifp->if_capenable ^= IFCAP_RXCSUM; 1505 if (mask & IFCAP_RXCSUM_IPV6) 1506 ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1507 1508 /* 1509 * Note that we leave CSUM_TSO alone (it is always set). The 1510 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1511 * sending a TSO request our way, so it's sufficient to toggle 1512 * IFCAP_TSOx only. 1513 */ 1514 if (mask & IFCAP_TSO4) { 1515 if (!(IFCAP_TSO4 & ifp->if_capenable) && 1516 !(IFCAP_TXCSUM & ifp->if_capenable)) { 1517 if_printf(ifp, "enable txcsum first.\n"); 1518 rc = EAGAIN; 1519 goto fail; 1520 } 1521 ifp->if_capenable ^= IFCAP_TSO4; 1522 } 1523 if (mask & IFCAP_TSO6) { 1524 if (!(IFCAP_TSO6 & ifp->if_capenable) && 1525 !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1526 if_printf(ifp, "enable txcsum6 first.\n"); 1527 rc = EAGAIN; 1528 goto fail; 1529 } 1530 ifp->if_capenable ^= IFCAP_TSO6; 1531 } 1532 if (mask & IFCAP_LRO) { 1533#if defined(INET) || defined(INET6) 1534 int i; 1535 struct sge_rxq *rxq; 1536 1537 ifp->if_capenable ^= IFCAP_LRO; 1538 for_each_rxq(vi, i, rxq) { 1539 if (ifp->if_capenable & IFCAP_LRO) 1540 rxq->iq.flags |= IQ_LRO_ENABLED; 1541 else 1542 rxq->iq.flags &= ~IQ_LRO_ENABLED; 1543 } 1544#endif 1545 } 1546#ifdef TCP_OFFLOAD 1547 if (mask & IFCAP_TOE) { 1548 int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1549 1550 rc = toe_capability(vi, enable); 1551 if (rc != 0) 1552 goto fail; 1553 1554 ifp->if_capenable ^= mask; 1555 } 1556#endif 1557 if (mask & IFCAP_VLAN_HWTAGGING) { 1558 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1559 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1560 rc = update_mac_settings(ifp, XGMAC_VLANEX); 1561 } 1562 if (mask & IFCAP_VLAN_MTU) { 1563 ifp->if_capenable ^= IFCAP_VLAN_MTU; 1564 1565 /* Need to find out how to disable auto-mtu-inflation */ 1566 } 1567 if (mask & IFCAP_VLAN_HWTSO) 1568 ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1569 if (mask & IFCAP_VLAN_HWCSUM) 1570 ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1571 1572#ifdef VLAN_CAPABILITIES 1573 VLAN_CAPABILITIES(ifp); 1574#endif 1575fail: 1576 end_synchronized_op(sc, 0); 1577 break; 1578 1579 case SIOCSIFMEDIA: 1580 case SIOCGIFMEDIA: 1581 ifmedia_ioctl(ifp, ifr, &vi->media, cmd); 1582 break; 1583 1584 case SIOCGI2C: { 1585 struct ifi2creq i2c; 1586 1587 rc = copyin(ifr->ifr_data, &i2c, sizeof(i2c)); 1588 if (rc != 0) 1589 break; 1590 if (i2c.dev_addr != 0xA0 && i2c.dev_addr != 0xA2) { 1591 rc = EPERM; 1592 break; 1593 } 1594 if (i2c.len > sizeof(i2c.data)) { 1595 rc = EINVAL; 1596 break; 1597 } 1598 rc = begin_synchronized_op(sc, vi, SLEEP_OK | INTR_OK, "t4i2c"); 1599 if (rc) 1600 return (rc); 1601 rc = -t4_i2c_rd(sc, sc->mbox, vi->pi->port_id, i2c.dev_addr, 1602 i2c.offset, i2c.len, &i2c.data[0]); 1603 end_synchronized_op(sc, 0); 1604 if (rc == 0) 1605 rc = copyout(&i2c, ifr->ifr_data, sizeof(i2c)); 1606 break; 1607 } 1608 1609 default: 1610 rc = ether_ioctl(ifp, cmd, data); 1611 } 1612 1613 return (rc); 1614} 1615 1616static int 1617cxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1618{ 1619 struct vi_info *vi = ifp->if_softc; 1620 struct port_info *pi = vi->pi; 1621 struct adapter *sc = pi->adapter; 1622 struct sge_txq *txq; 1623 void *items[1]; 1624 int rc; 1625 1626 M_ASSERTPKTHDR(m); 1627 MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */ 1628 1629 if (__predict_false(pi->link_cfg.link_ok == 0)) { 1630 m_freem(m); 1631 return (ENETDOWN); 1632 } 1633 1634 rc = parse_pkt(&m); 1635 if (__predict_false(rc != 0)) { 1636 MPASS(m == NULL); /* was freed already */ 1637 atomic_add_int(&pi->tx_parse_error, 1); /* rare, atomic is ok */ 1638 return (rc); 1639 } 1640 1641 /* Select a txq. */ 1642 txq = &sc->sge.txq[vi->first_txq]; 1643 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 1644 txq += ((m->m_pkthdr.flowid % (vi->ntxq - vi->rsrv_noflowq)) + 1645 vi->rsrv_noflowq); 1646 1647 items[0] = m; 1648 rc = mp_ring_enqueue(txq->r, items, 1, 4096); 1649 if (__predict_false(rc != 0)) 1650 m_freem(m); 1651 1652 return (rc); 1653} 1654 1655static void 1656cxgbe_qflush(struct ifnet *ifp) 1657{ 1658 struct vi_info *vi = ifp->if_softc; 1659 struct sge_txq *txq; 1660 int i; 1661 1662 /* queues do not exist if !VI_INIT_DONE. */ 1663 if (vi->flags & VI_INIT_DONE) { 1664 for_each_txq(vi, i, txq) { 1665 TXQ_LOCK(txq); 1666 txq->eq.flags &= ~EQ_ENABLED; 1667 TXQ_UNLOCK(txq); 1668 while (!mp_ring_is_idle(txq->r)) { 1669 mp_ring_check_drainage(txq->r, 0); 1670 pause("qflush", 1); 1671 } 1672 } 1673 } 1674 if_qflush(ifp); 1675} 1676 1677static int 1678cxgbe_media_change(struct ifnet *ifp) 1679{ 1680 struct vi_info *vi = ifp->if_softc; 1681 1682 device_printf(vi->dev, "%s unimplemented.\n", __func__); 1683 1684 return (EOPNOTSUPP); 1685} 1686 1687static void 1688cxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1689{ 1690 struct vi_info *vi = ifp->if_softc; 1691 struct port_info *pi = vi->pi; 1692 struct ifmedia_entry *cur; 1693 int speed = pi->link_cfg.speed; 1694 1695 cur = vi->media.ifm_cur; 1696 1697 ifmr->ifm_status = IFM_AVALID; 1698 if (!pi->link_cfg.link_ok) 1699 return; 1700 1701 ifmr->ifm_status |= IFM_ACTIVE; 1702 1703 /* active and current will differ iff current media is autoselect. */ 1704 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1705 return; 1706 1707 ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1708 if (speed == 10000) 1709 ifmr->ifm_active |= IFM_10G_T; 1710 else if (speed == 1000) 1711 ifmr->ifm_active |= IFM_1000_T; 1712 else if (speed == 100) 1713 ifmr->ifm_active |= IFM_100_TX; 1714 else if (speed == 10) 1715 ifmr->ifm_active |= IFM_10_T; 1716 else 1717 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1718 speed)); 1719} 1720 1721static int 1722vcxgbe_probe(device_t dev) 1723{ 1724 char buf[128]; 1725 struct vi_info *vi = device_get_softc(dev); 1726 1727 snprintf(buf, sizeof(buf), "port %d vi %td", vi->pi->port_id, 1728 vi - vi->pi->vi); 1729 device_set_desc_copy(dev, buf); 1730 1731 return (BUS_PROBE_DEFAULT); 1732} 1733 1734static int 1735vcxgbe_attach(device_t dev) 1736{ 1737 struct vi_info *vi; 1738 struct port_info *pi; 1739 struct adapter *sc; 1740 int func, index, rc; 1741 u32 param, val; 1742 1743 vi = device_get_softc(dev); 1744 pi = vi->pi; 1745 sc = pi->adapter; 1746 1747 index = vi - pi->vi; 1748 KASSERT(index < nitems(vi_mac_funcs), 1749 ("%s: VI %s doesn't have a MAC func", __func__, 1750 device_get_nameunit(dev))); 1751 func = vi_mac_funcs[index]; 1752 rc = t4_alloc_vi_func(sc, sc->mbox, pi->tx_chan, sc->pf, 0, 1, 1753 vi->hw_addr, &vi->rss_size, func, 0); 1754 if (rc < 0) { 1755 device_printf(dev, "Failed to allocate virtual interface " 1756 "for port %d: %d\n", pi->port_id, -rc); 1757 return (-rc); 1758 } 1759 vi->viid = rc; 1760 1761 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 1762 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_RSSINFO) | 1763 V_FW_PARAMS_PARAM_YZ(vi->viid); 1764 rc = t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1765 if (rc) 1766 vi->rss_base = 0xffff; 1767 else { 1768 /* MPASS((val >> 16) == rss_size); */ 1769 vi->rss_base = val & 0xffff; 1770 } 1771 1772 rc = cxgbe_vi_attach(dev, vi); 1773 if (rc) { 1774 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1775 return (rc); 1776 } 1777 return (0); 1778} 1779 1780static int 1781vcxgbe_detach(device_t dev) 1782{ 1783 struct vi_info *vi; 1784 struct adapter *sc; 1785 1786 vi = device_get_softc(dev); 1787 sc = vi->pi->adapter; 1788 1789 doom_vi(sc, vi); 1790 1791 cxgbe_vi_detach(vi); 1792 t4_free_vi(sc, sc->mbox, sc->pf, 0, vi->viid); 1793 1794 end_synchronized_op(sc, 0); 1795 1796 return (0); 1797} 1798 1799void 1800t4_fatal_err(struct adapter *sc) 1801{ 1802 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1803 t4_intr_disable(sc); 1804 log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1805 device_get_nameunit(sc->dev)); 1806} 1807 1808static int 1809map_bars_0_and_4(struct adapter *sc) 1810{ 1811 sc->regs_rid = PCIR_BAR(0); 1812 sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1813 &sc->regs_rid, RF_ACTIVE); 1814 if (sc->regs_res == NULL) { 1815 device_printf(sc->dev, "cannot map registers.\n"); 1816 return (ENXIO); 1817 } 1818 sc->bt = rman_get_bustag(sc->regs_res); 1819 sc->bh = rman_get_bushandle(sc->regs_res); 1820 sc->mmio_len = rman_get_size(sc->regs_res); 1821 setbit(&sc->doorbells, DOORBELL_KDB); 1822 1823 sc->msix_rid = PCIR_BAR(4); 1824 sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1825 &sc->msix_rid, RF_ACTIVE); 1826 if (sc->msix_res == NULL) { 1827 device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1828 return (ENXIO); 1829 } 1830 1831 return (0); 1832} 1833 1834static int 1835map_bar_2(struct adapter *sc) 1836{ 1837 1838 /* 1839 * T4: only iWARP driver uses the userspace doorbells. There is no need 1840 * to map it if RDMA is disabled. 1841 */ 1842 if (is_t4(sc) && sc->rdmacaps == 0) 1843 return (0); 1844 1845 sc->udbs_rid = PCIR_BAR(2); 1846 sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1847 &sc->udbs_rid, RF_ACTIVE); 1848 if (sc->udbs_res == NULL) { 1849 device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1850 return (ENXIO); 1851 } 1852 sc->udbs_base = rman_get_virtual(sc->udbs_res); 1853 1854 if (is_t5(sc)) { 1855 setbit(&sc->doorbells, DOORBELL_UDB); 1856#if defined(__i386__) || defined(__amd64__) 1857 if (t5_write_combine) { 1858 int rc; 1859 1860 /* 1861 * Enable write combining on BAR2. This is the 1862 * userspace doorbell BAR and is split into 128B 1863 * (UDBS_SEG_SIZE) doorbell regions, each associated 1864 * with an egress queue. The first 64B has the doorbell 1865 * and the second 64B can be used to submit a tx work 1866 * request with an implicit doorbell. 1867 */ 1868 1869 rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1870 rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1871 if (rc == 0) { 1872 clrbit(&sc->doorbells, DOORBELL_UDB); 1873 setbit(&sc->doorbells, DOORBELL_WCWR); 1874 setbit(&sc->doorbells, DOORBELL_UDBWC); 1875 } else { 1876 device_printf(sc->dev, 1877 "couldn't enable write combining: %d\n", 1878 rc); 1879 } 1880 1881 t4_write_reg(sc, A_SGE_STAT_CFG, 1882 V_STATSOURCE_T5(7) | V_STATMODE(0)); 1883 } 1884#endif 1885 } 1886 1887 return (0); 1888} 1889 1890struct memwin_init { 1891 uint32_t base; 1892 uint32_t aperture; 1893}; 1894 1895static const struct memwin_init t4_memwin[NUM_MEMWIN] = { 1896 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1897 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1898 { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1899}; 1900 1901static const struct memwin_init t5_memwin[NUM_MEMWIN] = { 1902 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1903 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1904 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1905}; 1906 1907static void 1908setup_memwin(struct adapter *sc) 1909{ 1910 const struct memwin_init *mw_init; 1911 struct memwin *mw; 1912 int i; 1913 uint32_t bar0; 1914 1915 if (is_t4(sc)) { 1916 /* 1917 * Read low 32b of bar0 indirectly via the hardware backdoor 1918 * mechanism. Works from within PCI passthrough environments 1919 * too, where rman_get_start() can return a different value. We 1920 * need to program the T4 memory window decoders with the actual 1921 * addresses that will be coming across the PCIe link. 1922 */ 1923 bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1924 bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1925 1926 mw_init = &t4_memwin[0]; 1927 } else { 1928 /* T5+ use the relative offset inside the PCIe BAR */ 1929 bar0 = 0; 1930 1931 mw_init = &t5_memwin[0]; 1932 } 1933 1934 for (i = 0, mw = &sc->memwin[0]; i < NUM_MEMWIN; i++, mw_init++, mw++) { 1935 rw_init(&mw->mw_lock, "memory window access"); 1936 mw->mw_base = mw_init->base; 1937 mw->mw_aperture = mw_init->aperture; 1938 mw->mw_curpos = 0; 1939 t4_write_reg(sc, 1940 PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1941 (mw->mw_base + bar0) | V_BIR(0) | 1942 V_WINDOW(ilog2(mw->mw_aperture) - 10)); 1943 rw_wlock(&mw->mw_lock); 1944 position_memwin(sc, i, 0); 1945 rw_wunlock(&mw->mw_lock); 1946 } 1947 1948 /* flush */ 1949 t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1950} 1951 1952/* 1953 * Positions the memory window at the given address in the card's address space. 1954 * There are some alignment requirements and the actual position may be at an 1955 * address prior to the requested address. mw->mw_curpos always has the actual 1956 * position of the window. 1957 */ 1958static void 1959position_memwin(struct adapter *sc, int idx, uint32_t addr) 1960{ 1961 struct memwin *mw; 1962 uint32_t pf; 1963 uint32_t reg; 1964 1965 MPASS(idx >= 0 && idx < NUM_MEMWIN); 1966 mw = &sc->memwin[idx]; 1967 rw_assert(&mw->mw_lock, RA_WLOCKED); 1968 1969 if (is_t4(sc)) { 1970 pf = 0; 1971 mw->mw_curpos = addr & ~0xf; /* start must be 16B aligned */ 1972 } else { 1973 pf = V_PFNUM(sc->pf); 1974 mw->mw_curpos = addr & ~0x7f; /* start must be 128B aligned */ 1975 } 1976 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, idx); 1977 t4_write_reg(sc, reg, mw->mw_curpos | pf); 1978 t4_read_reg(sc, reg); /* flush */ 1979} 1980 1981static int 1982rw_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, 1983 int len, int rw) 1984{ 1985 struct memwin *mw; 1986 uint32_t mw_end, v; 1987 1988 MPASS(idx >= 0 && idx < NUM_MEMWIN); 1989 1990 /* Memory can only be accessed in naturally aligned 4 byte units */ 1991 if (addr & 3 || len & 3 || len <= 0) 1992 return (EINVAL); 1993 1994 mw = &sc->memwin[idx]; 1995 while (len > 0) { 1996 rw_rlock(&mw->mw_lock); 1997 mw_end = mw->mw_curpos + mw->mw_aperture; 1998 if (addr >= mw_end || addr < mw->mw_curpos) { 1999 /* Will need to reposition the window */ 2000 if (!rw_try_upgrade(&mw->mw_lock)) { 2001 rw_runlock(&mw->mw_lock); 2002 rw_wlock(&mw->mw_lock); 2003 } 2004 rw_assert(&mw->mw_lock, RA_WLOCKED); 2005 position_memwin(sc, idx, addr); 2006 rw_downgrade(&mw->mw_lock); 2007 mw_end = mw->mw_curpos + mw->mw_aperture; 2008 } 2009 rw_assert(&mw->mw_lock, RA_RLOCKED); 2010 while (addr < mw_end && len > 0) { 2011 if (rw == 0) { 2012 v = t4_read_reg(sc, mw->mw_base + addr - 2013 mw->mw_curpos); 2014 *val++ = le32toh(v); 2015 } else { 2016 v = *val++; 2017 t4_write_reg(sc, mw->mw_base + addr - 2018 mw->mw_curpos, htole32(v));; 2019 } 2020 addr += 4; 2021 len -= 4; 2022 } 2023 rw_runlock(&mw->mw_lock); 2024 } 2025 2026 return (0); 2027} 2028 2029static inline int 2030read_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val, 2031 int len) 2032{ 2033 2034 return (rw_via_memwin(sc, idx, addr, val, len, 0)); 2035} 2036 2037static inline int 2038write_via_memwin(struct adapter *sc, int idx, uint32_t addr, 2039 const uint32_t *val, int len) 2040{ 2041 2042 return (rw_via_memwin(sc, idx, addr, (void *)(uintptr_t)val, len, 1)); 2043} 2044 2045static int 2046t4_range_cmp(const void *a, const void *b) 2047{ 2048 return ((const struct t4_range *)a)->start - 2049 ((const struct t4_range *)b)->start; 2050} 2051 2052/* 2053 * Verify that the memory range specified by the addr/len pair is valid within 2054 * the card's address space. 2055 */ 2056static int 2057validate_mem_range(struct adapter *sc, uint32_t addr, int len) 2058{ 2059 struct t4_range mem_ranges[4], *r, *next; 2060 uint32_t em, addr_len; 2061 int i, n, remaining; 2062 2063 /* Memory can only be accessed in naturally aligned 4 byte units */ 2064 if (addr & 3 || len & 3 || len <= 0) 2065 return (EINVAL); 2066 2067 /* Enabled memories */ 2068 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2069 2070 r = &mem_ranges[0]; 2071 n = 0; 2072 bzero(r, sizeof(mem_ranges)); 2073 if (em & F_EDRAM0_ENABLE) { 2074 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2075 r->size = G_EDRAM0_SIZE(addr_len) << 20; 2076 if (r->size > 0) { 2077 r->start = G_EDRAM0_BASE(addr_len) << 20; 2078 if (addr >= r->start && 2079 addr + len <= r->start + r->size) 2080 return (0); 2081 r++; 2082 n++; 2083 } 2084 } 2085 if (em & F_EDRAM1_ENABLE) { 2086 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2087 r->size = G_EDRAM1_SIZE(addr_len) << 20; 2088 if (r->size > 0) { 2089 r->start = G_EDRAM1_BASE(addr_len) << 20; 2090 if (addr >= r->start && 2091 addr + len <= r->start + r->size) 2092 return (0); 2093 r++; 2094 n++; 2095 } 2096 } 2097 if (em & F_EXT_MEM_ENABLE) { 2098 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2099 r->size = G_EXT_MEM_SIZE(addr_len) << 20; 2100 if (r->size > 0) { 2101 r->start = G_EXT_MEM_BASE(addr_len) << 20; 2102 if (addr >= r->start && 2103 addr + len <= r->start + r->size) 2104 return (0); 2105 r++; 2106 n++; 2107 } 2108 } 2109 if (is_t5(sc) && em & F_EXT_MEM1_ENABLE) { 2110 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2111 r->size = G_EXT_MEM1_SIZE(addr_len) << 20; 2112 if (r->size > 0) { 2113 r->start = G_EXT_MEM1_BASE(addr_len) << 20; 2114 if (addr >= r->start && 2115 addr + len <= r->start + r->size) 2116 return (0); 2117 r++; 2118 n++; 2119 } 2120 } 2121 MPASS(n <= nitems(mem_ranges)); 2122 2123 if (n > 1) { 2124 /* Sort and merge the ranges. */ 2125 qsort(mem_ranges, n, sizeof(struct t4_range), t4_range_cmp); 2126 2127 /* Start from index 0 and examine the next n - 1 entries. */ 2128 r = &mem_ranges[0]; 2129 for (remaining = n - 1; remaining > 0; remaining--, r++) { 2130 2131 MPASS(r->size > 0); /* r is a valid entry. */ 2132 next = r + 1; 2133 MPASS(next->size > 0); /* and so is the next one. */ 2134 2135 while (r->start + r->size >= next->start) { 2136 /* Merge the next one into the current entry. */ 2137 r->size = max(r->start + r->size, 2138 next->start + next->size) - r->start; 2139 n--; /* One fewer entry in total. */ 2140 if (--remaining == 0) 2141 goto done; /* short circuit */ 2142 next++; 2143 } 2144 if (next != r + 1) { 2145 /* 2146 * Some entries were merged into r and next 2147 * points to the first valid entry that couldn't 2148 * be merged. 2149 */ 2150 MPASS(next->size > 0); /* must be valid */ 2151 memcpy(r + 1, next, remaining * sizeof(*r)); 2152#ifdef INVARIANTS 2153 /* 2154 * This so that the foo->size assertion in the 2155 * next iteration of the loop do the right 2156 * thing for entries that were pulled up and are 2157 * no longer valid. 2158 */ 2159 MPASS(n < nitems(mem_ranges)); 2160 bzero(&mem_ranges[n], (nitems(mem_ranges) - n) * 2161 sizeof(struct t4_range)); 2162#endif 2163 } 2164 } 2165done: 2166 /* Done merging the ranges. */ 2167 MPASS(n > 0); 2168 r = &mem_ranges[0]; 2169 for (i = 0; i < n; i++, r++) { 2170 if (addr >= r->start && 2171 addr + len <= r->start + r->size) 2172 return (0); 2173 } 2174 } 2175 2176 return (EFAULT); 2177} 2178 2179static int 2180fwmtype_to_hwmtype(int mtype) 2181{ 2182 2183 switch (mtype) { 2184 case FW_MEMTYPE_EDC0: 2185 return (MEM_EDC0); 2186 case FW_MEMTYPE_EDC1: 2187 return (MEM_EDC1); 2188 case FW_MEMTYPE_EXTMEM: 2189 return (MEM_MC0); 2190 case FW_MEMTYPE_EXTMEM1: 2191 return (MEM_MC1); 2192 default: 2193 panic("%s: cannot translate fw mtype %d.", __func__, mtype); 2194 } 2195} 2196 2197/* 2198 * Verify that the memory range specified by the memtype/offset/len pair is 2199 * valid and lies entirely within the memtype specified. The global address of 2200 * the start of the range is returned in addr. 2201 */ 2202static int 2203validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 2204 uint32_t *addr) 2205{ 2206 uint32_t em, addr_len, maddr; 2207 2208 /* Memory can only be accessed in naturally aligned 4 byte units */ 2209 if (off & 3 || len & 3 || len == 0) 2210 return (EINVAL); 2211 2212 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 2213 switch (fwmtype_to_hwmtype(mtype)) { 2214 case MEM_EDC0: 2215 if (!(em & F_EDRAM0_ENABLE)) 2216 return (EINVAL); 2217 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 2218 maddr = G_EDRAM0_BASE(addr_len) << 20; 2219 break; 2220 case MEM_EDC1: 2221 if (!(em & F_EDRAM1_ENABLE)) 2222 return (EINVAL); 2223 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 2224 maddr = G_EDRAM1_BASE(addr_len) << 20; 2225 break; 2226 case MEM_MC: 2227 if (!(em & F_EXT_MEM_ENABLE)) 2228 return (EINVAL); 2229 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 2230 maddr = G_EXT_MEM_BASE(addr_len) << 20; 2231 break; 2232 case MEM_MC1: 2233 if (!is_t5(sc) || !(em & F_EXT_MEM1_ENABLE)) 2234 return (EINVAL); 2235 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 2236 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 2237 break; 2238 default: 2239 return (EINVAL); 2240 } 2241 2242 *addr = maddr + off; /* global address */ 2243 return (validate_mem_range(sc, *addr, len)); 2244} 2245 2246static int 2247fixup_devlog_params(struct adapter *sc) 2248{ 2249 struct devlog_params *dparams = &sc->params.devlog; 2250 int rc; 2251 2252 rc = validate_mt_off_len(sc, dparams->memtype, dparams->start, 2253 dparams->size, &dparams->addr); 2254 2255 return (rc); 2256} 2257 2258static int 2259cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, int num_vis, 2260 struct intrs_and_queues *iaq) 2261{ 2262 int rc, itype, navail, nrxq10g, nrxq1g, n; 2263 int nofldrxq10g = 0, nofldrxq1g = 0; 2264 2265 bzero(iaq, sizeof(*iaq)); 2266 2267 iaq->ntxq10g = t4_ntxq10g; 2268 iaq->ntxq1g = t4_ntxq1g; 2269 iaq->ntxq_vi = t4_ntxq_vi; 2270 iaq->nrxq10g = nrxq10g = t4_nrxq10g; 2271 iaq->nrxq1g = nrxq1g = t4_nrxq1g; 2272 iaq->nrxq_vi = t4_nrxq_vi; 2273 iaq->rsrv_noflowq = t4_rsrv_noflowq; 2274#ifdef TCP_OFFLOAD 2275 if (is_offload(sc)) { 2276 iaq->nofldtxq10g = t4_nofldtxq10g; 2277 iaq->nofldtxq1g = t4_nofldtxq1g; 2278 iaq->nofldtxq_vi = t4_nofldtxq_vi; 2279 iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 2280 iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 2281 iaq->nofldrxq_vi = t4_nofldrxq_vi; 2282 } 2283#endif 2284#ifdef DEV_NETMAP 2285 iaq->nnmtxq_vi = t4_nnmtxq_vi; 2286 iaq->nnmrxq_vi = t4_nnmrxq_vi; 2287#endif 2288 2289 for (itype = INTR_MSIX; itype; itype >>= 1) { 2290 2291 if ((itype & t4_intr_types) == 0) 2292 continue; /* not allowed */ 2293 2294 if (itype == INTR_MSIX) 2295 navail = pci_msix_count(sc->dev); 2296 else if (itype == INTR_MSI) 2297 navail = pci_msi_count(sc->dev); 2298 else 2299 navail = 1; 2300restart: 2301 if (navail == 0) 2302 continue; 2303 2304 iaq->intr_type = itype; 2305 iaq->intr_flags_10g = 0; 2306 iaq->intr_flags_1g = 0; 2307 2308 /* 2309 * Best option: an interrupt vector for errors, one for the 2310 * firmware event queue, and one for every rxq (NIC and TOE) of 2311 * every VI. The VIs that support netmap use the same 2312 * interrupts for the NIC rx queues and the netmap rx queues 2313 * because only one set of queues is active at a time. 2314 */ 2315 iaq->nirq = T4_EXTRA_INTR; 2316 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 2317 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 2318 iaq->nirq += (n10g + n1g) * (num_vis - 1) * 2319 max(iaq->nrxq_vi, iaq->nnmrxq_vi); /* See comment above. */ 2320 iaq->nirq += (n10g + n1g) * (num_vis - 1) * iaq->nofldrxq_vi; 2321 if (iaq->nirq <= navail && 2322 (itype != INTR_MSI || powerof2(iaq->nirq))) { 2323 iaq->intr_flags_10g = INTR_ALL; 2324 iaq->intr_flags_1g = INTR_ALL; 2325 goto allocate; 2326 } 2327 2328 /* Disable the VIs (and netmap) if there aren't enough intrs */ 2329 if (num_vis > 1) { 2330 device_printf(sc->dev, "virtual interfaces disabled " 2331 "because num_vis=%u with current settings " 2332 "(nrxq10g=%u, nrxq1g=%u, nofldrxq10g=%u, " 2333 "nofldrxq1g=%u, nrxq_vi=%u nofldrxq_vi=%u, " 2334 "nnmrxq_vi=%u) would need %u interrupts but " 2335 "only %u are available.\n", num_vis, nrxq10g, 2336 nrxq1g, nofldrxq10g, nofldrxq1g, iaq->nrxq_vi, 2337 iaq->nofldrxq_vi, iaq->nnmrxq_vi, iaq->nirq, 2338 navail); 2339 num_vis = 1; 2340 iaq->ntxq_vi = iaq->nrxq_vi = 0; 2341 iaq->nofldtxq_vi = iaq->nofldrxq_vi = 0; 2342 iaq->nnmtxq_vi = iaq->nnmrxq_vi = 0; 2343 goto restart; 2344 } 2345 2346 /* 2347 * Second best option: a vector for errors, one for the firmware 2348 * event queue, and vectors for either all the NIC rx queues or 2349 * all the TOE rx queues. The queues that don't get vectors 2350 * will forward their interrupts to those that do. 2351 */ 2352 iaq->nirq = T4_EXTRA_INTR; 2353 if (nrxq10g >= nofldrxq10g) { 2354 iaq->intr_flags_10g = INTR_RXQ; 2355 iaq->nirq += n10g * nrxq10g; 2356 } else { 2357 iaq->intr_flags_10g = INTR_OFLD_RXQ; 2358 iaq->nirq += n10g * nofldrxq10g; 2359 } 2360 if (nrxq1g >= nofldrxq1g) { 2361 iaq->intr_flags_1g = INTR_RXQ; 2362 iaq->nirq += n1g * nrxq1g; 2363 } else { 2364 iaq->intr_flags_1g = INTR_OFLD_RXQ; 2365 iaq->nirq += n1g * nofldrxq1g; 2366 } 2367 if (iaq->nirq <= navail && 2368 (itype != INTR_MSI || powerof2(iaq->nirq))) 2369 goto allocate; 2370 2371 /* 2372 * Next best option: an interrupt vector for errors, one for the 2373 * firmware event queue, and at least one per main-VI. At this 2374 * point we know we'll have to downsize nrxq and/or nofldrxq to 2375 * fit what's available to us. 2376 */ 2377 iaq->nirq = T4_EXTRA_INTR; 2378 iaq->nirq += n10g + n1g; 2379 if (iaq->nirq <= navail) { 2380 int leftover = navail - iaq->nirq; 2381 2382 if (n10g > 0) { 2383 int target = max(nrxq10g, nofldrxq10g); 2384 2385 iaq->intr_flags_10g = nrxq10g >= nofldrxq10g ? 2386 INTR_RXQ : INTR_OFLD_RXQ; 2387 2388 n = 1; 2389 while (n < target && leftover >= n10g) { 2390 leftover -= n10g; 2391 iaq->nirq += n10g; 2392 n++; 2393 } 2394 iaq->nrxq10g = min(n, nrxq10g); 2395#ifdef TCP_OFFLOAD 2396 iaq->nofldrxq10g = min(n, nofldrxq10g); 2397#endif 2398 } 2399 2400 if (n1g > 0) { 2401 int target = max(nrxq1g, nofldrxq1g); 2402 2403 iaq->intr_flags_1g = nrxq1g >= nofldrxq1g ? 2404 INTR_RXQ : INTR_OFLD_RXQ; 2405 2406 n = 1; 2407 while (n < target && leftover >= n1g) { 2408 leftover -= n1g; 2409 iaq->nirq += n1g; 2410 n++; 2411 } 2412 iaq->nrxq1g = min(n, nrxq1g); 2413#ifdef TCP_OFFLOAD 2414 iaq->nofldrxq1g = min(n, nofldrxq1g); 2415#endif 2416 } 2417 2418 if (itype != INTR_MSI || powerof2(iaq->nirq)) 2419 goto allocate; 2420 } 2421 2422 /* 2423 * Least desirable option: one interrupt vector for everything. 2424 */ 2425 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2426 iaq->intr_flags_10g = iaq->intr_flags_1g = 0; 2427#ifdef TCP_OFFLOAD 2428 if (is_offload(sc)) 2429 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2430#endif 2431allocate: 2432 navail = iaq->nirq; 2433 rc = 0; 2434 if (itype == INTR_MSIX) 2435 rc = pci_alloc_msix(sc->dev, &navail); 2436 else if (itype == INTR_MSI) 2437 rc = pci_alloc_msi(sc->dev, &navail); 2438 2439 if (rc == 0) { 2440 if (navail == iaq->nirq) 2441 return (0); 2442 2443 /* 2444 * Didn't get the number requested. Use whatever number 2445 * the kernel is willing to allocate (it's in navail). 2446 */ 2447 device_printf(sc->dev, "fewer vectors than requested, " 2448 "type=%d, req=%d, rcvd=%d; will downshift req.\n", 2449 itype, iaq->nirq, navail); 2450 pci_release_msi(sc->dev); 2451 goto restart; 2452 } 2453 2454 device_printf(sc->dev, 2455 "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 2456 itype, rc, iaq->nirq, navail); 2457 } 2458 2459 device_printf(sc->dev, 2460 "failed to find a usable interrupt type. " 2461 "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 2462 pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 2463 2464 return (ENXIO); 2465} 2466 2467#define FW_VERSION(chip) ( \ 2468 V_FW_HDR_FW_VER_MAJOR(chip##FW_VERSION_MAJOR) | \ 2469 V_FW_HDR_FW_VER_MINOR(chip##FW_VERSION_MINOR) | \ 2470 V_FW_HDR_FW_VER_MICRO(chip##FW_VERSION_MICRO) | \ 2471 V_FW_HDR_FW_VER_BUILD(chip##FW_VERSION_BUILD)) 2472#define FW_INTFVER(chip, intf) (chip##FW_HDR_INTFVER_##intf) 2473 2474struct fw_info { 2475 uint8_t chip; 2476 char *kld_name; 2477 char *fw_mod_name; 2478 struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 2479} fw_info[] = { 2480 { 2481 .chip = CHELSIO_T4, 2482 .kld_name = "t4fw_cfg", 2483 .fw_mod_name = "t4fw", 2484 .fw_hdr = { 2485 .chip = FW_HDR_CHIP_T4, 2486 .fw_ver = htobe32_const(FW_VERSION(T4)), 2487 .intfver_nic = FW_INTFVER(T4, NIC), 2488 .intfver_vnic = FW_INTFVER(T4, VNIC), 2489 .intfver_ofld = FW_INTFVER(T4, OFLD), 2490 .intfver_ri = FW_INTFVER(T4, RI), 2491 .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 2492 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 2493 .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 2494 .intfver_fcoe = FW_INTFVER(T4, FCOE), 2495 }, 2496 }, { 2497 .chip = CHELSIO_T5, 2498 .kld_name = "t5fw_cfg", 2499 .fw_mod_name = "t5fw", 2500 .fw_hdr = { 2501 .chip = FW_HDR_CHIP_T5, 2502 .fw_ver = htobe32_const(FW_VERSION(T5)), 2503 .intfver_nic = FW_INTFVER(T5, NIC), 2504 .intfver_vnic = FW_INTFVER(T5, VNIC), 2505 .intfver_ofld = FW_INTFVER(T5, OFLD), 2506 .intfver_ri = FW_INTFVER(T5, RI), 2507 .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 2508 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 2509 .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 2510 .intfver_fcoe = FW_INTFVER(T5, FCOE), 2511 }, 2512 } 2513}; 2514 2515static struct fw_info * 2516find_fw_info(int chip) 2517{ 2518 int i; 2519 2520 for (i = 0; i < nitems(fw_info); i++) { 2521 if (fw_info[i].chip == chip) 2522 return (&fw_info[i]); 2523 } 2524 return (NULL); 2525} 2526 2527/* 2528 * Is the given firmware API compatible with the one the driver was compiled 2529 * with? 2530 */ 2531static int 2532fw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 2533{ 2534 2535 /* short circuit if it's the exact same firmware version */ 2536 if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 2537 return (1); 2538 2539 /* 2540 * XXX: Is this too conservative? Perhaps I should limit this to the 2541 * features that are supported in the driver. 2542 */ 2543#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 2544 if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 2545 SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 2546 SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 2547 return (1); 2548#undef SAME_INTF 2549 2550 return (0); 2551} 2552 2553/* 2554 * The firmware in the KLD is usable, but should it be installed? This routine 2555 * explains itself in detail if it indicates the KLD firmware should be 2556 * installed. 2557 */ 2558static int 2559should_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 2560{ 2561 const char *reason; 2562 2563 if (!card_fw_usable) { 2564 reason = "incompatible or unusable"; 2565 goto install; 2566 } 2567 2568 if (k > c) { 2569 reason = "older than the version bundled with this driver"; 2570 goto install; 2571 } 2572 2573 if (t4_fw_install == 2 && k != c) { 2574 reason = "different than the version bundled with this driver"; 2575 goto install; 2576 } 2577 2578 return (0); 2579 2580install: 2581 if (t4_fw_install == 0) { 2582 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2583 "but the driver is prohibited from installing a different " 2584 "firmware on the card.\n", 2585 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2586 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason); 2587 2588 return (0); 2589 } 2590 2591 device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 2592 "installing firmware %u.%u.%u.%u on card.\n", 2593 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2594 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 2595 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2596 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2597 2598 return (1); 2599} 2600/* 2601 * Establish contact with the firmware and determine if we are the master driver 2602 * or not, and whether we are responsible for chip initialization. 2603 */ 2604static int 2605prep_firmware(struct adapter *sc) 2606{ 2607 const struct firmware *fw = NULL, *default_cfg; 2608 int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 2609 enum dev_state state; 2610 struct fw_info *fw_info; 2611 struct fw_hdr *card_fw; /* fw on the card */ 2612 const struct fw_hdr *kld_fw; /* fw in the KLD */ 2613 const struct fw_hdr *drv_fw; /* fw header the driver was compiled 2614 against */ 2615 2616 /* Contact firmware. */ 2617 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 2618 if (rc < 0 || state == DEV_STATE_ERR) { 2619 rc = -rc; 2620 device_printf(sc->dev, 2621 "failed to connect to the firmware: %d, %d.\n", rc, state); 2622 return (rc); 2623 } 2624 pf = rc; 2625 if (pf == sc->mbox) 2626 sc->flags |= MASTER_PF; 2627 else if (state == DEV_STATE_UNINIT) { 2628 /* 2629 * We didn't get to be the master so we definitely won't be 2630 * configuring the chip. It's a bug if someone else hasn't 2631 * configured it already. 2632 */ 2633 device_printf(sc->dev, "couldn't be master(%d), " 2634 "device not already initialized either(%d).\n", rc, state); 2635 return (EDOOFUS); 2636 } 2637 2638 /* This is the firmware whose headers the driver was compiled against */ 2639 fw_info = find_fw_info(chip_id(sc)); 2640 if (fw_info == NULL) { 2641 device_printf(sc->dev, 2642 "unable to look up firmware information for chip %d.\n", 2643 chip_id(sc)); 2644 return (EINVAL); 2645 } 2646 drv_fw = &fw_info->fw_hdr; 2647 2648 /* 2649 * The firmware KLD contains many modules. The KLD name is also the 2650 * name of the module that contains the default config file. 2651 */ 2652 default_cfg = firmware_get(fw_info->kld_name); 2653 2654 /* Read the header of the firmware on the card */ 2655 card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 2656 rc = -t4_read_flash(sc, FLASH_FW_START, 2657 sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 2658 if (rc == 0) 2659 card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 2660 else { 2661 device_printf(sc->dev, 2662 "Unable to read card's firmware header: %d\n", rc); 2663 card_fw_usable = 0; 2664 } 2665 2666 /* This is the firmware in the KLD */ 2667 fw = firmware_get(fw_info->fw_mod_name); 2668 if (fw != NULL) { 2669 kld_fw = (const void *)fw->data; 2670 kld_fw_usable = fw_compatible(drv_fw, kld_fw); 2671 } else { 2672 kld_fw = NULL; 2673 kld_fw_usable = 0; 2674 } 2675 2676 if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2677 (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver)) { 2678 /* 2679 * Common case: the firmware on the card is an exact match and 2680 * the KLD is an exact match too, or the KLD is 2681 * absent/incompatible. Note that t4_fw_install = 2 is ignored 2682 * here -- use cxgbetool loadfw if you want to reinstall the 2683 * same firmware as the one on the card. 2684 */ 2685 } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2686 should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2687 be32toh(card_fw->fw_ver))) { 2688 2689 rc = -t4_fw_upgrade(sc, sc->mbox, fw->data, fw->datasize, 0); 2690 if (rc != 0) { 2691 device_printf(sc->dev, 2692 "failed to install firmware: %d\n", rc); 2693 goto done; 2694 } 2695 2696 /* Installed successfully, update the cached header too. */ 2697 memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2698 card_fw_usable = 1; 2699 need_fw_reset = 0; /* already reset as part of load_fw */ 2700 } 2701 2702 if (!card_fw_usable) { 2703 uint32_t d, c, k; 2704 2705 d = ntohl(drv_fw->fw_ver); 2706 c = ntohl(card_fw->fw_ver); 2707 k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2708 2709 device_printf(sc->dev, "Cannot find a usable firmware: " 2710 "fw_install %d, chip state %d, " 2711 "driver compiled with %d.%d.%d.%d, " 2712 "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2713 t4_fw_install, state, 2714 G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2715 G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2716 G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2717 G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2718 G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2719 G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2720 rc = EINVAL; 2721 goto done; 2722 } 2723 2724 /* We're using whatever's on the card and it's known to be good. */ 2725 sc->params.fw_vers = ntohl(card_fw->fw_ver); 2726 snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2727 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2728 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2729 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2730 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2731 2732 t4_get_tp_version(sc, &sc->params.tp_vers); 2733 snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u", 2734 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), 2735 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), 2736 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), 2737 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); 2738 2739 if (t4_get_exprom_version(sc, &sc->params.exprom_vers) != 0) 2740 sc->params.exprom_vers = 0; 2741 else { 2742 snprintf(sc->exprom_version, sizeof(sc->exprom_version), 2743 "%u.%u.%u.%u", 2744 G_FW_HDR_FW_VER_MAJOR(sc->params.exprom_vers), 2745 G_FW_HDR_FW_VER_MINOR(sc->params.exprom_vers), 2746 G_FW_HDR_FW_VER_MICRO(sc->params.exprom_vers), 2747 G_FW_HDR_FW_VER_BUILD(sc->params.exprom_vers)); 2748 } 2749 2750 /* Reset device */ 2751 if (need_fw_reset && 2752 (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2753 device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2754 if (rc != ETIMEDOUT && rc != EIO) 2755 t4_fw_bye(sc, sc->mbox); 2756 goto done; 2757 } 2758 sc->flags |= FW_OK; 2759 2760 rc = get_params__pre_init(sc); 2761 if (rc != 0) 2762 goto done; /* error message displayed already */ 2763 2764 /* Partition adapter resources as specified in the config file. */ 2765 if (state == DEV_STATE_UNINIT) { 2766 2767 KASSERT(sc->flags & MASTER_PF, 2768 ("%s: trying to change chip settings when not master.", 2769 __func__)); 2770 2771 rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2772 if (rc != 0) 2773 goto done; /* error message displayed already */ 2774 2775 t4_tweak_chip_settings(sc); 2776 2777 /* get basic stuff going */ 2778 rc = -t4_fw_initialize(sc, sc->mbox); 2779 if (rc != 0) { 2780 device_printf(sc->dev, "fw init failed: %d.\n", rc); 2781 goto done; 2782 } 2783 } else { 2784 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2785 sc->cfcsum = 0; 2786 } 2787 2788done: 2789 free(card_fw, M_CXGBE); 2790 if (fw != NULL) 2791 firmware_put(fw, FIRMWARE_UNLOAD); 2792 if (default_cfg != NULL) 2793 firmware_put(default_cfg, FIRMWARE_UNLOAD); 2794 2795 return (rc); 2796} 2797 2798#define FW_PARAM_DEV(param) \ 2799 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2800 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2801#define FW_PARAM_PFVF(param) \ 2802 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2803 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2804 2805/* 2806 * Partition chip resources for use between various PFs, VFs, etc. 2807 */ 2808static int 2809partition_resources(struct adapter *sc, const struct firmware *default_cfg, 2810 const char *name_prefix) 2811{ 2812 const struct firmware *cfg = NULL; 2813 int rc = 0; 2814 struct fw_caps_config_cmd caps; 2815 uint32_t mtype, moff, finicsum, cfcsum; 2816 2817 /* 2818 * Figure out what configuration file to use. Pick the default config 2819 * file for the card if the user hasn't specified one explicitly. 2820 */ 2821 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2822 if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2823 /* Card specific overrides go here. */ 2824 if (pci_get_device(sc->dev) == 0x440a) 2825 snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2826 if (is_fpga(sc)) 2827 snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2828 } 2829 2830 /* 2831 * We need to load another module if the profile is anything except 2832 * "default" or "flash". 2833 */ 2834 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2835 strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2836 char s[32]; 2837 2838 snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2839 cfg = firmware_get(s); 2840 if (cfg == NULL) { 2841 if (default_cfg != NULL) { 2842 device_printf(sc->dev, 2843 "unable to load module \"%s\" for " 2844 "configuration profile \"%s\", will use " 2845 "the default config file instead.\n", 2846 s, sc->cfg_file); 2847 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2848 "%s", DEFAULT_CF); 2849 } else { 2850 device_printf(sc->dev, 2851 "unable to load module \"%s\" for " 2852 "configuration profile \"%s\", will use " 2853 "the config file on the card's flash " 2854 "instead.\n", s, sc->cfg_file); 2855 snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2856 "%s", FLASH_CF); 2857 } 2858 } 2859 } 2860 2861 if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2862 default_cfg == NULL) { 2863 device_printf(sc->dev, 2864 "default config file not available, will use the config " 2865 "file on the card's flash instead.\n"); 2866 snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2867 } 2868 2869 if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2870 u_int cflen; 2871 const uint32_t *cfdata; 2872 uint32_t param, val, addr; 2873 2874 KASSERT(cfg != NULL || default_cfg != NULL, 2875 ("%s: no config to upload", __func__)); 2876 2877 /* 2878 * Ask the firmware where it wants us to upload the config file. 2879 */ 2880 param = FW_PARAM_DEV(CF); 2881 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2882 if (rc != 0) { 2883 /* No support for config file? Shouldn't happen. */ 2884 device_printf(sc->dev, 2885 "failed to query config file location: %d.\n", rc); 2886 goto done; 2887 } 2888 mtype = G_FW_PARAMS_PARAM_Y(val); 2889 moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2890 2891 /* 2892 * XXX: sheer laziness. We deliberately added 4 bytes of 2893 * useless stuffing/comments at the end of the config file so 2894 * it's ok to simply throw away the last remaining bytes when 2895 * the config file is not an exact multiple of 4. This also 2896 * helps with the validate_mt_off_len check. 2897 */ 2898 if (cfg != NULL) { 2899 cflen = cfg->datasize & ~3; 2900 cfdata = cfg->data; 2901 } else { 2902 cflen = default_cfg->datasize & ~3; 2903 cfdata = default_cfg->data; 2904 } 2905 2906 if (cflen > FLASH_CFG_MAX_SIZE) { 2907 device_printf(sc->dev, 2908 "config file too long (%d, max allowed is %d). " 2909 "Will try to use the config on the card, if any.\n", 2910 cflen, FLASH_CFG_MAX_SIZE); 2911 goto use_config_on_flash; 2912 } 2913 2914 rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2915 if (rc != 0) { 2916 device_printf(sc->dev, 2917 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2918 "Will try to use the config on the card, if any.\n", 2919 __func__, mtype, moff, cflen, rc); 2920 goto use_config_on_flash; 2921 } 2922 write_via_memwin(sc, 2, addr, cfdata, cflen); 2923 } else { 2924use_config_on_flash: 2925 mtype = FW_MEMTYPE_FLASH; 2926 moff = t4_flash_cfg_addr(sc); 2927 } 2928 2929 bzero(&caps, sizeof(caps)); 2930 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2931 F_FW_CMD_REQUEST | F_FW_CMD_READ); 2932 caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2933 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2934 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2935 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2936 if (rc != 0) { 2937 device_printf(sc->dev, 2938 "failed to pre-process config file: %d " 2939 "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2940 goto done; 2941 } 2942 2943 finicsum = be32toh(caps.finicsum); 2944 cfcsum = be32toh(caps.cfcsum); 2945 if (finicsum != cfcsum) { 2946 device_printf(sc->dev, 2947 "WARNING: config file checksum mismatch: %08x %08x\n", 2948 finicsum, cfcsum); 2949 } 2950 sc->cfcsum = cfcsum; 2951 2952#define LIMIT_CAPS(x) do { \ 2953 caps.x &= htobe16(t4_##x##_allowed); \ 2954} while (0) 2955 2956 /* 2957 * Let the firmware know what features will (not) be used so it can tune 2958 * things accordingly. 2959 */ 2960 LIMIT_CAPS(nbmcaps); 2961 LIMIT_CAPS(linkcaps); 2962 LIMIT_CAPS(switchcaps); 2963 LIMIT_CAPS(niccaps); 2964 LIMIT_CAPS(toecaps); 2965 LIMIT_CAPS(rdmacaps); 2966 LIMIT_CAPS(tlscaps); 2967 LIMIT_CAPS(iscsicaps); 2968 LIMIT_CAPS(fcoecaps); 2969#undef LIMIT_CAPS 2970 2971 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2972 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2973 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2974 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2975 if (rc != 0) { 2976 device_printf(sc->dev, 2977 "failed to process config file: %d.\n", rc); 2978 } 2979done: 2980 if (cfg != NULL) 2981 firmware_put(cfg, FIRMWARE_UNLOAD); 2982 return (rc); 2983} 2984 2985/* 2986 * Retrieve parameters that are needed (or nice to have) very early. 2987 */ 2988static int 2989get_params__pre_init(struct adapter *sc) 2990{ 2991 int rc; 2992 uint32_t param[2], val[2]; 2993 2994 param[0] = FW_PARAM_DEV(PORTVEC); 2995 param[1] = FW_PARAM_DEV(CCLK); 2996 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2997 if (rc != 0) { 2998 device_printf(sc->dev, 2999 "failed to query parameters (pre_init): %d.\n", rc); 3000 return (rc); 3001 } 3002 3003 sc->params.portvec = val[0]; 3004 sc->params.nports = bitcount32(val[0]); 3005 sc->params.vpd.cclk = val[1]; 3006 3007 /* Read device log parameters. */ 3008 rc = -t4_init_devlog_params(sc, 1); 3009 if (rc == 0) 3010 fixup_devlog_params(sc); 3011 else { 3012 device_printf(sc->dev, 3013 "failed to get devlog parameters: %d.\n", rc); 3014 rc = 0; /* devlog isn't critical for device operation */ 3015 } 3016 3017 return (rc); 3018} 3019 3020/* 3021 * Retrieve various parameters that are of interest to the driver. The device 3022 * has been initialized by the firmware at this point. 3023 */ 3024static int 3025get_params__post_init(struct adapter *sc) 3026{ 3027 int rc; 3028 uint32_t param[7], val[7]; 3029 struct fw_caps_config_cmd caps; 3030 3031 param[0] = FW_PARAM_PFVF(IQFLINT_START); 3032 param[1] = FW_PARAM_PFVF(EQ_START); 3033 param[2] = FW_PARAM_PFVF(FILTER_START); 3034 param[3] = FW_PARAM_PFVF(FILTER_END); 3035 param[4] = FW_PARAM_PFVF(L2T_START); 3036 param[5] = FW_PARAM_PFVF(L2T_END); 3037 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3038 if (rc != 0) { 3039 device_printf(sc->dev, 3040 "failed to query parameters (post_init): %d.\n", rc); 3041 return (rc); 3042 } 3043 3044 sc->sge.iq_start = val[0]; 3045 sc->sge.eq_start = val[1]; 3046 sc->tids.ftid_base = val[2]; 3047 sc->tids.nftids = val[3] - val[2] + 1; 3048 sc->params.ftid_min = val[2]; 3049 sc->params.ftid_max = val[3]; 3050 sc->vres.l2t.start = val[4]; 3051 sc->vres.l2t.size = val[5] - val[4] + 1; 3052 KASSERT(sc->vres.l2t.size <= L2T_SIZE, 3053 ("%s: L2 table size (%u) larger than expected (%u)", 3054 __func__, sc->vres.l2t.size, L2T_SIZE)); 3055 3056 /* get capabilites */ 3057 bzero(&caps, sizeof(caps)); 3058 caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 3059 F_FW_CMD_REQUEST | F_FW_CMD_READ); 3060 caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 3061 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 3062 if (rc != 0) { 3063 device_printf(sc->dev, 3064 "failed to get card capabilities: %d.\n", rc); 3065 return (rc); 3066 } 3067 3068#define READ_CAPS(x) do { \ 3069 sc->x = htobe16(caps.x); \ 3070} while (0) 3071 READ_CAPS(nbmcaps); 3072 READ_CAPS(linkcaps); 3073 READ_CAPS(switchcaps); 3074 READ_CAPS(niccaps); 3075 READ_CAPS(toecaps); 3076 READ_CAPS(rdmacaps); 3077 READ_CAPS(tlscaps); 3078 READ_CAPS(iscsicaps); 3079 READ_CAPS(fcoecaps); 3080 3081 if (sc->niccaps & FW_CAPS_CONFIG_NIC_ETHOFLD) { 3082 param[0] = FW_PARAM_PFVF(ETHOFLD_START); 3083 param[1] = FW_PARAM_PFVF(ETHOFLD_END); 3084 param[2] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3085 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 3, param, val); 3086 if (rc != 0) { 3087 device_printf(sc->dev, 3088 "failed to query NIC parameters: %d.\n", rc); 3089 return (rc); 3090 } 3091 sc->tids.etid_base = val[0]; 3092 sc->params.etid_min = val[0]; 3093 sc->tids.netids = val[1] - val[0] + 1; 3094 sc->params.netids = sc->tids.netids; 3095 sc->params.eo_wr_cred = val[2]; 3096 sc->params.ethoffload = 1; 3097 } 3098 3099 if (sc->toecaps) { 3100 /* query offload-related parameters */ 3101 param[0] = FW_PARAM_DEV(NTID); 3102 param[1] = FW_PARAM_PFVF(SERVER_START); 3103 param[2] = FW_PARAM_PFVF(SERVER_END); 3104 param[3] = FW_PARAM_PFVF(TDDP_START); 3105 param[4] = FW_PARAM_PFVF(TDDP_END); 3106 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3107 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3108 if (rc != 0) { 3109 device_printf(sc->dev, 3110 "failed to query TOE parameters: %d.\n", rc); 3111 return (rc); 3112 } 3113 sc->tids.ntids = val[0]; 3114 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 3115 sc->tids.stid_base = val[1]; 3116 sc->tids.nstids = val[2] - val[1] + 1; 3117 sc->vres.ddp.start = val[3]; 3118 sc->vres.ddp.size = val[4] - val[3] + 1; 3119 sc->params.ofldq_wr_cred = val[5]; 3120 sc->params.offload = 1; 3121 } 3122 if (sc->rdmacaps) { 3123 param[0] = FW_PARAM_PFVF(STAG_START); 3124 param[1] = FW_PARAM_PFVF(STAG_END); 3125 param[2] = FW_PARAM_PFVF(RQ_START); 3126 param[3] = FW_PARAM_PFVF(RQ_END); 3127 param[4] = FW_PARAM_PFVF(PBL_START); 3128 param[5] = FW_PARAM_PFVF(PBL_END); 3129 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3130 if (rc != 0) { 3131 device_printf(sc->dev, 3132 "failed to query RDMA parameters(1): %d.\n", rc); 3133 return (rc); 3134 } 3135 sc->vres.stag.start = val[0]; 3136 sc->vres.stag.size = val[1] - val[0] + 1; 3137 sc->vres.rq.start = val[2]; 3138 sc->vres.rq.size = val[3] - val[2] + 1; 3139 sc->vres.pbl.start = val[4]; 3140 sc->vres.pbl.size = val[5] - val[4] + 1; 3141 3142 param[0] = FW_PARAM_PFVF(SQRQ_START); 3143 param[1] = FW_PARAM_PFVF(SQRQ_END); 3144 param[2] = FW_PARAM_PFVF(CQ_START); 3145 param[3] = FW_PARAM_PFVF(CQ_END); 3146 param[4] = FW_PARAM_PFVF(OCQ_START); 3147 param[5] = FW_PARAM_PFVF(OCQ_END); 3148 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 3149 if (rc != 0) { 3150 device_printf(sc->dev, 3151 "failed to query RDMA parameters(2): %d.\n", rc); 3152 return (rc); 3153 } 3154 sc->vres.qp.start = val[0]; 3155 sc->vres.qp.size = val[1] - val[0] + 1; 3156 sc->vres.cq.start = val[2]; 3157 sc->vres.cq.size = val[3] - val[2] + 1; 3158 sc->vres.ocq.start = val[4]; 3159 sc->vres.ocq.size = val[5] - val[4] + 1; 3160 } 3161 if (sc->iscsicaps) { 3162 param[0] = FW_PARAM_PFVF(ISCSI_START); 3163 param[1] = FW_PARAM_PFVF(ISCSI_END); 3164 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 3165 if (rc != 0) { 3166 device_printf(sc->dev, 3167 "failed to query iSCSI parameters: %d.\n", rc); 3168 return (rc); 3169 } 3170 sc->vres.iscsi.start = val[0]; 3171 sc->vres.iscsi.size = val[1] - val[0] + 1; 3172 } 3173 3174 /* 3175 * We've got the params we wanted to query via the firmware. Now grab 3176 * some others directly from the chip. 3177 */ 3178 rc = t4_read_chip_settings(sc); 3179 3180 return (rc); 3181} 3182 3183static int 3184set_params__post_init(struct adapter *sc) 3185{ 3186 uint32_t param, val; 3187 3188 /* ask for encapsulated CPLs */ 3189 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 3190 val = 1; 3191 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 3192 3193 return (0); 3194} 3195 3196#undef FW_PARAM_PFVF 3197#undef FW_PARAM_DEV 3198 3199static void 3200t4_set_desc(struct adapter *sc) 3201{ 3202 char buf[128]; 3203 struct adapter_params *p = &sc->params; 3204 3205 snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, " 3206 "P/N:%s, E/C:%s", p->vpd.id, is_offload(sc) ? "R" : "", 3207 chip_rev(sc), p->vpd.sn, p->vpd.pn, p->vpd.ec); 3208 3209 device_set_desc_copy(sc->dev, buf); 3210} 3211 3212static void 3213build_medialist(struct port_info *pi, struct ifmedia *media) 3214{ 3215 int m; 3216 3217 PORT_LOCK(pi); 3218 3219 ifmedia_removeall(media); 3220 3221 m = IFM_ETHER | IFM_FDX; 3222 3223 switch(pi->port_type) { 3224 case FW_PORT_TYPE_BT_XFI: 3225 case FW_PORT_TYPE_BT_XAUI: 3226 ifmedia_add(media, m | IFM_10G_T, 0, NULL); 3227 /* fall through */ 3228 3229 case FW_PORT_TYPE_BT_SGMII: 3230 ifmedia_add(media, m | IFM_1000_T, 0, NULL); 3231 ifmedia_add(media, m | IFM_100_TX, 0, NULL); 3232 ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL); 3233 ifmedia_set(media, IFM_ETHER | IFM_AUTO); 3234 break; 3235 3236 case FW_PORT_TYPE_CX4: 3237 ifmedia_add(media, m | IFM_10G_CX4, 0, NULL); 3238 ifmedia_set(media, m | IFM_10G_CX4); 3239 break; 3240 3241 case FW_PORT_TYPE_QSFP_10G: 3242 case FW_PORT_TYPE_SFP: 3243 case FW_PORT_TYPE_FIBER_XFI: 3244 case FW_PORT_TYPE_FIBER_XAUI: 3245 switch (pi->mod_type) { 3246 3247 case FW_PORT_MOD_TYPE_LR: 3248 ifmedia_add(media, m | IFM_10G_LR, 0, NULL); 3249 ifmedia_set(media, m | IFM_10G_LR); 3250 break; 3251 3252 case FW_PORT_MOD_TYPE_SR: 3253 ifmedia_add(media, m | IFM_10G_SR, 0, NULL); 3254 ifmedia_set(media, m | IFM_10G_SR); 3255 break; 3256 3257 case FW_PORT_MOD_TYPE_LRM: 3258 ifmedia_add(media, m | IFM_10G_LRM, 0, NULL); 3259 ifmedia_set(media, m | IFM_10G_LRM); 3260 break; 3261 3262 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3263 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3264 ifmedia_add(media, m | IFM_10G_TWINAX, 0, NULL); 3265 ifmedia_set(media, m | IFM_10G_TWINAX); 3266 break; 3267 3268 case FW_PORT_MOD_TYPE_NONE: 3269 m &= ~IFM_FDX; 3270 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3271 ifmedia_set(media, m | IFM_NONE); 3272 break; 3273 3274 case FW_PORT_MOD_TYPE_NA: 3275 case FW_PORT_MOD_TYPE_ER: 3276 default: 3277 device_printf(pi->dev, 3278 "unknown port_type (%d), mod_type (%d)\n", 3279 pi->port_type, pi->mod_type); 3280 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3281 ifmedia_set(media, m | IFM_UNKNOWN); 3282 break; 3283 } 3284 break; 3285 3286 case FW_PORT_TYPE_QSFP: 3287 switch (pi->mod_type) { 3288 3289 case FW_PORT_MOD_TYPE_LR: 3290 ifmedia_add(media, m | IFM_40G_LR4, 0, NULL); 3291 ifmedia_set(media, m | IFM_40G_LR4); 3292 break; 3293 3294 case FW_PORT_MOD_TYPE_SR: 3295 ifmedia_add(media, m | IFM_40G_SR4, 0, NULL); 3296 ifmedia_set(media, m | IFM_40G_SR4); 3297 break; 3298 3299 case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 3300 case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 3301 ifmedia_add(media, m | IFM_40G_CR4, 0, NULL); 3302 ifmedia_set(media, m | IFM_40G_CR4); 3303 break; 3304 3305 case FW_PORT_MOD_TYPE_NONE: 3306 m &= ~IFM_FDX; 3307 ifmedia_add(media, m | IFM_NONE, 0, NULL); 3308 ifmedia_set(media, m | IFM_NONE); 3309 break; 3310 3311 default: 3312 device_printf(pi->dev, 3313 "unknown port_type (%d), mod_type (%d)\n", 3314 pi->port_type, pi->mod_type); 3315 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3316 ifmedia_set(media, m | IFM_UNKNOWN); 3317 break; 3318 } 3319 break; 3320 3321 default: 3322 device_printf(pi->dev, 3323 "unknown port_type (%d), mod_type (%d)\n", pi->port_type, 3324 pi->mod_type); 3325 ifmedia_add(media, m | IFM_UNKNOWN, 0, NULL); 3326 ifmedia_set(media, m | IFM_UNKNOWN); 3327 break; 3328 } 3329 3330 PORT_UNLOCK(pi); 3331} 3332 3333#define FW_MAC_EXACT_CHUNK 7 3334 3335/* 3336 * Program the port's XGMAC based on parameters in ifnet. The caller also 3337 * indicates which parameters should be programmed (the rest are left alone). 3338 */ 3339int 3340update_mac_settings(struct ifnet *ifp, int flags) 3341{ 3342 int rc = 0; 3343 struct vi_info *vi = ifp->if_softc; 3344 struct port_info *pi = vi->pi; 3345 struct adapter *sc = pi->adapter; 3346 int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 3347 3348 ASSERT_SYNCHRONIZED_OP(sc); 3349 KASSERT(flags, ("%s: not told what to update.", __func__)); 3350 3351 if (flags & XGMAC_MTU) 3352 mtu = ifp->if_mtu; 3353 3354 if (flags & XGMAC_PROMISC) 3355 promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 3356 3357 if (flags & XGMAC_ALLMULTI) 3358 allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 3359 3360 if (flags & XGMAC_VLANEX) 3361 vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 3362 3363 if (flags & (XGMAC_MTU|XGMAC_PROMISC|XGMAC_ALLMULTI|XGMAC_VLANEX)) { 3364 rc = -t4_set_rxmode(sc, sc->mbox, vi->viid, mtu, promisc, 3365 allmulti, 1, vlanex, false); 3366 if (rc) { 3367 if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, 3368 rc); 3369 return (rc); 3370 } 3371 } 3372 3373 if (flags & XGMAC_UCADDR) { 3374 uint8_t ucaddr[ETHER_ADDR_LEN]; 3375 3376 bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 3377 rc = t4_change_mac(sc, sc->mbox, vi->viid, vi->xact_addr_filt, 3378 ucaddr, true, true); 3379 if (rc < 0) { 3380 rc = -rc; 3381 if_printf(ifp, "change_mac failed: %d\n", rc); 3382 return (rc); 3383 } else { 3384 vi->xact_addr_filt = rc; 3385 rc = 0; 3386 } 3387 } 3388 3389 if (flags & XGMAC_MCADDRS) { 3390 const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 3391 int del = 1; 3392 uint64_t hash = 0; 3393 struct ifmultiaddr *ifma; 3394 int i = 0, j; 3395 3396 if_maddr_rlock(ifp); 3397 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 3398 if (ifma->ifma_addr->sa_family != AF_LINK) 3399 continue; 3400 mcaddr[i] = 3401 LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 3402 MPASS(ETHER_IS_MULTICAST(mcaddr[i])); 3403 i++; 3404 3405 if (i == FW_MAC_EXACT_CHUNK) { 3406 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, 3407 del, i, mcaddr, NULL, &hash, 0); 3408 if (rc < 0) { 3409 rc = -rc; 3410 for (j = 0; j < i; j++) { 3411 if_printf(ifp, 3412 "failed to add mc address" 3413 " %02x:%02x:%02x:" 3414 "%02x:%02x:%02x rc=%d\n", 3415 mcaddr[j][0], mcaddr[j][1], 3416 mcaddr[j][2], mcaddr[j][3], 3417 mcaddr[j][4], mcaddr[j][5], 3418 rc); 3419 } 3420 goto mcfail; 3421 } 3422 del = 0; 3423 i = 0; 3424 } 3425 } 3426 if (i > 0) { 3427 rc = t4_alloc_mac_filt(sc, sc->mbox, vi->viid, del, i, 3428 mcaddr, NULL, &hash, 0); 3429 if (rc < 0) { 3430 rc = -rc; 3431 for (j = 0; j < i; j++) { 3432 if_printf(ifp, 3433 "failed to add mc address" 3434 " %02x:%02x:%02x:" 3435 "%02x:%02x:%02x rc=%d\n", 3436 mcaddr[j][0], mcaddr[j][1], 3437 mcaddr[j][2], mcaddr[j][3], 3438 mcaddr[j][4], mcaddr[j][5], 3439 rc); 3440 } 3441 goto mcfail; 3442 } 3443 } 3444 3445 rc = -t4_set_addr_hash(sc, sc->mbox, vi->viid, 0, hash, 0); 3446 if (rc != 0) 3447 if_printf(ifp, "failed to set mc address hash: %d", rc); 3448mcfail: 3449 if_maddr_runlock(ifp); 3450 } 3451 3452 return (rc); 3453} 3454 3455/* 3456 * {begin|end}_synchronized_op must be called from the same thread. 3457 */ 3458int 3459begin_synchronized_op(struct adapter *sc, struct vi_info *vi, int flags, 3460 char *wmesg) 3461{ 3462 int rc, pri; 3463 3464#ifdef WITNESS 3465 /* the caller thinks it's ok to sleep, but is it really? */ 3466 if (flags & SLEEP_OK) 3467 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 3468 "begin_synchronized_op"); 3469#endif 3470 3471 if (INTR_OK) 3472 pri = PCATCH; 3473 else 3474 pri = 0; 3475 3476 ADAPTER_LOCK(sc); 3477 for (;;) { 3478 3479 if (vi && IS_DOOMED(vi)) { 3480 rc = ENXIO; 3481 goto done; 3482 } 3483 3484 if (!IS_BUSY(sc)) { 3485 rc = 0; 3486 break; 3487 } 3488 3489 if (!(flags & SLEEP_OK)) { 3490 rc = EBUSY; 3491 goto done; 3492 } 3493 3494 if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 3495 rc = EINTR; 3496 goto done; 3497 } 3498 } 3499 3500 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 3501 SET_BUSY(sc); 3502#ifdef INVARIANTS 3503 sc->last_op = wmesg; 3504 sc->last_op_thr = curthread; 3505 sc->last_op_flags = flags; 3506#endif 3507 3508done: 3509 if (!(flags & HOLD_LOCK) || rc) 3510 ADAPTER_UNLOCK(sc); 3511 3512 return (rc); 3513} 3514 3515/* 3516 * Tell if_ioctl and if_init that the VI is going away. This is 3517 * special variant of begin_synchronized_op and must be paired with a 3518 * call to end_synchronized_op. 3519 */ 3520void 3521doom_vi(struct adapter *sc, struct vi_info *vi) 3522{ 3523 3524 ADAPTER_LOCK(sc); 3525 SET_DOOMED(vi); 3526 wakeup(&sc->flags); 3527 while (IS_BUSY(sc)) 3528 mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 3529 SET_BUSY(sc); 3530#ifdef INVARIANTS 3531 sc->last_op = "t4detach"; 3532 sc->last_op_thr = curthread; 3533 sc->last_op_flags = 0; 3534#endif 3535 ADAPTER_UNLOCK(sc); 3536} 3537 3538/* 3539 * {begin|end}_synchronized_op must be called from the same thread. 3540 */ 3541void 3542end_synchronized_op(struct adapter *sc, int flags) 3543{ 3544 3545 if (flags & LOCK_HELD) 3546 ADAPTER_LOCK_ASSERT_OWNED(sc); 3547 else 3548 ADAPTER_LOCK(sc); 3549 3550 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 3551 CLR_BUSY(sc); 3552 wakeup(&sc->flags); 3553 ADAPTER_UNLOCK(sc); 3554} 3555 3556static int 3557cxgbe_init_synchronized(struct vi_info *vi) 3558{ 3559 struct port_info *pi = vi->pi; 3560 struct adapter *sc = pi->adapter; 3561 struct ifnet *ifp = vi->ifp; 3562 int rc = 0, i; 3563 struct sge_txq *txq; 3564 3565 ASSERT_SYNCHRONIZED_OP(sc); 3566 3567 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 3568 return (0); /* already running */ 3569 3570 if (!(sc->flags & FULL_INIT_DONE) && 3571 ((rc = adapter_full_init(sc)) != 0)) 3572 return (rc); /* error message displayed already */ 3573 3574 if (!(vi->flags & VI_INIT_DONE) && 3575 ((rc = vi_full_init(vi)) != 0)) 3576 return (rc); /* error message displayed already */ 3577 3578 rc = update_mac_settings(ifp, XGMAC_ALL); 3579 if (rc) 3580 goto done; /* error message displayed already */ 3581 3582 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, true, true); 3583 if (rc != 0) { 3584 if_printf(ifp, "enable_vi failed: %d\n", rc); 3585 goto done; 3586 } 3587 3588 /* 3589 * Can't fail from this point onwards. Review cxgbe_uninit_synchronized 3590 * if this changes. 3591 */ 3592 3593 for_each_txq(vi, i, txq) { 3594 TXQ_LOCK(txq); 3595 txq->eq.flags |= EQ_ENABLED; 3596 TXQ_UNLOCK(txq); 3597 } 3598 3599 /* 3600 * The first iq of the first port to come up is used for tracing. 3601 */ 3602 if (sc->traceq < 0 && IS_MAIN_VI(vi)) { 3603 sc->traceq = sc->sge.rxq[vi->first_rxq].iq.abs_id; 3604 t4_write_reg(sc, is_t4(sc) ? A_MPS_TRC_RSS_CONTROL : 3605 A_MPS_T5_TRC_RSS_CONTROL, V_RSSCONTROL(pi->tx_chan) | 3606 V_QUEUENUMBER(sc->traceq)); 3607 pi->flags |= HAS_TRACEQ; 3608 } 3609 3610 /* all ok */ 3611 PORT_LOCK(pi); 3612 ifp->if_drv_flags |= IFF_DRV_RUNNING; 3613 pi->up_vis++; 3614 3615 if (pi->nvi > 1) 3616 callout_reset(&vi->tick, hz, vi_tick, vi); 3617 else 3618 callout_reset(&pi->tick, hz, cxgbe_tick, pi); 3619 PORT_UNLOCK(pi); 3620done: 3621 if (rc != 0) 3622 cxgbe_uninit_synchronized(vi); 3623 3624 return (rc); 3625} 3626 3627/* 3628 * Idempotent. 3629 */ 3630static int 3631cxgbe_uninit_synchronized(struct vi_info *vi) 3632{ 3633 struct port_info *pi = vi->pi; 3634 struct adapter *sc = pi->adapter; 3635 struct ifnet *ifp = vi->ifp; 3636 int rc, i; 3637 struct sge_txq *txq; 3638 3639 ASSERT_SYNCHRONIZED_OP(sc); 3640 3641 if (!(vi->flags & VI_INIT_DONE)) { 3642 KASSERT(!(ifp->if_drv_flags & IFF_DRV_RUNNING), 3643 ("uninited VI is running")); 3644 return (0); 3645 } 3646 3647 /* 3648 * Disable the VI so that all its data in either direction is discarded 3649 * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 3650 * tick) intact as the TP can deliver negative advice or data that it's 3651 * holding in its RAM (for an offloaded connection) even after the VI is 3652 * disabled. 3653 */ 3654 rc = -t4_enable_vi(sc, sc->mbox, vi->viid, false, false); 3655 if (rc) { 3656 if_printf(ifp, "disable_vi failed: %d\n", rc); 3657 return (rc); 3658 } 3659 3660 for_each_txq(vi, i, txq) { 3661 TXQ_LOCK(txq); 3662 txq->eq.flags &= ~EQ_ENABLED; 3663 TXQ_UNLOCK(txq); 3664 } 3665 3666 PORT_LOCK(pi); 3667 if (pi->nvi == 1) 3668 callout_stop(&pi->tick); 3669 else 3670 callout_stop(&vi->tick); 3671 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3672 PORT_UNLOCK(pi); 3673 return (0); 3674 } 3675 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 3676 pi->up_vis--; 3677 if (pi->up_vis > 0) { 3678 PORT_UNLOCK(pi); 3679 return (0); 3680 } 3681 PORT_UNLOCK(pi); 3682 3683 pi->link_cfg.link_ok = 0; 3684 pi->link_cfg.speed = 0; 3685 pi->linkdnrc = -1; 3686 t4_os_link_changed(sc, pi->port_id, 0, -1); 3687 3688 return (0); 3689} 3690 3691/* 3692 * It is ok for this function to fail midway and return right away. t4_detach 3693 * will walk the entire sc->irq list and clean up whatever is valid. 3694 */ 3695static int 3696setup_intr_handlers(struct adapter *sc) 3697{ 3698 int rc, rid, p, q, v; 3699 char s[8]; 3700 struct irq *irq; 3701 struct port_info *pi; 3702 struct vi_info *vi; 3703 struct sge *sge = &sc->sge; 3704 struct sge_rxq *rxq; 3705#ifdef TCP_OFFLOAD 3706 struct sge_ofld_rxq *ofld_rxq; 3707#endif 3708#ifdef DEV_NETMAP 3709 struct sge_nm_rxq *nm_rxq; 3710#endif 3711 3712 /* 3713 * Setup interrupts. 3714 */ 3715 irq = &sc->irq[0]; 3716 rid = sc->intr_type == INTR_INTX ? 0 : 1; 3717 if (sc->intr_count == 1) 3718 return (t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all")); 3719 3720 /* Multiple interrupts. */ 3721 KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 3722 ("%s: too few intr.", __func__)); 3723 3724 /* The first one is always error intr */ 3725 rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 3726 if (rc != 0) 3727 return (rc); 3728 irq++; 3729 rid++; 3730 3731 /* The second one is always the firmware event queue */ 3732 rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sge->fwq, "evt"); 3733 if (rc != 0) 3734 return (rc); 3735 irq++; 3736 rid++; 3737 3738 for_each_port(sc, p) { 3739 pi = sc->port[p]; 3740 for_each_vi(pi, v, vi) { 3741 vi->first_intr = rid - 1; 3742 3743 if (vi->nnmrxq > 0) { 3744 int n = max(vi->nrxq, vi->nnmrxq); 3745 3746 MPASS(vi->flags & INTR_RXQ); 3747 3748 rxq = &sge->rxq[vi->first_rxq]; 3749#ifdef DEV_NETMAP 3750 nm_rxq = &sge->nm_rxq[vi->first_nm_rxq]; 3751#endif 3752 for (q = 0; q < n; q++) { 3753 snprintf(s, sizeof(s), "%x%c%x", p, 3754 'a' + v, q); 3755 if (q < vi->nrxq) 3756 irq->rxq = rxq++; 3757#ifdef DEV_NETMAP 3758 if (q < vi->nnmrxq) 3759 irq->nm_rxq = nm_rxq++; 3760#endif 3761 rc = t4_alloc_irq(sc, irq, rid, 3762 t4_vi_intr, irq, s); 3763 if (rc != 0) 3764 return (rc); 3765 irq++; 3766 rid++; 3767 vi->nintr++; 3768 } 3769 } else if (vi->flags & INTR_RXQ) { 3770 for_each_rxq(vi, q, rxq) { 3771 snprintf(s, sizeof(s), "%x%c%x", p, 3772 'a' + v, q); 3773 rc = t4_alloc_irq(sc, irq, rid, 3774 t4_intr, rxq, s); 3775 if (rc != 0) 3776 return (rc); 3777 irq++; 3778 rid++; 3779 vi->nintr++; 3780 } 3781 } 3782#ifdef TCP_OFFLOAD 3783 if (vi->flags & INTR_OFLD_RXQ) { 3784 for_each_ofld_rxq(vi, q, ofld_rxq) { 3785 snprintf(s, sizeof(s), "%x%c%x", p, 3786 'A' + v, q); 3787 rc = t4_alloc_irq(sc, irq, rid, 3788 t4_intr, ofld_rxq, s); 3789 if (rc != 0) 3790 return (rc); 3791 irq++; 3792 rid++; 3793 vi->nintr++; 3794 } 3795 } 3796#endif 3797 } 3798 } 3799 MPASS(irq == &sc->irq[sc->intr_count]); 3800 3801 return (0); 3802} 3803 3804int 3805adapter_full_init(struct adapter *sc) 3806{ 3807 int rc, i; 3808 3809 ASSERT_SYNCHRONIZED_OP(sc); 3810 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3811 KASSERT((sc->flags & FULL_INIT_DONE) == 0, 3812 ("%s: FULL_INIT_DONE already", __func__)); 3813 3814 /* 3815 * queues that belong to the adapter (not any particular port). 3816 */ 3817 rc = t4_setup_adapter_queues(sc); 3818 if (rc != 0) 3819 goto done; 3820 3821 for (i = 0; i < nitems(sc->tq); i++) { 3822 sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3823 taskqueue_thread_enqueue, &sc->tq[i]); 3824 if (sc->tq[i] == NULL) { 3825 device_printf(sc->dev, 3826 "failed to allocate task queue %d\n", i); 3827 rc = ENOMEM; 3828 goto done; 3829 } 3830 taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3831 device_get_nameunit(sc->dev), i); 3832 } 3833 3834 t4_intr_enable(sc); 3835 sc->flags |= FULL_INIT_DONE; 3836done: 3837 if (rc != 0) 3838 adapter_full_uninit(sc); 3839 3840 return (rc); 3841} 3842 3843int 3844adapter_full_uninit(struct adapter *sc) 3845{ 3846 int i; 3847 3848 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3849 3850 t4_teardown_adapter_queues(sc); 3851 3852 for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3853 taskqueue_free(sc->tq[i]); 3854 sc->tq[i] = NULL; 3855 } 3856 3857 sc->flags &= ~FULL_INIT_DONE; 3858 3859 return (0); 3860} 3861 3862#ifdef RSS 3863#define SUPPORTED_RSS_HASHTYPES (RSS_HASHTYPE_RSS_IPV4 | \ 3864 RSS_HASHTYPE_RSS_TCP_IPV4 | RSS_HASHTYPE_RSS_IPV6 | \ 3865 RSS_HASHTYPE_RSS_TCP_IPV6 | RSS_HASHTYPE_RSS_UDP_IPV4 | \ 3866 RSS_HASHTYPE_RSS_UDP_IPV6) 3867 3868/* Translates kernel hash types to hardware. */ 3869static int 3870hashconfig_to_hashen(int hashconfig) 3871{ 3872 int hashen = 0; 3873 3874 if (hashconfig & RSS_HASHTYPE_RSS_IPV4) 3875 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN; 3876 if (hashconfig & RSS_HASHTYPE_RSS_IPV6) 3877 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN; 3878 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV4) { 3879 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3880 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3881 } 3882 if (hashconfig & RSS_HASHTYPE_RSS_UDP_IPV6) { 3883 hashen |= F_FW_RSS_VI_CONFIG_CMD_UDPEN | 3884 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3885 } 3886 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV4) 3887 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN; 3888 if (hashconfig & RSS_HASHTYPE_RSS_TCP_IPV6) 3889 hashen |= F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN; 3890 3891 return (hashen); 3892} 3893 3894/* Translates hardware hash types to kernel. */ 3895static int 3896hashen_to_hashconfig(int hashen) 3897{ 3898 int hashconfig = 0; 3899 3900 if (hashen & F_FW_RSS_VI_CONFIG_CMD_UDPEN) { 3901 /* 3902 * If UDP hashing was enabled it must have been enabled for 3903 * either IPv4 or IPv6 (inclusive or). Enabling UDP without 3904 * enabling any 4-tuple hash is nonsense configuration. 3905 */ 3906 MPASS(hashen & (F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 3907 F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)); 3908 3909 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3910 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV4; 3911 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3912 hashconfig |= RSS_HASHTYPE_RSS_UDP_IPV6; 3913 } 3914 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) 3915 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV4; 3916 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) 3917 hashconfig |= RSS_HASHTYPE_RSS_TCP_IPV6; 3918 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN) 3919 hashconfig |= RSS_HASHTYPE_RSS_IPV4; 3920 if (hashen & F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN) 3921 hashconfig |= RSS_HASHTYPE_RSS_IPV6; 3922 3923 return (hashconfig); 3924} 3925#endif 3926 3927int 3928vi_full_init(struct vi_info *vi) 3929{ 3930 struct adapter *sc = vi->pi->adapter; 3931 struct ifnet *ifp = vi->ifp; 3932 uint16_t *rss; 3933 struct sge_rxq *rxq; 3934 int rc, i, j, hashen; 3935#ifdef RSS 3936 int nbuckets = rss_getnumbuckets(); 3937 int hashconfig = rss_gethashconfig(); 3938 int extra; 3939 uint32_t raw_rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3940 uint32_t rss_key[RSS_KEYSIZE / sizeof(uint32_t)]; 3941#endif 3942 3943 ASSERT_SYNCHRONIZED_OP(sc); 3944 KASSERT((vi->flags & VI_INIT_DONE) == 0, 3945 ("%s: VI_INIT_DONE already", __func__)); 3946 3947 sysctl_ctx_init(&vi->ctx); 3948 vi->flags |= VI_SYSCTL_CTX; 3949 3950 /* 3951 * Allocate tx/rx/fl queues for this VI. 3952 */ 3953 rc = t4_setup_vi_queues(vi); 3954 if (rc != 0) 3955 goto done; /* error message displayed already */ 3956 3957 /* 3958 * Setup RSS for this VI. Save a copy of the RSS table for later use. 3959 */ 3960 if (vi->nrxq > vi->rss_size) { 3961 if_printf(ifp, "nrxq (%d) > hw RSS table size (%d); " 3962 "some queues will never receive traffic.\n", vi->nrxq, 3963 vi->rss_size); 3964 } else if (vi->rss_size % vi->nrxq) { 3965 if_printf(ifp, "nrxq (%d), hw RSS table size (%d); " 3966 "expect uneven traffic distribution.\n", vi->nrxq, 3967 vi->rss_size); 3968 } 3969#ifdef RSS 3970 MPASS(RSS_KEYSIZE == 40); 3971 if (vi->nrxq != nbuckets) { 3972 if_printf(ifp, "nrxq (%d) != kernel RSS buckets (%d);" 3973 "performance will be impacted.\n", vi->nrxq, nbuckets); 3974 } 3975 3976 rss_getkey((void *)&raw_rss_key[0]); 3977 for (i = 0; i < nitems(rss_key); i++) { 3978 rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]); 3979 } 3980 t4_write_rss_key(sc, &rss_key[0], -1); 3981#endif 3982 rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK); 3983 for (i = 0; i < vi->rss_size;) { 3984#ifdef RSS 3985 j = rss_get_indirection_to_bucket(i); 3986 j %= vi->nrxq; 3987 rxq = &sc->sge.rxq[vi->first_rxq + j]; 3988 rss[i++] = rxq->iq.abs_id; 3989#else 3990 for_each_rxq(vi, j, rxq) { 3991 rss[i++] = rxq->iq.abs_id; 3992 if (i == vi->rss_size) 3993 break; 3994 } 3995#endif 3996 } 3997 3998 rc = -t4_config_rss_range(sc, sc->mbox, vi->viid, 0, vi->rss_size, rss, 3999 vi->rss_size); 4000 if (rc != 0) { 4001 if_printf(ifp, "rss_config failed: %d\n", rc); 4002 goto done; 4003 } 4004 4005#ifdef RSS 4006 hashen = hashconfig_to_hashen(hashconfig); 4007 4008 /* 4009 * We may have had to enable some hashes even though the global config 4010 * wants them disabled. This is a potential problem that must be 4011 * reported to the user. 4012 */ 4013 extra = hashen_to_hashconfig(hashen) ^ hashconfig; 4014 4015 /* 4016 * If we consider only the supported hash types, then the enabled hashes 4017 * are a superset of the requested hashes. In other words, there cannot 4018 * be any supported hash that was requested but not enabled, but there 4019 * can be hashes that were not requested but had to be enabled. 4020 */ 4021 extra &= SUPPORTED_RSS_HASHTYPES; 4022 MPASS((extra & hashconfig) == 0); 4023 4024 if (extra) { 4025 if_printf(ifp, 4026 "global RSS config (0x%x) cannot be accomodated.\n", 4027 hashconfig); 4028 } 4029 if (extra & RSS_HASHTYPE_RSS_IPV4) 4030 if_printf(ifp, "IPv4 2-tuple hashing forced on.\n"); 4031 if (extra & RSS_HASHTYPE_RSS_TCP_IPV4) 4032 if_printf(ifp, "TCP/IPv4 4-tuple hashing forced on.\n"); 4033 if (extra & RSS_HASHTYPE_RSS_IPV6) 4034 if_printf(ifp, "IPv6 2-tuple hashing forced on.\n"); 4035 if (extra & RSS_HASHTYPE_RSS_TCP_IPV6) 4036 if_printf(ifp, "TCP/IPv6 4-tuple hashing forced on.\n"); 4037 if (extra & RSS_HASHTYPE_RSS_UDP_IPV4) 4038 if_printf(ifp, "UDP/IPv4 4-tuple hashing forced on.\n"); 4039 if (extra & RSS_HASHTYPE_RSS_UDP_IPV6) 4040 if_printf(ifp, "UDP/IPv6 4-tuple hashing forced on.\n"); 4041#else 4042 hashen = F_FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN | 4043 F_FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN | 4044 F_FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN | 4045 F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN | F_FW_RSS_VI_CONFIG_CMD_UDPEN; 4046#endif 4047 rc = -t4_config_vi_rss(sc, sc->mbox, vi->viid, hashen, rss[0]); 4048 if (rc != 0) { 4049 if_printf(ifp, "rss hash/defaultq config failed: %d\n", rc); 4050 goto done; 4051 } 4052 4053 vi->rss = rss; 4054 vi->flags |= VI_INIT_DONE; 4055done: 4056 if (rc != 0) 4057 vi_full_uninit(vi); 4058 4059 return (rc); 4060} 4061 4062/* 4063 * Idempotent. 4064 */ 4065int 4066vi_full_uninit(struct vi_info *vi) 4067{ 4068 struct port_info *pi = vi->pi; 4069 struct adapter *sc = pi->adapter; 4070 int i; 4071 struct sge_rxq *rxq; 4072 struct sge_txq *txq; 4073#ifdef TCP_OFFLOAD 4074 struct sge_ofld_rxq *ofld_rxq; 4075 struct sge_wrq *ofld_txq; 4076#endif 4077 4078 if (vi->flags & VI_INIT_DONE) { 4079 4080 /* Need to quiesce queues. */ 4081 4082 /* XXX: Only for the first VI? */ 4083 if (IS_MAIN_VI(vi)) 4084 quiesce_wrq(sc, &sc->sge.ctrlq[pi->port_id]); 4085 4086 for_each_txq(vi, i, txq) { 4087 quiesce_txq(sc, txq); 4088 } 4089 4090#ifdef TCP_OFFLOAD 4091 for_each_ofld_txq(vi, i, ofld_txq) { 4092 quiesce_wrq(sc, ofld_txq); 4093 } 4094#endif 4095 4096 for_each_rxq(vi, i, rxq) { 4097 quiesce_iq(sc, &rxq->iq); 4098 quiesce_fl(sc, &rxq->fl); 4099 } 4100 4101#ifdef TCP_OFFLOAD 4102 for_each_ofld_rxq(vi, i, ofld_rxq) { 4103 quiesce_iq(sc, &ofld_rxq->iq); 4104 quiesce_fl(sc, &ofld_rxq->fl); 4105 } 4106#endif 4107 free(vi->rss, M_CXGBE); 4108 free(vi->nm_rss, M_CXGBE); 4109 } 4110 4111 t4_teardown_vi_queues(vi); 4112 vi->flags &= ~VI_INIT_DONE; 4113 4114 return (0); 4115} 4116 4117static void 4118quiesce_txq(struct adapter *sc, struct sge_txq *txq) 4119{ 4120 struct sge_eq *eq = &txq->eq; 4121 struct sge_qstat *spg = (void *)&eq->desc[eq->sidx]; 4122 4123 (void) sc; /* unused */ 4124 4125#ifdef INVARIANTS 4126 TXQ_LOCK(txq); 4127 MPASS((eq->flags & EQ_ENABLED) == 0); 4128 TXQ_UNLOCK(txq); 4129#endif 4130 4131 /* Wait for the mp_ring to empty. */ 4132 while (!mp_ring_is_idle(txq->r)) { 4133 mp_ring_check_drainage(txq->r, 0); 4134 pause("rquiesce", 1); 4135 } 4136 4137 /* Then wait for the hardware to finish. */ 4138 while (spg->cidx != htobe16(eq->pidx)) 4139 pause("equiesce", 1); 4140 4141 /* Finally, wait for the driver to reclaim all descriptors. */ 4142 while (eq->cidx != eq->pidx) 4143 pause("dquiesce", 1); 4144} 4145 4146static void 4147quiesce_wrq(struct adapter *sc, struct sge_wrq *wrq) 4148{ 4149 4150 /* XXXTX */ 4151} 4152 4153static void 4154quiesce_iq(struct adapter *sc, struct sge_iq *iq) 4155{ 4156 (void) sc; /* unused */ 4157 4158 /* Synchronize with the interrupt handler */ 4159 while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 4160 pause("iqfree", 1); 4161} 4162 4163static void 4164quiesce_fl(struct adapter *sc, struct sge_fl *fl) 4165{ 4166 mtx_lock(&sc->sfl_lock); 4167 FL_LOCK(fl); 4168 fl->flags |= FL_DOOMED; 4169 FL_UNLOCK(fl); 4170 callout_stop(&sc->sfl_callout); 4171 mtx_unlock(&sc->sfl_lock); 4172 4173 KASSERT((fl->flags & FL_STARVING) == 0, 4174 ("%s: still starving", __func__)); 4175} 4176 4177static int 4178t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 4179 driver_intr_t *handler, void *arg, char *name) 4180{ 4181 int rc; 4182 4183 irq->rid = rid; 4184 irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 4185 RF_SHAREABLE | RF_ACTIVE); 4186 if (irq->res == NULL) { 4187 device_printf(sc->dev, 4188 "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 4189 return (ENOMEM); 4190 } 4191 4192 rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 4193 NULL, handler, arg, &irq->tag); 4194 if (rc != 0) { 4195 device_printf(sc->dev, 4196 "failed to setup interrupt for rid %d, name %s: %d\n", 4197 rid, name, rc); 4198 } else if (name) 4199 bus_describe_intr(sc->dev, irq->res, irq->tag, name); 4200 4201 return (rc); 4202} 4203 4204static int 4205t4_free_irq(struct adapter *sc, struct irq *irq) 4206{ 4207 if (irq->tag) 4208 bus_teardown_intr(sc->dev, irq->res, irq->tag); 4209 if (irq->res) 4210 bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 4211 4212 bzero(irq, sizeof(*irq)); 4213 4214 return (0); 4215} 4216 4217static void 4218get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 4219{ 4220 4221 regs->version = chip_id(sc) | chip_rev(sc) << 10; 4222 t4_get_regs(sc, buf, regs->len); 4223} 4224 4225#define A_PL_INDIR_CMD 0x1f8 4226 4227#define S_PL_AUTOINC 31 4228#define M_PL_AUTOINC 0x1U 4229#define V_PL_AUTOINC(x) ((x) << S_PL_AUTOINC) 4230#define G_PL_AUTOINC(x) (((x) >> S_PL_AUTOINC) & M_PL_AUTOINC) 4231 4232#define S_PL_VFID 20 4233#define M_PL_VFID 0xffU 4234#define V_PL_VFID(x) ((x) << S_PL_VFID) 4235#define G_PL_VFID(x) (((x) >> S_PL_VFID) & M_PL_VFID) 4236 4237#define S_PL_ADDR 0 4238#define M_PL_ADDR 0xfffffU 4239#define V_PL_ADDR(x) ((x) << S_PL_ADDR) 4240#define G_PL_ADDR(x) (((x) >> S_PL_ADDR) & M_PL_ADDR) 4241 4242#define A_PL_INDIR_DATA 0x1fc 4243 4244static uint64_t 4245read_vf_stat(struct adapter *sc, unsigned int viid, int reg) 4246{ 4247 u32 stats[2]; 4248 4249 mtx_assert(&sc->reg_lock, MA_OWNED); 4250 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4251 V_PL_VFID(G_FW_VIID_VIN(viid)) | V_PL_ADDR(VF_MPS_REG(reg))); 4252 stats[0] = t4_read_reg(sc, A_PL_INDIR_DATA); 4253 stats[1] = t4_read_reg(sc, A_PL_INDIR_DATA); 4254 return (((uint64_t)stats[1]) << 32 | stats[0]); 4255} 4256 4257static void 4258t4_get_vi_stats(struct adapter *sc, unsigned int viid, 4259 struct fw_vi_stats_vf *stats) 4260{ 4261 4262#define GET_STAT(name) \ 4263 read_vf_stat(sc, viid, A_MPS_VF_STAT_##name##_L) 4264 4265 stats->tx_bcast_bytes = GET_STAT(TX_VF_BCAST_BYTES); 4266 stats->tx_bcast_frames = GET_STAT(TX_VF_BCAST_FRAMES); 4267 stats->tx_mcast_bytes = GET_STAT(TX_VF_MCAST_BYTES); 4268 stats->tx_mcast_frames = GET_STAT(TX_VF_MCAST_FRAMES); 4269 stats->tx_ucast_bytes = GET_STAT(TX_VF_UCAST_BYTES); 4270 stats->tx_ucast_frames = GET_STAT(TX_VF_UCAST_FRAMES); 4271 stats->tx_drop_frames = GET_STAT(TX_VF_DROP_FRAMES); 4272 stats->tx_offload_bytes = GET_STAT(TX_VF_OFFLOAD_BYTES); 4273 stats->tx_offload_frames = GET_STAT(TX_VF_OFFLOAD_FRAMES); 4274 stats->rx_bcast_bytes = GET_STAT(RX_VF_BCAST_BYTES); 4275 stats->rx_bcast_frames = GET_STAT(RX_VF_BCAST_FRAMES); 4276 stats->rx_mcast_bytes = GET_STAT(RX_VF_MCAST_BYTES); 4277 stats->rx_mcast_frames = GET_STAT(RX_VF_MCAST_FRAMES); 4278 stats->rx_ucast_bytes = GET_STAT(RX_VF_UCAST_BYTES); 4279 stats->rx_ucast_frames = GET_STAT(RX_VF_UCAST_FRAMES); 4280 stats->rx_err_frames = GET_STAT(RX_VF_ERR_FRAMES); 4281 4282#undef GET_STAT 4283} 4284 4285static void 4286t4_clr_vi_stats(struct adapter *sc, unsigned int viid) 4287{ 4288 int reg; 4289 4290 t4_write_reg(sc, A_PL_INDIR_CMD, V_PL_AUTOINC(1) | 4291 V_PL_VFID(G_FW_VIID_VIN(viid)) | 4292 V_PL_ADDR(VF_MPS_REG(A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L))); 4293 for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; 4294 reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) 4295 t4_write_reg(sc, A_PL_INDIR_DATA, 0); 4296} 4297 4298static void 4299vi_refresh_stats(struct adapter *sc, struct vi_info *vi) 4300{ 4301 struct ifnet *ifp = vi->ifp; 4302 struct sge_txq *txq; 4303 int i, drops; 4304 struct fw_vi_stats_vf *s = &vi->stats; 4305 struct timeval tv; 4306 const struct timeval interval = {0, 250000}; /* 250ms */ 4307 4308 if (!(vi->flags & VI_INIT_DONE)) 4309 return; 4310 4311 getmicrotime(&tv); 4312 timevalsub(&tv, &interval); 4313 if (timevalcmp(&tv, &vi->last_refreshed, <)) 4314 return; 4315 4316 mtx_lock(&sc->reg_lock); 4317 t4_get_vi_stats(sc, vi->viid, &vi->stats); 4318 4319 ifp->if_ipackets = s->rx_bcast_frames + s->rx_mcast_frames + 4320 s->rx_ucast_frames; 4321 ifp->if_ierrors = s->rx_err_frames; 4322 ifp->if_opackets = s->tx_bcast_frames + s->tx_mcast_frames + 4323 s->tx_ucast_frames + s->tx_offload_frames; 4324 ifp->if_oerrors = s->tx_drop_frames; 4325 ifp->if_ibytes = s->rx_bcast_bytes + s->rx_mcast_bytes + 4326 s->rx_ucast_bytes; 4327 ifp->if_obytes = s->tx_bcast_bytes + s->tx_mcast_bytes + 4328 s->tx_ucast_bytes + s->tx_offload_bytes; 4329 ifp->if_imcasts = s->rx_mcast_frames; 4330 ifp->if_omcasts = s->tx_mcast_frames; 4331 4332 drops = 0; 4333 for_each_txq(vi, i, txq) 4334 drops += counter_u64_fetch(txq->r->drops); 4335 ifp->if_snd.ifq_drops = drops; 4336 4337 getmicrotime(&vi->last_refreshed); 4338 mtx_unlock(&sc->reg_lock); 4339} 4340 4341static void 4342cxgbe_refresh_stats(struct adapter *sc, struct port_info *pi) 4343{ 4344 struct vi_info *vi = &pi->vi[0]; 4345 struct ifnet *ifp = vi->ifp; 4346 struct sge_txq *txq; 4347 int i, drops; 4348 struct port_stats *s = &pi->stats; 4349 struct timeval tv; 4350 const struct timeval interval = {0, 250000}; /* 250ms */ 4351 4352 getmicrotime(&tv); 4353 timevalsub(&tv, &interval); 4354 if (timevalcmp(&tv, &pi->last_refreshed, <)) 4355 return; 4356 4357 t4_get_port_stats(sc, pi->tx_chan, s); 4358 4359 ifp->if_opackets = s->tx_frames; 4360 ifp->if_ipackets = s->rx_frames; 4361 ifp->if_obytes = s->tx_octets; 4362 ifp->if_ibytes = s->rx_octets; 4363 ifp->if_omcasts = s->tx_mcast_frames; 4364 ifp->if_imcasts = s->rx_mcast_frames; 4365 ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 4366 s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 4367 s->rx_trunc3; 4368 for (i = 0; i < sc->chip_params->nchan; i++) { 4369 if (pi->rx_chan_map & (1 << i)) { 4370 uint32_t v; 4371 4372 mtx_lock(&sc->reg_lock); 4373 t4_read_indirect(sc, A_TP_MIB_INDEX, A_TP_MIB_DATA, &v, 4374 1, A_TP_MIB_TNL_CNG_DROP_0 + i); 4375 mtx_unlock(&sc->reg_lock); 4376 ifp->if_iqdrops += v; 4377 } 4378 } 4379 4380 drops = s->tx_drop; 4381 for_each_txq(vi, i, txq) 4382 drops += counter_u64_fetch(txq->r->drops); 4383 ifp->if_snd.ifq_drops = drops; 4384 4385 ifp->if_oerrors = s->tx_error_frames; 4386 ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 4387 s->rx_fcs_err + s->rx_len_err; 4388 4389 getmicrotime(&pi->last_refreshed); 4390} 4391 4392static void 4393cxgbe_tick(void *arg) 4394{ 4395 struct port_info *pi = arg; 4396 struct adapter *sc = pi->adapter; 4397 4398 PORT_LOCK_ASSERT_OWNED(pi); 4399 cxgbe_refresh_stats(sc, pi); 4400 4401 callout_schedule(&pi->tick, hz); 4402} 4403 4404void 4405vi_tick(void *arg) 4406{ 4407 struct vi_info *vi = arg; 4408 struct adapter *sc = vi->pi->adapter; 4409 4410 vi_refresh_stats(sc, vi); 4411 4412 callout_schedule(&vi->tick, hz); 4413} 4414 4415static void 4416cxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 4417{ 4418 struct ifnet *vlan; 4419 4420 if (arg != ifp || ifp->if_type != IFT_ETHER) 4421 return; 4422 4423 vlan = VLAN_DEVAT(ifp, vid); 4424 VLAN_SETCOOKIE(vlan, ifp); 4425} 4426 4427static int 4428cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 4429{ 4430 4431#ifdef INVARIANTS 4432 panic("%s: opcode 0x%02x on iq %p with payload %p", 4433 __func__, rss->opcode, iq, m); 4434#else 4435 log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 4436 __func__, rss->opcode, iq, m); 4437 m_freem(m); 4438#endif 4439 return (EDOOFUS); 4440} 4441 4442int 4443t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 4444{ 4445 uintptr_t *loc, new; 4446 4447 if (opcode >= nitems(sc->cpl_handler)) 4448 return (EINVAL); 4449 4450 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 4451 loc = (uintptr_t *) &sc->cpl_handler[opcode]; 4452 atomic_store_rel_ptr(loc, new); 4453 4454 return (0); 4455} 4456 4457static int 4458an_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 4459{ 4460 4461#ifdef INVARIANTS 4462 panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 4463#else 4464 log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4465 __func__, iq, ctrl); 4466#endif 4467 return (EDOOFUS); 4468} 4469 4470int 4471t4_register_an_handler(struct adapter *sc, an_handler_t h) 4472{ 4473 uintptr_t *loc, new; 4474 4475 new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4476 loc = (uintptr_t *) &sc->an_handler; 4477 atomic_store_rel_ptr(loc, new); 4478 4479 return (0); 4480} 4481 4482static int 4483fw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4484{ 4485 const struct cpl_fw6_msg *cpl = 4486 __containerof(rpl, struct cpl_fw6_msg, data[0]); 4487 4488#ifdef INVARIANTS 4489 panic("%s: fw_msg type %d", __func__, cpl->type); 4490#else 4491 log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4492#endif 4493 return (EDOOFUS); 4494} 4495 4496int 4497t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4498{ 4499 uintptr_t *loc, new; 4500 4501 if (type >= nitems(sc->fw_msg_handler)) 4502 return (EINVAL); 4503 4504 /* 4505 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4506 * handler dispatch table. Reject any attempt to install a handler for 4507 * this subtype. 4508 */ 4509 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4510 return (EINVAL); 4511 4512 new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4513 loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4514 atomic_store_rel_ptr(loc, new); 4515 4516 return (0); 4517} 4518 4519/* 4520 * Should match fw_caps_config_<foo> enums in t4fw_interface.h 4521 */ 4522static char *caps_decoder[] = { 4523 "\20\001IPMI\002NCSI", /* 0: NBM */ 4524 "\20\001PPP\002QFC\003DCBX", /* 1: link */ 4525 "\20\001INGRESS\002EGRESS", /* 2: switch */ 4526 "\20\001NIC\002VM\003IDS\004UM\005UM_ISGL" /* 3: NIC */ 4527 "\006HASHFILTER\007ETHOFLD", 4528 "\20\001TOE", /* 4: TOE */ 4529 "\20\001RDDP\002RDMAC", /* 5: RDMA */ 4530 "\20\001INITIATOR_PDU\002TARGET_PDU" /* 6: iSCSI */ 4531 "\003INITIATOR_CNXOFLD\004TARGET_CNXOFLD" 4532 "\005INITIATOR_SSNOFLD\006TARGET_SSNOFLD" 4533 "\007T10DIF" 4534 "\010INITIATOR_CMDOFLD\011TARGET_CMDOFLD", 4535 "\20\00KEYS", /* 7: TLS */ 4536 "\20\001INITIATOR\002TARGET\003CTRL_OFLD" /* 8: FCoE */ 4537 "\004PO_INITIATOR\005PO_TARGET", 4538}; 4539 4540static void 4541t4_sysctls(struct adapter *sc) 4542{ 4543 struct sysctl_ctx_list *ctx; 4544 struct sysctl_oid *oid; 4545 struct sysctl_oid_list *children, *c0; 4546 static char *doorbells = {"\20\1UDB\2WCWR\3UDBWC\4KDB"}; 4547 4548 ctx = device_get_sysctl_ctx(sc->dev); 4549 4550 /* 4551 * dev.t4nex.X. 4552 */ 4553 oid = device_get_sysctl_tree(sc->dev); 4554 c0 = children = SYSCTL_CHILDREN(oid); 4555 4556 sc->sc_do_rxcopy = 1; 4557 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "do_rx_copy", CTLFLAG_RW, 4558 &sc->sc_do_rxcopy, 1, "Do RX copy of small frames"); 4559 4560 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4561 sc->params.nports, "# of ports"); 4562 4563 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4564 NULL, chip_rev(sc), "chip hardware revision"); 4565 4566 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "tp_version", 4567 CTLFLAG_RD, sc->tp_version, 0, "TP microcode version"); 4568 4569 if (sc->params.exprom_vers != 0) { 4570 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "exprom_version", 4571 CTLFLAG_RD, sc->exprom_version, 0, "expansion ROM version"); 4572 } 4573 4574 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4575 CTLFLAG_RD, sc->fw_version, 0, "firmware version"); 4576 4577 SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4578 CTLFLAG_RD, sc->cfg_file, 0, "configuration file"); 4579 4580 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4581 sc->cfcsum, "config file checksum"); 4582 4583 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4584 CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4585 sysctl_bitfield, "A", "available doorbells"); 4586 4587#define SYSCTL_CAP(name, n, text) \ 4588 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, #name, \ 4589 CTLTYPE_STRING | CTLFLAG_RD, caps_decoder[n], sc->name, \ 4590 sysctl_bitfield, "A", "available " text "capabilities") 4591 4592 SYSCTL_CAP(nbmcaps, 0, "NBM"); 4593 SYSCTL_CAP(linkcaps, 1, "link"); 4594 SYSCTL_CAP(switchcaps, 2, "switch"); 4595 SYSCTL_CAP(niccaps, 3, "NIC"); 4596 SYSCTL_CAP(toecaps, 4, "TCP offload"); 4597 SYSCTL_CAP(rdmacaps, 5, "RDMA"); 4598 SYSCTL_CAP(iscsicaps, 6, "iSCSI"); 4599 SYSCTL_CAP(tlscaps, 7, "TLS"); 4600 SYSCTL_CAP(fcoecaps, 8, "FCoE"); 4601#undef SYSCTL_CAP 4602 4603 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4604 sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4605 4606 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4607 CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.timer_val, 4608 sizeof(sc->params.sge.timer_val), sysctl_int_array, "A", 4609 "interrupt holdoff timer values (us)"); 4610 4611 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4612 CTLTYPE_STRING | CTLFLAG_RD, sc->params.sge.counter_val, 4613 sizeof(sc->params.sge.counter_val), sysctl_int_array, "A", 4614 "interrupt holdoff packet counter values"); 4615 4616 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nfilters", CTLFLAG_RD, 4617 NULL, sc->tids.nftids, "number of filters"); 4618 4619 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", CTLTYPE_INT | 4620 CTLFLAG_RD, sc, 0, sysctl_temperature, "I", 4621 "chip temperature (in Celsius)"); 4622 4623 t4_sge_sysctls(sc, ctx, children); 4624 4625 sc->lro_timeout = 100; 4626 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "lro_timeout", CTLFLAG_RW, 4627 &sc->lro_timeout, 0, "lro inactive-flush timeout (in us)"); 4628 4629 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "debug_flags", CTLFLAG_RW, 4630 &sc->debug_flags, 0, "flags to enable runtime debugging"); 4631 4632#ifdef SBUF_DRAIN 4633 /* 4634 * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4635 */ 4636 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4637 CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4638 "logs and miscellaneous information"); 4639 children = SYSCTL_CHILDREN(oid); 4640 4641 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4642 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4643 sysctl_cctrl, "A", "congestion control"); 4644 4645 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4646 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4647 sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4648 4649 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4650 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4651 sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4652 4653 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4654 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4655 sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4656 4657 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4658 CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4659 sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4660 4661 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4662 CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4663 sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4664 4665 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4666 CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4667 sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4668 4669 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4670 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4671 chip_id(sc) <= CHELSIO_T5 ? sysctl_cim_la : sysctl_cim_la_t6, 4672 "A", "CIM logic analyzer"); 4673 4674 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ma_la", 4675 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4676 sysctl_cim_ma_la, "A", "CIM MA logic analyzer"); 4677 4678 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4679 CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4680 sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4681 4682 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4683 CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4684 sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4685 4686 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4687 CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4688 sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4689 4690 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4691 CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4692 sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4693 4694 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4695 CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4696 sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4697 4698 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4699 CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4700 sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4701 4702 if (chip_id(sc) > CHELSIO_T4) { 4703 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4704 CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4705 sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4706 4707 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4708 CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4709 sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4710 } 4711 4712 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_pif_la", 4713 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4714 sysctl_cim_pif_la, "A", "CIM PIF logic analyzer"); 4715 4716 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4717 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4718 sysctl_cim_qcfg, "A", "CIM queue configuration"); 4719 4720 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4721 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4722 sysctl_cpl_stats, "A", "CPL statistics"); 4723 4724 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4725 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4726 sysctl_ddp_stats, "A", "non-TCP DDP statistics"); 4727 4728 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4729 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4730 sysctl_devlog, "A", "firmware's device log"); 4731 4732 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4733 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4734 sysctl_fcoe_stats, "A", "FCoE statistics"); 4735 4736 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4737 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4738 sysctl_hw_sched, "A", "hardware scheduler "); 4739 4740 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4741 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4742 sysctl_l2t, "A", "hardware L2 table"); 4743 4744 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4745 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4746 sysctl_lb_stats, "A", "loopback statistics"); 4747 4748 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4749 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4750 sysctl_meminfo, "A", "memory regions"); 4751 4752 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "mps_tcam", 4753 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4754 chip_id(sc) <= CHELSIO_T5 ? sysctl_mps_tcam : sysctl_mps_tcam_t6, 4755 "A", "MPS TCAM entries"); 4756 4757 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4758 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4759 sysctl_path_mtus, "A", "path MTUs"); 4760 4761 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4762 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4763 sysctl_pm_stats, "A", "PM statistics"); 4764 4765 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4766 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4767 sysctl_rdma_stats, "A", "RDMA statistics"); 4768 4769 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4770 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4771 sysctl_tcp_stats, "A", "TCP statistics"); 4772 4773 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4774 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4775 sysctl_tids, "A", "TID information"); 4776 4777 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4778 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4779 sysctl_tp_err_stats, "A", "TP error statistics"); 4780 4781 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la_mask", 4782 CTLTYPE_INT | CTLFLAG_RW, sc, 0, sysctl_tp_la_mask, "I", 4783 "TP logic analyzer event capture mask"); 4784 4785 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_la", 4786 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4787 sysctl_tp_la, "A", "TP logic analyzer"); 4788 4789 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4790 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4791 sysctl_tx_rate, "A", "Tx rate"); 4792 4793 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ulprx_la", 4794 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4795 sysctl_ulprx_la, "A", "ULPRX logic analyzer"); 4796 4797 if (is_t5(sc)) { 4798 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wcwr_stats", 4799 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4800 sysctl_wcwr_stats, "A", "write combined work requests"); 4801 } 4802#endif 4803 4804#ifdef TCP_OFFLOAD 4805 if (is_offload(sc)) { 4806 /* 4807 * dev.t4nex.X.toe. 4808 */ 4809 oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4810 NULL, "TOE parameters"); 4811 children = SYSCTL_CHILDREN(oid); 4812 4813 sc->tt.sndbuf = 256 * 1024; 4814 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4815 &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4816 4817 sc->tt.ddp = 0; 4818 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4819 &sc->tt.ddp, 0, "DDP allowed"); 4820 4821 sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4822 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4823 &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4824 4825 sc->tt.ddp_thres = 4826 G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4827 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4828 &sc->tt.ddp_thres, 0, "DDP threshold"); 4829 4830 sc->tt.rx_coalesce = 1; 4831 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "rx_coalesce", 4832 CTLFLAG_RW, &sc->tt.rx_coalesce, 0, "receive coalescing"); 4833 4834 sc->tt.tx_align = 1; 4835 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "tx_align", 4836 CTLFLAG_RW, &sc->tt.tx_align, 0, "chop and align payload"); 4837 4838 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timer_tick", 4839 CTLTYPE_STRING | CTLFLAG_RD, sc, 0, sysctl_tp_tick, "A", 4840 "TP timer tick (us)"); 4841 4842 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "timestamp_tick", 4843 CTLTYPE_STRING | CTLFLAG_RD, sc, 1, sysctl_tp_tick, "A", 4844 "TCP timestamp tick (us)"); 4845 4846 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_tick", 4847 CTLTYPE_STRING | CTLFLAG_RD, sc, 2, sysctl_tp_tick, "A", 4848 "DACK tick (us)"); 4849 4850 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "dack_timer", 4851 CTLTYPE_UINT | CTLFLAG_RD, sc, 0, sysctl_tp_dack_timer, 4852 "IU", "DACK timer (us)"); 4853 4854 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_min", 4855 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MIN, 4856 sysctl_tp_timer, "LU", "Retransmit min (us)"); 4857 4858 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rexmt_max", 4859 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_RXT_MAX, 4860 sysctl_tp_timer, "LU", "Retransmit max (us)"); 4861 4862 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_min", 4863 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MIN, 4864 sysctl_tp_timer, "LU", "Persist timer min (us)"); 4865 4866 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "persist_max", 4867 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_PERS_MAX, 4868 sysctl_tp_timer, "LU", "Persist timer max (us)"); 4869 4870 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_idle", 4871 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_IDLE, 4872 sysctl_tp_timer, "LU", "Keepidle idle timer (us)"); 4873 4874 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "keepalive_intvl", 4875 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_KEEP_INTVL, 4876 sysctl_tp_timer, "LU", "Keepidle interval (us)"); 4877 4878 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "initial_srtt", 4879 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_INIT_SRTT, 4880 sysctl_tp_timer, "LU", "Initial SRTT (us)"); 4881 4882 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "finwait2_timer", 4883 CTLTYPE_ULONG | CTLFLAG_RD, sc, A_TP_FINWAIT2_TIMER, 4884 sysctl_tp_timer, "LU", "FINWAIT2 timer (us)"); 4885 } 4886#endif 4887} 4888 4889void 4890vi_sysctls(struct vi_info *vi) 4891{ 4892 struct sysctl_ctx_list *ctx; 4893 struct sysctl_oid *oid; 4894 struct sysctl_oid_list *children; 4895 4896 ctx = device_get_sysctl_ctx(vi->dev); 4897 4898 /* 4899 * dev.v?(cxgbe|cxl).X. 4900 */ 4901 oid = device_get_sysctl_tree(vi->dev); 4902 children = SYSCTL_CHILDREN(oid); 4903 4904 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "viid", CTLFLAG_RD, NULL, 4905 vi->viid, "VI identifer"); 4906 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4907 &vi->nrxq, 0, "# of rx queues"); 4908 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4909 &vi->ntxq, 0, "# of tx queues"); 4910 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4911 &vi->first_rxq, 0, "index of first rx queue"); 4912 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4913 &vi->first_txq, 0, "index of first tx queue"); 4914 4915 if (IS_MAIN_VI(vi)) { 4916 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rsrv_noflowq", 4917 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_noflowq, "IU", 4918 "Reserve queue 0 for non-flowid packets"); 4919 } 4920 4921#ifdef TCP_OFFLOAD 4922 if (vi->nofldrxq != 0) { 4923 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4924 &vi->nofldrxq, 0, 4925 "# of rx queues for offloaded TCP connections"); 4926 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4927 &vi->nofldtxq, 0, 4928 "# of tx queues for offloaded TCP connections"); 4929 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4930 CTLFLAG_RD, &vi->first_ofld_rxq, 0, 4931 "index of first TOE rx queue"); 4932 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4933 CTLFLAG_RD, &vi->first_ofld_txq, 0, 4934 "index of first TOE tx queue"); 4935 } 4936#endif 4937#ifdef DEV_NETMAP 4938 if (vi->nnmrxq != 0) { 4939 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmrxq", CTLFLAG_RD, 4940 &vi->nnmrxq, 0, "# of netmap rx queues"); 4941 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nnmtxq", CTLFLAG_RD, 4942 &vi->nnmtxq, 0, "# of netmap tx queues"); 4943 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_rxq", 4944 CTLFLAG_RD, &vi->first_nm_rxq, 0, 4945 "index of first netmap rx queue"); 4946 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_nm_txq", 4947 CTLFLAG_RD, &vi->first_nm_txq, 0, 4948 "index of first netmap tx queue"); 4949 } 4950#endif 4951 4952 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4953 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_tmr_idx, "I", 4954 "holdoff timer index"); 4955 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4956 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_holdoff_pktc_idx, "I", 4957 "holdoff packet counter index"); 4958 4959 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4960 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_rxq, "I", 4961 "rx queue size"); 4962 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4963 CTLTYPE_INT | CTLFLAG_RW, vi, 0, sysctl_qsize_txq, "I", 4964 "tx queue size"); 4965} 4966 4967static void 4968cxgbe_sysctls(struct port_info *pi) 4969{ 4970 struct sysctl_ctx_list *ctx; 4971 struct sysctl_oid *oid; 4972 struct sysctl_oid_list *children, *children2; 4973 struct adapter *sc = pi->adapter; 4974 int i; 4975 char name[16]; 4976 4977 ctx = device_get_sysctl_ctx(pi->dev); 4978 4979 /* 4980 * dev.cxgbe.X. 4981 */ 4982 oid = device_get_sysctl_tree(pi->dev); 4983 children = SYSCTL_CHILDREN(oid); 4984 4985 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkdnrc", CTLTYPE_STRING | 4986 CTLFLAG_RD, pi, 0, sysctl_linkdnrc, "A", "reason why link is down"); 4987 if (pi->port_type == FW_PORT_TYPE_BT_XAUI) { 4988 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "temperature", 4989 CTLTYPE_INT | CTLFLAG_RD, pi, 0, sysctl_btphy, "I", 4990 "PHY temperature (in Celsius)"); 4991 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fw_version", 4992 CTLTYPE_INT | CTLFLAG_RD, pi, 1, sysctl_btphy, "I", 4993 "PHY firmware version"); 4994 } 4995 4996 SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pause_settings", 4997 CTLTYPE_STRING | CTLFLAG_RW, pi, PAUSE_TX, sysctl_pause_settings, 4998 "A", "PAUSE settings (bit 0 = rx_pause, bit 1 = tx_pause)"); 4999 5000 SYSCTL_ADD_INT(ctx, children, OID_AUTO, "max_speed", CTLFLAG_RD, NULL, 5001 port_top_speed(pi), "max speed (in Gbps)"); 5002 5003 /* 5004 * dev.(cxgbe|cxl).X.tc. 5005 */ 5006 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "tc", CTLFLAG_RD, NULL, 5007 "Tx scheduler traffic classes"); 5008 for (i = 0; i < sc->chip_params->nsched_cls; i++) { 5009 struct tx_sched_class *tc = &pi->tc[i]; 5010 5011 snprintf(name, sizeof(name), "%d", i); 5012 children2 = SYSCTL_CHILDREN(SYSCTL_ADD_NODE(ctx, 5013 SYSCTL_CHILDREN(oid), OID_AUTO, name, CTLFLAG_RD, NULL, 5014 "traffic class")); 5015 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "flags", CTLFLAG_RD, 5016 &tc->flags, 0, "flags"); 5017 SYSCTL_ADD_UINT(ctx, children2, OID_AUTO, "refcount", 5018 CTLFLAG_RD, &tc->refcount, 0, "references to this class"); 5019#ifdef SBUF_DRAIN 5020 SYSCTL_ADD_PROC(ctx, children2, OID_AUTO, "params", 5021 CTLTYPE_STRING | CTLFLAG_RD, sc, (pi->port_id << 16) | i, 5022 sysctl_tc_params, "A", "traffic class parameters"); 5023#endif 5024 } 5025 5026 /* 5027 * dev.cxgbe.X.stats. 5028 */ 5029 oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 5030 NULL, "port statistics"); 5031 children = SYSCTL_CHILDREN(oid); 5032 SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "tx_parse_error", CTLFLAG_RD, 5033 &pi->tx_parse_error, 0, 5034 "# of tx packets with invalid length or # of segments"); 5035 5036#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 5037 SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 5038 CTLTYPE_U64 | CTLFLAG_RD, sc, reg, \ 5039 sysctl_handle_t4_reg64, "QU", desc) 5040 5041 SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 5042 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 5043 SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 5044 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 5045 SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 5046 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 5047 SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 5048 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 5049 SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 5050 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 5051 SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 5052 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 5053 SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 5054 "# of tx frames in this range", 5055 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 5056 SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 5057 "# of tx frames in this range", 5058 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 5059 SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 5060 "# of tx frames in this range", 5061 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 5062 SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 5063 "# of tx frames in this range", 5064 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 5065 SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 5066 "# of tx frames in this range", 5067 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 5068 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 5069 "# of tx frames in this range", 5070 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 5071 SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 5072 "# of tx frames in this range", 5073 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 5074 SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 5075 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 5076 SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 5077 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 5078 SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 5079 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 5080 SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 5081 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 5082 SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 5083 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 5084 SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 5085 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 5086 SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 5087 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 5088 SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 5089 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 5090 SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 5091 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 5092 SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 5093 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 5094 5095 SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 5096 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 5097 SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 5098 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 5099 SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 5100 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 5101 SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 5102 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 5103 SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 5104 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 5105 SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 5106 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 5107 SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 5108 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 5109 SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 5110 "# of frames received with bad FCS", 5111 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 5112 SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 5113 "# of frames received with length error", 5114 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 5115 SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 5116 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 5117 SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 5118 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 5119 SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 5120 "# of rx frames in this range", 5121 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 5122 SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 5123 "# of rx frames in this range", 5124 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 5125 SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 5126 "# of rx frames in this range", 5127 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 5128 SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 5129 "# of rx frames in this range", 5130 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 5131 SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 5132 "# of rx frames in this range", 5133 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 5134 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 5135 "# of rx frames in this range", 5136 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 5137 SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 5138 "# of rx frames in this range", 5139 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 5140 SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 5141 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 5142 SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 5143 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 5144 SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 5145 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 5146 SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 5147 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 5148 SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 5149 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 5150 SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 5151 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 5152 SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 5153 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 5154 SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 5155 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 5156 SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 5157 PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 5158 5159#undef SYSCTL_ADD_T4_REG64 5160 5161#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 5162 SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 5163 &pi->stats.name, desc) 5164 5165 /* We get these from port_stats and they may be stale by upto 1s */ 5166 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 5167 "# drops due to buffer-group 0 overflows"); 5168 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 5169 "# drops due to buffer-group 1 overflows"); 5170 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 5171 "# drops due to buffer-group 2 overflows"); 5172 SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 5173 "# drops due to buffer-group 3 overflows"); 5174 SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 5175 "# of buffer-group 0 truncated packets"); 5176 SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 5177 "# of buffer-group 1 truncated packets"); 5178 SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 5179 "# of buffer-group 2 truncated packets"); 5180 SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 5181 "# of buffer-group 3 truncated packets"); 5182 5183#undef SYSCTL_ADD_T4_PORTSTAT 5184} 5185 5186static int 5187sysctl_int_array(SYSCTL_HANDLER_ARGS) 5188{ 5189 int rc, *i, space = 0; 5190 struct sbuf sb; 5191 5192 sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 5193 for (i = arg1; arg2; arg2 -= sizeof(int), i++) { 5194 if (space) 5195 sbuf_printf(&sb, " "); 5196 sbuf_printf(&sb, "%d", *i); 5197 space = 1; 5198 } 5199 sbuf_finish(&sb); 5200 rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 5201 sbuf_delete(&sb); 5202 return (rc); 5203} 5204 5205static int 5206sysctl_bitfield(SYSCTL_HANDLER_ARGS) 5207{ 5208 int rc; 5209 struct sbuf *sb; 5210 5211 rc = sysctl_wire_old_buffer(req, 0); 5212 if (rc != 0) 5213 return(rc); 5214 5215 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5216 if (sb == NULL) 5217 return (ENOMEM); 5218 5219 sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 5220 rc = sbuf_finish(sb); 5221 sbuf_delete(sb); 5222 5223 return (rc); 5224} 5225 5226static int 5227sysctl_btphy(SYSCTL_HANDLER_ARGS) 5228{ 5229 struct port_info *pi = arg1; 5230 int op = arg2; 5231 struct adapter *sc = pi->adapter; 5232 u_int v; 5233 int rc; 5234 5235 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, "t4btt"); 5236 if (rc) 5237 return (rc); 5238 /* XXX: magic numbers */ 5239 rc = -t4_mdio_rd(sc, sc->mbox, pi->mdio_addr, 0x1e, op ? 0x20 : 0xc820, 5240 &v); 5241 end_synchronized_op(sc, 0); 5242 if (rc) 5243 return (rc); 5244 if (op == 0) 5245 v /= 256; 5246 5247 rc = sysctl_handle_int(oidp, &v, 0, req); 5248 return (rc); 5249} 5250 5251static int 5252sysctl_noflowq(SYSCTL_HANDLER_ARGS) 5253{ 5254 struct vi_info *vi = arg1; 5255 int rc, val; 5256 5257 val = vi->rsrv_noflowq; 5258 rc = sysctl_handle_int(oidp, &val, 0, req); 5259 if (rc != 0 || req->newptr == NULL) 5260 return (rc); 5261 5262 if ((val >= 1) && (vi->ntxq > 1)) 5263 vi->rsrv_noflowq = 1; 5264 else 5265 vi->rsrv_noflowq = 0; 5266 5267 return (rc); 5268} 5269 5270static int 5271sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 5272{ 5273 struct vi_info *vi = arg1; 5274 struct adapter *sc = vi->pi->adapter; 5275 int idx, rc, i; 5276 struct sge_rxq *rxq; 5277#ifdef TCP_OFFLOAD 5278 struct sge_ofld_rxq *ofld_rxq; 5279#endif 5280 uint8_t v; 5281 5282 idx = vi->tmr_idx; 5283 5284 rc = sysctl_handle_int(oidp, &idx, 0, req); 5285 if (rc != 0 || req->newptr == NULL) 5286 return (rc); 5287 5288 if (idx < 0 || idx >= SGE_NTIMERS) 5289 return (EINVAL); 5290 5291 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5292 "t4tmr"); 5293 if (rc) 5294 return (rc); 5295 5296 v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(vi->pktc_idx != -1); 5297 for_each_rxq(vi, i, rxq) { 5298#ifdef atomic_store_rel_8 5299 atomic_store_rel_8(&rxq->iq.intr_params, v); 5300#else 5301 rxq->iq.intr_params = v; 5302#endif 5303 } 5304#ifdef TCP_OFFLOAD 5305 for_each_ofld_rxq(vi, i, ofld_rxq) { 5306#ifdef atomic_store_rel_8 5307 atomic_store_rel_8(&ofld_rxq->iq.intr_params, v); 5308#else 5309 ofld_rxq->iq.intr_params = v; 5310#endif 5311 } 5312#endif 5313 vi->tmr_idx = idx; 5314 5315 end_synchronized_op(sc, LOCK_HELD); 5316 return (0); 5317} 5318 5319static int 5320sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 5321{ 5322 struct vi_info *vi = arg1; 5323 struct adapter *sc = vi->pi->adapter; 5324 int idx, rc; 5325 5326 idx = vi->pktc_idx; 5327 5328 rc = sysctl_handle_int(oidp, &idx, 0, req); 5329 if (rc != 0 || req->newptr == NULL) 5330 return (rc); 5331 5332 if (idx < -1 || idx >= SGE_NCOUNTERS) 5333 return (EINVAL); 5334 5335 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5336 "t4pktc"); 5337 if (rc) 5338 return (rc); 5339 5340 if (vi->flags & VI_INIT_DONE) 5341 rc = EBUSY; /* cannot be changed once the queues are created */ 5342 else 5343 vi->pktc_idx = idx; 5344 5345 end_synchronized_op(sc, LOCK_HELD); 5346 return (rc); 5347} 5348 5349static int 5350sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 5351{ 5352 struct vi_info *vi = arg1; 5353 struct adapter *sc = vi->pi->adapter; 5354 int qsize, rc; 5355 5356 qsize = vi->qsize_rxq; 5357 5358 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5359 if (rc != 0 || req->newptr == NULL) 5360 return (rc); 5361 5362 if (qsize < 128 || (qsize & 7)) 5363 return (EINVAL); 5364 5365 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5366 "t4rxqs"); 5367 if (rc) 5368 return (rc); 5369 5370 if (vi->flags & VI_INIT_DONE) 5371 rc = EBUSY; /* cannot be changed once the queues are created */ 5372 else 5373 vi->qsize_rxq = qsize; 5374 5375 end_synchronized_op(sc, LOCK_HELD); 5376 return (rc); 5377} 5378 5379static int 5380sysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 5381{ 5382 struct vi_info *vi = arg1; 5383 struct adapter *sc = vi->pi->adapter; 5384 int qsize, rc; 5385 5386 qsize = vi->qsize_txq; 5387 5388 rc = sysctl_handle_int(oidp, &qsize, 0, req); 5389 if (rc != 0 || req->newptr == NULL) 5390 return (rc); 5391 5392 if (qsize < 128 || qsize > 65536) 5393 return (EINVAL); 5394 5395 rc = begin_synchronized_op(sc, vi, HOLD_LOCK | SLEEP_OK | INTR_OK, 5396 "t4txqs"); 5397 if (rc) 5398 return (rc); 5399 5400 if (vi->flags & VI_INIT_DONE) 5401 rc = EBUSY; /* cannot be changed once the queues are created */ 5402 else 5403 vi->qsize_txq = qsize; 5404 5405 end_synchronized_op(sc, LOCK_HELD); 5406 return (rc); 5407} 5408 5409static int 5410sysctl_pause_settings(SYSCTL_HANDLER_ARGS) 5411{ 5412 struct port_info *pi = arg1; 5413 struct adapter *sc = pi->adapter; 5414 struct link_config *lc = &pi->link_cfg; 5415 int rc; 5416 5417 if (req->newptr == NULL) { 5418 struct sbuf *sb; 5419 static char *bits = "\20\1PAUSE_RX\2PAUSE_TX"; 5420 5421 rc = sysctl_wire_old_buffer(req, 0); 5422 if (rc != 0) 5423 return(rc); 5424 5425 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 5426 if (sb == NULL) 5427 return (ENOMEM); 5428 5429 sbuf_printf(sb, "%b", lc->fc & (PAUSE_TX | PAUSE_RX), bits); 5430 rc = sbuf_finish(sb); 5431 sbuf_delete(sb); 5432 } else { 5433 char s[2]; 5434 int n; 5435 5436 s[0] = '0' + (lc->requested_fc & (PAUSE_TX | PAUSE_RX)); 5437 s[1] = 0; 5438 5439 rc = sysctl_handle_string(oidp, s, sizeof(s), req); 5440 if (rc != 0) 5441 return(rc); 5442 5443 if (s[1] != 0) 5444 return (EINVAL); 5445 if (s[0] < '0' || s[0] > '9') 5446 return (EINVAL); /* not a number */ 5447 n = s[0] - '0'; 5448 if (n & ~(PAUSE_TX | PAUSE_RX)) 5449 return (EINVAL); /* some other bit is set too */ 5450 5451 rc = begin_synchronized_op(sc, &pi->vi[0], SLEEP_OK | INTR_OK, 5452 "t4PAUSE"); 5453 if (rc) 5454 return (rc); 5455 if ((lc->requested_fc & (PAUSE_TX | PAUSE_RX)) != n) { 5456 int link_ok = lc->link_ok; 5457 5458 lc->requested_fc &= ~(PAUSE_TX | PAUSE_RX); 5459 lc->requested_fc |= n; 5460 rc = -t4_link_l1cfg(sc, sc->mbox, pi->tx_chan, lc); 5461 lc->link_ok = link_ok; /* restore */ 5462 } 5463 end_synchronized_op(sc, 0); 5464 } 5465 5466 return (rc); 5467} 5468 5469static int 5470sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 5471{ 5472 struct adapter *sc = arg1; 5473 int reg = arg2; 5474 uint64_t val; 5475 5476 val = t4_read_reg64(sc, reg); 5477 5478 return (sysctl_handle_64(oidp, &val, 0, req)); 5479} 5480 5481static int 5482sysctl_temperature(SYSCTL_HANDLER_ARGS) 5483{ 5484 struct adapter *sc = arg1; 5485 int rc, t; 5486 uint32_t param, val; 5487 5488 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4temp"); 5489 if (rc) 5490 return (rc); 5491 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 5492 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 5493 V_FW_PARAMS_PARAM_Y(FW_PARAM_DEV_DIAG_TMP); 5494 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 5495 end_synchronized_op(sc, 0); 5496 if (rc) 5497 return (rc); 5498 5499 /* unknown is returned as 0 but we display -1 in that case */ 5500 t = val == 0 ? -1 : val; 5501 5502 rc = sysctl_handle_int(oidp, &t, 0, req); 5503 return (rc); 5504} 5505 5506#ifdef SBUF_DRAIN 5507static int 5508sysctl_cctrl(SYSCTL_HANDLER_ARGS) 5509{ 5510 struct adapter *sc = arg1; 5511 struct sbuf *sb; 5512 int rc, i; 5513 uint16_t incr[NMTUS][NCCTRL_WIN]; 5514 static const char *dec_fac[] = { 5515 "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 5516 "0.9375" 5517 }; 5518 5519 rc = sysctl_wire_old_buffer(req, 0); 5520 if (rc != 0) 5521 return (rc); 5522 5523 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5524 if (sb == NULL) 5525 return (ENOMEM); 5526 5527 t4_read_cong_tbl(sc, incr); 5528 5529 for (i = 0; i < NCCTRL_WIN; ++i) { 5530 sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 5531 incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 5532 incr[5][i], incr[6][i], incr[7][i]); 5533 sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 5534 incr[8][i], incr[9][i], incr[10][i], incr[11][i], 5535 incr[12][i], incr[13][i], incr[14][i], incr[15][i], 5536 sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 5537 } 5538 5539 rc = sbuf_finish(sb); 5540 sbuf_delete(sb); 5541 5542 return (rc); 5543} 5544 5545static const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 5546 "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 5547 "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 5548 "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 5549}; 5550 5551static int 5552sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 5553{ 5554 struct adapter *sc = arg1; 5555 struct sbuf *sb; 5556 int rc, i, n, qid = arg2; 5557 uint32_t *buf, *p; 5558 char *qtype; 5559 u_int cim_num_obq = sc->chip_params->cim_num_obq; 5560 5561 KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 5562 ("%s: bad qid %d\n", __func__, qid)); 5563 5564 if (qid < CIM_NUM_IBQ) { 5565 /* inbound queue */ 5566 qtype = "IBQ"; 5567 n = 4 * CIM_IBQ_SIZE; 5568 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5569 rc = t4_read_cim_ibq(sc, qid, buf, n); 5570 } else { 5571 /* outbound queue */ 5572 qtype = "OBQ"; 5573 qid -= CIM_NUM_IBQ; 5574 n = 4 * cim_num_obq * CIM_OBQ_SIZE; 5575 buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 5576 rc = t4_read_cim_obq(sc, qid, buf, n); 5577 } 5578 5579 if (rc < 0) { 5580 rc = -rc; 5581 goto done; 5582 } 5583 n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 5584 5585 rc = sysctl_wire_old_buffer(req, 0); 5586 if (rc != 0) 5587 goto done; 5588 5589 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5590 if (sb == NULL) { 5591 rc = ENOMEM; 5592 goto done; 5593 } 5594 5595 sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 5596 for (i = 0, p = buf; i < n; i += 16, p += 4) 5597 sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 5598 p[2], p[3]); 5599 5600 rc = sbuf_finish(sb); 5601 sbuf_delete(sb); 5602done: 5603 free(buf, M_CXGBE); 5604 return (rc); 5605} 5606 5607static int 5608sysctl_cim_la(SYSCTL_HANDLER_ARGS) 5609{ 5610 struct adapter *sc = arg1; 5611 u_int cfg; 5612 struct sbuf *sb; 5613 uint32_t *buf, *p; 5614 int rc; 5615 5616 MPASS(chip_id(sc) <= CHELSIO_T5); 5617 5618 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5619 if (rc != 0) 5620 return (rc); 5621 5622 rc = sysctl_wire_old_buffer(req, 0); 5623 if (rc != 0) 5624 return (rc); 5625 5626 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5627 if (sb == NULL) 5628 return (ENOMEM); 5629 5630 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5631 M_ZERO | M_WAITOK); 5632 5633 rc = -t4_cim_read_la(sc, buf, NULL); 5634 if (rc != 0) 5635 goto done; 5636 5637 sbuf_printf(sb, "Status Data PC%s", 5638 cfg & F_UPDBGLACAPTPCONLY ? "" : 5639 " LS0Stat LS0Addr LS0Data"); 5640 5641 for (p = buf; p <= &buf[sc->params.cim_la_size - 8]; p += 8) { 5642 if (cfg & F_UPDBGLACAPTPCONLY) { 5643 sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 5644 p[6], p[7]); 5645 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 5646 (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 5647 p[4] & 0xff, p[5] >> 8); 5648 sbuf_printf(sb, "\n %02x %x%07x %x%07x", 5649 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5650 p[1] & 0xf, p[2] >> 4); 5651 } else { 5652 sbuf_printf(sb, 5653 "\n %02x %x%07x %x%07x %08x %08x " 5654 "%08x%08x%08x%08x", 5655 (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 5656 p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 5657 p[6], p[7]); 5658 } 5659 } 5660 5661 rc = sbuf_finish(sb); 5662 sbuf_delete(sb); 5663done: 5664 free(buf, M_CXGBE); 5665 return (rc); 5666} 5667 5668static int 5669sysctl_cim_la_t6(SYSCTL_HANDLER_ARGS) 5670{ 5671 struct adapter *sc = arg1; 5672 u_int cfg; 5673 struct sbuf *sb; 5674 uint32_t *buf, *p; 5675 int rc; 5676 5677 MPASS(chip_id(sc) > CHELSIO_T5); 5678 5679 rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 5680 if (rc != 0) 5681 return (rc); 5682 5683 rc = sysctl_wire_old_buffer(req, 0); 5684 if (rc != 0) 5685 return (rc); 5686 5687 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5688 if (sb == NULL) 5689 return (ENOMEM); 5690 5691 buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 5692 M_ZERO | M_WAITOK); 5693 5694 rc = -t4_cim_read_la(sc, buf, NULL); 5695 if (rc != 0) 5696 goto done; 5697 5698 sbuf_printf(sb, "Status Inst Data PC%s", 5699 cfg & F_UPDBGLACAPTPCONLY ? "" : 5700 " LS0Stat LS0Addr LS0Data LS1Stat LS1Addr LS1Data"); 5701 5702 for (p = buf; p <= &buf[sc->params.cim_la_size - 10]; p += 10) { 5703 if (cfg & F_UPDBGLACAPTPCONLY) { 5704 sbuf_printf(sb, "\n %02x %08x %08x %08x", 5705 p[3] & 0xff, p[2], p[1], p[0]); 5706 sbuf_printf(sb, "\n %02x %02x%06x %02x%06x %02x%06x", 5707 (p[6] >> 8) & 0xff, p[6] & 0xff, p[5] >> 8, 5708 p[5] & 0xff, p[4] >> 8, p[4] & 0xff, p[3] >> 8); 5709 sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x", 5710 (p[9] >> 16) & 0xff, p[9] & 0xffff, p[8] >> 16, 5711 p[8] & 0xffff, p[7] >> 16, p[7] & 0xffff, 5712 p[6] >> 16); 5713 } else { 5714 sbuf_printf(sb, "\n %02x %04x%04x %04x%04x %04x%04x " 5715 "%08x %08x %08x %08x %08x %08x", 5716 (p[9] >> 16) & 0xff, 5717 p[9] & 0xffff, p[8] >> 16, 5718 p[8] & 0xffff, p[7] >> 16, 5719 p[7] & 0xffff, p[6] >> 16, 5720 p[2], p[1], p[0], p[5], p[4], p[3]); 5721 } 5722 } 5723 5724 rc = sbuf_finish(sb); 5725 sbuf_delete(sb); 5726done: 5727 free(buf, M_CXGBE); 5728 return (rc); 5729} 5730 5731static int 5732sysctl_cim_ma_la(SYSCTL_HANDLER_ARGS) 5733{ 5734 struct adapter *sc = arg1; 5735 u_int i; 5736 struct sbuf *sb; 5737 uint32_t *buf, *p; 5738 int rc; 5739 5740 rc = sysctl_wire_old_buffer(req, 0); 5741 if (rc != 0) 5742 return (rc); 5743 5744 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5745 if (sb == NULL) 5746 return (ENOMEM); 5747 5748 buf = malloc(2 * CIM_MALA_SIZE * 5 * sizeof(uint32_t), M_CXGBE, 5749 M_ZERO | M_WAITOK); 5750 5751 t4_cim_read_ma_la(sc, buf, buf + 5 * CIM_MALA_SIZE); 5752 p = buf; 5753 5754 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5755 sbuf_printf(sb, "\n%02x%08x%08x%08x%08x", p[4], p[3], p[2], 5756 p[1], p[0]); 5757 } 5758 5759 sbuf_printf(sb, "\n\nCnt ID Tag UE Data RDY VLD"); 5760 for (i = 0; i < CIM_MALA_SIZE; i++, p += 5) { 5761 sbuf_printf(sb, "\n%3u %2u %x %u %08x%08x %u %u", 5762 (p[2] >> 10) & 0xff, (p[2] >> 7) & 7, 5763 (p[2] >> 3) & 0xf, (p[2] >> 2) & 1, 5764 (p[1] >> 2) | ((p[2] & 3) << 30), 5765 (p[0] >> 2) | ((p[1] & 3) << 30), (p[0] >> 1) & 1, 5766 p[0] & 1); 5767 } 5768 5769 rc = sbuf_finish(sb); 5770 sbuf_delete(sb); 5771 free(buf, M_CXGBE); 5772 return (rc); 5773} 5774 5775static int 5776sysctl_cim_pif_la(SYSCTL_HANDLER_ARGS) 5777{ 5778 struct adapter *sc = arg1; 5779 u_int i; 5780 struct sbuf *sb; 5781 uint32_t *buf, *p; 5782 int rc; 5783 5784 rc = sysctl_wire_old_buffer(req, 0); 5785 if (rc != 0) 5786 return (rc); 5787 5788 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5789 if (sb == NULL) 5790 return (ENOMEM); 5791 5792 buf = malloc(2 * CIM_PIFLA_SIZE * 6 * sizeof(uint32_t), M_CXGBE, 5793 M_ZERO | M_WAITOK); 5794 5795 t4_cim_read_pif_la(sc, buf, buf + 6 * CIM_PIFLA_SIZE, NULL, NULL); 5796 p = buf; 5797 5798 sbuf_printf(sb, "Cntl ID DataBE Addr Data"); 5799 for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { 5800 sbuf_printf(sb, "\n %02x %02x %04x %08x %08x%08x%08x%08x", 5801 (p[5] >> 22) & 0xff, (p[5] >> 16) & 0x3f, p[5] & 0xffff, 5802 p[4], p[3], p[2], p[1], p[0]); 5803 } 5804 5805 sbuf_printf(sb, "\n\nCntl ID Data"); 5806 for (i = 0; i < CIM_PIFLA_SIZE; i++, p += 6) { 5807 sbuf_printf(sb, "\n %02x %02x %08x%08x%08x%08x", 5808 (p[4] >> 6) & 0xff, p[4] & 0x3f, p[3], p[2], p[1], p[0]); 5809 } 5810 5811 rc = sbuf_finish(sb); 5812 sbuf_delete(sb); 5813 free(buf, M_CXGBE); 5814 return (rc); 5815} 5816 5817static int 5818sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 5819{ 5820 struct adapter *sc = arg1; 5821 struct sbuf *sb; 5822 int rc, i; 5823 uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5824 uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 5825 uint16_t thres[CIM_NUM_IBQ]; 5826 uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 5827 uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 5828 u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 5829 5830 cim_num_obq = sc->chip_params->cim_num_obq; 5831 if (is_t4(sc)) { 5832 ibq_rdaddr = A_UP_IBQ_0_RDADDR; 5833 obq_rdaddr = A_UP_OBQ_0_REALADDR; 5834 } else { 5835 ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 5836 obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 5837 } 5838 nq = CIM_NUM_IBQ + cim_num_obq; 5839 5840 rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 5841 if (rc == 0) 5842 rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 5843 if (rc != 0) 5844 return (rc); 5845 5846 t4_read_cimq_cfg(sc, base, size, thres); 5847 5848 rc = sysctl_wire_old_buffer(req, 0); 5849 if (rc != 0) 5850 return (rc); 5851 5852 sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 5853 if (sb == NULL) 5854 return (ENOMEM); 5855 5856 sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 5857 5858 for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 5859 sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 5860 qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 5861 G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5862 G_QUEREMFLITS(p[2]) * 16); 5863 for ( ; i < nq; i++, p += 4, wr += 2) 5864 sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 5865 base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 5866 wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 5867 G_QUEREMFLITS(p[2]) * 16); 5868 5869 rc = sbuf_finish(sb); 5870 sbuf_delete(sb); 5871 5872 return (rc); 5873} 5874 5875static int 5876sysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 5877{ 5878 struct adapter *sc = arg1; 5879 struct sbuf *sb; 5880 int rc; 5881 struct tp_cpl_stats stats; 5882 5883 rc = sysctl_wire_old_buffer(req, 0); 5884 if (rc != 0) 5885 return (rc); 5886 5887 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5888 if (sb == NULL) 5889 return (ENOMEM); 5890 5891 mtx_lock(&sc->reg_lock); 5892 t4_tp_get_cpl_stats(sc, &stats); 5893 mtx_unlock(&sc->reg_lock); 5894 5895 if (sc->chip_params->nchan > 2) { 5896 sbuf_printf(sb, " channel 0 channel 1" 5897 " channel 2 channel 3"); 5898 sbuf_printf(sb, "\nCPL requests: %10u %10u %10u %10u", 5899 stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 5900 sbuf_printf(sb, "\nCPL responses: %10u %10u %10u %10u", 5901 stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 5902 } else { 5903 sbuf_printf(sb, " channel 0 channel 1"); 5904 sbuf_printf(sb, "\nCPL requests: %10u %10u", 5905 stats.req[0], stats.req[1]); 5906 sbuf_printf(sb, "\nCPL responses: %10u %10u", 5907 stats.rsp[0], stats.rsp[1]); 5908 } 5909 5910 rc = sbuf_finish(sb); 5911 sbuf_delete(sb); 5912 5913 return (rc); 5914} 5915 5916static int 5917sysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 5918{ 5919 struct adapter *sc = arg1; 5920 struct sbuf *sb; 5921 int rc; 5922 struct tp_usm_stats stats; 5923 5924 rc = sysctl_wire_old_buffer(req, 0); 5925 if (rc != 0) 5926 return(rc); 5927 5928 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5929 if (sb == NULL) 5930 return (ENOMEM); 5931 5932 t4_get_usm_stats(sc, &stats); 5933 5934 sbuf_printf(sb, "Frames: %u\n", stats.frames); 5935 sbuf_printf(sb, "Octets: %ju\n", stats.octets); 5936 sbuf_printf(sb, "Drops: %u", stats.drops); 5937 5938 rc = sbuf_finish(sb); 5939 sbuf_delete(sb); 5940 5941 return (rc); 5942} 5943 5944static const char * const devlog_level_strings[] = { 5945 [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 5946 [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 5947 [FW_DEVLOG_LEVEL_ERR] = "ERR", 5948 [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 5949 [FW_DEVLOG_LEVEL_INFO] = "INFO", 5950 [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 5951}; 5952 5953static const char * const devlog_facility_strings[] = { 5954 [FW_DEVLOG_FACILITY_CORE] = "CORE", 5955 [FW_DEVLOG_FACILITY_CF] = "CF", 5956 [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 5957 [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5958 [FW_DEVLOG_FACILITY_RES] = "RES", 5959 [FW_DEVLOG_FACILITY_HW] = "HW", 5960 [FW_DEVLOG_FACILITY_FLR] = "FLR", 5961 [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5962 [FW_DEVLOG_FACILITY_PHY] = "PHY", 5963 [FW_DEVLOG_FACILITY_MAC] = "MAC", 5964 [FW_DEVLOG_FACILITY_PORT] = "PORT", 5965 [FW_DEVLOG_FACILITY_VI] = "VI", 5966 [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5967 [FW_DEVLOG_FACILITY_ACL] = "ACL", 5968 [FW_DEVLOG_FACILITY_TM] = "TM", 5969 [FW_DEVLOG_FACILITY_QFC] = "QFC", 5970 [FW_DEVLOG_FACILITY_DCB] = "DCB", 5971 [FW_DEVLOG_FACILITY_ETH] = "ETH", 5972 [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5973 [FW_DEVLOG_FACILITY_RI] = "RI", 5974 [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5975 [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5976 [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5977 [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE", 5978 [FW_DEVLOG_FACILITY_CHNET] = "CHNET", 5979}; 5980 5981static int 5982sysctl_devlog(SYSCTL_HANDLER_ARGS) 5983{ 5984 struct adapter *sc = arg1; 5985 struct devlog_params *dparams = &sc->params.devlog; 5986 struct fw_devlog_e *buf, *e; 5987 int i, j, rc, nentries, first = 0; 5988 struct sbuf *sb; 5989 uint64_t ftstamp = UINT64_MAX; 5990 5991 if (dparams->addr == 0) 5992 return (ENXIO); 5993 5994 buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5995 if (buf == NULL) 5996 return (ENOMEM); 5997 5998 rc = read_via_memwin(sc, 1, dparams->addr, (void *)buf, dparams->size); 5999 if (rc != 0) 6000 goto done; 6001 6002 nentries = dparams->size / sizeof(struct fw_devlog_e); 6003 for (i = 0; i < nentries; i++) { 6004 e = &buf[i]; 6005 6006 if (e->timestamp == 0) 6007 break; /* end */ 6008 6009 e->timestamp = be64toh(e->timestamp); 6010 e->seqno = be32toh(e->seqno); 6011 for (j = 0; j < 8; j++) 6012 e->params[j] = be32toh(e->params[j]); 6013 6014 if (e->timestamp < ftstamp) { 6015 ftstamp = e->timestamp; 6016 first = i; 6017 } 6018 } 6019 6020 if (buf[first].timestamp == 0) 6021 goto done; /* nothing in the log */ 6022 6023 rc = sysctl_wire_old_buffer(req, 0); 6024 if (rc != 0) 6025 goto done; 6026 6027 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6028 if (sb == NULL) { 6029 rc = ENOMEM; 6030 goto done; 6031 } 6032 sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 6033 "Seq#", "Tstamp", "Level", "Facility", "Message"); 6034 6035 i = first; 6036 do { 6037 e = &buf[i]; 6038 if (e->timestamp == 0) 6039 break; /* end */ 6040 6041 sbuf_printf(sb, "%10d %15ju %8s %8s ", 6042 e->seqno, e->timestamp, 6043 (e->level < nitems(devlog_level_strings) ? 6044 devlog_level_strings[e->level] : "UNKNOWN"), 6045 (e->facility < nitems(devlog_facility_strings) ? 6046 devlog_facility_strings[e->facility] : "UNKNOWN")); 6047 sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 6048 e->params[2], e->params[3], e->params[4], 6049 e->params[5], e->params[6], e->params[7]); 6050 6051 if (++i == nentries) 6052 i = 0; 6053 } while (i != first); 6054 6055 rc = sbuf_finish(sb); 6056 sbuf_delete(sb); 6057done: 6058 free(buf, M_CXGBE); 6059 return (rc); 6060} 6061 6062static int 6063sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 6064{ 6065 struct adapter *sc = arg1; 6066 struct sbuf *sb; 6067 int rc; 6068 struct tp_fcoe_stats stats[MAX_NCHAN]; 6069 int i, nchan = sc->chip_params->nchan; 6070 6071 rc = sysctl_wire_old_buffer(req, 0); 6072 if (rc != 0) 6073 return (rc); 6074 6075 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6076 if (sb == NULL) 6077 return (ENOMEM); 6078 6079 for (i = 0; i < nchan; i++) 6080 t4_get_fcoe_stats(sc, i, &stats[i]); 6081 6082 if (nchan > 2) { 6083 sbuf_printf(sb, " channel 0 channel 1" 6084 " channel 2 channel 3"); 6085 sbuf_printf(sb, "\noctetsDDP: %16ju %16ju %16ju %16ju", 6086 stats[0].octets_ddp, stats[1].octets_ddp, 6087 stats[2].octets_ddp, stats[3].octets_ddp); 6088 sbuf_printf(sb, "\nframesDDP: %16u %16u %16u %16u", 6089 stats[0].frames_ddp, stats[1].frames_ddp, 6090 stats[2].frames_ddp, stats[3].frames_ddp); 6091 sbuf_printf(sb, "\nframesDrop: %16u %16u %16u %16u", 6092 stats[0].frames_drop, stats[1].frames_drop, 6093 stats[2].frames_drop, stats[3].frames_drop); 6094 } else { 6095 sbuf_printf(sb, " channel 0 channel 1"); 6096 sbuf_printf(sb, "\noctetsDDP: %16ju %16ju", 6097 stats[0].octets_ddp, stats[1].octets_ddp); 6098 sbuf_printf(sb, "\nframesDDP: %16u %16u", 6099 stats[0].frames_ddp, stats[1].frames_ddp); 6100 sbuf_printf(sb, "\nframesDrop: %16u %16u", 6101 stats[0].frames_drop, stats[1].frames_drop); 6102 } 6103 6104 rc = sbuf_finish(sb); 6105 sbuf_delete(sb); 6106 6107 return (rc); 6108} 6109 6110static int 6111sysctl_hw_sched(SYSCTL_HANDLER_ARGS) 6112{ 6113 struct adapter *sc = arg1; 6114 struct sbuf *sb; 6115 int rc, i; 6116 unsigned int map, kbps, ipg, mode; 6117 unsigned int pace_tab[NTX_SCHED]; 6118 6119 rc = sysctl_wire_old_buffer(req, 0); 6120 if (rc != 0) 6121 return (rc); 6122 6123 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6124 if (sb == NULL) 6125 return (ENOMEM); 6126 6127 map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 6128 mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 6129 t4_read_pace_tbl(sc, pace_tab); 6130 6131 sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 6132 "Class IPG (0.1 ns) Flow IPG (us)"); 6133 6134 for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 6135 t4_get_tx_sched(sc, i, &kbps, &ipg); 6136 sbuf_printf(sb, "\n %u %-5s %u ", i, 6137 (mode & (1 << i)) ? "flow" : "class", map & 3); 6138 if (kbps) 6139 sbuf_printf(sb, "%9u ", kbps); 6140 else 6141 sbuf_printf(sb, " disabled "); 6142 6143 if (ipg) 6144 sbuf_printf(sb, "%13u ", ipg); 6145 else 6146 sbuf_printf(sb, " disabled "); 6147 6148 if (pace_tab[i]) 6149 sbuf_printf(sb, "%10u", pace_tab[i]); 6150 else 6151 sbuf_printf(sb, " disabled"); 6152 } 6153 6154 rc = sbuf_finish(sb); 6155 sbuf_delete(sb); 6156 6157 return (rc); 6158} 6159 6160static int 6161sysctl_lb_stats(SYSCTL_HANDLER_ARGS) 6162{ 6163 struct adapter *sc = arg1; 6164 struct sbuf *sb; 6165 int rc, i, j; 6166 uint64_t *p0, *p1; 6167 struct lb_port_stats s[2]; 6168 static const char *stat_name[] = { 6169 "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 6170 "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 6171 "Frames128To255:", "Frames256To511:", "Frames512To1023:", 6172 "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 6173 "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 6174 "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 6175 "BG2FramesTrunc:", "BG3FramesTrunc:" 6176 }; 6177 6178 rc = sysctl_wire_old_buffer(req, 0); 6179 if (rc != 0) 6180 return (rc); 6181 6182 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6183 if (sb == NULL) 6184 return (ENOMEM); 6185 6186 memset(s, 0, sizeof(s)); 6187 6188 for (i = 0; i < sc->chip_params->nchan; i += 2) { 6189 t4_get_lb_stats(sc, i, &s[0]); 6190 t4_get_lb_stats(sc, i + 1, &s[1]); 6191 6192 p0 = &s[0].octets; 6193 p1 = &s[1].octets; 6194 sbuf_printf(sb, "%s Loopback %u" 6195 " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 6196 6197 for (j = 0; j < nitems(stat_name); j++) 6198 sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 6199 *p0++, *p1++); 6200 } 6201 6202 rc = sbuf_finish(sb); 6203 sbuf_delete(sb); 6204 6205 return (rc); 6206} 6207 6208static int 6209sysctl_linkdnrc(SYSCTL_HANDLER_ARGS) 6210{ 6211 int rc = 0; 6212 struct port_info *pi = arg1; 6213 struct sbuf *sb; 6214 6215 rc = sysctl_wire_old_buffer(req, 0); 6216 if (rc != 0) 6217 return(rc); 6218 sb = sbuf_new_for_sysctl(NULL, NULL, 64, req); 6219 if (sb == NULL) 6220 return (ENOMEM); 6221 6222 if (pi->linkdnrc < 0) 6223 sbuf_printf(sb, "n/a"); 6224 else 6225 sbuf_printf(sb, "%s", t4_link_down_rc_str(pi->linkdnrc)); 6226 6227 rc = sbuf_finish(sb); 6228 sbuf_delete(sb); 6229 6230 return (rc); 6231} 6232 6233struct mem_desc { 6234 unsigned int base; 6235 unsigned int limit; 6236 unsigned int idx; 6237}; 6238 6239static int 6240mem_desc_cmp(const void *a, const void *b) 6241{ 6242 return ((const struct mem_desc *)a)->base - 6243 ((const struct mem_desc *)b)->base; 6244} 6245 6246static void 6247mem_region_show(struct sbuf *sb, const char *name, unsigned int from, 6248 unsigned int to) 6249{ 6250 unsigned int size; 6251 6252 if (from == to) 6253 return; 6254 6255 size = to - from + 1; 6256 if (size == 0) 6257 return; 6258 6259 /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 6260 sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 6261} 6262 6263static int 6264sysctl_meminfo(SYSCTL_HANDLER_ARGS) 6265{ 6266 struct adapter *sc = arg1; 6267 struct sbuf *sb; 6268 int rc, i, n; 6269 uint32_t lo, hi, used, alloc; 6270 static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 6271 static const char *region[] = { 6272 "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 6273 "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 6274 "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 6275 "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 6276 "RQUDP region:", "PBL region:", "TXPBL region:", 6277 "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 6278 "On-chip queues:" 6279 }; 6280 struct mem_desc avail[4]; 6281 struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 6282 struct mem_desc *md = mem; 6283 6284 rc = sysctl_wire_old_buffer(req, 0); 6285 if (rc != 0) 6286 return (rc); 6287 6288 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6289 if (sb == NULL) 6290 return (ENOMEM); 6291 6292 for (i = 0; i < nitems(mem); i++) { 6293 mem[i].limit = 0; 6294 mem[i].idx = i; 6295 } 6296 6297 /* Find and sort the populated memory ranges */ 6298 i = 0; 6299 lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 6300 if (lo & F_EDRAM0_ENABLE) { 6301 hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 6302 avail[i].base = G_EDRAM0_BASE(hi) << 20; 6303 avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 6304 avail[i].idx = 0; 6305 i++; 6306 } 6307 if (lo & F_EDRAM1_ENABLE) { 6308 hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 6309 avail[i].base = G_EDRAM1_BASE(hi) << 20; 6310 avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 6311 avail[i].idx = 1; 6312 i++; 6313 } 6314 if (lo & F_EXT_MEM_ENABLE) { 6315 hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 6316 avail[i].base = G_EXT_MEM_BASE(hi) << 20; 6317 avail[i].limit = avail[i].base + 6318 (G_EXT_MEM_SIZE(hi) << 20); 6319 avail[i].idx = is_t5(sc) ? 3 : 2; /* Call it MC0 for T5 */ 6320 i++; 6321 } 6322 if (is_t5(sc) && lo & F_EXT_MEM1_ENABLE) { 6323 hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 6324 avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 6325 avail[i].limit = avail[i].base + 6326 (G_EXT_MEM1_SIZE(hi) << 20); 6327 avail[i].idx = 4; 6328 i++; 6329 } 6330 if (!i) /* no memory available */ 6331 return 0; 6332 qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 6333 6334 (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 6335 (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 6336 (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 6337 (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6338 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 6339 (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 6340 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 6341 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 6342 (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 6343 6344 /* the next few have explicit upper bounds */ 6345 md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 6346 md->limit = md->base - 1 + 6347 t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 6348 G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 6349 md++; 6350 6351 md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 6352 md->limit = md->base - 1 + 6353 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 6354 G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 6355 md++; 6356 6357 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6358 if (chip_id(sc) <= CHELSIO_T5) 6359 md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 6360 else 6361 md->base = t4_read_reg(sc, A_LE_DB_HASH_TBL_BASE_ADDR); 6362 md->limit = 0; 6363 } else { 6364 md->base = 0; 6365 md->idx = nitems(region); /* hide it */ 6366 } 6367 md++; 6368 6369#define ulp_region(reg) \ 6370 md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 6371 (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 6372 6373 ulp_region(RX_ISCSI); 6374 ulp_region(RX_TDDP); 6375 ulp_region(TX_TPT); 6376 ulp_region(RX_STAG); 6377 ulp_region(RX_RQ); 6378 ulp_region(RX_RQUDP); 6379 ulp_region(RX_PBL); 6380 ulp_region(TX_PBL); 6381#undef ulp_region 6382 6383 md->base = 0; 6384 md->idx = nitems(region); 6385 if (!is_t4(sc)) { 6386 uint32_t size = 0; 6387 uint32_t sge_ctrl = t4_read_reg(sc, A_SGE_CONTROL2); 6388 uint32_t fifo_size = t4_read_reg(sc, A_SGE_DBVFIFO_SIZE); 6389 6390 if (is_t5(sc)) { 6391 if (sge_ctrl & F_VFIFO_ENABLE) 6392 size = G_DBVFIFO_SIZE(fifo_size); 6393 } else 6394 size = G_T6_DBVFIFO_SIZE(fifo_size); 6395 6396 if (size) { 6397 md->base = G_BASEADDR(t4_read_reg(sc, 6398 A_SGE_DBVFIFO_BADDR)); 6399 md->limit = md->base + (size << 2) - 1; 6400 } 6401 } 6402 md++; 6403 6404 md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 6405 md->limit = 0; 6406 md++; 6407 md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 6408 md->limit = 0; 6409 md++; 6410 6411 md->base = sc->vres.ocq.start; 6412 if (sc->vres.ocq.size) 6413 md->limit = md->base + sc->vres.ocq.size - 1; 6414 else 6415 md->idx = nitems(region); /* hide it */ 6416 md++; 6417 6418 /* add any address-space holes, there can be up to 3 */ 6419 for (n = 0; n < i - 1; n++) 6420 if (avail[n].limit < avail[n + 1].base) 6421 (md++)->base = avail[n].limit; 6422 if (avail[n].limit) 6423 (md++)->base = avail[n].limit; 6424 6425 n = md - mem; 6426 qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 6427 6428 for (lo = 0; lo < i; lo++) 6429 mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 6430 avail[lo].limit - 1); 6431 6432 sbuf_printf(sb, "\n"); 6433 for (i = 0; i < n; i++) { 6434 if (mem[i].idx >= nitems(region)) 6435 continue; /* skip holes */ 6436 if (!mem[i].limit) 6437 mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 6438 mem_region_show(sb, region[mem[i].idx], mem[i].base, 6439 mem[i].limit); 6440 } 6441 6442 sbuf_printf(sb, "\n"); 6443 lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 6444 hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 6445 mem_region_show(sb, "uP RAM:", lo, hi); 6446 6447 lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 6448 hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 6449 mem_region_show(sb, "uP Extmem2:", lo, hi); 6450 6451 lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 6452 sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 6453 G_PMRXMAXPAGE(lo), 6454 t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 6455 (lo & F_PMRXNUMCHN) ? 2 : 1); 6456 6457 lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 6458 hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 6459 sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 6460 G_PMTXMAXPAGE(lo), 6461 hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 6462 hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 6463 sbuf_printf(sb, "%u p-structs\n", 6464 t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 6465 6466 for (i = 0; i < 4; i++) { 6467 if (chip_id(sc) > CHELSIO_T5) 6468 lo = t4_read_reg(sc, A_MPS_RX_MAC_BG_PG_CNT0 + i * 4); 6469 else 6470 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 6471 if (is_t5(sc)) { 6472 used = G_T5_USED(lo); 6473 alloc = G_T5_ALLOC(lo); 6474 } else { 6475 used = G_USED(lo); 6476 alloc = G_ALLOC(lo); 6477 } 6478 /* For T6 these are MAC buffer groups */ 6479 sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 6480 i, used, alloc); 6481 } 6482 for (i = 0; i < sc->chip_params->nchan; i++) { 6483 if (chip_id(sc) > CHELSIO_T5) 6484 lo = t4_read_reg(sc, A_MPS_RX_LPBK_BG_PG_CNT0 + i * 4); 6485 else 6486 lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 6487 if (is_t5(sc)) { 6488 used = G_T5_USED(lo); 6489 alloc = G_T5_ALLOC(lo); 6490 } else { 6491 used = G_USED(lo); 6492 alloc = G_ALLOC(lo); 6493 } 6494 /* For T6 these are MAC buffer groups */ 6495 sbuf_printf(sb, 6496 "\nLoopback %d using %u pages out of %u allocated", 6497 i, used, alloc); 6498 } 6499 6500 rc = sbuf_finish(sb); 6501 sbuf_delete(sb); 6502 6503 return (rc); 6504} 6505 6506static inline void 6507tcamxy2valmask(uint64_t x, uint64_t y, uint8_t *addr, uint64_t *mask) 6508{ 6509 *mask = x | y; 6510 y = htobe64(y); 6511 memcpy(addr, (char *)&y + 2, ETHER_ADDR_LEN); 6512} 6513 6514static int 6515sysctl_mps_tcam(SYSCTL_HANDLER_ARGS) 6516{ 6517 struct adapter *sc = arg1; 6518 struct sbuf *sb; 6519 int rc, i; 6520 6521 MPASS(chip_id(sc) <= CHELSIO_T5); 6522 6523 rc = sysctl_wire_old_buffer(req, 0); 6524 if (rc != 0) 6525 return (rc); 6526 6527 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6528 if (sb == NULL) 6529 return (ENOMEM); 6530 6531 sbuf_printf(sb, 6532 "Idx Ethernet address Mask Vld Ports PF" 6533 " VF Replication P0 P1 P2 P3 ML"); 6534 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { 6535 uint64_t tcamx, tcamy, mask; 6536 uint32_t cls_lo, cls_hi; 6537 uint8_t addr[ETHER_ADDR_LEN]; 6538 6539 tcamy = t4_read_reg64(sc, MPS_CLS_TCAM_Y_L(i)); 6540 tcamx = t4_read_reg64(sc, MPS_CLS_TCAM_X_L(i)); 6541 if (tcamx & tcamy) 6542 continue; 6543 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6544 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6545 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6546 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x %012jx" 6547 " %c %#x%4u%4d", i, addr[0], addr[1], addr[2], 6548 addr[3], addr[4], addr[5], (uintmax_t)mask, 6549 (cls_lo & F_SRAM_VLD) ? 'Y' : 'N', 6550 G_PORTMAP(cls_hi), G_PF(cls_lo), 6551 (cls_lo & F_VF_VALID) ? G_VF(cls_lo) : -1); 6552 6553 if (cls_lo & F_REPLICATE) { 6554 struct fw_ldst_cmd ldst_cmd; 6555 6556 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6557 ldst_cmd.op_to_addrspace = 6558 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6559 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6560 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6561 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6562 ldst_cmd.u.mps.rplc.fid_idx = 6563 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6564 V_FW_LDST_CMD_IDX(i)); 6565 6566 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6567 "t4mps"); 6568 if (rc) 6569 break; 6570 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6571 sizeof(ldst_cmd), &ldst_cmd); 6572 end_synchronized_op(sc, 0); 6573 6574 if (rc != 0) { 6575 sbuf_printf(sb, "%36d", rc); 6576 rc = 0; 6577 } else { 6578 sbuf_printf(sb, " %08x %08x %08x %08x", 6579 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6580 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6581 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6582 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6583 } 6584 } else 6585 sbuf_printf(sb, "%36s", ""); 6586 6587 sbuf_printf(sb, "%4u%3u%3u%3u %#3x", G_SRAM_PRIO0(cls_lo), 6588 G_SRAM_PRIO1(cls_lo), G_SRAM_PRIO2(cls_lo), 6589 G_SRAM_PRIO3(cls_lo), (cls_lo >> S_MULTILISTEN0) & 0xf); 6590 } 6591 6592 if (rc) 6593 (void) sbuf_finish(sb); 6594 else 6595 rc = sbuf_finish(sb); 6596 sbuf_delete(sb); 6597 6598 return (rc); 6599} 6600 6601static int 6602sysctl_mps_tcam_t6(SYSCTL_HANDLER_ARGS) 6603{ 6604 struct adapter *sc = arg1; 6605 struct sbuf *sb; 6606 int rc, i; 6607 6608 MPASS(chip_id(sc) > CHELSIO_T5); 6609 6610 rc = sysctl_wire_old_buffer(req, 0); 6611 if (rc != 0) 6612 return (rc); 6613 6614 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 6615 if (sb == NULL) 6616 return (ENOMEM); 6617 6618 sbuf_printf(sb, "Idx Ethernet address Mask VNI Mask" 6619 " IVLAN Vld DIP_Hit Lookup Port Vld Ports PF VF" 6620 " Replication" 6621 " P0 P1 P2 P3 ML\n"); 6622 6623 for (i = 0; i < sc->chip_params->mps_tcam_size; i++) { 6624 uint8_t dip_hit, vlan_vld, lookup_type, port_num; 6625 uint16_t ivlan; 6626 uint64_t tcamx, tcamy, val, mask; 6627 uint32_t cls_lo, cls_hi, ctl, data2, vnix, vniy; 6628 uint8_t addr[ETHER_ADDR_LEN]; 6629 6630 ctl = V_CTLREQID(1) | V_CTLCMDTYPE(0) | V_CTLXYBITSEL(0); 6631 if (i < 256) 6632 ctl |= V_CTLTCAMINDEX(i) | V_CTLTCAMSEL(0); 6633 else 6634 ctl |= V_CTLTCAMINDEX(i - 256) | V_CTLTCAMSEL(1); 6635 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); 6636 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); 6637 tcamy = G_DMACH(val) << 32; 6638 tcamy |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); 6639 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); 6640 lookup_type = G_DATALKPTYPE(data2); 6641 port_num = G_DATAPORTNUM(data2); 6642 if (lookup_type && lookup_type != M_DATALKPTYPE) { 6643 /* Inner header VNI */ 6644 vniy = ((data2 & F_DATAVIDH2) << 23) | 6645 (G_DATAVIDH1(data2) << 16) | G_VIDL(val); 6646 dip_hit = data2 & F_DATADIPHIT; 6647 vlan_vld = 0; 6648 } else { 6649 vniy = 0; 6650 dip_hit = 0; 6651 vlan_vld = data2 & F_DATAVIDH2; 6652 ivlan = G_VIDL(val); 6653 } 6654 6655 ctl |= V_CTLXYBITSEL(1); 6656 t4_write_reg(sc, A_MPS_CLS_TCAM_DATA2_CTL, ctl); 6657 val = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA1_REQ_ID1); 6658 tcamx = G_DMACH(val) << 32; 6659 tcamx |= t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA0_REQ_ID1); 6660 data2 = t4_read_reg(sc, A_MPS_CLS_TCAM_RDATA2_REQ_ID1); 6661 if (lookup_type && lookup_type != M_DATALKPTYPE) { 6662 /* Inner header VNI mask */ 6663 vnix = ((data2 & F_DATAVIDH2) << 23) | 6664 (G_DATAVIDH1(data2) << 16) | G_VIDL(val); 6665 } else 6666 vnix = 0; 6667 6668 if (tcamx & tcamy) 6669 continue; 6670 tcamxy2valmask(tcamx, tcamy, addr, &mask); 6671 6672 cls_lo = t4_read_reg(sc, MPS_CLS_SRAM_L(i)); 6673 cls_hi = t4_read_reg(sc, MPS_CLS_SRAM_H(i)); 6674 6675 if (lookup_type && lookup_type != M_DATALKPTYPE) { 6676 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " 6677 "%012jx %06x %06x - - %3c" 6678 " 'I' %4x %3c %#x%4u%4d", i, addr[0], 6679 addr[1], addr[2], addr[3], addr[4], addr[5], 6680 (uintmax_t)mask, vniy, vnix, dip_hit ? 'Y' : 'N', 6681 port_num, cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', 6682 G_PORTMAP(cls_hi), G_T6_PF(cls_lo), 6683 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); 6684 } else { 6685 sbuf_printf(sb, "\n%3u %02x:%02x:%02x:%02x:%02x:%02x " 6686 "%012jx - - ", i, addr[0], addr[1], 6687 addr[2], addr[3], addr[4], addr[5], 6688 (uintmax_t)mask); 6689 6690 if (vlan_vld) 6691 sbuf_printf(sb, "%4u Y ", ivlan); 6692 else 6693 sbuf_printf(sb, " - N "); 6694 6695 sbuf_printf(sb, "- %3c %4x %3c %#x%4u%4d", 6696 lookup_type ? 'I' : 'O', port_num, 6697 cls_lo & F_T6_SRAM_VLD ? 'Y' : 'N', 6698 G_PORTMAP(cls_hi), G_T6_PF(cls_lo), 6699 cls_lo & F_T6_VF_VALID ? G_T6_VF(cls_lo) : -1); 6700 } 6701 6702 6703 if (cls_lo & F_T6_REPLICATE) { 6704 struct fw_ldst_cmd ldst_cmd; 6705 6706 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 6707 ldst_cmd.op_to_addrspace = 6708 htobe32(V_FW_CMD_OP(FW_LDST_CMD) | 6709 F_FW_CMD_REQUEST | F_FW_CMD_READ | 6710 V_FW_LDST_CMD_ADDRSPACE(FW_LDST_ADDRSPC_MPS)); 6711 ldst_cmd.cycles_to_len16 = htobe32(FW_LEN16(ldst_cmd)); 6712 ldst_cmd.u.mps.rplc.fid_idx = 6713 htobe16(V_FW_LDST_CMD_FID(FW_LDST_MPS_RPLC) | 6714 V_FW_LDST_CMD_IDX(i)); 6715 6716 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, 6717 "t6mps"); 6718 if (rc) 6719 break; 6720 rc = -t4_wr_mbox(sc, sc->mbox, &ldst_cmd, 6721 sizeof(ldst_cmd), &ldst_cmd); 6722 end_synchronized_op(sc, 0); 6723 6724 if (rc != 0) { 6725 sbuf_printf(sb, "%72d", rc); 6726 rc = 0; 6727 } else { 6728 sbuf_printf(sb, " %08x %08x %08x %08x" 6729 " %08x %08x %08x %08x", 6730 be32toh(ldst_cmd.u.mps.rplc.rplc255_224), 6731 be32toh(ldst_cmd.u.mps.rplc.rplc223_192), 6732 be32toh(ldst_cmd.u.mps.rplc.rplc191_160), 6733 be32toh(ldst_cmd.u.mps.rplc.rplc159_128), 6734 be32toh(ldst_cmd.u.mps.rplc.rplc127_96), 6735 be32toh(ldst_cmd.u.mps.rplc.rplc95_64), 6736 be32toh(ldst_cmd.u.mps.rplc.rplc63_32), 6737 be32toh(ldst_cmd.u.mps.rplc.rplc31_0)); 6738 } 6739 } else 6740 sbuf_printf(sb, "%72s", ""); 6741 6742 sbuf_printf(sb, "%4u%3u%3u%3u %#x", 6743 G_T6_SRAM_PRIO0(cls_lo), G_T6_SRAM_PRIO1(cls_lo), 6744 G_T6_SRAM_PRIO2(cls_lo), G_T6_SRAM_PRIO3(cls_lo), 6745 (cls_lo >> S_T6_MULTILISTEN0) & 0xf); 6746 } 6747 6748 if (rc) 6749 (void) sbuf_finish(sb); 6750 else 6751 rc = sbuf_finish(sb); 6752 sbuf_delete(sb); 6753 6754 return (rc); 6755} 6756 6757static int 6758sysctl_path_mtus(SYSCTL_HANDLER_ARGS) 6759{ 6760 struct adapter *sc = arg1; 6761 struct sbuf *sb; 6762 int rc; 6763 uint16_t mtus[NMTUS]; 6764 6765 rc = sysctl_wire_old_buffer(req, 0); 6766 if (rc != 0) 6767 return (rc); 6768 6769 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6770 if (sb == NULL) 6771 return (ENOMEM); 6772 6773 t4_read_mtu_tbl(sc, mtus, NULL); 6774 6775 sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 6776 mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 6777 mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 6778 mtus[14], mtus[15]); 6779 6780 rc = sbuf_finish(sb); 6781 sbuf_delete(sb); 6782 6783 return (rc); 6784} 6785 6786static int 6787sysctl_pm_stats(SYSCTL_HANDLER_ARGS) 6788{ 6789 struct adapter *sc = arg1; 6790 struct sbuf *sb; 6791 int rc, i; 6792 uint32_t tx_cnt[MAX_PM_NSTATS], rx_cnt[MAX_PM_NSTATS]; 6793 uint64_t tx_cyc[MAX_PM_NSTATS], rx_cyc[MAX_PM_NSTATS]; 6794 static const char *tx_stats[MAX_PM_NSTATS] = { 6795 "Read:", "Write bypass:", "Write mem:", "Bypass + mem:", 6796 "Tx FIFO wait", NULL, "Tx latency" 6797 }; 6798 static const char *rx_stats[MAX_PM_NSTATS] = { 6799 "Read:", "Write bypass:", "Write mem:", "Flush:", 6800 " Rx FIFO wait", NULL, "Rx latency" 6801 }; 6802 6803 rc = sysctl_wire_old_buffer(req, 0); 6804 if (rc != 0) 6805 return (rc); 6806 6807 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6808 if (sb == NULL) 6809 return (ENOMEM); 6810 6811 t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 6812 t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 6813 6814 sbuf_printf(sb, " Tx pcmds Tx bytes"); 6815 for (i = 0; i < 4; i++) { 6816 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 6817 tx_cyc[i]); 6818 } 6819 6820 sbuf_printf(sb, "\n Rx pcmds Rx bytes"); 6821 for (i = 0; i < 4; i++) { 6822 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 6823 rx_cyc[i]); 6824 } 6825 6826 if (chip_id(sc) > CHELSIO_T5) { 6827 sbuf_printf(sb, 6828 "\n Total wait Total occupancy"); 6829 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 6830 tx_cyc[i]); 6831 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 6832 rx_cyc[i]); 6833 6834 i += 2; 6835 MPASS(i < nitems(tx_stats)); 6836 6837 sbuf_printf(sb, 6838 "\n Reads Total wait"); 6839 sbuf_printf(sb, "\n%-13s %10u %20ju", tx_stats[i], tx_cnt[i], 6840 tx_cyc[i]); 6841 sbuf_printf(sb, "\n%-13s %10u %20ju", rx_stats[i], rx_cnt[i], 6842 rx_cyc[i]); 6843 } 6844 6845 rc = sbuf_finish(sb); 6846 sbuf_delete(sb); 6847 6848 return (rc); 6849} 6850 6851static int 6852sysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 6853{ 6854 struct adapter *sc = arg1; 6855 struct sbuf *sb; 6856 int rc; 6857 struct tp_rdma_stats stats; 6858 6859 rc = sysctl_wire_old_buffer(req, 0); 6860 if (rc != 0) 6861 return (rc); 6862 6863 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6864 if (sb == NULL) 6865 return (ENOMEM); 6866 6867 mtx_lock(&sc->reg_lock); 6868 t4_tp_get_rdma_stats(sc, &stats); 6869 mtx_unlock(&sc->reg_lock); 6870 6871 sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 6872 sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 6873 6874 rc = sbuf_finish(sb); 6875 sbuf_delete(sb); 6876 6877 return (rc); 6878} 6879 6880static int 6881sysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 6882{ 6883 struct adapter *sc = arg1; 6884 struct sbuf *sb; 6885 int rc; 6886 struct tp_tcp_stats v4, v6; 6887 6888 rc = sysctl_wire_old_buffer(req, 0); 6889 if (rc != 0) 6890 return (rc); 6891 6892 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6893 if (sb == NULL) 6894 return (ENOMEM); 6895 6896 mtx_lock(&sc->reg_lock); 6897 t4_tp_get_tcp_stats(sc, &v4, &v6); 6898 mtx_unlock(&sc->reg_lock); 6899 6900 sbuf_printf(sb, 6901 " IP IPv6\n"); 6902 sbuf_printf(sb, "OutRsts: %20u %20u\n", 6903 v4.tcp_out_rsts, v6.tcp_out_rsts); 6904 sbuf_printf(sb, "InSegs: %20ju %20ju\n", 6905 v4.tcp_in_segs, v6.tcp_in_segs); 6906 sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 6907 v4.tcp_out_segs, v6.tcp_out_segs); 6908 sbuf_printf(sb, "RetransSegs: %20ju %20ju", 6909 v4.tcp_retrans_segs, v6.tcp_retrans_segs); 6910 6911 rc = sbuf_finish(sb); 6912 sbuf_delete(sb); 6913 6914 return (rc); 6915} 6916 6917static int 6918sysctl_tids(SYSCTL_HANDLER_ARGS) 6919{ 6920 struct adapter *sc = arg1; 6921 struct sbuf *sb; 6922 int rc; 6923 struct tid_info *t = &sc->tids; 6924 6925 rc = sysctl_wire_old_buffer(req, 0); 6926 if (rc != 0) 6927 return (rc); 6928 6929 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6930 if (sb == NULL) 6931 return (ENOMEM); 6932 6933 if (t->natids) { 6934 sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 6935 t->atids_in_use); 6936 } 6937 6938 if (t->ntids) { 6939 if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 6940 uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 6941 6942 if (b) { 6943 sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 6944 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6945 t->ntids - 1); 6946 } else { 6947 sbuf_printf(sb, "TID range: %u-%u", 6948 t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 6949 t->ntids - 1); 6950 } 6951 } else 6952 sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 6953 sbuf_printf(sb, ", in use: %u\n", 6954 atomic_load_acq_int(&t->tids_in_use)); 6955 } 6956 6957 if (t->nstids) { 6958 sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 6959 t->stid_base + t->nstids - 1, t->stids_in_use); 6960 } 6961 6962 if (t->nftids) { 6963 sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 6964 t->ftid_base + t->nftids - 1); 6965 } 6966 6967 if (t->netids) { 6968 sbuf_printf(sb, "ETID range: %u-%u\n", t->etid_base, 6969 t->etid_base + t->netids - 1); 6970 } 6971 6972 sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 6973 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 6974 t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 6975 6976 rc = sbuf_finish(sb); 6977 sbuf_delete(sb); 6978 6979 return (rc); 6980} 6981 6982static int 6983sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 6984{ 6985 struct adapter *sc = arg1; 6986 struct sbuf *sb; 6987 int rc; 6988 struct tp_err_stats stats; 6989 6990 rc = sysctl_wire_old_buffer(req, 0); 6991 if (rc != 0) 6992 return (rc); 6993 6994 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 6995 if (sb == NULL) 6996 return (ENOMEM); 6997 6998 mtx_lock(&sc->reg_lock); 6999 t4_tp_get_err_stats(sc, &stats); 7000 mtx_unlock(&sc->reg_lock); 7001 7002 if (sc->chip_params->nchan > 2) { 7003 sbuf_printf(sb, " channel 0 channel 1" 7004 " channel 2 channel 3\n"); 7005 sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 7006 stats.mac_in_errs[0], stats.mac_in_errs[1], 7007 stats.mac_in_errs[2], stats.mac_in_errs[3]); 7008 sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 7009 stats.hdr_in_errs[0], stats.hdr_in_errs[1], 7010 stats.hdr_in_errs[2], stats.hdr_in_errs[3]); 7011 sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 7012 stats.tcp_in_errs[0], stats.tcp_in_errs[1], 7013 stats.tcp_in_errs[2], stats.tcp_in_errs[3]); 7014 sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 7015 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1], 7016 stats.tcp6_in_errs[2], stats.tcp6_in_errs[3]); 7017 sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 7018 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1], 7019 stats.tnl_cong_drops[2], stats.tnl_cong_drops[3]); 7020 sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 7021 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1], 7022 stats.tnl_tx_drops[2], stats.tnl_tx_drops[3]); 7023 sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 7024 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1], 7025 stats.ofld_vlan_drops[2], stats.ofld_vlan_drops[3]); 7026 sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 7027 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1], 7028 stats.ofld_chan_drops[2], stats.ofld_chan_drops[3]); 7029 } else { 7030 sbuf_printf(sb, " channel 0 channel 1\n"); 7031 sbuf_printf(sb, "macInErrs: %10u %10u\n", 7032 stats.mac_in_errs[0], stats.mac_in_errs[1]); 7033 sbuf_printf(sb, "hdrInErrs: %10u %10u\n", 7034 stats.hdr_in_errs[0], stats.hdr_in_errs[1]); 7035 sbuf_printf(sb, "tcpInErrs: %10u %10u\n", 7036 stats.tcp_in_errs[0], stats.tcp_in_errs[1]); 7037 sbuf_printf(sb, "tcp6InErrs: %10u %10u\n", 7038 stats.tcp6_in_errs[0], stats.tcp6_in_errs[1]); 7039 sbuf_printf(sb, "tnlCongDrops: %10u %10u\n", 7040 stats.tnl_cong_drops[0], stats.tnl_cong_drops[1]); 7041 sbuf_printf(sb, "tnlTxDrops: %10u %10u\n", 7042 stats.tnl_tx_drops[0], stats.tnl_tx_drops[1]); 7043 sbuf_printf(sb, "ofldVlanDrops: %10u %10u\n", 7044 stats.ofld_vlan_drops[0], stats.ofld_vlan_drops[1]); 7045 sbuf_printf(sb, "ofldChanDrops: %10u %10u\n\n", 7046 stats.ofld_chan_drops[0], stats.ofld_chan_drops[1]); 7047 } 7048 7049 sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 7050 stats.ofld_no_neigh, stats.ofld_cong_defer); 7051 7052 rc = sbuf_finish(sb); 7053 sbuf_delete(sb); 7054 7055 return (rc); 7056} 7057 7058static int 7059sysctl_tp_la_mask(SYSCTL_HANDLER_ARGS) 7060{ 7061 struct adapter *sc = arg1; 7062 struct tp_params *tpp = &sc->params.tp; 7063 u_int mask; 7064 int rc; 7065 7066 mask = tpp->la_mask >> 16; 7067 rc = sysctl_handle_int(oidp, &mask, 0, req); 7068 if (rc != 0 || req->newptr == NULL) 7069 return (rc); 7070 if (mask > 0xffff) 7071 return (EINVAL); 7072 tpp->la_mask = mask << 16; 7073 t4_set_reg_field(sc, A_TP_DBG_LA_CONFIG, 0xffff0000U, tpp->la_mask); 7074 7075 return (0); 7076} 7077 7078struct field_desc { 7079 const char *name; 7080 u_int start; 7081 u_int width; 7082}; 7083 7084static void 7085field_desc_show(struct sbuf *sb, uint64_t v, const struct field_desc *f) 7086{ 7087 char buf[32]; 7088 int line_size = 0; 7089 7090 while (f->name) { 7091 uint64_t mask = (1ULL << f->width) - 1; 7092 int len = snprintf(buf, sizeof(buf), "%s: %ju", f->name, 7093 ((uintmax_t)v >> f->start) & mask); 7094 7095 if (line_size + len >= 79) { 7096 line_size = 8; 7097 sbuf_printf(sb, "\n "); 7098 } 7099 sbuf_printf(sb, "%s ", buf); 7100 line_size += len + 1; 7101 f++; 7102 } 7103 sbuf_printf(sb, "\n"); 7104} 7105 7106static const struct field_desc tp_la0[] = { 7107 { "RcfOpCodeOut", 60, 4 }, 7108 { "State", 56, 4 }, 7109 { "WcfState", 52, 4 }, 7110 { "RcfOpcSrcOut", 50, 2 }, 7111 { "CRxError", 49, 1 }, 7112 { "ERxError", 48, 1 }, 7113 { "SanityFailed", 47, 1 }, 7114 { "SpuriousMsg", 46, 1 }, 7115 { "FlushInputMsg", 45, 1 }, 7116 { "FlushInputCpl", 44, 1 }, 7117 { "RssUpBit", 43, 1 }, 7118 { "RssFilterHit", 42, 1 }, 7119 { "Tid", 32, 10 }, 7120 { "InitTcb", 31, 1 }, 7121 { "LineNumber", 24, 7 }, 7122 { "Emsg", 23, 1 }, 7123 { "EdataOut", 22, 1 }, 7124 { "Cmsg", 21, 1 }, 7125 { "CdataOut", 20, 1 }, 7126 { "EreadPdu", 19, 1 }, 7127 { "CreadPdu", 18, 1 }, 7128 { "TunnelPkt", 17, 1 }, 7129 { "RcfPeerFin", 16, 1 }, 7130 { "RcfReasonOut", 12, 4 }, 7131 { "TxCchannel", 10, 2 }, 7132 { "RcfTxChannel", 8, 2 }, 7133 { "RxEchannel", 6, 2 }, 7134 { "RcfRxChannel", 5, 1 }, 7135 { "RcfDataOutSrdy", 4, 1 }, 7136 { "RxDvld", 3, 1 }, 7137 { "RxOoDvld", 2, 1 }, 7138 { "RxCongestion", 1, 1 }, 7139 { "TxCongestion", 0, 1 }, 7140 { NULL } 7141}; 7142 7143static const struct field_desc tp_la1[] = { 7144 { "CplCmdIn", 56, 8 }, 7145 { "CplCmdOut", 48, 8 }, 7146 { "ESynOut", 47, 1 }, 7147 { "EAckOut", 46, 1 }, 7148 { "EFinOut", 45, 1 }, 7149 { "ERstOut", 44, 1 }, 7150 { "SynIn", 43, 1 }, 7151 { "AckIn", 42, 1 }, 7152 { "FinIn", 41, 1 }, 7153 { "RstIn", 40, 1 }, 7154 { "DataIn", 39, 1 }, 7155 { "DataInVld", 38, 1 }, 7156 { "PadIn", 37, 1 }, 7157 { "RxBufEmpty", 36, 1 }, 7158 { "RxDdp", 35, 1 }, 7159 { "RxFbCongestion", 34, 1 }, 7160 { "TxFbCongestion", 33, 1 }, 7161 { "TxPktSumSrdy", 32, 1 }, 7162 { "RcfUlpType", 28, 4 }, 7163 { "Eread", 27, 1 }, 7164 { "Ebypass", 26, 1 }, 7165 { "Esave", 25, 1 }, 7166 { "Static0", 24, 1 }, 7167 { "Cread", 23, 1 }, 7168 { "Cbypass", 22, 1 }, 7169 { "Csave", 21, 1 }, 7170 { "CPktOut", 20, 1 }, 7171 { "RxPagePoolFull", 18, 2 }, 7172 { "RxLpbkPkt", 17, 1 }, 7173 { "TxLpbkPkt", 16, 1 }, 7174 { "RxVfValid", 15, 1 }, 7175 { "SynLearned", 14, 1 }, 7176 { "SetDelEntry", 13, 1 }, 7177 { "SetInvEntry", 12, 1 }, 7178 { "CpcmdDvld", 11, 1 }, 7179 { "CpcmdSave", 10, 1 }, 7180 { "RxPstructsFull", 8, 2 }, 7181 { "EpcmdDvld", 7, 1 }, 7182 { "EpcmdFlush", 6, 1 }, 7183 { "EpcmdTrimPrefix", 5, 1 }, 7184 { "EpcmdTrimPostfix", 4, 1 }, 7185 { "ERssIp4Pkt", 3, 1 }, 7186 { "ERssIp6Pkt", 2, 1 }, 7187 { "ERssTcpUdpPkt", 1, 1 }, 7188 { "ERssFceFipPkt", 0, 1 }, 7189 { NULL } 7190}; 7191 7192static const struct field_desc tp_la2[] = { 7193 { "CplCmdIn", 56, 8 }, 7194 { "MpsVfVld", 55, 1 }, 7195 { "MpsPf", 52, 3 }, 7196 { "MpsVf", 44, 8 }, 7197 { "SynIn", 43, 1 }, 7198 { "AckIn", 42, 1 }, 7199 { "FinIn", 41, 1 }, 7200 { "RstIn", 40, 1 }, 7201 { "DataIn", 39, 1 }, 7202 { "DataInVld", 38, 1 }, 7203 { "PadIn", 37, 1 }, 7204 { "RxBufEmpty", 36, 1 }, 7205 { "RxDdp", 35, 1 }, 7206 { "RxFbCongestion", 34, 1 }, 7207 { "TxFbCongestion", 33, 1 }, 7208 { "TxPktSumSrdy", 32, 1 }, 7209 { "RcfUlpType", 28, 4 }, 7210 { "Eread", 27, 1 }, 7211 { "Ebypass", 26, 1 }, 7212 { "Esave", 25, 1 }, 7213 { "Static0", 24, 1 }, 7214 { "Cread", 23, 1 }, 7215 { "Cbypass", 22, 1 }, 7216 { "Csave", 21, 1 }, 7217 { "CPktOut", 20, 1 }, 7218 { "RxPagePoolFull", 18, 2 }, 7219 { "RxLpbkPkt", 17, 1 }, 7220 { "TxLpbkPkt", 16, 1 }, 7221 { "RxVfValid", 15, 1 }, 7222 { "SynLearned", 14, 1 }, 7223 { "SetDelEntry", 13, 1 }, 7224 { "SetInvEntry", 12, 1 }, 7225 { "CpcmdDvld", 11, 1 }, 7226 { "CpcmdSave", 10, 1 }, 7227 { "RxPstructsFull", 8, 2 }, 7228 { "EpcmdDvld", 7, 1 }, 7229 { "EpcmdFlush", 6, 1 }, 7230 { "EpcmdTrimPrefix", 5, 1 }, 7231 { "EpcmdTrimPostfix", 4, 1 }, 7232 { "ERssIp4Pkt", 3, 1 }, 7233 { "ERssIp6Pkt", 2, 1 }, 7234 { "ERssTcpUdpPkt", 1, 1 }, 7235 { "ERssFceFipPkt", 0, 1 }, 7236 { NULL } 7237}; 7238 7239static void 7240tp_la_show(struct sbuf *sb, uint64_t *p, int idx) 7241{ 7242 7243 field_desc_show(sb, *p, tp_la0); 7244} 7245 7246static void 7247tp_la_show2(struct sbuf *sb, uint64_t *p, int idx) 7248{ 7249 7250 if (idx) 7251 sbuf_printf(sb, "\n"); 7252 field_desc_show(sb, p[0], tp_la0); 7253 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7254 field_desc_show(sb, p[1], tp_la0); 7255} 7256 7257static void 7258tp_la_show3(struct sbuf *sb, uint64_t *p, int idx) 7259{ 7260 7261 if (idx) 7262 sbuf_printf(sb, "\n"); 7263 field_desc_show(sb, p[0], tp_la0); 7264 if (idx < (TPLA_SIZE / 2 - 1) || p[1] != ~0ULL) 7265 field_desc_show(sb, p[1], (p[0] & (1 << 17)) ? tp_la2 : tp_la1); 7266} 7267 7268static int 7269sysctl_tp_la(SYSCTL_HANDLER_ARGS) 7270{ 7271 struct adapter *sc = arg1; 7272 struct sbuf *sb; 7273 uint64_t *buf, *p; 7274 int rc; 7275 u_int i, inc; 7276 void (*show_func)(struct sbuf *, uint64_t *, int); 7277 7278 rc = sysctl_wire_old_buffer(req, 0); 7279 if (rc != 0) 7280 return (rc); 7281 7282 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7283 if (sb == NULL) 7284 return (ENOMEM); 7285 7286 buf = malloc(TPLA_SIZE * sizeof(uint64_t), M_CXGBE, M_ZERO | M_WAITOK); 7287 7288 t4_tp_read_la(sc, buf, NULL); 7289 p = buf; 7290 7291 switch (G_DBGLAMODE(t4_read_reg(sc, A_TP_DBG_LA_CONFIG))) { 7292 case 2: 7293 inc = 2; 7294 show_func = tp_la_show2; 7295 break; 7296 case 3: 7297 inc = 2; 7298 show_func = tp_la_show3; 7299 break; 7300 default: 7301 inc = 1; 7302 show_func = tp_la_show; 7303 } 7304 7305 for (i = 0; i < TPLA_SIZE / inc; i++, p += inc) 7306 (*show_func)(sb, p, i); 7307 7308 rc = sbuf_finish(sb); 7309 sbuf_delete(sb); 7310 free(buf, M_CXGBE); 7311 return (rc); 7312} 7313 7314static int 7315sysctl_tx_rate(SYSCTL_HANDLER_ARGS) 7316{ 7317 struct adapter *sc = arg1; 7318 struct sbuf *sb; 7319 int rc; 7320 u64 nrate[MAX_NCHAN], orate[MAX_NCHAN]; 7321 7322 rc = sysctl_wire_old_buffer(req, 0); 7323 if (rc != 0) 7324 return (rc); 7325 7326 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 7327 if (sb == NULL) 7328 return (ENOMEM); 7329 7330 t4_get_chan_txrate(sc, nrate, orate); 7331 7332 if (sc->chip_params->nchan > 2) { 7333 sbuf_printf(sb, " channel 0 channel 1" 7334 " channel 2 channel 3\n"); 7335 sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 7336 nrate[0], nrate[1], nrate[2], nrate[3]); 7337 sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 7338 orate[0], orate[1], orate[2], orate[3]); 7339 } else { 7340 sbuf_printf(sb, " channel 0 channel 1\n"); 7341 sbuf_printf(sb, "NIC B/s: %10ju %10ju\n", 7342 nrate[0], nrate[1]); 7343 sbuf_printf(sb, "Offload B/s: %10ju %10ju", 7344 orate[0], orate[1]); 7345 } 7346 7347 rc = sbuf_finish(sb); 7348 sbuf_delete(sb); 7349 7350 return (rc); 7351} 7352 7353static int 7354sysctl_ulprx_la(SYSCTL_HANDLER_ARGS) 7355{ 7356 struct adapter *sc = arg1; 7357 struct sbuf *sb; 7358 uint32_t *buf, *p; 7359 int rc, i; 7360 7361 rc = sysctl_wire_old_buffer(req, 0); 7362 if (rc != 0) 7363 return (rc); 7364 7365 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7366 if (sb == NULL) 7367 return (ENOMEM); 7368 7369 buf = malloc(ULPRX_LA_SIZE * 8 * sizeof(uint32_t), M_CXGBE, 7370 M_ZERO | M_WAITOK); 7371 7372 t4_ulprx_read_la(sc, buf); 7373 p = buf; 7374 7375 sbuf_printf(sb, " Pcmd Type Message" 7376 " Data"); 7377 for (i = 0; i < ULPRX_LA_SIZE; i++, p += 8) { 7378 sbuf_printf(sb, "\n%08x%08x %4x %08x %08x%08x%08x%08x", 7379 p[1], p[0], p[2], p[3], p[7], p[6], p[5], p[4]); 7380 } 7381 7382 rc = sbuf_finish(sb); 7383 sbuf_delete(sb); 7384 free(buf, M_CXGBE); 7385 return (rc); 7386} 7387 7388static int 7389sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS) 7390{ 7391 struct adapter *sc = arg1; 7392 struct sbuf *sb; 7393 int rc, v; 7394 7395 rc = sysctl_wire_old_buffer(req, 0); 7396 if (rc != 0) 7397 return (rc); 7398 7399 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7400 if (sb == NULL) 7401 return (ENOMEM); 7402 7403 v = t4_read_reg(sc, A_SGE_STAT_CFG); 7404 if (G_STATSOURCE_T5(v) == 7) { 7405 if (G_STATMODE(v) == 0) { 7406 sbuf_printf(sb, "total %d, incomplete %d", 7407 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7408 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7409 } else if (G_STATMODE(v) == 1) { 7410 sbuf_printf(sb, "total %d, data overflow %d", 7411 t4_read_reg(sc, A_SGE_STAT_TOTAL), 7412 t4_read_reg(sc, A_SGE_STAT_MATCH)); 7413 } 7414 } 7415 rc = sbuf_finish(sb); 7416 sbuf_delete(sb); 7417 7418 return (rc); 7419} 7420 7421static int 7422sysctl_tc_params(SYSCTL_HANDLER_ARGS) 7423{ 7424 struct adapter *sc = arg1; 7425 struct tx_sched_class *tc; 7426 struct t4_sched_class_params p; 7427 struct sbuf *sb; 7428 int i, rc, port_id, flags, mbps, gbps; 7429 7430 rc = sysctl_wire_old_buffer(req, 0); 7431 if (rc != 0) 7432 return (rc); 7433 7434 sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 7435 if (sb == NULL) 7436 return (ENOMEM); 7437 7438 port_id = arg2 >> 16; 7439 MPASS(port_id < sc->params.nports); 7440 MPASS(sc->port[port_id] != NULL); 7441 i = arg2 & 0xffff; 7442 MPASS(i < sc->chip_params->nsched_cls); 7443 tc = &sc->port[port_id]->tc[i]; 7444 7445 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7446 "t4tc_p"); 7447 if (rc) 7448 goto done; 7449 flags = tc->flags; 7450 p = tc->params; 7451 end_synchronized_op(sc, LOCK_HELD); 7452 7453 if ((flags & TX_SC_OK) == 0) { 7454 sbuf_printf(sb, "none"); 7455 goto done; 7456 } 7457 7458 if (p.level == SCHED_CLASS_LEVEL_CL_WRR) { 7459 sbuf_printf(sb, "cl-wrr weight %u", p.weight); 7460 goto done; 7461 } else if (p.level == SCHED_CLASS_LEVEL_CL_RL) 7462 sbuf_printf(sb, "cl-rl"); 7463 else if (p.level == SCHED_CLASS_LEVEL_CH_RL) 7464 sbuf_printf(sb, "ch-rl"); 7465 else { 7466 rc = ENXIO; 7467 goto done; 7468 } 7469 7470 if (p.ratemode == SCHED_CLASS_RATEMODE_REL) { 7471 /* XXX: top speed or actual link speed? */ 7472 gbps = port_top_speed(sc->port[port_id]); 7473 sbuf_printf(sb, " %u%% of %uGbps", p.maxrate, gbps); 7474 } 7475 else if (p.ratemode == SCHED_CLASS_RATEMODE_ABS) { 7476 switch (p.rateunit) { 7477 case SCHED_CLASS_RATEUNIT_BITS: 7478 mbps = p.maxrate / 1000; 7479 gbps = p.maxrate / 1000000; 7480 if (p.maxrate == gbps * 1000000) 7481 sbuf_printf(sb, " %uGbps", gbps); 7482 else if (p.maxrate == mbps * 1000) 7483 sbuf_printf(sb, " %uMbps", mbps); 7484 else 7485 sbuf_printf(sb, " %uKbps", p.maxrate); 7486 break; 7487 case SCHED_CLASS_RATEUNIT_PKTS: 7488 sbuf_printf(sb, " %upps", p.maxrate); 7489 break; 7490 default: 7491 rc = ENXIO; 7492 goto done; 7493 } 7494 } 7495 7496 switch (p.mode) { 7497 case SCHED_CLASS_MODE_CLASS: 7498 sbuf_printf(sb, " aggregate"); 7499 break; 7500 case SCHED_CLASS_MODE_FLOW: 7501 sbuf_printf(sb, " per-flow"); 7502 break; 7503 default: 7504 rc = ENXIO; 7505 goto done; 7506 } 7507 7508done: 7509 if (rc == 0) 7510 rc = sbuf_finish(sb); 7511 sbuf_delete(sb); 7512 7513 return (rc); 7514} 7515#endif 7516 7517#ifdef TCP_OFFLOAD 7518static void 7519unit_conv(char *buf, size_t len, u_int val, u_int factor) 7520{ 7521 u_int rem = val % factor; 7522 7523 if (rem == 0) 7524 snprintf(buf, len, "%u", val / factor); 7525 else { 7526 while (rem % 10 == 0) 7527 rem /= 10; 7528 snprintf(buf, len, "%u.%u", val / factor, rem); 7529 } 7530} 7531 7532static int 7533sysctl_tp_tick(SYSCTL_HANDLER_ARGS) 7534{ 7535 struct adapter *sc = arg1; 7536 char buf[16]; 7537 u_int res, re; 7538 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; 7539 7540 res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 7541 switch (arg2) { 7542 case 0: 7543 /* timer_tick */ 7544 re = G_TIMERRESOLUTION(res); 7545 break; 7546 case 1: 7547 /* TCP timestamp tick */ 7548 re = G_TIMESTAMPRESOLUTION(res); 7549 break; 7550 case 2: 7551 /* DACK tick */ 7552 re = G_DELAYEDACKRESOLUTION(res); 7553 break; 7554 default: 7555 return (EDOOFUS); 7556 } 7557 7558 unit_conv(buf, sizeof(buf), (cclk_ps << re), 1000000); 7559 7560 return (sysctl_handle_string(oidp, buf, sizeof(buf), req)); 7561} 7562 7563static int 7564sysctl_tp_dack_timer(SYSCTL_HANDLER_ARGS) 7565{ 7566 struct adapter *sc = arg1; 7567 u_int res, dack_re, v; 7568 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; 7569 7570 res = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 7571 dack_re = G_DELAYEDACKRESOLUTION(res); 7572 v = ((cclk_ps << dack_re) / 1000000) * t4_read_reg(sc, A_TP_DACK_TIMER); 7573 7574 return (sysctl_handle_int(oidp, &v, 0, req)); 7575} 7576 7577static int 7578sysctl_tp_timer(SYSCTL_HANDLER_ARGS) 7579{ 7580 struct adapter *sc = arg1; 7581 int reg = arg2; 7582 u_int tre; 7583 u_long tp_tick_us, v; 7584 u_int cclk_ps = 1000000000 / sc->params.vpd.cclk; 7585 7586 MPASS(reg == A_TP_RXT_MIN || reg == A_TP_RXT_MAX || 7587 reg == A_TP_PERS_MIN || reg == A_TP_PERS_MAX || 7588 reg == A_TP_KEEP_IDLE || A_TP_KEEP_INTVL || reg == A_TP_INIT_SRTT || 7589 reg == A_TP_FINWAIT2_TIMER); 7590 7591 tre = G_TIMERRESOLUTION(t4_read_reg(sc, A_TP_TIMER_RESOLUTION)); 7592 tp_tick_us = (cclk_ps << tre) / 1000000; 7593 7594 if (reg == A_TP_INIT_SRTT) 7595 v = tp_tick_us * G_INITSRTT(t4_read_reg(sc, reg)); 7596 else 7597 v = tp_tick_us * t4_read_reg(sc, reg); 7598 7599 return (sysctl_handle_long(oidp, &v, 0, req)); 7600} 7601#endif 7602 7603static uint32_t 7604fconf_iconf_to_mode(uint32_t fconf, uint32_t iconf) 7605{ 7606 uint32_t mode; 7607 7608 mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 7609 T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 7610 7611 if (fconf & F_FRAGMENTATION) 7612 mode |= T4_FILTER_IP_FRAGMENT; 7613 7614 if (fconf & F_MPSHITTYPE) 7615 mode |= T4_FILTER_MPS_HIT_TYPE; 7616 7617 if (fconf & F_MACMATCH) 7618 mode |= T4_FILTER_MAC_IDX; 7619 7620 if (fconf & F_ETHERTYPE) 7621 mode |= T4_FILTER_ETH_TYPE; 7622 7623 if (fconf & F_PROTOCOL) 7624 mode |= T4_FILTER_IP_PROTO; 7625 7626 if (fconf & F_TOS) 7627 mode |= T4_FILTER_IP_TOS; 7628 7629 if (fconf & F_VLAN) 7630 mode |= T4_FILTER_VLAN; 7631 7632 if (fconf & F_VNIC_ID) { 7633 mode |= T4_FILTER_VNIC; 7634 if (iconf & F_VNIC) 7635 mode |= T4_FILTER_IC_VNIC; 7636 } 7637 7638 if (fconf & F_PORT) 7639 mode |= T4_FILTER_PORT; 7640 7641 if (fconf & F_FCOE) 7642 mode |= T4_FILTER_FCoE; 7643 7644 return (mode); 7645} 7646 7647static uint32_t 7648mode_to_fconf(uint32_t mode) 7649{ 7650 uint32_t fconf = 0; 7651 7652 if (mode & T4_FILTER_IP_FRAGMENT) 7653 fconf |= F_FRAGMENTATION; 7654 7655 if (mode & T4_FILTER_MPS_HIT_TYPE) 7656 fconf |= F_MPSHITTYPE; 7657 7658 if (mode & T4_FILTER_MAC_IDX) 7659 fconf |= F_MACMATCH; 7660 7661 if (mode & T4_FILTER_ETH_TYPE) 7662 fconf |= F_ETHERTYPE; 7663 7664 if (mode & T4_FILTER_IP_PROTO) 7665 fconf |= F_PROTOCOL; 7666 7667 if (mode & T4_FILTER_IP_TOS) 7668 fconf |= F_TOS; 7669 7670 if (mode & T4_FILTER_VLAN) 7671 fconf |= F_VLAN; 7672 7673 if (mode & T4_FILTER_VNIC) 7674 fconf |= F_VNIC_ID; 7675 7676 if (mode & T4_FILTER_PORT) 7677 fconf |= F_PORT; 7678 7679 if (mode & T4_FILTER_FCoE) 7680 fconf |= F_FCOE; 7681 7682 return (fconf); 7683} 7684 7685static uint32_t 7686mode_to_iconf(uint32_t mode) 7687{ 7688 7689 if (mode & T4_FILTER_IC_VNIC) 7690 return (F_VNIC); 7691 return (0); 7692} 7693 7694static int check_fspec_against_fconf_iconf(struct adapter *sc, 7695 struct t4_filter_specification *fs) 7696{ 7697 struct tp_params *tpp = &sc->params.tp; 7698 uint32_t fconf = 0; 7699 7700 if (fs->val.frag || fs->mask.frag) 7701 fconf |= F_FRAGMENTATION; 7702 7703 if (fs->val.matchtype || fs->mask.matchtype) 7704 fconf |= F_MPSHITTYPE; 7705 7706 if (fs->val.macidx || fs->mask.macidx) 7707 fconf |= F_MACMATCH; 7708 7709 if (fs->val.ethtype || fs->mask.ethtype) 7710 fconf |= F_ETHERTYPE; 7711 7712 if (fs->val.proto || fs->mask.proto) 7713 fconf |= F_PROTOCOL; 7714 7715 if (fs->val.tos || fs->mask.tos) 7716 fconf |= F_TOS; 7717 7718 if (fs->val.vlan_vld || fs->mask.vlan_vld) 7719 fconf |= F_VLAN; 7720 7721 if (fs->val.ovlan_vld || fs->mask.ovlan_vld) { 7722 fconf |= F_VNIC_ID; 7723 if (tpp->ingress_config & F_VNIC) 7724 return (EINVAL); 7725 } 7726 7727 if (fs->val.pfvf_vld || fs->mask.pfvf_vld) { 7728 fconf |= F_VNIC_ID; 7729 if ((tpp->ingress_config & F_VNIC) == 0) 7730 return (EINVAL); 7731 } 7732 7733 if (fs->val.iport || fs->mask.iport) 7734 fconf |= F_PORT; 7735 7736 if (fs->val.fcoe || fs->mask.fcoe) 7737 fconf |= F_FCOE; 7738 7739 if ((tpp->vlan_pri_map | fconf) != tpp->vlan_pri_map) 7740 return (E2BIG); 7741 7742 return (0); 7743} 7744 7745static int 7746get_filter_mode(struct adapter *sc, uint32_t *mode) 7747{ 7748 struct tp_params *tpp = &sc->params.tp; 7749 7750 /* 7751 * We trust the cached values of the relevant TP registers. This means 7752 * things work reliably only if writes to those registers are always via 7753 * t4_set_filter_mode. 7754 */ 7755 *mode = fconf_iconf_to_mode(tpp->vlan_pri_map, tpp->ingress_config); 7756 7757 return (0); 7758} 7759 7760static int 7761set_filter_mode(struct adapter *sc, uint32_t mode) 7762{ 7763 struct tp_params *tpp = &sc->params.tp; 7764 uint32_t fconf, iconf; 7765 int rc; 7766 7767 iconf = mode_to_iconf(mode); 7768 if ((iconf ^ tpp->ingress_config) & F_VNIC) { 7769 /* 7770 * For now we just complain if A_TP_INGRESS_CONFIG is not 7771 * already set to the correct value for the requested filter 7772 * mode. It's not clear if it's safe to write to this register 7773 * on the fly. (And we trust the cached value of the register). 7774 */ 7775 return (EBUSY); 7776 } 7777 7778 fconf = mode_to_fconf(mode); 7779 7780 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7781 "t4setfm"); 7782 if (rc) 7783 return (rc); 7784 7785 if (sc->tids.ftids_in_use > 0) { 7786 rc = EBUSY; 7787 goto done; 7788 } 7789 7790#ifdef TCP_OFFLOAD 7791 if (uld_active(sc, ULD_TOM)) { 7792 rc = EBUSY; 7793 goto done; 7794 } 7795#endif 7796 7797 rc = -t4_set_filter_mode(sc, fconf); 7798done: 7799 end_synchronized_op(sc, LOCK_HELD); 7800 return (rc); 7801} 7802 7803static inline uint64_t 7804get_filter_hits(struct adapter *sc, uint32_t fid) 7805{ 7806 uint32_t tcb_addr; 7807 7808 tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE) + 7809 (fid + sc->tids.ftid_base) * TCB_SIZE; 7810 7811 if (is_t4(sc)) { 7812 uint64_t hits; 7813 7814 read_via_memwin(sc, 0, tcb_addr + 16, (uint32_t *)&hits, 8); 7815 return (be64toh(hits)); 7816 } else { 7817 uint32_t hits; 7818 7819 read_via_memwin(sc, 0, tcb_addr + 24, &hits, 4); 7820 return (be32toh(hits)); 7821 } 7822} 7823 7824static int 7825get_filter(struct adapter *sc, struct t4_filter *t) 7826{ 7827 int i, rc, nfilters = sc->tids.nftids; 7828 struct filter_entry *f; 7829 7830 rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 7831 "t4getf"); 7832 if (rc) 7833 return (rc); 7834 7835 if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 7836 t->idx >= nfilters) { 7837 t->idx = 0xffffffff; 7838 goto done; 7839 } 7840 7841 f = &sc->tids.ftid_tab[t->idx]; 7842 for (i = t->idx; i < nfilters; i++, f++) { 7843 if (f->valid) { 7844 t->idx = i; 7845 t->l2tidx = f->l2t ? f->l2t->idx : 0; 7846 t->smtidx = f->smtidx; 7847 if (f->fs.hitcnts) 7848 t->hits = get_filter_hits(sc, t->idx); 7849 else 7850 t->hits = UINT64_MAX; 7851 t->fs = f->fs; 7852 7853 goto done; 7854 } 7855 } 7856 7857 t->idx = 0xffffffff; 7858done: 7859 end_synchronized_op(sc, LOCK_HELD); 7860 return (0); 7861} 7862 7863static int 7864set_filter(struct adapter *sc, struct t4_filter *t) 7865{ 7866 unsigned int nfilters, nports; 7867 struct filter_entry *f; 7868 int i, rc; 7869 7870 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 7871 if (rc) 7872 return (rc); 7873 7874 nfilters = sc->tids.nftids; 7875 nports = sc->params.nports; 7876 7877 if (nfilters == 0) { 7878 rc = ENOTSUP; 7879 goto done; 7880 } 7881 7882 if (!(sc->flags & FULL_INIT_DONE)) { 7883 rc = EAGAIN; 7884 goto done; 7885 } 7886 7887 if (t->idx >= nfilters) { 7888 rc = EINVAL; 7889 goto done; 7890 } 7891 7892 /* Validate against the global filter mode and ingress config */ 7893 rc = check_fspec_against_fconf_iconf(sc, &t->fs); 7894 if (rc != 0) 7895 goto done; 7896 7897 if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 7898 rc = EINVAL; 7899 goto done; 7900 } 7901 7902 if (t->fs.val.iport >= nports) { 7903 rc = EINVAL; 7904 goto done; 7905 } 7906 7907 /* Can't specify an iq if not steering to it */ 7908 if (!t->fs.dirsteer && t->fs.iq) { 7909 rc = EINVAL; 7910 goto done; 7911 } 7912 7913 /* IPv6 filter idx must be 4 aligned */ 7914 if (t->fs.type == 1 && 7915 ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 7916 rc = EINVAL; 7917 goto done; 7918 } 7919 7920 if (sc->tids.ftid_tab == NULL) { 7921 KASSERT(sc->tids.ftids_in_use == 0, 7922 ("%s: no memory allocated but filters_in_use > 0", 7923 __func__)); 7924 7925 sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 7926 nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 7927 if (sc->tids.ftid_tab == NULL) { 7928 rc = ENOMEM; 7929 goto done; 7930 } 7931 mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 7932 } 7933 7934 for (i = 0; i < 4; i++) { 7935 f = &sc->tids.ftid_tab[t->idx + i]; 7936 7937 if (f->pending || f->valid) { 7938 rc = EBUSY; 7939 goto done; 7940 } 7941 if (f->locked) { 7942 rc = EPERM; 7943 goto done; 7944 } 7945 7946 if (t->fs.type == 0) 7947 break; 7948 } 7949 7950 f = &sc->tids.ftid_tab[t->idx]; 7951 f->fs = t->fs; 7952 7953 rc = set_filter_wr(sc, t->idx); 7954done: 7955 end_synchronized_op(sc, 0); 7956 7957 if (rc == 0) { 7958 mtx_lock(&sc->tids.ftid_lock); 7959 for (;;) { 7960 if (f->pending == 0) { 7961 rc = f->valid ? 0 : EIO; 7962 break; 7963 } 7964 7965 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 7966 PCATCH, "t4setfw", 0)) { 7967 rc = EINPROGRESS; 7968 break; 7969 } 7970 } 7971 mtx_unlock(&sc->tids.ftid_lock); 7972 } 7973 return (rc); 7974} 7975 7976static int 7977del_filter(struct adapter *sc, struct t4_filter *t) 7978{ 7979 unsigned int nfilters; 7980 struct filter_entry *f; 7981 int rc; 7982 7983 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 7984 if (rc) 7985 return (rc); 7986 7987 nfilters = sc->tids.nftids; 7988 7989 if (nfilters == 0) { 7990 rc = ENOTSUP; 7991 goto done; 7992 } 7993 7994 if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 7995 t->idx >= nfilters) { 7996 rc = EINVAL; 7997 goto done; 7998 } 7999 8000 if (!(sc->flags & FULL_INIT_DONE)) { 8001 rc = EAGAIN; 8002 goto done; 8003 } 8004 8005 f = &sc->tids.ftid_tab[t->idx]; 8006 8007 if (f->pending) { 8008 rc = EBUSY; 8009 goto done; 8010 } 8011 if (f->locked) { 8012 rc = EPERM; 8013 goto done; 8014 } 8015 8016 if (f->valid) { 8017 t->fs = f->fs; /* extra info for the caller */ 8018 rc = del_filter_wr(sc, t->idx); 8019 } 8020 8021done: 8022 end_synchronized_op(sc, 0); 8023 8024 if (rc == 0) { 8025 mtx_lock(&sc->tids.ftid_lock); 8026 for (;;) { 8027 if (f->pending == 0) { 8028 rc = f->valid ? EIO : 0; 8029 break; 8030 } 8031 8032 if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 8033 PCATCH, "t4delfw", 0)) { 8034 rc = EINPROGRESS; 8035 break; 8036 } 8037 } 8038 mtx_unlock(&sc->tids.ftid_lock); 8039 } 8040 8041 return (rc); 8042} 8043 8044static void 8045clear_filter(struct filter_entry *f) 8046{ 8047 if (f->l2t) 8048 t4_l2t_release(f->l2t); 8049 8050 bzero(f, sizeof (*f)); 8051} 8052 8053static int 8054set_filter_wr(struct adapter *sc, int fidx) 8055{ 8056 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 8057 struct fw_filter_wr *fwr; 8058 unsigned int ftid, vnic_vld, vnic_vld_mask; 8059 struct wrq_cookie cookie; 8060 8061 ASSERT_SYNCHRONIZED_OP(sc); 8062 8063 if (f->fs.newdmac || f->fs.newvlan) { 8064 /* This filter needs an L2T entry; allocate one. */ 8065 f->l2t = t4_l2t_alloc_switching(sc->l2t); 8066 if (f->l2t == NULL) 8067 return (EAGAIN); 8068 if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 8069 f->fs.dmac)) { 8070 t4_l2t_release(f->l2t); 8071 f->l2t = NULL; 8072 return (ENOMEM); 8073 } 8074 } 8075 8076 /* Already validated against fconf, iconf */ 8077 MPASS((f->fs.val.pfvf_vld & f->fs.val.ovlan_vld) == 0); 8078 MPASS((f->fs.mask.pfvf_vld & f->fs.mask.ovlan_vld) == 0); 8079 if (f->fs.val.pfvf_vld || f->fs.val.ovlan_vld) 8080 vnic_vld = 1; 8081 else 8082 vnic_vld = 0; 8083 if (f->fs.mask.pfvf_vld || f->fs.mask.ovlan_vld) 8084 vnic_vld_mask = 1; 8085 else 8086 vnic_vld_mask = 0; 8087 8088 ftid = sc->tids.ftid_base + fidx; 8089 8090 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8091 if (fwr == NULL) 8092 return (ENOMEM); 8093 bzero(fwr, sizeof(*fwr)); 8094 8095 fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 8096 fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 8097 fwr->tid_to_iq = 8098 htobe32(V_FW_FILTER_WR_TID(ftid) | 8099 V_FW_FILTER_WR_RQTYPE(f->fs.type) | 8100 V_FW_FILTER_WR_NOREPLY(0) | 8101 V_FW_FILTER_WR_IQ(f->fs.iq)); 8102 fwr->del_filter_to_l2tix = 8103 htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 8104 V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 8105 V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 8106 V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 8107 V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 8108 V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 8109 V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 8110 V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 8111 V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 8112 f->fs.newvlan == VLAN_REWRITE) | 8113 V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 8114 f->fs.newvlan == VLAN_REWRITE) | 8115 V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 8116 V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 8117 V_FW_FILTER_WR_PRIO(f->fs.prio) | 8118 V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 8119 fwr->ethtype = htobe16(f->fs.val.ethtype); 8120 fwr->ethtypem = htobe16(f->fs.mask.ethtype); 8121 fwr->frag_to_ovlan_vldm = 8122 (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 8123 V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 8124 V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 8125 V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) | 8126 V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 8127 V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask)); 8128 fwr->smac_sel = 0; 8129 fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 8130 V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 8131 fwr->maci_to_matchtypem = 8132 htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 8133 V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 8134 V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 8135 V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 8136 V_FW_FILTER_WR_PORT(f->fs.val.iport) | 8137 V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 8138 V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 8139 V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 8140 fwr->ptcl = f->fs.val.proto; 8141 fwr->ptclm = f->fs.mask.proto; 8142 fwr->ttyp = f->fs.val.tos; 8143 fwr->ttypm = f->fs.mask.tos; 8144 fwr->ivlan = htobe16(f->fs.val.vlan); 8145 fwr->ivlanm = htobe16(f->fs.mask.vlan); 8146 fwr->ovlan = htobe16(f->fs.val.vnic); 8147 fwr->ovlanm = htobe16(f->fs.mask.vnic); 8148 bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 8149 bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 8150 bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 8151 bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 8152 fwr->lp = htobe16(f->fs.val.dport); 8153 fwr->lpm = htobe16(f->fs.mask.dport); 8154 fwr->fp = htobe16(f->fs.val.sport); 8155 fwr->fpm = htobe16(f->fs.mask.sport); 8156 if (f->fs.newsmac) 8157 bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 8158 8159 f->pending = 1; 8160 sc->tids.ftids_in_use++; 8161 8162 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8163 return (0); 8164} 8165 8166static int 8167del_filter_wr(struct adapter *sc, int fidx) 8168{ 8169 struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 8170 struct fw_filter_wr *fwr; 8171 unsigned int ftid; 8172 struct wrq_cookie cookie; 8173 8174 ftid = sc->tids.ftid_base + fidx; 8175 8176 fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie); 8177 if (fwr == NULL) 8178 return (ENOMEM); 8179 bzero(fwr, sizeof (*fwr)); 8180 8181 t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 8182 8183 f->pending = 1; 8184 commit_wrq_wr(&sc->sge.mgmtq, fwr, &cookie); 8185 return (0); 8186} 8187 8188int 8189t4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 8190{ 8191 struct adapter *sc = iq->adapter; 8192 const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 8193 unsigned int idx = GET_TID(rpl); 8194 unsigned int rc; 8195 struct filter_entry *f; 8196 8197 KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 8198 rss->opcode)); 8199 8200 if (is_ftid(sc, idx)) { 8201 8202 idx -= sc->tids.ftid_base; 8203 f = &sc->tids.ftid_tab[idx]; 8204 rc = G_COOKIE(rpl->cookie); 8205 8206 mtx_lock(&sc->tids.ftid_lock); 8207 if (rc == FW_FILTER_WR_FLT_ADDED) { 8208 KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 8209 __func__, idx)); 8210 f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 8211 f->pending = 0; /* asynchronous setup completed */ 8212 f->valid = 1; 8213 } else { 8214 if (rc != FW_FILTER_WR_FLT_DELETED) { 8215 /* Add or delete failed, display an error */ 8216 log(LOG_ERR, 8217 "filter %u setup failed with error %u\n", 8218 idx, rc); 8219 } 8220 8221 clear_filter(f); 8222 sc->tids.ftids_in_use--; 8223 } 8224 wakeup(&sc->tids.ftid_tab); 8225 mtx_unlock(&sc->tids.ftid_lock); 8226 } 8227 8228 return (0); 8229} 8230 8231static int 8232get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 8233{ 8234 int rc; 8235 8236 if (cntxt->cid > M_CTXTQID) 8237 return (EINVAL); 8238 8239 if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 8240 cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 8241 return (EINVAL); 8242 8243 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 8244 if (rc) 8245 return (rc); 8246 8247 if (sc->flags & FW_OK) { 8248 rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 8249 &cntxt->data[0]); 8250 if (rc == 0) 8251 goto done; 8252 } 8253 8254 /* 8255 * Read via firmware failed or wasn't even attempted. Read directly via 8256 * the backdoor. 8257 */ 8258 rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 8259done: 8260 end_synchronized_op(sc, 0); 8261 return (rc); 8262} 8263 8264static int 8265load_fw(struct adapter *sc, struct t4_data *fw) 8266{ 8267 int rc; 8268 uint8_t *fw_data; 8269 8270 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 8271 if (rc) 8272 return (rc); 8273 8274 if (sc->flags & FULL_INIT_DONE) { 8275 rc = EBUSY; 8276 goto done; 8277 } 8278 8279 fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 8280 if (fw_data == NULL) { 8281 rc = ENOMEM; 8282 goto done; 8283 } 8284 8285 rc = copyin(fw->data, fw_data, fw->len); 8286 if (rc == 0) 8287 rc = -t4_load_fw(sc, fw_data, fw->len); 8288 8289 free(fw_data, M_CXGBE); 8290done: 8291 end_synchronized_op(sc, 0); 8292 return (rc); 8293} 8294 8295#define MAX_READ_BUF_SIZE (128 * 1024) 8296static int 8297read_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 8298{ 8299 uint32_t addr, remaining, n; 8300 uint32_t *buf; 8301 int rc; 8302 uint8_t *dst; 8303 8304 rc = validate_mem_range(sc, mr->addr, mr->len); 8305 if (rc != 0) 8306 return (rc); 8307 8308 buf = malloc(min(mr->len, MAX_READ_BUF_SIZE), M_CXGBE, M_WAITOK); 8309 addr = mr->addr; 8310 remaining = mr->len; 8311 dst = (void *)mr->data; 8312 8313 while (remaining) { 8314 n = min(remaining, MAX_READ_BUF_SIZE); 8315 read_via_memwin(sc, 2, addr, buf, n); 8316 8317 rc = copyout(buf, dst, n); 8318 if (rc != 0) 8319 break; 8320 8321 dst += n; 8322 remaining -= n; 8323 addr += n; 8324 } 8325 8326 free(buf, M_CXGBE); 8327 return (rc); 8328} 8329#undef MAX_READ_BUF_SIZE 8330 8331static int 8332read_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 8333{ 8334 int rc; 8335 8336 if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 8337 return (EINVAL); 8338 8339 if (i2cd->len > sizeof(i2cd->data)) 8340 return (EFBIG); 8341 8342 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 8343 if (rc) 8344 return (rc); 8345 rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 8346 i2cd->offset, i2cd->len, &i2cd->data[0]); 8347 end_synchronized_op(sc, 0); 8348 8349 return (rc); 8350} 8351 8352static int 8353in_range(int val, int lo, int hi) 8354{ 8355 8356 return (val < 0 || (val <= hi && val >= lo)); 8357} 8358 8359static int 8360set_sched_class_config(struct adapter *sc, int minmax) 8361{ 8362 int rc; 8363 8364 if (minmax < 0) 8365 return (EINVAL); 8366 8367 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4sscc"); 8368 if (rc) 8369 return (rc); 8370 rc = -t4_sched_config(sc, FW_SCHED_TYPE_PKTSCHED, minmax, 1); 8371 end_synchronized_op(sc, 0); 8372 8373 return (rc); 8374} 8375 8376static int 8377set_sched_class_params(struct adapter *sc, struct t4_sched_class_params *p, 8378 int sleep_ok) 8379{ 8380 int rc, top_speed, fw_level, fw_mode, fw_rateunit, fw_ratemode; 8381 struct port_info *pi; 8382 struct tx_sched_class *tc; 8383 8384 if (p->level == SCHED_CLASS_LEVEL_CL_RL) 8385 fw_level = FW_SCHED_PARAMS_LEVEL_CL_RL; 8386 else if (p->level == SCHED_CLASS_LEVEL_CL_WRR) 8387 fw_level = FW_SCHED_PARAMS_LEVEL_CL_WRR; 8388 else if (p->level == SCHED_CLASS_LEVEL_CH_RL) 8389 fw_level = FW_SCHED_PARAMS_LEVEL_CH_RL; 8390 else 8391 return (EINVAL); 8392 8393 if (p->mode == SCHED_CLASS_MODE_CLASS) 8394 fw_mode = FW_SCHED_PARAMS_MODE_CLASS; 8395 else if (p->mode == SCHED_CLASS_MODE_FLOW) 8396 fw_mode = FW_SCHED_PARAMS_MODE_FLOW; 8397 else 8398 return (EINVAL); 8399 8400 if (p->rateunit == SCHED_CLASS_RATEUNIT_BITS) 8401 fw_rateunit = FW_SCHED_PARAMS_UNIT_BITRATE; 8402 else if (p->rateunit == SCHED_CLASS_RATEUNIT_PKTS) 8403 fw_rateunit = FW_SCHED_PARAMS_UNIT_PKTRATE; 8404 else 8405 return (EINVAL); 8406 8407 if (p->ratemode == SCHED_CLASS_RATEMODE_REL) 8408 fw_ratemode = FW_SCHED_PARAMS_RATE_REL; 8409 else if (p->ratemode == SCHED_CLASS_RATEMODE_ABS) 8410 fw_ratemode = FW_SCHED_PARAMS_RATE_ABS; 8411 else 8412 return (EINVAL); 8413 8414 /* Vet our parameters ... */ 8415 if (!in_range(p->channel, 0, sc->chip_params->nchan - 1)) 8416 return (ERANGE); 8417 8418 pi = sc->port[sc->chan_map[p->channel]]; 8419 if (pi == NULL) 8420 return (ENXIO); 8421 MPASS(pi->tx_chan == p->channel); 8422 top_speed = port_top_speed(pi) * 1000000; /* Gbps -> Kbps */ 8423 8424 if (!in_range(p->cl, 0, sc->chip_params->nsched_cls) || 8425 !in_range(p->minrate, 0, top_speed) || 8426 !in_range(p->maxrate, 0, top_speed) || 8427 !in_range(p->weight, 0, 100)) 8428 return (ERANGE); 8429 8430 /* 8431 * Translate any unset parameters into the firmware's 8432 * nomenclature and/or fail the call if the parameters 8433 * are required ... 8434 */ 8435 if (p->rateunit < 0 || p->ratemode < 0 || p->channel < 0 || p->cl < 0) 8436 return (EINVAL); 8437 8438 if (p->minrate < 0) 8439 p->minrate = 0; 8440 if (p->maxrate < 0) { 8441 if (p->level == SCHED_CLASS_LEVEL_CL_RL || 8442 p->level == SCHED_CLASS_LEVEL_CH_RL) 8443 return (EINVAL); 8444 else 8445 p->maxrate = 0; 8446 } 8447 if (p->weight < 0) { 8448 if (p->level == SCHED_CLASS_LEVEL_CL_WRR) 8449 return (EINVAL); 8450 else 8451 p->weight = 0; 8452 } 8453 if (p->pktsize < 0) { 8454 if (p->level == SCHED_CLASS_LEVEL_CL_RL || 8455 p->level == SCHED_CLASS_LEVEL_CH_RL) 8456 return (EINVAL); 8457 else 8458 p->pktsize = 0; 8459 } 8460 8461 rc = begin_synchronized_op(sc, NULL, 8462 sleep_ok ? (SLEEP_OK | INTR_OK) : HOLD_LOCK, "t4sscp"); 8463 if (rc) 8464 return (rc); 8465 tc = &pi->tc[p->cl]; 8466 tc->params = *p; 8467 rc = -t4_sched_params(sc, FW_SCHED_TYPE_PKTSCHED, fw_level, fw_mode, 8468 fw_rateunit, fw_ratemode, p->channel, p->cl, p->minrate, p->maxrate, 8469 p->weight, p->pktsize, sleep_ok); 8470 if (rc == 0) 8471 tc->flags |= TX_SC_OK; 8472 else { 8473 /* 8474 * Unknown state at this point, see tc->params for what was 8475 * attempted. 8476 */ 8477 tc->flags &= ~TX_SC_OK; 8478 } 8479 end_synchronized_op(sc, sleep_ok ? 0 : LOCK_HELD); 8480 8481 return (rc); 8482} 8483 8484static int 8485set_sched_class(struct adapter *sc, struct t4_sched_params *p) 8486{ 8487 8488 if (p->type != SCHED_CLASS_TYPE_PACKET) 8489 return (EINVAL); 8490 8491 if (p->subcmd == SCHED_CLASS_SUBCMD_CONFIG) 8492 return (set_sched_class_config(sc, p->u.config.minmax)); 8493 8494 if (p->subcmd == SCHED_CLASS_SUBCMD_PARAMS) 8495 return (set_sched_class_params(sc, &p->u.params, 1)); 8496 8497 return (EINVAL); 8498} 8499 8500static int 8501set_sched_queue(struct adapter *sc, struct t4_sched_queue *p) 8502{ 8503 struct port_info *pi = NULL; 8504 struct vi_info *vi; 8505 struct sge_txq *txq; 8506 uint32_t fw_mnem, fw_queue, fw_class; 8507 int i, rc; 8508 8509 rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setsq"); 8510 if (rc) 8511 return (rc); 8512 8513 if (p->port >= sc->params.nports) { 8514 rc = EINVAL; 8515 goto done; 8516 } 8517 8518 /* XXX: Only supported for the main VI. */ 8519 pi = sc->port[p->port]; 8520 vi = &pi->vi[0]; 8521 if (!(vi->flags & VI_INIT_DONE)) { 8522 /* tx queues not set up yet */ 8523 rc = EAGAIN; 8524 goto done; 8525 } 8526 8527 if (!in_range(p->queue, 0, vi->ntxq - 1) || 8528 !in_range(p->cl, 0, sc->chip_params->nsched_cls - 1)) { 8529 rc = EINVAL; 8530 goto done; 8531 } 8532 8533 /* 8534 * Create a template for the FW_PARAMS_CMD mnemonic and value (TX 8535 * Scheduling Class in this case). 8536 */ 8537 fw_mnem = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) | 8538 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH)); 8539 fw_class = p->cl < 0 ? 0xffffffff : p->cl; 8540 8541 /* 8542 * If op.queue is non-negative, then we're only changing the scheduling 8543 * on a single specified TX queue. 8544 */ 8545 if (p->queue >= 0) { 8546 txq = &sc->sge.txq[vi->first_txq + p->queue]; 8547 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8548 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8549 &fw_class); 8550 goto done; 8551 } 8552 8553 /* 8554 * Change the scheduling on all the TX queues for the 8555 * interface. 8556 */ 8557 for_each_txq(vi, i, txq) { 8558 fw_queue = (fw_mnem | V_FW_PARAMS_PARAM_YZ(txq->eq.cntxt_id)); 8559 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &fw_queue, 8560 &fw_class); 8561 if (rc) 8562 goto done; 8563 } 8564 8565 rc = 0; 8566done: 8567 end_synchronized_op(sc, 0); 8568 return (rc); 8569} 8570 8571int 8572t4_os_find_pci_capability(struct adapter *sc, int cap) 8573{ 8574 int i; 8575 8576 return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 8577} 8578 8579int 8580t4_os_pci_save_state(struct adapter *sc) 8581{ 8582 device_t dev; 8583 struct pci_devinfo *dinfo; 8584 8585 dev = sc->dev; 8586 dinfo = device_get_ivars(dev); 8587 8588 pci_cfg_save(dev, dinfo, 0); 8589 return (0); 8590} 8591 8592int 8593t4_os_pci_restore_state(struct adapter *sc) 8594{ 8595 device_t dev; 8596 struct pci_devinfo *dinfo; 8597 8598 dev = sc->dev; 8599 dinfo = device_get_ivars(dev); 8600 8601 pci_cfg_restore(dev, dinfo); 8602 return (0); 8603} 8604 8605void 8606t4_os_portmod_changed(const struct adapter *sc, int idx) 8607{ 8608 struct port_info *pi = sc->port[idx]; 8609 struct vi_info *vi; 8610 struct ifnet *ifp; 8611 int v; 8612 static const char *mod_str[] = { 8613 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 8614 }; 8615 8616 for_each_vi(pi, v, vi) { 8617 build_medialist(pi, &vi->media); 8618 } 8619 8620 ifp = pi->vi[0].ifp; 8621 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 8622 if_printf(ifp, "transceiver unplugged.\n"); 8623 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 8624 if_printf(ifp, "unknown transceiver inserted.\n"); 8625 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 8626 if_printf(ifp, "unsupported transceiver inserted.\n"); 8627 else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 8628 if_printf(ifp, "%s transceiver inserted.\n", 8629 mod_str[pi->mod_type]); 8630 } else { 8631 if_printf(ifp, "transceiver (type %d) inserted.\n", 8632 pi->mod_type); 8633 } 8634} 8635 8636void 8637t4_os_link_changed(struct adapter *sc, int idx, int link_stat, int reason) 8638{ 8639 struct port_info *pi = sc->port[idx]; 8640 struct vi_info *vi; 8641 struct ifnet *ifp; 8642 int v; 8643 8644 if (link_stat) 8645 pi->linkdnrc = -1; 8646 else { 8647 if (reason >= 0) 8648 pi->linkdnrc = reason; 8649 } 8650 for_each_vi(pi, v, vi) { 8651 ifp = vi->ifp; 8652 if (ifp == NULL) 8653 continue; 8654 8655 if (link_stat) { 8656 ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 8657 if_link_state_change(ifp, LINK_STATE_UP); 8658 } else { 8659 if_link_state_change(ifp, LINK_STATE_DOWN); 8660 } 8661 } 8662} 8663 8664void 8665t4_iterate(void (*func)(struct adapter *, void *), void *arg) 8666{ 8667 struct adapter *sc; 8668 8669 sx_slock(&t4_list_lock); 8670 SLIST_FOREACH(sc, &t4_list, link) { 8671 /* 8672 * func should not make any assumptions about what state sc is 8673 * in - the only guarantee is that sc->sc_lock is a valid lock. 8674 */ 8675 func(sc, arg); 8676 } 8677 sx_sunlock(&t4_list_lock); 8678} 8679 8680static int 8681t4_open(struct cdev *dev, int flags, int type, struct thread *td) 8682{ 8683 return (0); 8684} 8685 8686static int 8687t4_close(struct cdev *dev, int flags, int type, struct thread *td) 8688{ 8689 return (0); 8690} 8691 8692static int 8693t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 8694 struct thread *td) 8695{ 8696 int rc; 8697 struct adapter *sc = dev->si_drv1; 8698 8699 rc = priv_check(td, PRIV_DRIVER); 8700 if (rc != 0) 8701 return (rc); 8702 8703 switch (cmd) { 8704 case CHELSIO_T4_GETREG: { 8705 struct t4_reg *edata = (struct t4_reg *)data; 8706 8707 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8708 return (EFAULT); 8709 8710 if (edata->size == 4) 8711 edata->val = t4_read_reg(sc, edata->addr); 8712 else if (edata->size == 8) 8713 edata->val = t4_read_reg64(sc, edata->addr); 8714 else 8715 return (EINVAL); 8716 8717 break; 8718 } 8719 case CHELSIO_T4_SETREG: { 8720 struct t4_reg *edata = (struct t4_reg *)data; 8721 8722 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 8723 return (EFAULT); 8724 8725 if (edata->size == 4) { 8726 if (edata->val & 0xffffffff00000000) 8727 return (EINVAL); 8728 t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 8729 } else if (edata->size == 8) 8730 t4_write_reg64(sc, edata->addr, edata->val); 8731 else 8732 return (EINVAL); 8733 break; 8734 } 8735 case CHELSIO_T4_REGDUMP: { 8736 struct t4_regdump *regs = (struct t4_regdump *)data; 8737 int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 8738 uint8_t *buf; 8739 8740 if (regs->len < reglen) { 8741 regs->len = reglen; /* hint to the caller */ 8742 return (ENOBUFS); 8743 } 8744 8745 regs->len = reglen; 8746 buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 8747 get_regs(sc, regs, buf); 8748 rc = copyout(buf, regs->data, reglen); 8749 free(buf, M_CXGBE); 8750 break; 8751 } 8752 case CHELSIO_T4_GET_FILTER_MODE: 8753 rc = get_filter_mode(sc, (uint32_t *)data); 8754 break; 8755 case CHELSIO_T4_SET_FILTER_MODE: 8756 rc = set_filter_mode(sc, *(uint32_t *)data); 8757 break; 8758 case CHELSIO_T4_GET_FILTER: 8759 rc = get_filter(sc, (struct t4_filter *)data); 8760 break; 8761 case CHELSIO_T4_SET_FILTER: 8762 rc = set_filter(sc, (struct t4_filter *)data); 8763 break; 8764 case CHELSIO_T4_DEL_FILTER: 8765 rc = del_filter(sc, (struct t4_filter *)data); 8766 break; 8767 case CHELSIO_T4_GET_SGE_CONTEXT: 8768 rc = get_sge_context(sc, (struct t4_sge_context *)data); 8769 break; 8770 case CHELSIO_T4_LOAD_FW: 8771 rc = load_fw(sc, (struct t4_data *)data); 8772 break; 8773 case CHELSIO_T4_GET_MEM: 8774 rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 8775 break; 8776 case CHELSIO_T4_GET_I2C: 8777 rc = read_i2c(sc, (struct t4_i2c_data *)data); 8778 break; 8779 case CHELSIO_T4_CLEAR_STATS: { 8780 int i, v; 8781 u_int port_id = *(uint32_t *)data; 8782 struct port_info *pi; 8783 struct vi_info *vi; 8784 8785 if (port_id >= sc->params.nports) 8786 return (EINVAL); 8787 pi = sc->port[port_id]; 8788 8789 /* MAC stats */ 8790 t4_clr_port_stats(sc, pi->tx_chan); 8791 pi->tx_parse_error = 0; 8792 mtx_lock(&sc->reg_lock); 8793 for_each_vi(pi, v, vi) { 8794 if (vi->flags & VI_INIT_DONE) 8795 t4_clr_vi_stats(sc, vi->viid); 8796 } 8797 mtx_unlock(&sc->reg_lock); 8798 8799 /* 8800 * Since this command accepts a port, clear stats for 8801 * all VIs on this port. 8802 */ 8803 for_each_vi(pi, v, vi) { 8804 if (vi->flags & VI_INIT_DONE) { 8805 struct sge_rxq *rxq; 8806 struct sge_txq *txq; 8807 struct sge_wrq *wrq; 8808 8809 for_each_rxq(vi, i, rxq) { 8810#if defined(INET) || defined(INET6) 8811 rxq->lro.lro_queued = 0; 8812 rxq->lro.lro_flushed = 0; 8813#endif 8814 rxq->rxcsum = 0; 8815 rxq->vlan_extraction = 0; 8816 } 8817 8818 for_each_txq(vi, i, txq) { 8819 txq->txcsum = 0; 8820 txq->tso_wrs = 0; 8821 txq->vlan_insertion = 0; 8822 txq->imm_wrs = 0; 8823 txq->sgl_wrs = 0; 8824 txq->txpkt_wrs = 0; 8825 txq->txpkts0_wrs = 0; 8826 txq->txpkts1_wrs = 0; 8827 txq->txpkts0_pkts = 0; 8828 txq->txpkts1_pkts = 0; 8829 mp_ring_reset_stats(txq->r); 8830 } 8831 8832#ifdef TCP_OFFLOAD 8833 /* nothing to clear for each ofld_rxq */ 8834 8835 for_each_ofld_txq(vi, i, wrq) { 8836 wrq->tx_wrs_direct = 0; 8837 wrq->tx_wrs_copied = 0; 8838 } 8839#endif 8840 8841 if (IS_MAIN_VI(vi)) { 8842 wrq = &sc->sge.ctrlq[pi->port_id]; 8843 wrq->tx_wrs_direct = 0; 8844 wrq->tx_wrs_copied = 0; 8845 } 8846 } 8847 } 8848 break; 8849 } 8850 case CHELSIO_T4_SCHED_CLASS: 8851 rc = set_sched_class(sc, (struct t4_sched_params *)data); 8852 break; 8853 case CHELSIO_T4_SCHED_QUEUE: 8854 rc = set_sched_queue(sc, (struct t4_sched_queue *)data); 8855 break; 8856 case CHELSIO_T4_GET_TRACER: 8857 rc = t4_get_tracer(sc, (struct t4_tracer *)data); 8858 break; 8859 case CHELSIO_T4_SET_TRACER: 8860 rc = t4_set_tracer(sc, (struct t4_tracer *)data); 8861 break; 8862 default: 8863 rc = EINVAL; 8864 } 8865 8866 return (rc); 8867} 8868 8869void 8870t4_db_full(struct adapter *sc) 8871{ 8872 8873 CXGBE_UNIMPLEMENTED(__func__); 8874} 8875 8876void 8877t4_db_dropped(struct adapter *sc) 8878{ 8879 8880 CXGBE_UNIMPLEMENTED(__func__); 8881} 8882 8883#ifdef TCP_OFFLOAD 8884void 8885t4_iscsi_init(struct ifnet *ifp, unsigned int tag_mask, 8886 const unsigned int *pgsz_order) 8887{ 8888 struct vi_info *vi = ifp->if_softc; 8889 struct adapter *sc = vi->pi->adapter; 8890 8891 t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); 8892 t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | 8893 V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | 8894 V_HPZ3(pgsz_order[3])); 8895} 8896 8897static int 8898toe_capability(struct vi_info *vi, int enable) 8899{ 8900 int rc; 8901 struct port_info *pi = vi->pi; 8902 struct adapter *sc = pi->adapter; 8903 8904 ASSERT_SYNCHRONIZED_OP(sc); 8905 8906 if (!is_offload(sc)) 8907 return (ENODEV); 8908 8909 if (enable) { 8910 if ((vi->ifp->if_capenable & IFCAP_TOE) != 0) { 8911 /* TOE is already enabled. */ 8912 return (0); 8913 } 8914 8915 /* 8916 * We need the port's queues around so that we're able to send 8917 * and receive CPLs to/from the TOE even if the ifnet for this 8918 * port has never been UP'd administratively. 8919 */ 8920 if (!(vi->flags & VI_INIT_DONE)) { 8921 rc = vi_full_init(vi); 8922 if (rc) 8923 return (rc); 8924 } 8925 if (!(pi->vi[0].flags & VI_INIT_DONE)) { 8926 rc = vi_full_init(&pi->vi[0]); 8927 if (rc) 8928 return (rc); 8929 } 8930 8931 if (isset(&sc->offload_map, pi->port_id)) { 8932 /* TOE is enabled on another VI of this port. */ 8933 pi->uld_vis++; 8934 return (0); 8935 } 8936 8937 if (!uld_active(sc, ULD_TOM)) { 8938 rc = t4_activate_uld(sc, ULD_TOM); 8939 if (rc == EAGAIN) { 8940 log(LOG_WARNING, 8941 "You must kldload t4_tom.ko before trying " 8942 "to enable TOE on a cxgbe interface.\n"); 8943 } 8944 if (rc != 0) 8945 return (rc); 8946 KASSERT(sc->tom_softc != NULL, 8947 ("%s: TOM activated but softc NULL", __func__)); 8948 KASSERT(uld_active(sc, ULD_TOM), 8949 ("%s: TOM activated but flag not set", __func__)); 8950 } 8951 8952 /* Activate iWARP and iSCSI too, if the modules are loaded. */ 8953 if (!uld_active(sc, ULD_IWARP)) 8954 (void) t4_activate_uld(sc, ULD_IWARP); 8955 if (!uld_active(sc, ULD_ISCSI)) 8956 (void) t4_activate_uld(sc, ULD_ISCSI); 8957 8958 pi->uld_vis++; 8959 setbit(&sc->offload_map, pi->port_id); 8960 } else { 8961 pi->uld_vis--; 8962 8963 if (!isset(&sc->offload_map, pi->port_id) || pi->uld_vis > 0) 8964 return (0); 8965 8966 KASSERT(uld_active(sc, ULD_TOM), 8967 ("%s: TOM never initialized?", __func__)); 8968 clrbit(&sc->offload_map, pi->port_id); 8969 } 8970 8971 return (0); 8972} 8973 8974/* 8975 * Add an upper layer driver to the global list. 8976 */ 8977int 8978t4_register_uld(struct uld_info *ui) 8979{ 8980 int rc = 0; 8981 struct uld_info *u; 8982 8983 sx_xlock(&t4_uld_list_lock); 8984 SLIST_FOREACH(u, &t4_uld_list, link) { 8985 if (u->uld_id == ui->uld_id) { 8986 rc = EEXIST; 8987 goto done; 8988 } 8989 } 8990 8991 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 8992 ui->refcount = 0; 8993done: 8994 sx_xunlock(&t4_uld_list_lock); 8995 return (rc); 8996} 8997 8998int 8999t4_unregister_uld(struct uld_info *ui) 9000{ 9001 int rc = EINVAL; 9002 struct uld_info *u; 9003 9004 sx_xlock(&t4_uld_list_lock); 9005 9006 SLIST_FOREACH(u, &t4_uld_list, link) { 9007 if (u == ui) { 9008 if (ui->refcount > 0) { 9009 rc = EBUSY; 9010 goto done; 9011 } 9012 9013 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 9014 rc = 0; 9015 goto done; 9016 } 9017 } 9018done: 9019 sx_xunlock(&t4_uld_list_lock); 9020 return (rc); 9021} 9022 9023int 9024t4_activate_uld(struct adapter *sc, int id) 9025{ 9026 int rc; 9027 struct uld_info *ui; 9028 9029 ASSERT_SYNCHRONIZED_OP(sc); 9030 9031 if (id < 0 || id > ULD_MAX) 9032 return (EINVAL); 9033 rc = EAGAIN; /* kldoad the module with this ULD and try again. */ 9034 9035 sx_slock(&t4_uld_list_lock); 9036 9037 SLIST_FOREACH(ui, &t4_uld_list, link) { 9038 if (ui->uld_id == id) { 9039 if (!(sc->flags & FULL_INIT_DONE)) { 9040 rc = adapter_full_init(sc); 9041 if (rc != 0) 9042 break; 9043 } 9044 9045 rc = ui->activate(sc); 9046 if (rc == 0) { 9047 setbit(&sc->active_ulds, id); 9048 ui->refcount++; 9049 } 9050 break; 9051 } 9052 } 9053 9054 sx_sunlock(&t4_uld_list_lock); 9055 9056 return (rc); 9057} 9058 9059int 9060t4_deactivate_uld(struct adapter *sc, int id) 9061{ 9062 int rc; 9063 struct uld_info *ui; 9064 9065 ASSERT_SYNCHRONIZED_OP(sc); 9066 9067 if (id < 0 || id > ULD_MAX) 9068 return (EINVAL); 9069 rc = ENXIO; 9070 9071 sx_slock(&t4_uld_list_lock); 9072 9073 SLIST_FOREACH(ui, &t4_uld_list, link) { 9074 if (ui->uld_id == id) { 9075 rc = ui->deactivate(sc); 9076 if (rc == 0) { 9077 clrbit(&sc->active_ulds, id); 9078 ui->refcount--; 9079 } 9080 break; 9081 } 9082 } 9083 9084 sx_sunlock(&t4_uld_list_lock); 9085 9086 return (rc); 9087} 9088 9089int 9090uld_active(struct adapter *sc, int uld_id) 9091{ 9092 9093 MPASS(uld_id >= 0 && uld_id <= ULD_MAX); 9094 9095 return (isset(&sc->active_ulds, uld_id)); 9096} 9097#endif 9098 9099/* 9100 * Come up with reasonable defaults for some of the tunables, provided they're 9101 * not set by the user (in which case we'll use the values as is). 9102 */ 9103static void 9104tweak_tunables(void) 9105{ 9106 int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 9107 9108 if (t4_ntxq10g < 1) { 9109#ifdef RSS 9110 t4_ntxq10g = rss_getnumbuckets(); 9111#else 9112 t4_ntxq10g = min(nc, NTXQ_10G); 9113#endif 9114 } 9115 9116 if (t4_ntxq1g < 1) { 9117#ifdef RSS 9118 /* XXX: way too many for 1GbE? */ 9119 t4_ntxq1g = rss_getnumbuckets(); 9120#else 9121 t4_ntxq1g = min(nc, NTXQ_1G); 9122#endif 9123 } 9124 9125 if (t4_ntxq_vi < 1) 9126 t4_ntxq_vi = min(nc, NTXQ_VI); 9127 9128 if (t4_nrxq10g < 1) { 9129#ifdef RSS 9130 t4_nrxq10g = rss_getnumbuckets(); 9131#else 9132 t4_nrxq10g = min(nc, NRXQ_10G); 9133#endif 9134 } 9135 9136 if (t4_nrxq1g < 1) { 9137#ifdef RSS 9138 /* XXX: way too many for 1GbE? */ 9139 t4_nrxq1g = rss_getnumbuckets(); 9140#else 9141 t4_nrxq1g = min(nc, NRXQ_1G); 9142#endif 9143 } 9144 9145 if (t4_nrxq_vi < 1) 9146 t4_nrxq_vi = min(nc, NRXQ_VI); 9147 9148#ifdef TCP_OFFLOAD 9149 if (t4_nofldtxq10g < 1) 9150 t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 9151 9152 if (t4_nofldtxq1g < 1) 9153 t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 9154 9155 if (t4_nofldtxq_vi < 1) 9156 t4_nofldtxq_vi = min(nc, NOFLDTXQ_VI); 9157 9158 if (t4_nofldrxq10g < 1) 9159 t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 9160 9161 if (t4_nofldrxq1g < 1) 9162 t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 9163 9164 if (t4_nofldrxq_vi < 1) 9165 t4_nofldrxq_vi = min(nc, NOFLDRXQ_VI); 9166 9167 if (t4_toecaps_allowed == -1) 9168 t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 9169 9170 if (t4_rdmacaps_allowed == -1) { 9171 t4_rdmacaps_allowed = FW_CAPS_CONFIG_RDMA_RDDP | 9172 FW_CAPS_CONFIG_RDMA_RDMAC; 9173 } 9174 9175 if (t4_iscsicaps_allowed == -1) { 9176 t4_iscsicaps_allowed = FW_CAPS_CONFIG_ISCSI_INITIATOR_PDU | 9177 FW_CAPS_CONFIG_ISCSI_TARGET_PDU | 9178 FW_CAPS_CONFIG_ISCSI_T10DIF; 9179 } 9180#else 9181 if (t4_toecaps_allowed == -1) 9182 t4_toecaps_allowed = 0; 9183 9184 if (t4_rdmacaps_allowed == -1) 9185 t4_rdmacaps_allowed = 0; 9186 9187 if (t4_iscsicaps_allowed == -1) 9188 t4_iscsicaps_allowed = 0; 9189#endif 9190 9191#ifdef DEV_NETMAP 9192 if (t4_nnmtxq_vi < 1) 9193 t4_nnmtxq_vi = min(nc, NNMTXQ_VI); 9194 9195 if (t4_nnmrxq_vi < 1) 9196 t4_nnmrxq_vi = min(nc, NNMRXQ_VI); 9197#endif 9198 9199 if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 9200 t4_tmr_idx_10g = TMR_IDX_10G; 9201 9202 if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 9203 t4_pktc_idx_10g = PKTC_IDX_10G; 9204 9205 if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 9206 t4_tmr_idx_1g = TMR_IDX_1G; 9207 9208 if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 9209 t4_pktc_idx_1g = PKTC_IDX_1G; 9210 9211 if (t4_qsize_txq < 128) 9212 t4_qsize_txq = 128; 9213 9214 if (t4_qsize_rxq < 128) 9215 t4_qsize_rxq = 128; 9216 while (t4_qsize_rxq & 7) 9217 t4_qsize_rxq++; 9218 9219 t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 9220} 9221 9222#ifdef DDB 9223static void 9224t4_dump_tcb(struct adapter *sc, int tid) 9225{ 9226 uint32_t base, i, j, off, pf, reg, save, tcb_addr, win_pos; 9227 9228 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2); 9229 save = t4_read_reg(sc, reg); 9230 base = sc->memwin[2].mw_base; 9231 9232 /* Dump TCB for the tid */ 9233 tcb_addr = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 9234 tcb_addr += tid * TCB_SIZE; 9235 9236 if (is_t4(sc)) { 9237 pf = 0; 9238 win_pos = tcb_addr & ~0xf; /* start must be 16B aligned */ 9239 } else { 9240 pf = V_PFNUM(sc->pf); 9241 win_pos = tcb_addr & ~0x7f; /* start must be 128B aligned */ 9242 } 9243 t4_write_reg(sc, reg, win_pos | pf); 9244 t4_read_reg(sc, reg); 9245 9246 off = tcb_addr - win_pos; 9247 for (i = 0; i < 4; i++) { 9248 uint32_t buf[8]; 9249 for (j = 0; j < 8; j++, off += 4) 9250 buf[j] = htonl(t4_read_reg(sc, base + off)); 9251 9252 db_printf("%08x %08x %08x %08x %08x %08x %08x %08x\n", 9253 buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], 9254 buf[7]); 9255 } 9256 9257 t4_write_reg(sc, reg, save); 9258 t4_read_reg(sc, reg); 9259} 9260 9261static void 9262t4_dump_devlog(struct adapter *sc) 9263{ 9264 struct devlog_params *dparams = &sc->params.devlog; 9265 struct fw_devlog_e e; 9266 int i, first, j, m, nentries, rc; 9267 uint64_t ftstamp = UINT64_MAX; 9268 9269 if (dparams->start == 0) { 9270 db_printf("devlog params not valid\n"); 9271 return; 9272 } 9273 9274 nentries = dparams->size / sizeof(struct fw_devlog_e); 9275 m = fwmtype_to_hwmtype(dparams->memtype); 9276 9277 /* Find the first entry. */ 9278 first = -1; 9279 for (i = 0; i < nentries && !db_pager_quit; i++) { 9280 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e), 9281 sizeof(e), (void *)&e); 9282 if (rc != 0) 9283 break; 9284 9285 if (e.timestamp == 0) 9286 break; 9287 9288 e.timestamp = be64toh(e.timestamp); 9289 if (e.timestamp < ftstamp) { 9290 ftstamp = e.timestamp; 9291 first = i; 9292 } 9293 } 9294 9295 if (first == -1) 9296 return; 9297 9298 i = first; 9299 do { 9300 rc = -t4_mem_read(sc, m, dparams->start + i * sizeof(e), 9301 sizeof(e), (void *)&e); 9302 if (rc != 0) 9303 return; 9304 9305 if (e.timestamp == 0) 9306 return; 9307 9308 e.timestamp = be64toh(e.timestamp); 9309 e.seqno = be32toh(e.seqno); 9310 for (j = 0; j < 8; j++) 9311 e.params[j] = be32toh(e.params[j]); 9312 9313 db_printf("%10d %15ju %8s %8s ", 9314 e.seqno, e.timestamp, 9315 (e.level < nitems(devlog_level_strings) ? 9316 devlog_level_strings[e.level] : "UNKNOWN"), 9317 (e.facility < nitems(devlog_facility_strings) ? 9318 devlog_facility_strings[e.facility] : "UNKNOWN")); 9319 db_printf(e.fmt, e.params[0], e.params[1], e.params[2], 9320 e.params[3], e.params[4], e.params[5], e.params[6], 9321 e.params[7]); 9322 9323 if (++i == nentries) 9324 i = 0; 9325 } while (i != first && !db_pager_quit); 9326} 9327 9328static struct command_table db_t4_table = LIST_HEAD_INITIALIZER(db_t4_table); 9329_DB_SET(_show, t4, NULL, db_show_table, 0, &db_t4_table); 9330 9331DB_FUNC(devlog, db_show_devlog, db_t4_table, CS_OWN, NULL) 9332{ 9333 device_t dev; 9334 int t; 9335 bool valid; 9336 9337 valid = false; 9338 t = db_read_token(); 9339 if (t == tIDENT) { 9340 dev = device_lookup_by_name(db_tok_string); 9341 valid = true; 9342 } 9343 db_skip_to_eol(); 9344 if (!valid) { 9345 db_printf("usage: show t4 devlog <nexus>\n"); 9346 return; 9347 } 9348 9349 if (dev == NULL) { 9350 db_printf("device not found\n"); 9351 return; 9352 } 9353 9354 t4_dump_devlog(device_get_softc(dev)); 9355} 9356 9357DB_FUNC(tcb, db_show_t4tcb, db_t4_table, CS_OWN, NULL) 9358{ 9359 device_t dev; 9360 int radix, tid, t; 9361 bool valid; 9362 9363 valid = false; 9364 radix = db_radix; 9365 db_radix = 10; 9366 t = db_read_token(); 9367 if (t == tIDENT) { 9368 dev = device_lookup_by_name(db_tok_string); 9369 t = db_read_token(); 9370 if (t == tNUMBER) { 9371 tid = db_tok_number; 9372 valid = true; 9373 } 9374 } 9375 db_radix = radix; 9376 db_skip_to_eol(); 9377 if (!valid) { 9378 db_printf("usage: show t4 tcb <nexus> <tid>\n"); 9379 return; 9380 } 9381 9382 if (dev == NULL) { 9383 db_printf("device not found\n"); 9384 return; 9385 } 9386 if (tid < 0) { 9387 db_printf("invalid tid\n"); 9388 return; 9389 } 9390 9391 t4_dump_tcb(device_get_softc(dev), tid); 9392} 9393#endif 9394 9395static struct sx mlu; /* mod load unload */ 9396SX_SYSINIT(cxgbe_mlu, &mlu, "cxgbe mod load/unload"); 9397 9398static int 9399mod_event(module_t mod, int cmd, void *arg) 9400{ 9401 int rc = 0; 9402 static int loaded = 0; 9403 9404 switch (cmd) { 9405 case MOD_LOAD: 9406 sx_xlock(&mlu); 9407 if (loaded++ == 0) { 9408 t4_sge_modload(); 9409 sx_init(&t4_list_lock, "T4/T5 adapters"); 9410 SLIST_INIT(&t4_list); 9411#ifdef TCP_OFFLOAD 9412 sx_init(&t4_uld_list_lock, "T4/T5 ULDs"); 9413 SLIST_INIT(&t4_uld_list); 9414#endif 9415 t4_tracer_modload(); 9416 tweak_tunables(); 9417 } 9418 sx_xunlock(&mlu); 9419 break; 9420 9421 case MOD_UNLOAD: 9422 sx_xlock(&mlu); 9423 if (--loaded == 0) { 9424 int tries; 9425 9426 sx_slock(&t4_list_lock); 9427 if (!SLIST_EMPTY(&t4_list)) { 9428 rc = EBUSY; 9429 sx_sunlock(&t4_list_lock); 9430 goto done_unload; 9431 } 9432#ifdef TCP_OFFLOAD 9433 sx_slock(&t4_uld_list_lock); 9434 if (!SLIST_EMPTY(&t4_uld_list)) { 9435 rc = EBUSY; 9436 sx_sunlock(&t4_uld_list_lock); 9437 sx_sunlock(&t4_list_lock); 9438 goto done_unload; 9439 } 9440#endif 9441 tries = 0; 9442 while (tries++ < 5 && t4_sge_extfree_refs() != 0) { 9443 uprintf("%ju clusters with custom free routine " 9444 "still is use.\n", t4_sge_extfree_refs()); 9445 pause("t4unload", 2 * hz); 9446 } 9447#ifdef TCP_OFFLOAD 9448 sx_sunlock(&t4_uld_list_lock); 9449#endif 9450 sx_sunlock(&t4_list_lock); 9451 9452 if (t4_sge_extfree_refs() == 0) { 9453 t4_tracer_modunload(); 9454#ifdef TCP_OFFLOAD 9455 sx_destroy(&t4_uld_list_lock); 9456#endif 9457 sx_destroy(&t4_list_lock); 9458 t4_sge_modunload(); 9459 loaded = 0; 9460 } else { 9461 rc = EBUSY; 9462 loaded++; /* undo earlier decrement */ 9463 } 9464 } 9465done_unload: 9466 sx_xunlock(&mlu); 9467 break; 9468 } 9469 9470 return (rc); 9471} 9472 9473static devclass_t t4_devclass, t5_devclass; 9474static devclass_t cxgbe_devclass, cxl_devclass; 9475static devclass_t vcxgbe_devclass, vcxl_devclass; 9476 9477DRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 9478MODULE_VERSION(t4nex, 1); 9479MODULE_DEPEND(t4nex, firmware, 1, 1, 1); 9480 9481DRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 9482MODULE_VERSION(t5nex, 1); 9483MODULE_DEPEND(t5nex, firmware, 1, 1, 1); 9484 9485DRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 9486MODULE_VERSION(cxgbe, 1); 9487 9488DRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 9489MODULE_VERSION(cxl, 1); 9490 9491DRIVER_MODULE(vcxgbe, cxgbe, vcxgbe_driver, vcxgbe_devclass, 0, 0); 9492MODULE_VERSION(vcxgbe, 1); 9493 9494DRIVER_MODULE(vcxl, cxl, vcxl_driver, vcxl_devclass, 0, 0); 9495MODULE_VERSION(vcxl, 1); 9496