t4_main.c revision 249376
1218792Snp/*- 2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc. 3218792Snp * All rights reserved. 4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org> 5218792Snp * 6218792Snp * Redistribution and use in source and binary forms, with or without 7218792Snp * modification, are permitted provided that the following conditions 8218792Snp * are met: 9218792Snp * 1. Redistributions of source code must retain the above copyright 10218792Snp * notice, this list of conditions and the following disclaimer. 11218792Snp * 2. Redistributions in binary form must reproduce the above copyright 12218792Snp * notice, this list of conditions and the following disclaimer in the 13218792Snp * documentation and/or other materials provided with the distribution. 14218792Snp * 15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18218792Snp * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25218792Snp * SUCH DAMAGE. 26218792Snp */ 27218792Snp 28218792Snp#include <sys/cdefs.h> 29218792Snp__FBSDID("$FreeBSD: head/sys/dev/cxgbe/t4_main.c 249376 2013-04-11 19:39:40Z np $"); 30218792Snp 31218792Snp#include "opt_inet.h" 32237819Snp#include "opt_inet6.h" 33218792Snp 34218792Snp#include <sys/param.h> 35218792Snp#include <sys/conf.h> 36218792Snp#include <sys/priv.h> 37218792Snp#include <sys/kernel.h> 38218792Snp#include <sys/bus.h> 39218792Snp#include <sys/module.h> 40219286Snp#include <sys/malloc.h> 41219286Snp#include <sys/queue.h> 42219286Snp#include <sys/taskqueue.h> 43218792Snp#include <sys/pciio.h> 44218792Snp#include <dev/pci/pcireg.h> 45218792Snp#include <dev/pci/pcivar.h> 46218792Snp#include <dev/pci/pci_private.h> 47218792Snp#include <sys/firmware.h> 48219436Snp#include <sys/sbuf.h> 49218792Snp#include <sys/smp.h> 50218792Snp#include <sys/socket.h> 51218792Snp#include <sys/sockio.h> 52218792Snp#include <sys/sysctl.h> 53218792Snp#include <net/ethernet.h> 54218792Snp#include <net/if.h> 55218792Snp#include <net/if_types.h> 56218792Snp#include <net/if_dl.h> 57222003Snp#include <net/if_vlan_var.h> 58248925Snp#if defined(__i386__) || defined(__amd64__) 59248925Snp#include <vm/vm.h> 60248925Snp#include <vm/pmap.h> 61248925Snp#endif 62218792Snp 63218792Snp#include "common/common.h" 64221474Snp#include "common/t4_msg.h" 65218792Snp#include "common/t4_regs.h" 66218792Snp#include "common/t4_regs_values.h" 67218792Snp#include "t4_ioctl.h" 68222509Snp#include "t4_l2t.h" 69218792Snp 70218792Snp/* T4 bus driver interface */ 71218792Snpstatic int t4_probe(device_t); 72218792Snpstatic int t4_attach(device_t); 73218792Snpstatic int t4_detach(device_t); 74218792Snpstatic device_method_t t4_methods[] = { 75218792Snp DEVMETHOD(device_probe, t4_probe), 76218792Snp DEVMETHOD(device_attach, t4_attach), 77218792Snp DEVMETHOD(device_detach, t4_detach), 78218792Snp 79227843Smarius DEVMETHOD_END 80218792Snp}; 81218792Snpstatic driver_t t4_driver = { 82218792Snp "t4nex", 83218792Snp t4_methods, 84218792Snp sizeof(struct adapter) 85218792Snp}; 86218792Snp 87218792Snp 88218792Snp/* T4 port (cxgbe) interface */ 89218792Snpstatic int cxgbe_probe(device_t); 90218792Snpstatic int cxgbe_attach(device_t); 91218792Snpstatic int cxgbe_detach(device_t); 92218792Snpstatic device_method_t cxgbe_methods[] = { 93218792Snp DEVMETHOD(device_probe, cxgbe_probe), 94218792Snp DEVMETHOD(device_attach, cxgbe_attach), 95218792Snp DEVMETHOD(device_detach, cxgbe_detach), 96218792Snp { 0, 0 } 97218792Snp}; 98218792Snpstatic driver_t cxgbe_driver = { 99218792Snp "cxgbe", 100218792Snp cxgbe_methods, 101218792Snp sizeof(struct port_info) 102218792Snp}; 103218792Snp 104218792Snpstatic d_ioctl_t t4_ioctl; 105218792Snpstatic d_open_t t4_open; 106218792Snpstatic d_close_t t4_close; 107218792Snp 108218792Snpstatic struct cdevsw t4_cdevsw = { 109218792Snp .d_version = D_VERSION, 110218792Snp .d_flags = 0, 111218792Snp .d_open = t4_open, 112218792Snp .d_close = t4_close, 113218792Snp .d_ioctl = t4_ioctl, 114218792Snp .d_name = "t4nex", 115218792Snp}; 116218792Snp 117248925Snp/* T5 bus driver interface */ 118248925Snpstatic int t5_probe(device_t); 119248925Snpstatic device_method_t t5_methods[] = { 120248925Snp DEVMETHOD(device_probe, t5_probe), 121248925Snp DEVMETHOD(device_attach, t4_attach), 122248925Snp DEVMETHOD(device_detach, t4_detach), 123248925Snp 124248925Snp DEVMETHOD_END 125248925Snp}; 126248925Snpstatic driver_t t5_driver = { 127248925Snp "t5nex", 128248925Snp t5_methods, 129248925Snp sizeof(struct adapter) 130248925Snp}; 131248925Snp 132248925Snp 133248925Snp/* T5 port (cxl) interface */ 134248925Snpstatic driver_t cxl_driver = { 135248925Snp "cxl", 136248925Snp cxgbe_methods, 137248925Snp sizeof(struct port_info) 138248925Snp}; 139248925Snp 140248925Snpstatic struct cdevsw t5_cdevsw = { 141248925Snp .d_version = D_VERSION, 142248925Snp .d_flags = 0, 143248925Snp .d_open = t4_open, 144248925Snp .d_close = t4_close, 145248925Snp .d_ioctl = t4_ioctl, 146248925Snp .d_name = "t5nex", 147248925Snp}; 148248925Snp 149218792Snp/* ifnet + media interface */ 150218792Snpstatic void cxgbe_init(void *); 151218792Snpstatic int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t); 152218792Snpstatic int cxgbe_transmit(struct ifnet *, struct mbuf *); 153218792Snpstatic void cxgbe_qflush(struct ifnet *); 154218792Snpstatic int cxgbe_media_change(struct ifnet *); 155218792Snpstatic void cxgbe_media_status(struct ifnet *, struct ifmediareq *); 156218792Snp 157248925SnpMALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services"); 158218792Snp 159237263Snp/* 160237263Snp * Correct lock order when you need to acquire multiple locks is t4_list_lock, 161237263Snp * then ADAPTER_LOCK, then t4_uld_list_lock. 162237263Snp */ 163228561Snpstatic struct mtx t4_list_lock; 164228561Snpstatic SLIST_HEAD(, adapter) t4_list; 165237263Snp#ifdef TCP_OFFLOAD 166228561Snpstatic struct mtx t4_uld_list_lock; 167228561Snpstatic SLIST_HEAD(, uld_info) t4_uld_list; 168228561Snp#endif 169218792Snp 170218792Snp/* 171228561Snp * Tunables. See tweak_tunables() too. 172248925Snp * 173248925Snp * Each tunable is set to a default value here if it's known at compile-time. 174248925Snp * Otherwise it is set to -1 as an indication to tweak_tunables() that it should 175248925Snp * provide a reasonable default when the driver is loaded. 176248925Snp * 177248925Snp * Tunables applicable to both T4 and T5 are under hw.cxgbe. Those specific to 178248925Snp * T5 are under hw.cxl. 179218792Snp */ 180218792Snp 181218792Snp/* 182228561Snp * Number of queues for tx and rx, 10G and 1G, NIC and offload. 183218792Snp */ 184228561Snp#define NTXQ_10G 16 185228561Snpstatic int t4_ntxq10g = -1; 186228561SnpTUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g); 187218792Snp 188228561Snp#define NRXQ_10G 8 189228561Snpstatic int t4_nrxq10g = -1; 190228561SnpTUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g); 191218792Snp 192228561Snp#define NTXQ_1G 4 193228561Snpstatic int t4_ntxq1g = -1; 194228561SnpTUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g); 195218792Snp 196228561Snp#define NRXQ_1G 2 197228561Snpstatic int t4_nrxq1g = -1; 198228561SnpTUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g); 199218792Snp 200237263Snp#ifdef TCP_OFFLOAD 201228561Snp#define NOFLDTXQ_10G 8 202228561Snpstatic int t4_nofldtxq10g = -1; 203228561SnpTUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g); 204228561Snp 205228561Snp#define NOFLDRXQ_10G 2 206228561Snpstatic int t4_nofldrxq10g = -1; 207228561SnpTUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g); 208228561Snp 209228561Snp#define NOFLDTXQ_1G 2 210228561Snpstatic int t4_nofldtxq1g = -1; 211228561SnpTUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g); 212228561Snp 213228561Snp#define NOFLDRXQ_1G 1 214228561Snpstatic int t4_nofldrxq1g = -1; 215228561SnpTUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g); 216228561Snp#endif 217228561Snp 218218792Snp/* 219218792Snp * Holdoff parameters for 10G and 1G ports. 220218792Snp */ 221228561Snp#define TMR_IDX_10G 1 222228561Snpstatic int t4_tmr_idx_10g = TMR_IDX_10G; 223228561SnpTUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g); 224218792Snp 225234833Snp#define PKTC_IDX_10G (-1) 226228561Snpstatic int t4_pktc_idx_10g = PKTC_IDX_10G; 227228561SnpTUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g); 228218792Snp 229228561Snp#define TMR_IDX_1G 1 230228561Snpstatic int t4_tmr_idx_1g = TMR_IDX_1G; 231228561SnpTUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g); 232218792Snp 233234833Snp#define PKTC_IDX_1G (-1) 234228561Snpstatic int t4_pktc_idx_1g = PKTC_IDX_1G; 235228561SnpTUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g); 236218792Snp 237218792Snp/* 238218792Snp * Size (# of entries) of each tx and rx queue. 239218792Snp */ 240228561Snpstatic unsigned int t4_qsize_txq = TX_EQ_QSIZE; 241228561SnpTUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq); 242218792Snp 243228561Snpstatic unsigned int t4_qsize_rxq = RX_IQ_QSIZE; 244228561SnpTUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq); 245218792Snp 246218792Snp/* 247228561Snp * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively). 248218792Snp */ 249228561Snpstatic int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX; 250228561SnpTUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types); 251218792Snp 252218792Snp/* 253228561Snp * Configuration file. 254218792Snp */ 255248925Snp#define DEFAULT_CF "default" 256248925Snp#define FLASH_CF "flash" 257248925Snp#define UWIRE_CF "uwire" 258249376Snp#define FPGA_CF "fpga" 259248925Snpstatic char t4_cfg_file[32] = DEFAULT_CF; 260228561SnpTUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file)); 261218792Snp 262228561Snp/* 263247347Snp * Firmware auto-install by driver during attach (0, 1, 2 = prohibited, allowed, 264247347Snp * encouraged respectively). 265247347Snp */ 266247347Snpstatic unsigned int t4_fw_install = 1; 267247347SnpTUNABLE_INT("hw.cxgbe.fw_install", &t4_fw_install); 268247347Snp 269247347Snp/* 270228561Snp * ASIC features that will be used. Disable the ones you don't want so that the 271228561Snp * chip resources aren't wasted on features that will not be used. 272228561Snp */ 273228561Snpstatic int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */ 274228561SnpTUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed); 275221474Snp 276228561Snpstatic int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC; 277228561SnpTUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed); 278228561Snp 279238028Snpstatic int t4_toecaps_allowed = -1; 280228561SnpTUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed); 281228561Snp 282228561Snpstatic int t4_rdmacaps_allowed = 0; 283228561SnpTUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed); 284228561Snp 285228561Snpstatic int t4_iscsicaps_allowed = 0; 286228561SnpTUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed); 287228561Snp 288228561Snpstatic int t4_fcoecaps_allowed = 0; 289228561SnpTUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed); 290228561Snp 291248925Snpstatic int t5_write_combine = 0; 292248925SnpTUNABLE_INT("hw.cxl.write_combine", &t5_write_combine); 293248925Snp 294218792Snpstruct intrs_and_queues { 295219944Snp int intr_type; /* INTx, MSI, or MSI-X */ 296218792Snp int nirq; /* Number of vectors */ 297228561Snp int intr_flags; 298218792Snp int ntxq10g; /* # of NIC txq's for each 10G port */ 299218792Snp int nrxq10g; /* # of NIC rxq's for each 10G port */ 300218792Snp int ntxq1g; /* # of NIC txq's for each 1G port */ 301218792Snp int nrxq1g; /* # of NIC rxq's for each 1G port */ 302237263Snp#ifdef TCP_OFFLOAD 303228561Snp int nofldtxq10g; /* # of TOE txq's for each 10G port */ 304228561Snp int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 305228561Snp int nofldtxq1g; /* # of TOE txq's for each 1G port */ 306228561Snp int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 307228561Snp#endif 308218792Snp}; 309218792Snp 310221474Snpstruct filter_entry { 311221474Snp uint32_t valid:1; /* filter allocated and valid */ 312221474Snp uint32_t locked:1; /* filter is administratively locked */ 313221474Snp uint32_t pending:1; /* filter action is pending firmware reply */ 314221474Snp uint32_t smtidx:8; /* Source MAC Table index for smac */ 315222509Snp struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 316221474Snp 317221474Snp struct t4_filter_specification fs; 318221474Snp}; 319221474Snp 320218792Snpenum { 321218792Snp XGMAC_MTU = (1 << 0), 322218792Snp XGMAC_PROMISC = (1 << 1), 323218792Snp XGMAC_ALLMULTI = (1 << 2), 324218792Snp XGMAC_VLANEX = (1 << 3), 325218792Snp XGMAC_UCADDR = (1 << 4), 326218792Snp XGMAC_MCADDRS = (1 << 5), 327218792Snp 328218792Snp XGMAC_ALL = 0xffff 329218792Snp}; 330218792Snp 331248925Snpstatic int map_bars_0_and_4(struct adapter *); 332248925Snpstatic int map_bar_2(struct adapter *); 333218792Snpstatic void setup_memwin(struct adapter *); 334248925Snpstatic int validate_mem_range(struct adapter *, uint32_t, int); 335248925Snpstatic int validate_mt_off_len(struct adapter *, int, uint32_t, int, 336248925Snp uint32_t *); 337248925Snpstatic void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 338248925Snpstatic uint32_t position_memwin(struct adapter *, int, uint32_t); 339218792Snpstatic int cfg_itype_and_nqueues(struct adapter *, int, int, 340218792Snp struct intrs_and_queues *); 341218792Snpstatic int prep_firmware(struct adapter *); 342248925Snpstatic int partition_resources(struct adapter *, const struct firmware *, 343248925Snp const char *); 344228561Snpstatic int get_params__pre_init(struct adapter *); 345228561Snpstatic int get_params__post_init(struct adapter *); 346247291Snpstatic int set_params__post_init(struct adapter *); 347218792Snpstatic void t4_set_desc(struct adapter *); 348218792Snpstatic void build_medialist(struct port_info *); 349218792Snpstatic int update_mac_settings(struct port_info *, int); 350218792Snpstatic int cxgbe_init_synchronized(struct port_info *); 351218792Snpstatic int cxgbe_uninit_synchronized(struct port_info *); 352240453Snpstatic int setup_intr_handlers(struct adapter *); 353228561Snpstatic int adapter_full_init(struct adapter *); 354228561Snpstatic int adapter_full_uninit(struct adapter *); 355228561Snpstatic int port_full_init(struct port_info *); 356228561Snpstatic int port_full_uninit(struct port_info *); 357228561Snpstatic void quiesce_eq(struct adapter *, struct sge_eq *); 358228561Snpstatic void quiesce_iq(struct adapter *, struct sge_iq *); 359228561Snpstatic void quiesce_fl(struct adapter *, struct sge_fl *); 360218792Snpstatic int t4_alloc_irq(struct adapter *, struct irq *, int rid, 361228561Snp driver_intr_t *, void *, char *); 362218792Snpstatic int t4_free_irq(struct adapter *, struct irq *); 363218792Snpstatic void reg_block_dump(struct adapter *, uint8_t *, unsigned int, 364218792Snp unsigned int); 365218792Snpstatic void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *); 366218792Snpstatic void cxgbe_tick(void *); 367237263Snpstatic void cxgbe_vlan_config(void *, struct ifnet *, uint16_t); 368228561Snpstatic int cpl_not_handled(struct sge_iq *, const struct rss_header *, 369228561Snp struct mbuf *); 370237263Snpstatic int an_not_handled(struct sge_iq *, const struct rsp_ctrl *); 371239336Snpstatic int fw_msg_not_handled(struct adapter *, const __be64 *); 372218792Snpstatic int t4_sysctls(struct adapter *); 373218792Snpstatic int cxgbe_sysctls(struct port_info *); 374219436Snpstatic int sysctl_int_array(SYSCTL_HANDLER_ARGS); 375228561Snpstatic int sysctl_bitfield(SYSCTL_HANDLER_ARGS); 376218792Snpstatic int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS); 377218792Snpstatic int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS); 378218792Snpstatic int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS); 379218792Snpstatic int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS); 380218792Snpstatic int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS); 381231115Snp#ifdef SBUF_DRAIN 382228561Snpstatic int sysctl_cctrl(SYSCTL_HANDLER_ARGS); 383247122Snpstatic int sysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS); 384247122Snpstatic int sysctl_cim_la(SYSCTL_HANDLER_ARGS); 385247122Snpstatic int sysctl_cim_qcfg(SYSCTL_HANDLER_ARGS); 386228561Snpstatic int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS); 387228561Snpstatic int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS); 388222551Snpstatic int sysctl_devlog(SYSCTL_HANDLER_ARGS); 389228561Snpstatic int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS); 390228561Snpstatic int sysctl_hw_sched(SYSCTL_HANDLER_ARGS); 391228561Snpstatic int sysctl_lb_stats(SYSCTL_HANDLER_ARGS); 392228561Snpstatic int sysctl_meminfo(SYSCTL_HANDLER_ARGS); 393228561Snpstatic int sysctl_path_mtus(SYSCTL_HANDLER_ARGS); 394228561Snpstatic int sysctl_pm_stats(SYSCTL_HANDLER_ARGS); 395228561Snpstatic int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS); 396228561Snpstatic int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS); 397228561Snpstatic int sysctl_tids(SYSCTL_HANDLER_ARGS); 398228561Snpstatic int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS); 399228561Snpstatic int sysctl_tx_rate(SYSCTL_HANDLER_ARGS); 400248925Snpstatic int sysctl_wrwc_stats(SYSCTL_HANDLER_ARGS); 401231115Snp#endif 402219286Snpstatic inline void txq_start(struct ifnet *, struct sge_txq *); 403221474Snpstatic uint32_t fconf_to_mode(uint32_t); 404221474Snpstatic uint32_t mode_to_fconf(uint32_t); 405221474Snpstatic uint32_t fspec_to_fconf(struct t4_filter_specification *); 406221474Snpstatic int get_filter_mode(struct adapter *, uint32_t *); 407221474Snpstatic int set_filter_mode(struct adapter *, uint32_t); 408222552Snpstatic inline uint64_t get_filter_hits(struct adapter *, uint32_t); 409221474Snpstatic int get_filter(struct adapter *, struct t4_filter *); 410221474Snpstatic int set_filter(struct adapter *, struct t4_filter *); 411221474Snpstatic int del_filter(struct adapter *, struct t4_filter *); 412222509Snpstatic void clear_filter(struct filter_entry *); 413221474Snpstatic int set_filter_wr(struct adapter *, int); 414221474Snpstatic int del_filter_wr(struct adapter *, int); 415222973Snpstatic int get_sge_context(struct adapter *, struct t4_sge_context *); 416245274Snpstatic int load_fw(struct adapter *, struct t4_data *); 417248925Snpstatic int read_card_mem(struct adapter *, int, struct t4_mem_range *); 418241399Snpstatic int read_i2c(struct adapter *, struct t4_i2c_data *); 419237263Snp#ifdef TCP_OFFLOAD 420228561Snpstatic int toe_capability(struct port_info *, int); 421228561Snp#endif 422249370Snpstatic int mod_event(module_t, int, void *); 423218792Snp 424248925Snpstruct { 425218792Snp uint16_t device; 426218792Snp char *desc; 427218792Snp} t4_pciids[] = { 428237587Snp {0xa000, "Chelsio Terminator 4 FPGA"}, 429237587Snp {0x4400, "Chelsio T440-dbg"}, 430237587Snp {0x4401, "Chelsio T420-CR"}, 431237587Snp {0x4402, "Chelsio T422-CR"}, 432237587Snp {0x4403, "Chelsio T440-CR"}, 433237587Snp {0x4404, "Chelsio T420-BCH"}, 434237587Snp {0x4405, "Chelsio T440-BCH"}, 435237587Snp {0x4406, "Chelsio T440-CH"}, 436237587Snp {0x4407, "Chelsio T420-SO"}, 437237587Snp {0x4408, "Chelsio T420-CX"}, 438237587Snp {0x4409, "Chelsio T420-BT"}, 439237587Snp {0x440a, "Chelsio T404-BT"}, 440244580Snp {0x440e, "Chelsio T440-LP-CR"}, 441248925Snp}, t5_pciids[] = { 442248925Snp {0xb000, "Chelsio Terminator 5 FPGA"}, 443248925Snp {0x5400, "Chelsio T580-dbg"}, 444218792Snp}; 445218792Snp 446237263Snp#ifdef TCP_OFFLOAD 447237263Snp/* 448237263Snp * service_iq() has an iq and needs the fl. Offset of fl from the iq should be 449237263Snp * exactly the same for both rxq and ofld_rxq. 450237263Snp */ 451237263SnpCTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq)); 452228561SnpCTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl)); 453228561Snp#endif 454228561Snp 455239336Snp/* No easy way to include t4_msg.h before adapter.h so we check this way */ 456240452SnpCTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS); 457240452SnpCTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES); 458239336Snp 459218792Snpstatic int 460218792Snpt4_probe(device_t dev) 461218792Snp{ 462218792Snp int i; 463218792Snp uint16_t v = pci_get_vendor(dev); 464218792Snp uint16_t d = pci_get_device(dev); 465237587Snp uint8_t f = pci_get_function(dev); 466218792Snp 467218792Snp if (v != PCI_VENDOR_ID_CHELSIO) 468218792Snp return (ENXIO); 469218792Snp 470237587Snp /* Attach only to PF0 of the FPGA */ 471237587Snp if (d == 0xa000 && f != 0) 472237587Snp return (ENXIO); 473237587Snp 474240452Snp for (i = 0; i < nitems(t4_pciids); i++) { 475237587Snp if (d == t4_pciids[i].device) { 476218792Snp device_set_desc(dev, t4_pciids[i].desc); 477218792Snp return (BUS_PROBE_DEFAULT); 478218792Snp } 479218792Snp } 480218792Snp 481218792Snp return (ENXIO); 482218792Snp} 483218792Snp 484218792Snpstatic int 485248925Snpt5_probe(device_t dev) 486248925Snp{ 487248925Snp int i; 488248925Snp uint16_t v = pci_get_vendor(dev); 489248925Snp uint16_t d = pci_get_device(dev); 490248925Snp uint8_t f = pci_get_function(dev); 491248925Snp 492248925Snp if (v != PCI_VENDOR_ID_CHELSIO) 493248925Snp return (ENXIO); 494248925Snp 495248925Snp /* Attach only to PF0 of the FPGA */ 496248925Snp if (d == 0xb000 && f != 0) 497248925Snp return (ENXIO); 498248925Snp 499248925Snp for (i = 0; i < nitems(t5_pciids); i++) { 500248925Snp if (d == t5_pciids[i].device) { 501248925Snp device_set_desc(dev, t5_pciids[i].desc); 502248925Snp return (BUS_PROBE_DEFAULT); 503248925Snp } 504248925Snp } 505248925Snp 506248925Snp return (ENXIO); 507248925Snp} 508248925Snp 509248925Snpstatic int 510218792Snpt4_attach(device_t dev) 511218792Snp{ 512218792Snp struct adapter *sc; 513218792Snp int rc = 0, i, n10g, n1g, rqidx, tqidx; 514218792Snp struct intrs_and_queues iaq; 515218792Snp struct sge *s; 516237263Snp#ifdef TCP_OFFLOAD 517228561Snp int ofld_rqidx, ofld_tqidx; 518228561Snp#endif 519218792Snp 520218792Snp sc = device_get_softc(dev); 521218792Snp sc->dev = dev; 522218792Snp 523218792Snp pci_enable_busmaster(dev); 524222085Snp if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) { 525228561Snp uint32_t v; 526228561Snp 527222085Snp pci_set_max_read_req(dev, 4096); 528240680Sgavin v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2); 529240680Sgavin v |= PCIEM_CTL_RELAXED_ORD_ENABLE; 530240680Sgavin pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2); 531222085Snp } 532222085Snp 533218792Snp snprintf(sc->lockname, sizeof(sc->lockname), "%s", 534218792Snp device_get_nameunit(dev)); 535218792Snp mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); 536228561Snp mtx_lock(&t4_list_lock); 537228561Snp SLIST_INSERT_HEAD(&t4_list, sc, link); 538228561Snp mtx_unlock(&t4_list_lock); 539218792Snp 540228561Snp mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); 541228561Snp TAILQ_INIT(&sc->sfl); 542228561Snp callout_init(&sc->sfl_callout, CALLOUT_MPSAFE); 543228561Snp 544248925Snp rc = map_bars_0_and_4(sc); 545218792Snp if (rc != 0) 546218792Snp goto done; /* error message displayed already */ 547218792Snp 548237587Snp /* 549237587Snp * This is the real PF# to which we're attaching. Works from within PCI 550237587Snp * passthrough environments too, where pci_get_function() could return a 551237587Snp * different PF# depending on the passthrough configuration. We need to 552237587Snp * use the real PF# in all our communication with the firmware. 553237587Snp */ 554237587Snp sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI)); 555237587Snp sc->mbox = sc->pf; 556237587Snp 557218792Snp memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); 558237263Snp sc->an_handler = an_not_handled; 559240452Snp for (i = 0; i < nitems(sc->cpl_handler); i++) 560228561Snp sc->cpl_handler[i] = cpl_not_handled; 561240452Snp for (i = 0; i < nitems(sc->fw_msg_handler); i++) 562239336Snp sc->fw_msg_handler[i] = fw_msg_not_handled; 563239338Snp t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl); 564248925Snp t4_init_sge_cpl_handlers(sc); 565218792Snp 566218792Snp /* Prepare the adapter for operation */ 567218792Snp rc = -t4_prep_adapter(sc); 568218792Snp if (rc != 0) { 569218792Snp device_printf(dev, "failed to prepare adapter: %d.\n", rc); 570218792Snp goto done; 571218792Snp } 572218792Snp 573228561Snp /* 574228561Snp * Do this really early, with the memory windows set up even before the 575228561Snp * character device. The userland tool's register i/o and mem read 576228561Snp * will work even in "recovery mode". 577228561Snp */ 578228561Snp setup_memwin(sc); 579248925Snp sc->cdev = make_dev(is_t4(sc) ? &t4_cdevsw : &t5_cdevsw, 580248925Snp device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", 581248925Snp device_get_nameunit(dev)); 582248925Snp if (sc->cdev == NULL) 583248925Snp device_printf(dev, "failed to create nexus char device.\n"); 584248925Snp else 585248925Snp sc->cdev->si_drv1 = sc; 586218792Snp 587228561Snp /* Go no further if recovery mode has been requested. */ 588228561Snp if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) { 589228561Snp device_printf(dev, "recovery mode.\n"); 590228561Snp goto done; 591228561Snp } 592228561Snp 593218792Snp /* Prepare the firmware for operation */ 594218792Snp rc = prep_firmware(sc); 595218792Snp if (rc != 0) 596218792Snp goto done; /* error message displayed already */ 597218792Snp 598248925Snp rc = get_params__post_init(sc); 599228561Snp if (rc != 0) 600228561Snp goto done; /* error message displayed already */ 601222551Snp 602248925Snp rc = set_params__post_init(sc); 603228561Snp if (rc != 0) 604228561Snp goto done; /* error message displayed already */ 605218792Snp 606248925Snp rc = map_bar_2(sc); 607228561Snp if (rc != 0) 608228561Snp goto done; /* error message displayed already */ 609218792Snp 610228561Snp for (i = 0; i < NCHAN; i++) 611228561Snp sc->params.tp.tx_modq[i] = i; 612218792Snp 613218792Snp rc = t4_create_dma_tag(sc); 614218792Snp if (rc != 0) 615218792Snp goto done; /* error message displayed already */ 616218792Snp 617218792Snp /* 618218792Snp * First pass over all the ports - allocate VIs and initialize some 619218792Snp * basic parameters like mac address, port type, etc. We also figure 620218792Snp * out whether a port is 10G or 1G and use that information when 621218792Snp * calculating how many interrupts to attempt to allocate. 622218792Snp */ 623218792Snp n10g = n1g = 0; 624218792Snp for_each_port(sc, i) { 625218792Snp struct port_info *pi; 626218792Snp 627218792Snp pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); 628218792Snp sc->port[i] = pi; 629218792Snp 630218792Snp /* These must be set before t4_port_init */ 631218792Snp pi->adapter = sc; 632218792Snp pi->port_id = i; 633218792Snp 634218792Snp /* Allocate the vi and initialize parameters like mac addr */ 635218792Snp rc = -t4_port_init(pi, sc->mbox, sc->pf, 0); 636218792Snp if (rc != 0) { 637218792Snp device_printf(dev, "unable to initialize port %d: %d\n", 638218792Snp i, rc); 639218792Snp free(pi, M_CXGBE); 640222510Snp sc->port[i] = NULL; 641222510Snp goto done; 642218792Snp } 643218792Snp 644218792Snp snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", 645218792Snp device_get_nameunit(dev), i); 646218792Snp mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); 647218792Snp 648218792Snp if (is_10G_port(pi)) { 649218792Snp n10g++; 650228561Snp pi->tmr_idx = t4_tmr_idx_10g; 651228561Snp pi->pktc_idx = t4_pktc_idx_10g; 652218792Snp } else { 653218792Snp n1g++; 654228561Snp pi->tmr_idx = t4_tmr_idx_1g; 655228561Snp pi->pktc_idx = t4_pktc_idx_1g; 656218792Snp } 657218792Snp 658218792Snp pi->xact_addr_filt = -1; 659218792Snp 660228561Snp pi->qsize_rxq = t4_qsize_rxq; 661228561Snp pi->qsize_txq = t4_qsize_txq; 662218792Snp 663248925Snp pi->dev = device_add_child(dev, is_t4(sc) ? "cxgbe" : "cxl", -1); 664218792Snp if (pi->dev == NULL) { 665218792Snp device_printf(dev, 666218792Snp "failed to add device for port %d.\n", i); 667218792Snp rc = ENXIO; 668218792Snp goto done; 669218792Snp } 670218792Snp device_set_softc(pi->dev, pi); 671218792Snp } 672218792Snp 673218792Snp /* 674218792Snp * Interrupt type, # of interrupts, # of rx/tx queues, etc. 675218792Snp */ 676218792Snp rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq); 677218792Snp if (rc != 0) 678218792Snp goto done; /* error message displayed already */ 679218792Snp 680218792Snp sc->intr_type = iaq.intr_type; 681218792Snp sc->intr_count = iaq.nirq; 682228561Snp sc->flags |= iaq.intr_flags; 683218792Snp 684218792Snp s = &sc->sge; 685218792Snp s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g; 686218792Snp s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g; 687220873Snp s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ 688228561Snp s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */ 689218792Snp s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 690222510Snp 691237263Snp#ifdef TCP_OFFLOAD 692228561Snp if (is_offload(sc)) { 693228561Snp 694228561Snp s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 695228561Snp s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 696228561Snp s->neq += s->nofldtxq + s->nofldrxq; 697228561Snp s->niq += s->nofldrxq; 698228561Snp 699228561Snp s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq), 700228561Snp M_CXGBE, M_ZERO | M_WAITOK); 701228561Snp s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq), 702228561Snp M_CXGBE, M_ZERO | M_WAITOK); 703228561Snp } 704228561Snp#endif 705228561Snp 706228561Snp s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE, 707220873Snp M_ZERO | M_WAITOK); 708218792Snp s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, 709218792Snp M_ZERO | M_WAITOK); 710218792Snp s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, 711218792Snp M_ZERO | M_WAITOK); 712218792Snp s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE, 713218792Snp M_ZERO | M_WAITOK); 714218792Snp s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE, 715218792Snp M_ZERO | M_WAITOK); 716218792Snp 717218792Snp sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, 718218792Snp M_ZERO | M_WAITOK); 719218792Snp 720228561Snp t4_init_l2t(sc, M_WAITOK); 721222509Snp 722218792Snp /* 723218792Snp * Second pass over the ports. This time we know the number of rx and 724218792Snp * tx queues that each port should get. 725218792Snp */ 726218792Snp rqidx = tqidx = 0; 727237263Snp#ifdef TCP_OFFLOAD 728228561Snp ofld_rqidx = ofld_tqidx = 0; 729228561Snp#endif 730218792Snp for_each_port(sc, i) { 731218792Snp struct port_info *pi = sc->port[i]; 732218792Snp 733218792Snp if (pi == NULL) 734218792Snp continue; 735218792Snp 736218792Snp pi->first_rxq = rqidx; 737218792Snp pi->first_txq = tqidx; 738228561Snp if (is_10G_port(pi)) { 739228561Snp pi->nrxq = iaq.nrxq10g; 740228561Snp pi->ntxq = iaq.ntxq10g; 741228561Snp } else { 742228561Snp pi->nrxq = iaq.nrxq1g; 743228561Snp pi->ntxq = iaq.ntxq1g; 744228561Snp } 745218792Snp 746218792Snp rqidx += pi->nrxq; 747218792Snp tqidx += pi->ntxq; 748228561Snp 749237263Snp#ifdef TCP_OFFLOAD 750228561Snp if (is_offload(sc)) { 751228561Snp pi->first_ofld_rxq = ofld_rqidx; 752228561Snp pi->first_ofld_txq = ofld_tqidx; 753228561Snp if (is_10G_port(pi)) { 754228561Snp pi->nofldrxq = iaq.nofldrxq10g; 755228561Snp pi->nofldtxq = iaq.nofldtxq10g; 756228561Snp } else { 757228561Snp pi->nofldrxq = iaq.nofldrxq1g; 758228561Snp pi->nofldtxq = iaq.nofldtxq1g; 759228561Snp } 760228561Snp ofld_rqidx += pi->nofldrxq; 761228561Snp ofld_tqidx += pi->nofldtxq; 762228561Snp } 763228561Snp#endif 764218792Snp } 765218792Snp 766240453Snp rc = setup_intr_handlers(sc); 767240453Snp if (rc != 0) { 768240453Snp device_printf(dev, 769240453Snp "failed to setup interrupt handlers: %d\n", rc); 770240453Snp goto done; 771240453Snp } 772240453Snp 773218792Snp rc = bus_generic_attach(dev); 774218792Snp if (rc != 0) { 775218792Snp device_printf(dev, 776218792Snp "failed to attach all child ports: %d\n", rc); 777218792Snp goto done; 778218792Snp } 779218792Snp 780218792Snp device_printf(dev, 781228561Snp "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n", 782228561Snp sc->params.pci.width, sc->params.nports, sc->intr_count, 783228561Snp sc->intr_type == INTR_MSIX ? "MSI-X" : 784228561Snp (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), 785228561Snp sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); 786228561Snp 787218792Snp t4_set_desc(sc); 788218792Snp 789218792Snpdone: 790228561Snp if (rc != 0 && sc->cdev) { 791228561Snp /* cdev was created and so cxgbetool works; recover that way. */ 792228561Snp device_printf(dev, 793228561Snp "error during attach, adapter is now in recovery mode.\n"); 794228561Snp rc = 0; 795228561Snp } 796228561Snp 797218792Snp if (rc != 0) 798218792Snp t4_detach(dev); 799228561Snp else 800228561Snp t4_sysctls(sc); 801218792Snp 802218792Snp return (rc); 803218792Snp} 804218792Snp 805218792Snp/* 806218792Snp * Idempotent 807218792Snp */ 808218792Snpstatic int 809218792Snpt4_detach(device_t dev) 810218792Snp{ 811218792Snp struct adapter *sc; 812218792Snp struct port_info *pi; 813228561Snp int i, rc; 814218792Snp 815218792Snp sc = device_get_softc(dev); 816218792Snp 817228561Snp if (sc->flags & FULL_INIT_DONE) 818228561Snp t4_intr_disable(sc); 819228561Snp 820228561Snp if (sc->cdev) { 821218792Snp destroy_dev(sc->cdev); 822228561Snp sc->cdev = NULL; 823228561Snp } 824218792Snp 825228561Snp rc = bus_generic_detach(dev); 826228561Snp if (rc) { 827228561Snp device_printf(dev, 828228561Snp "failed to detach child devices: %d\n", rc); 829228561Snp return (rc); 830228561Snp } 831228561Snp 832240453Snp for (i = 0; i < sc->intr_count; i++) 833240453Snp t4_free_irq(sc, &sc->irq[i]); 834240453Snp 835218792Snp for (i = 0; i < MAX_NPORTS; i++) { 836218792Snp pi = sc->port[i]; 837218792Snp if (pi) { 838218792Snp t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid); 839218792Snp if (pi->dev) 840218792Snp device_delete_child(dev, pi->dev); 841218792Snp 842218792Snp mtx_destroy(&pi->pi_lock); 843218792Snp free(pi, M_CXGBE); 844218792Snp } 845218792Snp } 846218792Snp 847228561Snp if (sc->flags & FULL_INIT_DONE) 848228561Snp adapter_full_uninit(sc); 849228561Snp 850218792Snp if (sc->flags & FW_OK) 851218792Snp t4_fw_bye(sc, sc->mbox); 852218792Snp 853219944Snp if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX) 854218792Snp pci_release_msi(dev); 855218792Snp 856218792Snp if (sc->regs_res) 857218792Snp bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid, 858218792Snp sc->regs_res); 859218792Snp 860248925Snp if (sc->udbs_res) 861248925Snp bus_release_resource(dev, SYS_RES_MEMORY, sc->udbs_rid, 862248925Snp sc->udbs_res); 863248925Snp 864218792Snp if (sc->msix_res) 865218792Snp bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid, 866218792Snp sc->msix_res); 867218792Snp 868222509Snp if (sc->l2t) 869222509Snp t4_free_l2t(sc->l2t); 870222509Snp 871237263Snp#ifdef TCP_OFFLOAD 872228561Snp free(sc->sge.ofld_rxq, M_CXGBE); 873228561Snp free(sc->sge.ofld_txq, M_CXGBE); 874228561Snp#endif 875218792Snp free(sc->irq, M_CXGBE); 876218792Snp free(sc->sge.rxq, M_CXGBE); 877218792Snp free(sc->sge.txq, M_CXGBE); 878220873Snp free(sc->sge.ctrlq, M_CXGBE); 879218792Snp free(sc->sge.iqmap, M_CXGBE); 880218792Snp free(sc->sge.eqmap, M_CXGBE); 881221474Snp free(sc->tids.ftid_tab, M_CXGBE); 882218792Snp t4_destroy_dma_tag(sc); 883228561Snp if (mtx_initialized(&sc->sc_lock)) { 884228561Snp mtx_lock(&t4_list_lock); 885228561Snp SLIST_REMOVE(&t4_list, sc, adapter, link); 886228561Snp mtx_unlock(&t4_list_lock); 887228561Snp mtx_destroy(&sc->sc_lock); 888228561Snp } 889218792Snp 890245274Snp if (mtx_initialized(&sc->tids.ftid_lock)) 891245274Snp mtx_destroy(&sc->tids.ftid_lock); 892228561Snp if (mtx_initialized(&sc->sfl_lock)) 893228561Snp mtx_destroy(&sc->sfl_lock); 894228561Snp 895218792Snp bzero(sc, sizeof(*sc)); 896218792Snp 897218792Snp return (0); 898218792Snp} 899218792Snp 900218792Snp 901218792Snpstatic int 902218792Snpcxgbe_probe(device_t dev) 903218792Snp{ 904218792Snp char buf[128]; 905218792Snp struct port_info *pi = device_get_softc(dev); 906218792Snp 907228561Snp snprintf(buf, sizeof(buf), "port %d", pi->port_id); 908218792Snp device_set_desc_copy(dev, buf); 909218792Snp 910218792Snp return (BUS_PROBE_DEFAULT); 911218792Snp} 912218792Snp 913218792Snp#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \ 914218792Snp IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \ 915237831Snp IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6) 916237819Snp#define T4_CAP_ENABLE (T4_CAP) 917218792Snp 918218792Snpstatic int 919218792Snpcxgbe_attach(device_t dev) 920218792Snp{ 921218792Snp struct port_info *pi = device_get_softc(dev); 922218792Snp struct ifnet *ifp; 923218792Snp 924218792Snp /* Allocate an ifnet and set it up */ 925218792Snp ifp = if_alloc(IFT_ETHER); 926218792Snp if (ifp == NULL) { 927218792Snp device_printf(dev, "Cannot allocate ifnet\n"); 928218792Snp return (ENOMEM); 929218792Snp } 930218792Snp pi->ifp = ifp; 931218792Snp ifp->if_softc = pi; 932218792Snp 933218792Snp callout_init(&pi->tick, CALLOUT_MPSAFE); 934218792Snp 935218792Snp if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 936218792Snp ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 937218792Snp 938218792Snp ifp->if_init = cxgbe_init; 939218792Snp ifp->if_ioctl = cxgbe_ioctl; 940218792Snp ifp->if_transmit = cxgbe_transmit; 941218792Snp ifp->if_qflush = cxgbe_qflush; 942218792Snp 943218792Snp ifp->if_capabilities = T4_CAP; 944237263Snp#ifdef TCP_OFFLOAD 945228561Snp if (is_offload(pi->adapter)) 946245933Snp ifp->if_capabilities |= IFCAP_TOE; 947228561Snp#endif 948218792Snp ifp->if_capenable = T4_CAP_ENABLE; 949237799Snp ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO | 950237799Snp CSUM_UDP_IPV6 | CSUM_TCP_IPV6; 951218792Snp 952218792Snp /* Initialize ifmedia for this port */ 953218792Snp ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, 954218792Snp cxgbe_media_status); 955218792Snp build_medialist(pi); 956218792Snp 957237263Snp pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp, 958237263Snp EVENTHANDLER_PRI_ANY); 959237263Snp 960218792Snp ether_ifattach(ifp, pi->hw_addr); 961218792Snp 962237263Snp#ifdef TCP_OFFLOAD 963228561Snp if (is_offload(pi->adapter)) { 964228561Snp device_printf(dev, 965228561Snp "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n", 966228561Snp pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq); 967228561Snp } else 968218792Snp#endif 969228561Snp device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq); 970218792Snp 971218792Snp cxgbe_sysctls(pi); 972218792Snp 973218792Snp return (0); 974218792Snp} 975218792Snp 976218792Snpstatic int 977218792Snpcxgbe_detach(device_t dev) 978218792Snp{ 979218792Snp struct port_info *pi = device_get_softc(dev); 980218792Snp struct adapter *sc = pi->adapter; 981228561Snp struct ifnet *ifp = pi->ifp; 982218792Snp 983218792Snp /* Tell if_ioctl and if_init that the port is going away */ 984218792Snp ADAPTER_LOCK(sc); 985218792Snp SET_DOOMED(pi); 986218792Snp wakeup(&sc->flags); 987218792Snp while (IS_BUSY(sc)) 988218792Snp mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0); 989218792Snp SET_BUSY(sc); 990245274Snp#ifdef INVARIANTS 991245274Snp sc->last_op = "t4detach"; 992245274Snp sc->last_op_thr = curthread; 993245274Snp#endif 994218792Snp ADAPTER_UNLOCK(sc); 995218792Snp 996237263Snp if (pi->vlan_c) 997237263Snp EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c); 998237263Snp 999228561Snp PORT_LOCK(pi); 1000228561Snp ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1001228561Snp callout_stop(&pi->tick); 1002228561Snp PORT_UNLOCK(pi); 1003228561Snp callout_drain(&pi->tick); 1004218792Snp 1005228561Snp /* Let detach proceed even if these fail. */ 1006228561Snp cxgbe_uninit_synchronized(pi); 1007228561Snp port_full_uninit(pi); 1008219286Snp 1009218792Snp ifmedia_removeall(&pi->media); 1010218792Snp ether_ifdetach(pi->ifp); 1011218792Snp if_free(pi->ifp); 1012218792Snp 1013218792Snp ADAPTER_LOCK(sc); 1014218792Snp CLR_BUSY(sc); 1015245274Snp wakeup(&sc->flags); 1016218792Snp ADAPTER_UNLOCK(sc); 1017218792Snp 1018218792Snp return (0); 1019218792Snp} 1020218792Snp 1021218792Snpstatic void 1022218792Snpcxgbe_init(void *arg) 1023218792Snp{ 1024218792Snp struct port_info *pi = arg; 1025218792Snp struct adapter *sc = pi->adapter; 1026218792Snp 1027245274Snp if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0) 1028245274Snp return; 1029245274Snp cxgbe_init_synchronized(pi); 1030245274Snp end_synchronized_op(sc, 0); 1031218792Snp} 1032218792Snp 1033218792Snpstatic int 1034218792Snpcxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data) 1035218792Snp{ 1036218792Snp int rc = 0, mtu, flags; 1037218792Snp struct port_info *pi = ifp->if_softc; 1038218792Snp struct adapter *sc = pi->adapter; 1039218792Snp struct ifreq *ifr = (struct ifreq *)data; 1040218792Snp uint32_t mask; 1041218792Snp 1042218792Snp switch (cmd) { 1043218792Snp case SIOCSIFMTU: 1044245274Snp mtu = ifr->ifr_mtu; 1045245274Snp if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) 1046245274Snp return (EINVAL); 1047245274Snp 1048245274Snp rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu"); 1049245274Snp if (rc) 1050218792Snp return (rc); 1051245274Snp ifp->if_mtu = mtu; 1052245274Snp if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1053245274Snp t4_update_fl_bufsize(ifp); 1054245274Snp rc = update_mac_settings(pi, XGMAC_MTU); 1055218792Snp } 1056245274Snp end_synchronized_op(sc, 0); 1057218792Snp break; 1058218792Snp 1059218792Snp case SIOCSIFFLAGS: 1060245274Snp rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg"); 1061245274Snp if (rc) 1062245274Snp return (rc); 1063245274Snp 1064218792Snp if (ifp->if_flags & IFF_UP) { 1065218792Snp if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1066218792Snp flags = pi->if_flags; 1067218792Snp if ((ifp->if_flags ^ flags) & 1068218792Snp (IFF_PROMISC | IFF_ALLMULTI)) { 1069218792Snp rc = update_mac_settings(pi, 1070218792Snp XGMAC_PROMISC | XGMAC_ALLMULTI); 1071218792Snp } 1072218792Snp } else 1073245274Snp rc = cxgbe_init_synchronized(pi); 1074218792Snp pi->if_flags = ifp->if_flags; 1075218792Snp } else if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1076245274Snp rc = cxgbe_uninit_synchronized(pi); 1077245274Snp end_synchronized_op(sc, 0); 1078218792Snp break; 1079218792Snp 1080218792Snp case SIOCADDMULTI: 1081245274Snp case SIOCDELMULTI: /* these two are called with a mutex held :-( */ 1082245274Snp rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi"); 1083218792Snp if (rc) 1084245274Snp return (rc); 1085245274Snp if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1086218792Snp rc = update_mac_settings(pi, XGMAC_MCADDRS); 1087245274Snp end_synchronized_op(sc, LOCK_HELD); 1088218792Snp break; 1089218792Snp 1090218792Snp case SIOCSIFCAP: 1091245274Snp rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap"); 1092218792Snp if (rc) 1093245274Snp return (rc); 1094218792Snp 1095218792Snp mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1096218792Snp if (mask & IFCAP_TXCSUM) { 1097218792Snp ifp->if_capenable ^= IFCAP_TXCSUM; 1098218792Snp ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 1099218792Snp 1100237831Snp if (IFCAP_TSO4 & ifp->if_capenable && 1101218792Snp !(IFCAP_TXCSUM & ifp->if_capenable)) { 1102237799Snp ifp->if_capenable &= ~IFCAP_TSO4; 1103218792Snp if_printf(ifp, 1104237831Snp "tso4 disabled due to -txcsum.\n"); 1105218792Snp } 1106218792Snp } 1107237799Snp if (mask & IFCAP_TXCSUM_IPV6) { 1108237799Snp ifp->if_capenable ^= IFCAP_TXCSUM_IPV6; 1109237799Snp ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6); 1110237799Snp 1111237799Snp if (IFCAP_TSO6 & ifp->if_capenable && 1112237799Snp !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1113237799Snp ifp->if_capenable &= ~IFCAP_TSO6; 1114237799Snp if_printf(ifp, 1115237799Snp "tso6 disabled due to -txcsum6.\n"); 1116237799Snp } 1117237799Snp } 1118218792Snp if (mask & IFCAP_RXCSUM) 1119218792Snp ifp->if_capenable ^= IFCAP_RXCSUM; 1120237799Snp if (mask & IFCAP_RXCSUM_IPV6) 1121237799Snp ifp->if_capenable ^= IFCAP_RXCSUM_IPV6; 1122237799Snp 1123237799Snp /* 1124237799Snp * Note that we leave CSUM_TSO alone (it is always set). The 1125237799Snp * kernel takes both IFCAP_TSOx and CSUM_TSO into account before 1126237799Snp * sending a TSO request our way, so it's sufficient to toggle 1127237799Snp * IFCAP_TSOx only. 1128237799Snp */ 1129218792Snp if (mask & IFCAP_TSO4) { 1130237799Snp if (!(IFCAP_TSO4 & ifp->if_capenable) && 1131237799Snp !(IFCAP_TXCSUM & ifp->if_capenable)) { 1132237799Snp if_printf(ifp, "enable txcsum first.\n"); 1133237799Snp rc = EAGAIN; 1134237799Snp goto fail; 1135237799Snp } 1136218792Snp ifp->if_capenable ^= IFCAP_TSO4; 1137218792Snp } 1138237799Snp if (mask & IFCAP_TSO6) { 1139237799Snp if (!(IFCAP_TSO6 & ifp->if_capenable) && 1140237799Snp !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) { 1141237799Snp if_printf(ifp, "enable txcsum6 first.\n"); 1142237799Snp rc = EAGAIN; 1143237799Snp goto fail; 1144237799Snp } 1145237799Snp ifp->if_capenable ^= IFCAP_TSO6; 1146237799Snp } 1147218792Snp if (mask & IFCAP_LRO) { 1148237819Snp#if defined(INET) || defined(INET6) 1149218792Snp int i; 1150218792Snp struct sge_rxq *rxq; 1151218792Snp 1152218792Snp ifp->if_capenable ^= IFCAP_LRO; 1153218792Snp for_each_rxq(pi, i, rxq) { 1154218792Snp if (ifp->if_capenable & IFCAP_LRO) 1155228561Snp rxq->iq.flags |= IQ_LRO_ENABLED; 1156218792Snp else 1157228561Snp rxq->iq.flags &= ~IQ_LRO_ENABLED; 1158218792Snp } 1159218792Snp#endif 1160218792Snp } 1161237263Snp#ifdef TCP_OFFLOAD 1162228561Snp if (mask & IFCAP_TOE) { 1163228561Snp int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE; 1164228561Snp 1165228561Snp rc = toe_capability(pi, enable); 1166228561Snp if (rc != 0) 1167228561Snp goto fail; 1168228561Snp 1169228561Snp ifp->if_capenable ^= mask; 1170218792Snp } 1171218792Snp#endif 1172218792Snp if (mask & IFCAP_VLAN_HWTAGGING) { 1173218792Snp ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1174245274Snp if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1175218792Snp rc = update_mac_settings(pi, XGMAC_VLANEX); 1176218792Snp } 1177218792Snp if (mask & IFCAP_VLAN_MTU) { 1178218792Snp ifp->if_capenable ^= IFCAP_VLAN_MTU; 1179218792Snp 1180218792Snp /* Need to find out how to disable auto-mtu-inflation */ 1181218792Snp } 1182218792Snp if (mask & IFCAP_VLAN_HWTSO) 1183218792Snp ifp->if_capenable ^= IFCAP_VLAN_HWTSO; 1184218792Snp if (mask & IFCAP_VLAN_HWCSUM) 1185218792Snp ifp->if_capenable ^= IFCAP_VLAN_HWCSUM; 1186218792Snp 1187218792Snp#ifdef VLAN_CAPABILITIES 1188218792Snp VLAN_CAPABILITIES(ifp); 1189218792Snp#endif 1190245274Snpfail: 1191245274Snp end_synchronized_op(sc, 0); 1192218792Snp break; 1193218792Snp 1194218792Snp case SIOCSIFMEDIA: 1195218792Snp case SIOCGIFMEDIA: 1196218792Snp ifmedia_ioctl(ifp, ifr, &pi->media, cmd); 1197218792Snp break; 1198218792Snp 1199218792Snp default: 1200218792Snp rc = ether_ioctl(ifp, cmd, data); 1201218792Snp } 1202218792Snp 1203218792Snp return (rc); 1204218792Snp} 1205218792Snp 1206218792Snpstatic int 1207218792Snpcxgbe_transmit(struct ifnet *ifp, struct mbuf *m) 1208218792Snp{ 1209218792Snp struct port_info *pi = ifp->if_softc; 1210218792Snp struct adapter *sc = pi->adapter; 1211218792Snp struct sge_txq *txq = &sc->sge.txq[pi->first_txq]; 1212218792Snp struct buf_ring *br; 1213218792Snp int rc; 1214218792Snp 1215218792Snp M_ASSERTPKTHDR(m); 1216218792Snp 1217228561Snp if (__predict_false(pi->link_cfg.link_ok == 0)) { 1218218792Snp m_freem(m); 1219228561Snp return (ENETDOWN); 1220218792Snp } 1221218792Snp 1222218792Snp if (m->m_flags & M_FLOWID) 1223218792Snp txq += (m->m_pkthdr.flowid % pi->ntxq); 1224220873Snp br = txq->br; 1225218792Snp 1226218792Snp if (TXQ_TRYLOCK(txq) == 0) { 1227228561Snp struct sge_eq *eq = &txq->eq; 1228228561Snp 1229218792Snp /* 1230228561Snp * It is possible that t4_eth_tx finishes up and releases the 1231228561Snp * lock between the TRYLOCK above and the drbr_enqueue here. We 1232228561Snp * need to make sure that this mbuf doesn't just sit there in 1233228561Snp * the drbr. 1234218792Snp */ 1235218792Snp 1236228561Snp rc = drbr_enqueue(ifp, br, m); 1237228561Snp if (rc == 0 && callout_pending(&eq->tx_callout) == 0 && 1238228561Snp !(eq->flags & EQ_DOOMED)) 1239228561Snp callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq); 1240228561Snp return (rc); 1241218792Snp } 1242218792Snp 1243218792Snp /* 1244218792Snp * txq->m is the mbuf that is held up due to a temporary shortage of 1245218792Snp * resources and it should be put on the wire first. Then what's in 1246218792Snp * drbr and finally the mbuf that was just passed in to us. 1247218792Snp * 1248218792Snp * Return code should indicate the fate of the mbuf that was passed in 1249218792Snp * this time. 1250218792Snp */ 1251218792Snp 1252218792Snp TXQ_LOCK_ASSERT_OWNED(txq); 1253218792Snp if (drbr_needs_enqueue(ifp, br) || txq->m) { 1254218792Snp 1255218792Snp /* Queued for transmission. */ 1256218792Snp 1257218792Snp rc = drbr_enqueue(ifp, br, m); 1258218792Snp m = txq->m ? txq->m : drbr_dequeue(ifp, br); 1259218792Snp (void) t4_eth_tx(ifp, txq, m); 1260218792Snp TXQ_UNLOCK(txq); 1261218792Snp return (rc); 1262218792Snp } 1263218792Snp 1264218792Snp /* Direct transmission. */ 1265218792Snp rc = t4_eth_tx(ifp, txq, m); 1266218792Snp if (rc != 0 && txq->m) 1267218792Snp rc = 0; /* held, will be transmitted soon (hopefully) */ 1268218792Snp 1269218792Snp TXQ_UNLOCK(txq); 1270218792Snp return (rc); 1271218792Snp} 1272218792Snp 1273218792Snpstatic void 1274218792Snpcxgbe_qflush(struct ifnet *ifp) 1275218792Snp{ 1276218792Snp struct port_info *pi = ifp->if_softc; 1277220649Snp struct sge_txq *txq; 1278220649Snp int i; 1279220649Snp struct mbuf *m; 1280218792Snp 1281228561Snp /* queues do not exist if !PORT_INIT_DONE. */ 1282228561Snp if (pi->flags & PORT_INIT_DONE) { 1283220649Snp for_each_txq(pi, i, txq) { 1284220649Snp TXQ_LOCK(txq); 1285220649Snp m_freem(txq->m); 1286228561Snp txq->m = NULL; 1287220873Snp while ((m = buf_ring_dequeue_sc(txq->br)) != NULL) 1288220649Snp m_freem(m); 1289220649Snp TXQ_UNLOCK(txq); 1290220649Snp } 1291220649Snp } 1292220649Snp if_qflush(ifp); 1293218792Snp} 1294218792Snp 1295218792Snpstatic int 1296218792Snpcxgbe_media_change(struct ifnet *ifp) 1297218792Snp{ 1298218792Snp struct port_info *pi = ifp->if_softc; 1299218792Snp 1300218792Snp device_printf(pi->dev, "%s unimplemented.\n", __func__); 1301218792Snp 1302218792Snp return (EOPNOTSUPP); 1303218792Snp} 1304218792Snp 1305218792Snpstatic void 1306218792Snpcxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1307218792Snp{ 1308218792Snp struct port_info *pi = ifp->if_softc; 1309218792Snp struct ifmedia_entry *cur = pi->media.ifm_cur; 1310218792Snp int speed = pi->link_cfg.speed; 1311218792Snp int data = (pi->port_type << 8) | pi->mod_type; 1312218792Snp 1313218792Snp if (cur->ifm_data != data) { 1314218792Snp build_medialist(pi); 1315218792Snp cur = pi->media.ifm_cur; 1316218792Snp } 1317218792Snp 1318218792Snp ifmr->ifm_status = IFM_AVALID; 1319218792Snp if (!pi->link_cfg.link_ok) 1320218792Snp return; 1321218792Snp 1322218792Snp ifmr->ifm_status |= IFM_ACTIVE; 1323218792Snp 1324218792Snp /* active and current will differ iff current media is autoselect. */ 1325218792Snp if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO) 1326218792Snp return; 1327218792Snp 1328218792Snp ifmr->ifm_active = IFM_ETHER | IFM_FDX; 1329218792Snp if (speed == SPEED_10000) 1330218792Snp ifmr->ifm_active |= IFM_10G_T; 1331218792Snp else if (speed == SPEED_1000) 1332218792Snp ifmr->ifm_active |= IFM_1000_T; 1333218792Snp else if (speed == SPEED_100) 1334218792Snp ifmr->ifm_active |= IFM_100_TX; 1335218792Snp else if (speed == SPEED_10) 1336218792Snp ifmr->ifm_active |= IFM_10_T; 1337218792Snp else 1338218792Snp KASSERT(0, ("%s: link up but speed unknown (%u)", __func__, 1339218792Snp speed)); 1340218792Snp} 1341218792Snp 1342218792Snpvoid 1343218792Snpt4_fatal_err(struct adapter *sc) 1344218792Snp{ 1345218792Snp t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 1346218792Snp t4_intr_disable(sc); 1347218792Snp log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n", 1348218792Snp device_get_nameunit(sc->dev)); 1349218792Snp} 1350218792Snp 1351218792Snpstatic int 1352248925Snpmap_bars_0_and_4(struct adapter *sc) 1353218792Snp{ 1354218792Snp sc->regs_rid = PCIR_BAR(0); 1355218792Snp sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1356218792Snp &sc->regs_rid, RF_ACTIVE); 1357218792Snp if (sc->regs_res == NULL) { 1358218792Snp device_printf(sc->dev, "cannot map registers.\n"); 1359218792Snp return (ENXIO); 1360218792Snp } 1361218792Snp sc->bt = rman_get_bustag(sc->regs_res); 1362218792Snp sc->bh = rman_get_bushandle(sc->regs_res); 1363218792Snp sc->mmio_len = rman_get_size(sc->regs_res); 1364248925Snp setbit(&sc->doorbells, DOORBELL_KDB); 1365218792Snp 1366218792Snp sc->msix_rid = PCIR_BAR(4); 1367218792Snp sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1368218792Snp &sc->msix_rid, RF_ACTIVE); 1369218792Snp if (sc->msix_res == NULL) { 1370218792Snp device_printf(sc->dev, "cannot map MSI-X BAR.\n"); 1371218792Snp return (ENXIO); 1372218792Snp } 1373218792Snp 1374218792Snp return (0); 1375218792Snp} 1376218792Snp 1377248925Snpstatic int 1378248925Snpmap_bar_2(struct adapter *sc) 1379248925Snp{ 1380248925Snp 1381248925Snp /* 1382248925Snp * T4: only iWARP driver uses the userspace doorbells. There is no need 1383248925Snp * to map it if RDMA is disabled. 1384248925Snp */ 1385248925Snp if (is_t4(sc) && sc->rdmacaps == 0) 1386248925Snp return (0); 1387248925Snp 1388248925Snp sc->udbs_rid = PCIR_BAR(2); 1389248925Snp sc->udbs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, 1390248925Snp &sc->udbs_rid, RF_ACTIVE); 1391248925Snp if (sc->udbs_res == NULL) { 1392248925Snp device_printf(sc->dev, "cannot map doorbell BAR.\n"); 1393248925Snp return (ENXIO); 1394248925Snp } 1395248925Snp sc->udbs_base = rman_get_virtual(sc->udbs_res); 1396248925Snp 1397248925Snp if (is_t5(sc)) { 1398248925Snp setbit(&sc->doorbells, DOORBELL_UDB); 1399248925Snp#if defined(__i386__) || defined(__amd64__) 1400248925Snp if (t5_write_combine) { 1401248925Snp int rc; 1402248925Snp 1403248925Snp /* 1404248925Snp * Enable write combining on BAR2. This is the 1405248925Snp * userspace doorbell BAR and is split into 128B 1406248925Snp * (UDBS_SEG_SIZE) doorbell regions, each associated 1407248925Snp * with an egress queue. The first 64B has the doorbell 1408248925Snp * and the second 64B can be used to submit a tx work 1409248925Snp * request with an implicit doorbell. 1410248925Snp */ 1411248925Snp 1412248925Snp rc = pmap_change_attr((vm_offset_t)sc->udbs_base, 1413248925Snp rman_get_size(sc->udbs_res), PAT_WRITE_COMBINING); 1414248925Snp if (rc == 0) { 1415248925Snp clrbit(&sc->doorbells, DOORBELL_UDB); 1416248925Snp setbit(&sc->doorbells, DOORBELL_WRWC); 1417248925Snp setbit(&sc->doorbells, DOORBELL_UDBWC); 1418248925Snp } else { 1419248925Snp device_printf(sc->dev, 1420248925Snp "couldn't enable write combining: %d\n", 1421248925Snp rc); 1422248925Snp } 1423248925Snp 1424248925Snp t4_write_reg(sc, A_SGE_STAT_CFG, 1425248925Snp V_STATSOURCE_T5(7) | V_STATMODE(0)); 1426248925Snp } 1427248925Snp#endif 1428248925Snp } 1429248925Snp 1430248925Snp return (0); 1431248925Snp} 1432248925Snp 1433248925Snpstatic const struct memwin t4_memwin[] = { 1434248925Snp { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1435248925Snp { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1436248925Snp { MEMWIN2_BASE_T4, MEMWIN2_APERTURE_T4 } 1437248925Snp}; 1438248925Snp 1439248925Snpstatic const struct memwin t5_memwin[] = { 1440248925Snp { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1441248925Snp { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1442248925Snp { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1443248925Snp}; 1444248925Snp 1445218792Snpstatic void 1446218792Snpsetup_memwin(struct adapter *sc) 1447218792Snp{ 1448248925Snp const struct memwin *mw; 1449248925Snp int i, n; 1450237587Snp uint32_t bar0; 1451218792Snp 1452248925Snp if (is_t4(sc)) { 1453248925Snp /* 1454248925Snp * Read low 32b of bar0 indirectly via the hardware backdoor 1455248925Snp * mechanism. Works from within PCI passthrough environments 1456248925Snp * too, where rman_get_start() can return a different value. We 1457248925Snp * need to program the T4 memory window decoders with the actual 1458248925Snp * addresses that will be coming across the PCIe link. 1459248925Snp */ 1460248925Snp bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0)); 1461248925Snp bar0 &= (uint32_t) PCIM_BAR_MEM_BASE; 1462218792Snp 1463248925Snp mw = &t4_memwin[0]; 1464248925Snp n = nitems(t4_memwin); 1465248925Snp } else { 1466248925Snp /* T5 uses the relative offset inside the PCIe BAR */ 1467248925Snp bar0 = 0; 1468218792Snp 1469248925Snp mw = &t5_memwin[0]; 1470248925Snp n = nitems(t5_memwin); 1471248925Snp } 1472218792Snp 1473248925Snp for (i = 0; i < n; i++, mw++) { 1474248925Snp t4_write_reg(sc, 1475248925Snp PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, i), 1476248925Snp (mw->base + bar0) | V_BIR(0) | 1477248925Snp V_WINDOW(ilog2(mw->aperture) - 10)); 1478248925Snp } 1479237587Snp 1480237587Snp /* flush */ 1481237587Snp t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1482218792Snp} 1483218792Snp 1484248925Snp/* 1485248925Snp * Verify that the memory range specified by the addr/len pair is valid and lies 1486248925Snp * entirely within a single region (EDCx or MCx). 1487248925Snp */ 1488218792Snpstatic int 1489248925Snpvalidate_mem_range(struct adapter *sc, uint32_t addr, int len) 1490248925Snp{ 1491248925Snp uint32_t em, addr_len, maddr, mlen; 1492248925Snp 1493248925Snp /* Memory can only be accessed in naturally aligned 4 byte units */ 1494248925Snp if (addr & 3 || len & 3 || len == 0) 1495248925Snp return (EINVAL); 1496248925Snp 1497248925Snp /* Enabled memories */ 1498248925Snp em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1499248925Snp if (em & F_EDRAM0_ENABLE) { 1500248925Snp addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1501248925Snp maddr = G_EDRAM0_BASE(addr_len) << 20; 1502248925Snp mlen = G_EDRAM0_SIZE(addr_len) << 20; 1503248925Snp if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1504248925Snp addr + len <= maddr + mlen) 1505248925Snp return (0); 1506248925Snp } 1507248925Snp if (em & F_EDRAM1_ENABLE) { 1508248925Snp addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1509248925Snp maddr = G_EDRAM1_BASE(addr_len) << 20; 1510248925Snp mlen = G_EDRAM1_SIZE(addr_len) << 20; 1511248925Snp if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1512248925Snp addr + len <= maddr + mlen) 1513248925Snp return (0); 1514248925Snp } 1515248925Snp if (em & F_EXT_MEM_ENABLE) { 1516248925Snp addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1517248925Snp maddr = G_EXT_MEM_BASE(addr_len) << 20; 1518248925Snp mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1519248925Snp if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1520248925Snp addr + len <= maddr + mlen) 1521248925Snp return (0); 1522248925Snp } 1523248925Snp if (!is_t4(sc) && em & F_EXT_MEM1_ENABLE) { 1524248925Snp addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1525248925Snp maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1526248925Snp mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1527248925Snp if (mlen > 0 && addr >= maddr && addr < maddr + mlen && 1528248925Snp addr + len <= maddr + mlen) 1529248925Snp return (0); 1530248925Snp } 1531248925Snp 1532248925Snp return (EFAULT); 1533248925Snp} 1534248925Snp 1535248925Snp/* 1536248925Snp * Verify that the memory range specified by the memtype/offset/len pair is 1537248925Snp * valid and lies entirely within the memtype specified. The global address of 1538248925Snp * the start of the range is returned in addr. 1539248925Snp */ 1540248925Snpstatic int 1541248925Snpvalidate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1542248925Snp uint32_t *addr) 1543248925Snp{ 1544248925Snp uint32_t em, addr_len, maddr, mlen; 1545248925Snp 1546248925Snp /* Memory can only be accessed in naturally aligned 4 byte units */ 1547248925Snp if (off & 3 || len & 3 || len == 0) 1548248925Snp return (EINVAL); 1549248925Snp 1550248925Snp em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1551248925Snp switch (mtype) { 1552248925Snp case MEM_EDC0: 1553248925Snp if (!(em & F_EDRAM0_ENABLE)) 1554248925Snp return (EINVAL); 1555248925Snp addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1556248925Snp maddr = G_EDRAM0_BASE(addr_len) << 20; 1557248925Snp mlen = G_EDRAM0_SIZE(addr_len) << 20; 1558248925Snp break; 1559248925Snp case MEM_EDC1: 1560248925Snp if (!(em & F_EDRAM1_ENABLE)) 1561248925Snp return (EINVAL); 1562248925Snp addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1563248925Snp maddr = G_EDRAM1_BASE(addr_len) << 20; 1564248925Snp mlen = G_EDRAM1_SIZE(addr_len) << 20; 1565248925Snp break; 1566248925Snp case MEM_MC: 1567248925Snp if (!(em & F_EXT_MEM_ENABLE)) 1568248925Snp return (EINVAL); 1569248925Snp addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1570248925Snp maddr = G_EXT_MEM_BASE(addr_len) << 20; 1571248925Snp mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1572248925Snp break; 1573248925Snp case MEM_MC1: 1574248925Snp if (is_t4(sc) || !(em & F_EXT_MEM1_ENABLE)) 1575248925Snp return (EINVAL); 1576248925Snp addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1577248925Snp maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1578248925Snp mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1579248925Snp break; 1580248925Snp default: 1581248925Snp return (EINVAL); 1582248925Snp } 1583248925Snp 1584248925Snp if (mlen > 0 && off < mlen && off + len <= mlen) { 1585248925Snp *addr = maddr + off; /* global address */ 1586248925Snp return (0); 1587248925Snp } 1588248925Snp 1589248925Snp return (EFAULT); 1590248925Snp} 1591248925Snp 1592248925Snpstatic void 1593248925Snpmemwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1594248925Snp{ 1595248925Snp const struct memwin *mw; 1596248925Snp 1597248925Snp if (is_t4(sc)) { 1598248925Snp KASSERT(win >= 0 && win < nitems(t4_memwin), 1599248925Snp ("%s: incorrect memwin# (%d)", __func__, win)); 1600248925Snp mw = &t4_memwin[win]; 1601248925Snp } else { 1602248925Snp KASSERT(win >= 0 && win < nitems(t5_memwin), 1603248925Snp ("%s: incorrect memwin# (%d)", __func__, win)); 1604248925Snp mw = &t5_memwin[win]; 1605248925Snp } 1606248925Snp 1607248925Snp if (base != NULL) 1608248925Snp *base = mw->base; 1609248925Snp if (aperture != NULL) 1610248925Snp *aperture = mw->aperture; 1611248925Snp} 1612248925Snp 1613248925Snp/* 1614248925Snp * Positions the memory window such that it can be used to access the specified 1615248925Snp * address in the chip's address space. The return value is the offset of addr 1616248925Snp * from the start of the window. 1617248925Snp */ 1618248925Snpstatic uint32_t 1619248925Snpposition_memwin(struct adapter *sc, int n, uint32_t addr) 1620248925Snp{ 1621248925Snp uint32_t start, pf; 1622248925Snp uint32_t reg; 1623248925Snp 1624248925Snp KASSERT(n >= 0 && n <= 3, 1625248925Snp ("%s: invalid window %d.", __func__, n)); 1626248925Snp KASSERT((addr & 3) == 0, 1627248925Snp ("%s: addr (0x%x) is not at a 4B boundary.", __func__, addr)); 1628248925Snp 1629248925Snp if (is_t4(sc)) { 1630248925Snp pf = 0; 1631248925Snp start = addr & ~0xf; /* start must be 16B aligned */ 1632248925Snp } else { 1633248925Snp pf = V_PFNUM(sc->pf); 1634248925Snp start = addr & ~0x7f; /* start must be 128B aligned */ 1635248925Snp } 1636248925Snp reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1637248925Snp 1638248925Snp t4_write_reg(sc, reg, start | pf); 1639248925Snp t4_read_reg(sc, reg); 1640248925Snp 1641248925Snp return (addr - start); 1642248925Snp} 1643248925Snp 1644248925Snpstatic int 1645218792Snpcfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 1646218792Snp struct intrs_and_queues *iaq) 1647218792Snp{ 1648228561Snp int rc, itype, navail, nrxq10g, nrxq1g, n; 1649228561Snp int nofldrxq10g = 0, nofldrxq1g = 0; 1650218792Snp 1651218792Snp bzero(iaq, sizeof(*iaq)); 1652218792Snp 1653228561Snp iaq->ntxq10g = t4_ntxq10g; 1654228561Snp iaq->ntxq1g = t4_ntxq1g; 1655228561Snp iaq->nrxq10g = nrxq10g = t4_nrxq10g; 1656228561Snp iaq->nrxq1g = nrxq1g = t4_nrxq1g; 1657237263Snp#ifdef TCP_OFFLOAD 1658237463Snp if (is_offload(sc)) { 1659237463Snp iaq->nofldtxq10g = t4_nofldtxq10g; 1660237463Snp iaq->nofldtxq1g = t4_nofldtxq1g; 1661237463Snp iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g; 1662237463Snp iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g; 1663237463Snp } 1664228561Snp#endif 1665228561Snp 1666219944Snp for (itype = INTR_MSIX; itype; itype >>= 1) { 1667218792Snp 1668228561Snp if ((itype & t4_intr_types) == 0) 1669218792Snp continue; /* not allowed */ 1670218792Snp 1671219944Snp if (itype == INTR_MSIX) 1672218792Snp navail = pci_msix_count(sc->dev); 1673219944Snp else if (itype == INTR_MSI) 1674218792Snp navail = pci_msi_count(sc->dev); 1675218792Snp else 1676218792Snp navail = 1; 1677228561Snprestart: 1678218792Snp if (navail == 0) 1679218792Snp continue; 1680218792Snp 1681218792Snp iaq->intr_type = itype; 1682228561Snp iaq->intr_flags = 0; 1683218792Snp 1684228561Snp /* 1685228561Snp * Best option: an interrupt vector for errors, one for the 1686228561Snp * firmware event queue, and one each for each rxq (NIC as well 1687228561Snp * as offload). 1688228561Snp */ 1689228561Snp iaq->nirq = T4_EXTRA_INTR; 1690228561Snp iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 1691228561Snp iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 1692228561Snp if (iaq->nirq <= navail && 1693228561Snp (itype != INTR_MSI || powerof2(iaq->nirq))) { 1694228561Snp iaq->intr_flags |= INTR_DIRECT; 1695228561Snp goto allocate; 1696228561Snp } 1697218792Snp 1698228561Snp /* 1699228561Snp * Second best option: an interrupt vector for errors, one for 1700228561Snp * the firmware event queue, and one each for either NIC or 1701228561Snp * offload rxq's. 1702228561Snp */ 1703228561Snp iaq->nirq = T4_EXTRA_INTR; 1704228561Snp iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 1705228561Snp iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 1706228561Snp if (iaq->nirq <= navail && 1707228561Snp (itype != INTR_MSI || powerof2(iaq->nirq))) 1708228561Snp goto allocate; 1709218792Snp 1710228561Snp /* 1711228561Snp * Next best option: an interrupt vector for errors, one for the 1712228561Snp * firmware event queue, and at least one per port. At this 1713228561Snp * point we know we'll have to downsize nrxq or nofldrxq to fit 1714228561Snp * what's available to us. 1715228561Snp */ 1716228561Snp iaq->nirq = T4_EXTRA_INTR; 1717228561Snp iaq->nirq += n10g + n1g; 1718228561Snp if (iaq->nirq <= navail) { 1719228561Snp int leftover = navail - iaq->nirq; 1720218792Snp 1721228561Snp if (n10g > 0) { 1722228561Snp int target = max(nrxq10g, nofldrxq10g); 1723219944Snp 1724228561Snp n = 1; 1725228561Snp while (n < target && leftover >= n10g) { 1726228561Snp leftover -= n10g; 1727228561Snp iaq->nirq += n10g; 1728228561Snp n++; 1729228561Snp } 1730228561Snp iaq->nrxq10g = min(n, nrxq10g); 1731237263Snp#ifdef TCP_OFFLOAD 1732237463Snp if (is_offload(sc)) 1733237463Snp iaq->nofldrxq10g = min(n, nofldrxq10g); 1734228561Snp#endif 1735228561Snp } 1736218792Snp 1737228561Snp if (n1g > 0) { 1738228561Snp int target = max(nrxq1g, nofldrxq1g); 1739219944Snp 1740228561Snp n = 1; 1741228561Snp while (n < target && leftover >= n1g) { 1742228561Snp leftover -= n1g; 1743228561Snp iaq->nirq += n1g; 1744228561Snp n++; 1745219944Snp } 1746228561Snp iaq->nrxq1g = min(n, nrxq1g); 1747237263Snp#ifdef TCP_OFFLOAD 1748237463Snp if (is_offload(sc)) 1749237463Snp iaq->nofldrxq1g = min(n, nofldrxq1g); 1750228561Snp#endif 1751219944Snp } 1752219944Snp 1753228561Snp if (itype != INTR_MSI || powerof2(iaq->nirq)) 1754228561Snp goto allocate; 1755218792Snp } 1756218792Snp 1757228561Snp /* 1758228561Snp * Least desirable option: one interrupt vector for everything. 1759228561Snp */ 1760228561Snp iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 1761237263Snp#ifdef TCP_OFFLOAD 1762237463Snp if (is_offload(sc)) 1763237463Snp iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 1764228561Snp#endif 1765228561Snp 1766228561Snpallocate: 1767218792Snp navail = iaq->nirq; 1768218792Snp rc = 0; 1769219944Snp if (itype == INTR_MSIX) 1770218792Snp rc = pci_alloc_msix(sc->dev, &navail); 1771219944Snp else if (itype == INTR_MSI) 1772218792Snp rc = pci_alloc_msi(sc->dev, &navail); 1773218792Snp 1774218792Snp if (rc == 0) { 1775218792Snp if (navail == iaq->nirq) 1776218792Snp return (0); 1777218792Snp 1778218792Snp /* 1779218792Snp * Didn't get the number requested. Use whatever number 1780218792Snp * the kernel is willing to allocate (it's in navail). 1781218792Snp */ 1782228561Snp device_printf(sc->dev, "fewer vectors than requested, " 1783228561Snp "type=%d, req=%d, rcvd=%d; will downshift req.\n", 1784228561Snp itype, iaq->nirq, navail); 1785218792Snp pci_release_msi(sc->dev); 1786228561Snp goto restart; 1787218792Snp } 1788218792Snp 1789218792Snp device_printf(sc->dev, 1790218792Snp "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", 1791218792Snp itype, rc, iaq->nirq, navail); 1792218792Snp } 1793218792Snp 1794218792Snp device_printf(sc->dev, 1795218792Snp "failed to find a usable interrupt type. " 1796228561Snp "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, 1797218792Snp pci_msix_count(sc->dev), pci_msi_count(sc->dev)); 1798218792Snp 1799218792Snp return (ENXIO); 1800218792Snp} 1801218792Snp 1802248925Snp#define FW_VERSION(chip) ( \ 1803248925Snp V_FW_HDR_FW_VER_MAJOR(FW_VERSION_MAJOR_##chip) | \ 1804248925Snp V_FW_HDR_FW_VER_MINOR(FW_VERSION_MINOR_##chip) | \ 1805248925Snp V_FW_HDR_FW_VER_MICRO(FW_VERSION_MICRO_##chip) | \ 1806248925Snp V_FW_HDR_FW_VER_BUILD(FW_VERSION_BUILD_##chip)) 1807248925Snp#define FW_INTFVER(chip, intf) (FW_HDR_INTFVER_##intf) 1808248925Snp 1809248925Snpstruct fw_info { 1810248925Snp uint8_t chip; 1811248925Snp char *kld_name; 1812248925Snp char *fw_mod_name; 1813248925Snp struct fw_hdr fw_hdr; /* XXX: waste of space, need a sparse struct */ 1814248925Snp} fw_info[] = { 1815248925Snp { 1816248925Snp .chip = CHELSIO_T4, 1817248925Snp .kld_name = "t4fw_cfg", 1818248925Snp .fw_mod_name = "t4fw", 1819248925Snp .fw_hdr = { 1820248925Snp .chip = FW_HDR_CHIP_T4, 1821248925Snp .fw_ver = htobe32_const(FW_VERSION(T4)), 1822248925Snp .intfver_nic = FW_INTFVER(T4, NIC), 1823248925Snp .intfver_vnic = FW_INTFVER(T4, VNIC), 1824248925Snp .intfver_ofld = FW_INTFVER(T4, OFLD), 1825248925Snp .intfver_ri = FW_INTFVER(T4, RI), 1826248925Snp .intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU), 1827248925Snp .intfver_iscsi = FW_INTFVER(T4, ISCSI), 1828248925Snp .intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU), 1829248925Snp .intfver_fcoe = FW_INTFVER(T4, FCOE), 1830248925Snp }, 1831248925Snp }, { 1832248925Snp .chip = CHELSIO_T5, 1833248925Snp .kld_name = "t5fw_cfg", 1834248925Snp .fw_mod_name = "t5fw", 1835248925Snp .fw_hdr = { 1836248925Snp .chip = FW_HDR_CHIP_T5, 1837248925Snp .fw_ver = htobe32_const(FW_VERSION(T5)), 1838248925Snp .intfver_nic = FW_INTFVER(T5, NIC), 1839248925Snp .intfver_vnic = FW_INTFVER(T5, VNIC), 1840248925Snp .intfver_ofld = FW_INTFVER(T5, OFLD), 1841248925Snp .intfver_ri = FW_INTFVER(T5, RI), 1842248925Snp .intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU), 1843248925Snp .intfver_iscsi = FW_INTFVER(T5, ISCSI), 1844248925Snp .intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU), 1845248925Snp .intfver_fcoe = FW_INTFVER(T5, FCOE), 1846248925Snp }, 1847248925Snp } 1848248925Snp}; 1849248925Snp 1850248925Snpstatic struct fw_info * 1851248925Snpfind_fw_info(int chip) 1852248925Snp{ 1853248925Snp int i; 1854248925Snp 1855248925Snp for (i = 0; i < nitems(fw_info); i++) { 1856248925Snp if (fw_info[i].chip == chip) 1857248925Snp return (&fw_info[i]); 1858248925Snp } 1859248925Snp return (NULL); 1860248925Snp} 1861248925Snp 1862218792Snp/* 1863248925Snp * Is the given firmware API compatible with the one the driver was compiled 1864248925Snp * with? 1865247347Snp */ 1866247347Snpstatic int 1867248925Snpfw_compatible(const struct fw_hdr *hdr1, const struct fw_hdr *hdr2) 1868247347Snp{ 1869247347Snp 1870248925Snp /* short circuit if it's the exact same firmware version */ 1871248925Snp if (hdr1->chip == hdr2->chip && hdr1->fw_ver == hdr2->fw_ver) 1872247347Snp return (1); 1873247347Snp 1874247347Snp /* 1875247347Snp * XXX: Is this too conservative? Perhaps I should limit this to the 1876247347Snp * features that are supported in the driver. 1877247347Snp */ 1878248925Snp#define SAME_INTF(x) (hdr1->intfver_##x == hdr2->intfver_##x) 1879248925Snp if (hdr1->chip == hdr2->chip && SAME_INTF(nic) && SAME_INTF(vnic) && 1880248925Snp SAME_INTF(ofld) && SAME_INTF(ri) && SAME_INTF(iscsipdu) && 1881248925Snp SAME_INTF(iscsi) && SAME_INTF(fcoepdu) && SAME_INTF(fcoe)) 1882247347Snp return (1); 1883248925Snp#undef SAME_INTF 1884247347Snp 1885247347Snp return (0); 1886247347Snp} 1887247347Snp 1888247347Snp/* 1889249376Snp * The firmware in the KLD is usable and can be installed. But should it be? 1890249376Snp * This routine explains itself in detail if it indicates the KLD firmware 1891249376Snp * should be installed. 1892249376Snp */ 1893249376Snpstatic int 1894249376Snpshould_install_kld_fw(struct adapter *sc, int card_fw_usable, int k, int c) 1895249376Snp{ 1896249376Snp const char *reason; 1897249376Snp 1898249376Snp KASSERT(t4_fw_install != 0, ("%s: Can't install; shouldn't be asked " 1899249376Snp "to evaluate if install is a good idea.", __func__)); 1900249376Snp 1901249376Snp if (!card_fw_usable) { 1902249376Snp reason = "incompatible or unusable"; 1903249376Snp goto install; 1904249376Snp } 1905249376Snp 1906249376Snp if (k > c) { 1907249376Snp reason = "older than the version bundled with this driver"; 1908249376Snp goto install; 1909249376Snp } 1910249376Snp 1911249376Snp if (t4_fw_install == 2 && k != c) { 1912249376Snp reason = "different than the version bundled with this driver"; 1913249376Snp goto install; 1914249376Snp } 1915249376Snp 1916249376Snp return (0); 1917249376Snp 1918249376Snpinstall: 1919249376Snp device_printf(sc->dev, "firmware on card (%u.%u.%u.%u) is %s, " 1920249376Snp "installing firmware %u.%u.%u.%u on card.\n", 1921249376Snp G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 1922249376Snp G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), reason, 1923249376Snp G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 1924249376Snp G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 1925249376Snp 1926249376Snp return (1); 1927249376Snp} 1928249376Snp/* 1929248925Snp * Establish contact with the firmware and determine if we are the master driver 1930248925Snp * or not, and whether we are responsible for chip initialization. 1931218792Snp */ 1932218792Snpstatic int 1933218792Snpprep_firmware(struct adapter *sc) 1934218792Snp{ 1935248925Snp const struct firmware *fw = NULL, *default_cfg; 1936248925Snp int rc, pf, card_fw_usable, kld_fw_usable, need_fw_reset = 1; 1937218792Snp enum dev_state state; 1938248925Snp struct fw_info *fw_info; 1939248925Snp struct fw_hdr *card_fw; /* fw on the card */ 1940248925Snp const struct fw_hdr *kld_fw; /* fw in the KLD */ 1941248925Snp const struct fw_hdr *drv_fw; /* fw header the driver was compiled 1942248925Snp against */ 1943218792Snp 1944248925Snp /* Contact firmware. */ 1945248925Snp rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state); 1946248925Snp if (rc < 0 || state == DEV_STATE_ERR) { 1947248925Snp rc = -rc; 1948248925Snp device_printf(sc->dev, 1949248925Snp "failed to connect to the firmware: %d, %d.\n", rc, state); 1950248925Snp return (rc); 1951248925Snp } 1952248925Snp pf = rc; 1953248925Snp if (pf == sc->mbox) 1954248925Snp sc->flags |= MASTER_PF; 1955248925Snp else if (state == DEV_STATE_UNINIT) { 1956248925Snp /* 1957248925Snp * We didn't get to be the master so we definitely won't be 1958248925Snp * configuring the chip. It's a bug if someone else hasn't 1959248925Snp * configured it already. 1960248925Snp */ 1961248925Snp device_printf(sc->dev, "couldn't be master(%d), " 1962248925Snp "device not already initialized either(%d).\n", rc, state); 1963248925Snp return (EDOOFUS); 1964248925Snp } 1965228561Snp 1966248925Snp /* This is the firmware whose headers the driver was compiled against */ 1967248925Snp fw_info = find_fw_info(chip_id(sc)); 1968248925Snp if (fw_info == NULL) { 1969248925Snp device_printf(sc->dev, 1970248925Snp "unable to look up firmware information for chip %d.\n", 1971248925Snp chip_id(sc)); 1972248925Snp return (EINVAL); 1973248925Snp } 1974248925Snp drv_fw = &fw_info->fw_hdr; 1975248925Snp 1976248925Snp /* 1977248925Snp * The firmware KLD contains many modules. The KLD name is also the 1978248925Snp * name of the module that contains the default config file. 1979248925Snp */ 1980248925Snp default_cfg = firmware_get(fw_info->kld_name); 1981248925Snp 1982247347Snp /* Read the header of the firmware on the card */ 1983247347Snp card_fw = malloc(sizeof(*card_fw), M_CXGBE, M_ZERO | M_WAITOK); 1984247347Snp rc = -t4_read_flash(sc, FLASH_FW_START, 1985247347Snp sizeof (*card_fw) / sizeof (uint32_t), (uint32_t *)card_fw, 1); 1986247347Snp if (rc == 0) 1987248925Snp card_fw_usable = fw_compatible(drv_fw, (const void*)card_fw); 1988247347Snp else { 1989247347Snp device_printf(sc->dev, 1990247347Snp "Unable to read card's firmware header: %d\n", rc); 1991247347Snp card_fw_usable = 0; 1992247347Snp } 1993218792Snp 1994247347Snp /* This is the firmware in the KLD */ 1995248925Snp fw = firmware_get(fw_info->fw_mod_name); 1996247347Snp if (fw != NULL) { 1997247347Snp kld_fw = (const void *)fw->data; 1998248925Snp kld_fw_usable = fw_compatible(drv_fw, kld_fw); 1999247347Snp } else { 2000247347Snp kld_fw = NULL; 2001247347Snp kld_fw_usable = 0; 2002247347Snp } 2003219287Snp 2004248925Snp if (card_fw_usable && card_fw->fw_ver == drv_fw->fw_ver && 2005248925Snp (!kld_fw_usable || kld_fw->fw_ver == drv_fw->fw_ver || 2006248925Snp t4_fw_install == 0)) { 2007248925Snp /* 2008248925Snp * Common case: the firmware on the card is an exact match and 2009248925Snp * the KLD is an exact match too, or the KLD is 2010248925Snp * absent/incompatible, or we're prohibited from using it. Note 2011248925Snp * that t4_fw_install = 2 is ignored here -- use cxgbetool 2012248925Snp * loadfw if you want to reinstall the same firmware as the one 2013248925Snp * on the card. 2014248925Snp */ 2015248925Snp } else if (kld_fw_usable && state == DEV_STATE_UNINIT && 2016249376Snp should_install_kld_fw(sc, card_fw_usable, be32toh(kld_fw->fw_ver), 2017249376Snp be32toh(card_fw->fw_ver))) { 2018219287Snp 2019247347Snp rc = -t4_load_fw(sc, fw->data, fw->datasize); 2020247347Snp if (rc != 0) { 2021247347Snp device_printf(sc->dev, 2022247347Snp "failed to install firmware: %d\n", rc); 2023228561Snp goto done; 2024219287Snp } 2025219287Snp 2026247347Snp /* Installed successfully, update the cached header too. */ 2027247347Snp memcpy(card_fw, kld_fw, sizeof(*card_fw)); 2028247347Snp card_fw_usable = 1; 2029248925Snp need_fw_reset = 0; /* already reset as part of load_fw */ 2030247347Snp } 2031219287Snp 2032247347Snp if (!card_fw_usable) { 2033248925Snp uint32_t d, c, k; 2034247347Snp 2035248925Snp d = ntohl(drv_fw->fw_ver); 2036247347Snp c = ntohl(card_fw->fw_ver); 2037247347Snp k = kld_fw ? ntohl(kld_fw->fw_ver) : 0; 2038247347Snp 2039247347Snp device_printf(sc->dev, "Cannot find a usable firmware: " 2040248925Snp "fw_install %d, chip state %d, " 2041248925Snp "driver compiled with %d.%d.%d.%d, " 2042247347Snp "card has %d.%d.%d.%d, KLD has %d.%d.%d.%d\n", 2043248925Snp t4_fw_install, state, 2044248925Snp G_FW_HDR_FW_VER_MAJOR(d), G_FW_HDR_FW_VER_MINOR(d), 2045248925Snp G_FW_HDR_FW_VER_MICRO(d), G_FW_HDR_FW_VER_BUILD(d), 2046247347Snp G_FW_HDR_FW_VER_MAJOR(c), G_FW_HDR_FW_VER_MINOR(c), 2047247347Snp G_FW_HDR_FW_VER_MICRO(c), G_FW_HDR_FW_VER_BUILD(c), 2048247347Snp G_FW_HDR_FW_VER_MAJOR(k), G_FW_HDR_FW_VER_MINOR(k), 2049247347Snp G_FW_HDR_FW_VER_MICRO(k), G_FW_HDR_FW_VER_BUILD(k)); 2050248925Snp rc = EINVAL; 2051247347Snp goto done; 2052218792Snp } 2053218792Snp 2054247347Snp /* We're using whatever's on the card and it's known to be good. */ 2055247347Snp sc->params.fw_vers = ntohl(card_fw->fw_ver); 2056247347Snp snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", 2057247347Snp G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2058247347Snp G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2059247347Snp G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2060247347Snp G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2061247347Snp 2062218792Snp /* Reset device */ 2063248925Snp if (need_fw_reset && 2064248925Snp (rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST)) != 0) { 2065218792Snp device_printf(sc->dev, "firmware reset failed: %d.\n", rc); 2066218792Snp if (rc != ETIMEDOUT && rc != EIO) 2067218792Snp t4_fw_bye(sc, sc->mbox); 2068228561Snp goto done; 2069218792Snp } 2070248925Snp sc->flags |= FW_OK; 2071218792Snp 2072248925Snp rc = get_params__pre_init(sc); 2073248925Snp if (rc != 0) 2074248925Snp goto done; /* error message displayed already */ 2075248925Snp 2076228561Snp /* Partition adapter resources as specified in the config file. */ 2077248925Snp if (state == DEV_STATE_UNINIT) { 2078228561Snp 2079248925Snp KASSERT(sc->flags & MASTER_PF, 2080248925Snp ("%s: trying to change chip settings when not master.", 2081248925Snp __func__)); 2082228561Snp 2083248925Snp rc = partition_resources(sc, default_cfg, fw_info->kld_name); 2084228561Snp if (rc != 0) 2085228561Snp goto done; /* error message displayed already */ 2086248925Snp 2087248925Snp t4_tweak_chip_settings(sc); 2088248925Snp 2089248925Snp /* get basic stuff going */ 2090248925Snp rc = -t4_fw_initialize(sc, sc->mbox); 2091248925Snp if (rc != 0) { 2092248925Snp device_printf(sc->dev, "fw init failed: %d.\n", rc); 2093248925Snp goto done; 2094248925Snp } 2095245936Snp } else { 2096248925Snp snprintf(sc->cfg_file, sizeof(sc->cfg_file), "pf%d", pf); 2097248925Snp sc->cfcsum = 0; 2098228561Snp } 2099228561Snp 2100228561Snpdone: 2101247347Snp free(card_fw, M_CXGBE); 2102228561Snp if (fw != NULL) 2103228561Snp firmware_put(fw, FIRMWARE_UNLOAD); 2104228561Snp if (default_cfg != NULL) 2105228561Snp firmware_put(default_cfg, FIRMWARE_UNLOAD); 2106228561Snp 2107228561Snp return (rc); 2108218792Snp} 2109218792Snp 2110228561Snp#define FW_PARAM_DEV(param) \ 2111228561Snp (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 2112228561Snp V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 2113228561Snp#define FW_PARAM_PFVF(param) \ 2114228561Snp (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 2115228561Snp V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 2116228561Snp 2117228561Snp/* 2118248925Snp * Partition chip resources for use between various PFs, VFs, etc. 2119228561Snp */ 2120218792Snpstatic int 2121248925Snppartition_resources(struct adapter *sc, const struct firmware *default_cfg, 2122248925Snp const char *name_prefix) 2123222551Snp{ 2124248925Snp const struct firmware *cfg = NULL; 2125248925Snp int rc = 0; 2126248925Snp struct fw_caps_config_cmd caps; 2127248925Snp uint32_t mtype, moff, finicsum, cfcsum; 2128222551Snp 2129248925Snp /* 2130248925Snp * Figure out what configuration file to use. Pick the default config 2131248925Snp * file for the card if the user hasn't specified one explicitly. 2132248925Snp */ 2133248925Snp snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", t4_cfg_file); 2134248925Snp if (strncmp(t4_cfg_file, DEFAULT_CF, sizeof(t4_cfg_file)) == 0) { 2135248925Snp /* Card specific overrides go here. */ 2136248925Snp if (pci_get_device(sc->dev) == 0x440a) 2137248925Snp snprintf(sc->cfg_file, sizeof(sc->cfg_file), UWIRE_CF); 2138249376Snp if (is_fpga(sc)) 2139249376Snp snprintf(sc->cfg_file, sizeof(sc->cfg_file), FPGA_CF); 2140222551Snp } 2141222551Snp 2142248925Snp /* 2143248925Snp * We need to load another module if the profile is anything except 2144248925Snp * "default" or "flash". 2145248925Snp */ 2146248925Snp if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) != 0 && 2147248925Snp strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2148248925Snp char s[32]; 2149248925Snp 2150248925Snp snprintf(s, sizeof(s), "%s_%s", name_prefix, sc->cfg_file); 2151248925Snp cfg = firmware_get(s); 2152248925Snp if (cfg == NULL) { 2153248925Snp if (default_cfg != NULL) { 2154249376Snp device_printf(sc->dev, 2155249376Snp "unable to load module \"%s\" for " 2156249376Snp "configuration profile \"%s\", will use " 2157249376Snp "the default config file instead.\n", 2158249376Snp s, sc->cfg_file); 2159248925Snp snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2160248925Snp "%s", DEFAULT_CF); 2161248925Snp } else { 2162249376Snp device_printf(sc->dev, 2163249376Snp "unable to load module \"%s\" for " 2164249376Snp "configuration profile \"%s\", will use " 2165249376Snp "the config file on the card's flash " 2166249376Snp "instead.\n", s, sc->cfg_file); 2167248925Snp snprintf(sc->cfg_file, sizeof(sc->cfg_file), 2168248925Snp "%s", FLASH_CF); 2169248925Snp } 2170248925Snp } 2171228561Snp } 2172222551Snp 2173248925Snp if (strncmp(sc->cfg_file, DEFAULT_CF, sizeof(sc->cfg_file)) == 0 && 2174248925Snp default_cfg == NULL) { 2175228561Snp device_printf(sc->dev, 2176248925Snp "default config file not available, will use the config " 2177248925Snp "file on the card's flash instead.\n"); 2178248925Snp snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", FLASH_CF); 2179228561Snp } 2180228561Snp 2181248925Snp if (strncmp(sc->cfg_file, FLASH_CF, sizeof(sc->cfg_file)) != 0) { 2182248925Snp u_int cflen, i, n; 2183248925Snp const uint32_t *cfdata; 2184248925Snp uint32_t param, val, addr, off, mw_base, mw_aperture; 2185228561Snp 2186248925Snp KASSERT(cfg != NULL || default_cfg != NULL, 2187248925Snp ("%s: no config to upload", __func__)); 2188228561Snp 2189248925Snp /* 2190248925Snp * Ask the firmware where it wants us to upload the config file. 2191248925Snp */ 2192248925Snp param = FW_PARAM_DEV(CF); 2193248925Snp rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2194248925Snp if (rc != 0) { 2195248925Snp /* No support for config file? Shouldn't happen. */ 2196248925Snp device_printf(sc->dev, 2197248925Snp "failed to query config file location: %d.\n", rc); 2198248925Snp goto done; 2199248925Snp } 2200248925Snp mtype = G_FW_PARAMS_PARAM_Y(val); 2201248925Snp moff = G_FW_PARAMS_PARAM_Z(val) << 16; 2202228561Snp 2203248925Snp /* 2204248925Snp * XXX: sheer laziness. We deliberately added 4 bytes of 2205248925Snp * useless stuffing/comments at the end of the config file so 2206248925Snp * it's ok to simply throw away the last remaining bytes when 2207248925Snp * the config file is not an exact multiple of 4. This also 2208248925Snp * helps with the validate_mt_off_len check. 2209248925Snp */ 2210248925Snp if (cfg != NULL) { 2211248925Snp cflen = cfg->datasize & ~3; 2212248925Snp cfdata = cfg->data; 2213248925Snp } else { 2214248925Snp cflen = default_cfg->datasize & ~3; 2215248925Snp cfdata = default_cfg->data; 2216248925Snp } 2217222551Snp 2218248925Snp if (cflen > FLASH_CFG_MAX_SIZE) { 2219248925Snp device_printf(sc->dev, 2220248925Snp "config file too long (%d, max allowed is %d). " 2221248925Snp "Will try to use the config on the card, if any.\n", 2222248925Snp cflen, FLASH_CFG_MAX_SIZE); 2223248925Snp goto use_config_on_flash; 2224248925Snp } 2225218792Snp 2226248925Snp rc = validate_mt_off_len(sc, mtype, moff, cflen, &addr); 2227248925Snp if (rc != 0) { 2228248925Snp device_printf(sc->dev, 2229248925Snp "%s: addr (%d/0x%x) or len %d is not valid: %d. " 2230248925Snp "Will try to use the config on the card, if any.\n", 2231248925Snp __func__, mtype, moff, cflen, rc); 2232248925Snp goto use_config_on_flash; 2233248925Snp } 2234248925Snp 2235248925Snp memwin_info(sc, 2, &mw_base, &mw_aperture); 2236248925Snp while (cflen) { 2237248925Snp off = position_memwin(sc, 2, addr); 2238248925Snp n = min(cflen, mw_aperture - off); 2239248925Snp for (i = 0; i < n; i += 4) 2240248925Snp t4_write_reg(sc, mw_base + off + i, *cfdata++); 2241248925Snp cflen -= n; 2242248925Snp addr += n; 2243248925Snp } 2244248925Snp } else { 2245248925Snpuse_config_on_flash: 2246228561Snp mtype = FW_MEMTYPE_CF_FLASH; 2247248925Snp moff = t4_flash_cfg_addr(sc); 2248228561Snp } 2249228561Snp 2250228561Snp bzero(&caps, sizeof(caps)); 2251228561Snp caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2252218792Snp F_FW_CMD_REQUEST | F_FW_CMD_READ); 2253228561Snp caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID | 2254228561Snp V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 2255248925Snp V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(moff >> 16) | FW_LEN16(caps)); 2256228561Snp rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2257228561Snp if (rc != 0) { 2258228561Snp device_printf(sc->dev, 2259249376Snp "failed to pre-process config file: %d " 2260249376Snp "(mtype %d, moff 0x%x).\n", rc, mtype, moff); 2261248925Snp goto done; 2262228561Snp } 2263218792Snp 2264228561Snp finicsum = be32toh(caps.finicsum); 2265228561Snp cfcsum = be32toh(caps.cfcsum); 2266228561Snp if (finicsum != cfcsum) { 2267228561Snp device_printf(sc->dev, 2268228561Snp "WARNING: config file checksum mismatch: %08x %08x\n", 2269228561Snp finicsum, cfcsum); 2270228561Snp } 2271228561Snp sc->cfcsum = cfcsum; 2272218792Snp 2273228561Snp#define LIMIT_CAPS(x) do { \ 2274228561Snp caps.x &= htobe16(t4_##x##_allowed); \ 2275228561Snp sc->x = htobe16(caps.x); \ 2276228561Snp} while (0) 2277228561Snp 2278228561Snp /* 2279228561Snp * Let the firmware know what features will (not) be used so it can tune 2280228561Snp * things accordingly. 2281228561Snp */ 2282228561Snp LIMIT_CAPS(linkcaps); 2283228561Snp LIMIT_CAPS(niccaps); 2284228561Snp LIMIT_CAPS(toecaps); 2285228561Snp LIMIT_CAPS(rdmacaps); 2286228561Snp LIMIT_CAPS(iscsicaps); 2287228561Snp LIMIT_CAPS(fcoecaps); 2288228561Snp#undef LIMIT_CAPS 2289228561Snp 2290228561Snp caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2291218792Snp F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 2292228561Snp caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2293228561Snp rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL); 2294228561Snp if (rc != 0) { 2295228561Snp device_printf(sc->dev, 2296228561Snp "failed to process config file: %d.\n", rc); 2297228561Snp } 2298248925Snpdone: 2299248925Snp if (cfg != NULL) 2300248925Snp firmware_put(cfg, FIRMWARE_UNLOAD); 2301248925Snp return (rc); 2302218792Snp} 2303218792Snp 2304228561Snp/* 2305248925Snp * Retrieve parameters that are needed (or nice to have) very early. 2306228561Snp */ 2307218792Snpstatic int 2308228561Snpget_params__pre_init(struct adapter *sc) 2309218792Snp{ 2310218792Snp int rc; 2311228561Snp uint32_t param[2], val[2]; 2312228561Snp struct fw_devlog_cmd cmd; 2313228561Snp struct devlog_params *dlog = &sc->params.devlog; 2314218792Snp 2315228561Snp param[0] = FW_PARAM_DEV(PORTVEC); 2316228561Snp param[1] = FW_PARAM_DEV(CCLK); 2317228561Snp rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2318218792Snp if (rc != 0) { 2319218792Snp device_printf(sc->dev, 2320228561Snp "failed to query parameters (pre_init): %d.\n", rc); 2321228561Snp return (rc); 2322218792Snp } 2323218792Snp 2324218792Snp sc->params.portvec = val[0]; 2325240452Snp sc->params.nports = bitcount32(val[0]); 2326228561Snp sc->params.vpd.cclk = val[1]; 2327218792Snp 2328228561Snp /* Read device log parameters. */ 2329228561Snp bzero(&cmd, sizeof(cmd)); 2330228561Snp cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) | 2331228561Snp F_FW_CMD_REQUEST | F_FW_CMD_READ); 2332228561Snp cmd.retval_len16 = htobe32(FW_LEN16(cmd)); 2333228561Snp rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd); 2334228561Snp if (rc != 0) { 2335228561Snp device_printf(sc->dev, 2336228561Snp "failed to get devlog parameters: %d.\n", rc); 2337228561Snp bzero(dlog, sizeof (*dlog)); 2338228561Snp rc = 0; /* devlog isn't critical for device operation */ 2339228561Snp } else { 2340228561Snp val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog); 2341228561Snp dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 2342228561Snp dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 2343228561Snp dlog->size = be32toh(cmd.memsize_devlog); 2344228561Snp } 2345228561Snp 2346228561Snp return (rc); 2347228561Snp} 2348228561Snp 2349228561Snp/* 2350228561Snp * Retrieve various parameters that are of interest to the driver. The device 2351228561Snp * has been initialized by the firmware at this point. 2352228561Snp */ 2353228561Snpstatic int 2354228561Snpget_params__post_init(struct adapter *sc) 2355228561Snp{ 2356228561Snp int rc; 2357228561Snp uint32_t param[7], val[7]; 2358228561Snp struct fw_caps_config_cmd caps; 2359228561Snp 2360228561Snp param[0] = FW_PARAM_PFVF(IQFLINT_START); 2361228561Snp param[1] = FW_PARAM_PFVF(EQ_START); 2362228561Snp param[2] = FW_PARAM_PFVF(FILTER_START); 2363228561Snp param[3] = FW_PARAM_PFVF(FILTER_END); 2364245434Snp param[4] = FW_PARAM_PFVF(L2T_START); 2365245434Snp param[5] = FW_PARAM_PFVF(L2T_END); 2366245434Snp rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2367228561Snp if (rc != 0) { 2368228561Snp device_printf(sc->dev, 2369228561Snp "failed to query parameters (post_init): %d.\n", rc); 2370228561Snp return (rc); 2371228561Snp } 2372228561Snp 2373228561Snp sc->sge.iq_start = val[0]; 2374228561Snp sc->sge.eq_start = val[1]; 2375228561Snp sc->tids.ftid_base = val[2]; 2376228561Snp sc->tids.nftids = val[3] - val[2] + 1; 2377245434Snp sc->vres.l2t.start = val[4]; 2378245434Snp sc->vres.l2t.size = val[5] - val[4] + 1; 2379245434Snp KASSERT(sc->vres.l2t.size <= L2T_SIZE, 2380245434Snp ("%s: L2 table size (%u) larger than expected (%u)", 2381245434Snp __func__, sc->vres.l2t.size, L2T_SIZE)); 2382228561Snp 2383228561Snp /* get capabilites */ 2384228561Snp bzero(&caps, sizeof(caps)); 2385228561Snp caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 2386228561Snp F_FW_CMD_REQUEST | F_FW_CMD_READ); 2387228561Snp caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps)); 2388228561Snp rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps); 2389228561Snp if (rc != 0) { 2390228561Snp device_printf(sc->dev, 2391228561Snp "failed to get card capabilities: %d.\n", rc); 2392228561Snp return (rc); 2393228561Snp } 2394228561Snp 2395228561Snp if (caps.toecaps) { 2396218792Snp /* query offload-related parameters */ 2397228561Snp param[0] = FW_PARAM_DEV(NTID); 2398228561Snp param[1] = FW_PARAM_PFVF(SERVER_START); 2399228561Snp param[2] = FW_PARAM_PFVF(SERVER_END); 2400228561Snp param[3] = FW_PARAM_PFVF(TDDP_START); 2401228561Snp param[4] = FW_PARAM_PFVF(TDDP_END); 2402228561Snp param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 2403228561Snp rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2404218792Snp if (rc != 0) { 2405218792Snp device_printf(sc->dev, 2406218792Snp "failed to query TOE parameters: %d.\n", rc); 2407228561Snp return (rc); 2408218792Snp } 2409218792Snp sc->tids.ntids = val[0]; 2410218792Snp sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 2411218792Snp sc->tids.stid_base = val[1]; 2412218792Snp sc->tids.nstids = val[2] - val[1] + 1; 2413218792Snp sc->vres.ddp.start = val[3]; 2414218792Snp sc->vres.ddp.size = val[4] - val[3] + 1; 2415218792Snp sc->params.ofldq_wr_cred = val[5]; 2416218792Snp sc->params.offload = 1; 2417218792Snp } 2418228561Snp if (caps.rdmacaps) { 2419228561Snp param[0] = FW_PARAM_PFVF(STAG_START); 2420228561Snp param[1] = FW_PARAM_PFVF(STAG_END); 2421228561Snp param[2] = FW_PARAM_PFVF(RQ_START); 2422228561Snp param[3] = FW_PARAM_PFVF(RQ_END); 2423228561Snp param[4] = FW_PARAM_PFVF(PBL_START); 2424228561Snp param[5] = FW_PARAM_PFVF(PBL_END); 2425228561Snp rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 2426218792Snp if (rc != 0) { 2427218792Snp device_printf(sc->dev, 2428228561Snp "failed to query RDMA parameters(1): %d.\n", rc); 2429228561Snp return (rc); 2430218792Snp } 2431218792Snp sc->vres.stag.start = val[0]; 2432218792Snp sc->vres.stag.size = val[1] - val[0] + 1; 2433218792Snp sc->vres.rq.start = val[2]; 2434218792Snp sc->vres.rq.size = val[3] - val[2] + 1; 2435218792Snp sc->vres.pbl.start = val[4]; 2436218792Snp sc->vres.pbl.size = val[5] - val[4] + 1; 2437228561Snp 2438228561Snp param[0] = FW_PARAM_PFVF(SQRQ_START); 2439228561Snp param[1] = FW_PARAM_PFVF(SQRQ_END); 2440228561Snp param[2] = FW_PARAM_PFVF(CQ_START); 2441228561Snp param[3] = FW_PARAM_PFVF(CQ_END); 2442228561Snp param[4] = FW_PARAM_PFVF(OCQ_START); 2443228561Snp param[5] = FW_PARAM_PFVF(OCQ_END); 2444228561Snp rc = -t4_query_params(sc, 0, 0, 0, 6, param, val); 2445228561Snp if (rc != 0) { 2446228561Snp device_printf(sc->dev, 2447228561Snp "failed to query RDMA parameters(2): %d.\n", rc); 2448228561Snp return (rc); 2449228561Snp } 2450228561Snp sc->vres.qp.start = val[0]; 2451228561Snp sc->vres.qp.size = val[1] - val[0] + 1; 2452228561Snp sc->vres.cq.start = val[2]; 2453228561Snp sc->vres.cq.size = val[3] - val[2] + 1; 2454228561Snp sc->vres.ocq.start = val[4]; 2455228561Snp sc->vres.ocq.size = val[5] - val[4] + 1; 2456218792Snp } 2457228561Snp if (caps.iscsicaps) { 2458228561Snp param[0] = FW_PARAM_PFVF(ISCSI_START); 2459228561Snp param[1] = FW_PARAM_PFVF(ISCSI_END); 2460228561Snp rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 2461218792Snp if (rc != 0) { 2462218792Snp device_printf(sc->dev, 2463218792Snp "failed to query iSCSI parameters: %d.\n", rc); 2464228561Snp return (rc); 2465218792Snp } 2466218792Snp sc->vres.iscsi.start = val[0]; 2467218792Snp sc->vres.iscsi.size = val[1] - val[0] + 1; 2468218792Snp } 2469218792Snp 2470248925Snp /* 2471248925Snp * We've got the params we wanted to query via the firmware. Now grab 2472248925Snp * some others directly from the chip. 2473248925Snp */ 2474248925Snp rc = t4_read_chip_settings(sc); 2475228561Snp 2476218792Snp return (rc); 2477218792Snp} 2478218792Snp 2479247291Snpstatic int 2480247291Snpset_params__post_init(struct adapter *sc) 2481247291Snp{ 2482247291Snp uint32_t param, val; 2483247291Snp int rc; 2484247291Snp 2485247291Snp param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2486247291Snp rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2487247291Snp if (rc == 0) { 2488247291Snp /* ask for encapsulated CPLs */ 2489247291Snp param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 2490247291Snp val = 1; 2491247291Snp rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2492247291Snp if (rc != 0) { 2493247291Snp device_printf(sc->dev, 2494247291Snp "failed to set parameter (post_init): %d.\n", rc); 2495247291Snp return (rc); 2496247291Snp } 2497247291Snp } else if (rc != FW_EINVAL) { 2498247291Snp device_printf(sc->dev, 2499247291Snp "failed to check for encapsulated CPLs: %d.\n", rc); 2500247291Snp } else 2501247291Snp rc = 0; /* the firmware doesn't support the param, no worries */ 2502247291Snp 2503247291Snp return (rc); 2504247291Snp} 2505247291Snp 2506228561Snp#undef FW_PARAM_PFVF 2507228561Snp#undef FW_PARAM_DEV 2508228561Snp 2509218792Snpstatic void 2510218792Snpt4_set_desc(struct adapter *sc) 2511218792Snp{ 2512218792Snp char buf[128]; 2513218792Snp struct adapter_params *p = &sc->params; 2514218792Snp 2515228561Snp snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s", 2516248925Snp p->vpd.id, is_offload(sc) ? "R" : "", chip_rev(sc), p->vpd.sn, 2517248925Snp p->vpd.ec); 2518218792Snp 2519218792Snp device_set_desc_copy(sc->dev, buf); 2520218792Snp} 2521218792Snp 2522218792Snpstatic void 2523218792Snpbuild_medialist(struct port_info *pi) 2524218792Snp{ 2525218792Snp struct ifmedia *media = &pi->media; 2526218792Snp int data, m; 2527218792Snp 2528218792Snp PORT_LOCK(pi); 2529218792Snp 2530218792Snp ifmedia_removeall(media); 2531218792Snp 2532218792Snp m = IFM_ETHER | IFM_FDX; 2533218792Snp data = (pi->port_type << 8) | pi->mod_type; 2534218792Snp 2535218792Snp switch(pi->port_type) { 2536218792Snp case FW_PORT_TYPE_BT_XFI: 2537218792Snp ifmedia_add(media, m | IFM_10G_T, data, NULL); 2538218792Snp break; 2539218792Snp 2540218792Snp case FW_PORT_TYPE_BT_XAUI: 2541218792Snp ifmedia_add(media, m | IFM_10G_T, data, NULL); 2542218792Snp /* fall through */ 2543218792Snp 2544218792Snp case FW_PORT_TYPE_BT_SGMII: 2545218792Snp ifmedia_add(media, m | IFM_1000_T, data, NULL); 2546218792Snp ifmedia_add(media, m | IFM_100_TX, data, NULL); 2547218792Snp ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL); 2548218792Snp ifmedia_set(media, IFM_ETHER | IFM_AUTO); 2549218792Snp break; 2550218792Snp 2551218792Snp case FW_PORT_TYPE_CX4: 2552218792Snp ifmedia_add(media, m | IFM_10G_CX4, data, NULL); 2553218792Snp ifmedia_set(media, m | IFM_10G_CX4); 2554218792Snp break; 2555218792Snp 2556218792Snp case FW_PORT_TYPE_SFP: 2557218792Snp case FW_PORT_TYPE_FIBER_XFI: 2558218792Snp case FW_PORT_TYPE_FIBER_XAUI: 2559218792Snp switch (pi->mod_type) { 2560218792Snp 2561218792Snp case FW_PORT_MOD_TYPE_LR: 2562218792Snp ifmedia_add(media, m | IFM_10G_LR, data, NULL); 2563218792Snp ifmedia_set(media, m | IFM_10G_LR); 2564218792Snp break; 2565218792Snp 2566218792Snp case FW_PORT_MOD_TYPE_SR: 2567218792Snp ifmedia_add(media, m | IFM_10G_SR, data, NULL); 2568218792Snp ifmedia_set(media, m | IFM_10G_SR); 2569218792Snp break; 2570218792Snp 2571218792Snp case FW_PORT_MOD_TYPE_LRM: 2572218792Snp ifmedia_add(media, m | IFM_10G_LRM, data, NULL); 2573218792Snp ifmedia_set(media, m | IFM_10G_LRM); 2574218792Snp break; 2575218792Snp 2576218792Snp case FW_PORT_MOD_TYPE_TWINAX_PASSIVE: 2577218792Snp case FW_PORT_MOD_TYPE_TWINAX_ACTIVE: 2578218792Snp ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL); 2579218792Snp ifmedia_set(media, m | IFM_10G_TWINAX); 2580218792Snp break; 2581218792Snp 2582218792Snp case FW_PORT_MOD_TYPE_NONE: 2583218792Snp m &= ~IFM_FDX; 2584218792Snp ifmedia_add(media, m | IFM_NONE, data, NULL); 2585218792Snp ifmedia_set(media, m | IFM_NONE); 2586218792Snp break; 2587218792Snp 2588218792Snp case FW_PORT_MOD_TYPE_NA: 2589218792Snp case FW_PORT_MOD_TYPE_ER: 2590218792Snp default: 2591218792Snp ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2592218792Snp ifmedia_set(media, m | IFM_UNKNOWN); 2593218792Snp break; 2594218792Snp } 2595218792Snp break; 2596218792Snp 2597218792Snp case FW_PORT_TYPE_KX4: 2598218792Snp case FW_PORT_TYPE_KX: 2599218792Snp case FW_PORT_TYPE_KR: 2600218792Snp default: 2601218792Snp ifmedia_add(media, m | IFM_UNKNOWN, data, NULL); 2602218792Snp ifmedia_set(media, m | IFM_UNKNOWN); 2603218792Snp break; 2604218792Snp } 2605218792Snp 2606218792Snp PORT_UNLOCK(pi); 2607218792Snp} 2608218792Snp 2609231172Snp#define FW_MAC_EXACT_CHUNK 7 2610231172Snp 2611218792Snp/* 2612218792Snp * Program the port's XGMAC based on parameters in ifnet. The caller also 2613218792Snp * indicates which parameters should be programmed (the rest are left alone). 2614218792Snp */ 2615218792Snpstatic int 2616218792Snpupdate_mac_settings(struct port_info *pi, int flags) 2617218792Snp{ 2618218792Snp int rc; 2619218792Snp struct ifnet *ifp = pi->ifp; 2620218792Snp struct adapter *sc = pi->adapter; 2621218792Snp int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1; 2622218792Snp 2623245274Snp ASSERT_SYNCHRONIZED_OP(sc); 2624218792Snp KASSERT(flags, ("%s: not told what to update.", __func__)); 2625218792Snp 2626218792Snp if (flags & XGMAC_MTU) 2627218792Snp mtu = ifp->if_mtu; 2628218792Snp 2629218792Snp if (flags & XGMAC_PROMISC) 2630218792Snp promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0; 2631218792Snp 2632218792Snp if (flags & XGMAC_ALLMULTI) 2633218792Snp allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0; 2634218792Snp 2635218792Snp if (flags & XGMAC_VLANEX) 2636218792Snp vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0; 2637218792Snp 2638218792Snp rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1, 2639218792Snp vlanex, false); 2640218792Snp if (rc) { 2641218792Snp if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc); 2642218792Snp return (rc); 2643218792Snp } 2644218792Snp 2645218792Snp if (flags & XGMAC_UCADDR) { 2646218792Snp uint8_t ucaddr[ETHER_ADDR_LEN]; 2647218792Snp 2648218792Snp bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr)); 2649218792Snp rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt, 2650218792Snp ucaddr, true, true); 2651218792Snp if (rc < 0) { 2652218792Snp rc = -rc; 2653218792Snp if_printf(ifp, "change_mac failed: %d\n", rc); 2654218792Snp return (rc); 2655218792Snp } else { 2656218792Snp pi->xact_addr_filt = rc; 2657218792Snp rc = 0; 2658218792Snp } 2659218792Snp } 2660218792Snp 2661218792Snp if (flags & XGMAC_MCADDRS) { 2662231172Snp const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK]; 2663218792Snp int del = 1; 2664218792Snp uint64_t hash = 0; 2665218792Snp struct ifmultiaddr *ifma; 2666231172Snp int i = 0, j; 2667218792Snp 2668218792Snp if_maddr_rlock(ifp); 2669218792Snp TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2670238054Snp if (ifma->ifma_addr->sa_family != AF_LINK) 2671218792Snp continue; 2672231172Snp mcaddr[i++] = 2673231172Snp LLADDR((struct sockaddr_dl *)ifma->ifma_addr); 2674218792Snp 2675231172Snp if (i == FW_MAC_EXACT_CHUNK) { 2676231172Snp rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2677231172Snp del, i, mcaddr, NULL, &hash, 0); 2678231172Snp if (rc < 0) { 2679231172Snp rc = -rc; 2680231172Snp for (j = 0; j < i; j++) { 2681231172Snp if_printf(ifp, 2682231172Snp "failed to add mc address" 2683231172Snp " %02x:%02x:%02x:" 2684231172Snp "%02x:%02x:%02x rc=%d\n", 2685231172Snp mcaddr[j][0], mcaddr[j][1], 2686231172Snp mcaddr[j][2], mcaddr[j][3], 2687231172Snp mcaddr[j][4], mcaddr[j][5], 2688231172Snp rc); 2689231172Snp } 2690231172Snp goto mcfail; 2691231172Snp } 2692231172Snp del = 0; 2693231172Snp i = 0; 2694231172Snp } 2695231172Snp } 2696231172Snp if (i > 0) { 2697231172Snp rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid, 2698231172Snp del, i, mcaddr, NULL, &hash, 0); 2699218792Snp if (rc < 0) { 2700218792Snp rc = -rc; 2701231172Snp for (j = 0; j < i; j++) { 2702231172Snp if_printf(ifp, 2703231172Snp "failed to add mc address" 2704231172Snp " %02x:%02x:%02x:" 2705231172Snp "%02x:%02x:%02x rc=%d\n", 2706231172Snp mcaddr[j][0], mcaddr[j][1], 2707231172Snp mcaddr[j][2], mcaddr[j][3], 2708231172Snp mcaddr[j][4], mcaddr[j][5], 2709231172Snp rc); 2710231172Snp } 2711218792Snp goto mcfail; 2712218792Snp } 2713218792Snp } 2714218792Snp 2715218792Snp rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0); 2716218792Snp if (rc != 0) 2717218792Snp if_printf(ifp, "failed to set mc address hash: %d", rc); 2718218792Snpmcfail: 2719218792Snp if_maddr_runlock(ifp); 2720218792Snp } 2721218792Snp 2722218792Snp return (rc); 2723218792Snp} 2724218792Snp 2725245274Snpint 2726245274Snpbegin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags, 2727245274Snp char *wmesg) 2728218792Snp{ 2729245274Snp int rc, pri; 2730218792Snp 2731245274Snp#ifdef WITNESS 2732245274Snp /* the caller thinks it's ok to sleep, but is it really? */ 2733245274Snp if (flags & SLEEP_OK) 2734245274Snp pause("t4slptst", 1); 2735245274Snp#endif 2736218792Snp 2737245274Snp if (INTR_OK) 2738245274Snp pri = PCATCH; 2739245274Snp else 2740245274Snp pri = 0; 2741245274Snp 2742245274Snp ADAPTER_LOCK(sc); 2743245274Snp for (;;) { 2744245274Snp 2745245274Snp if (pi && IS_DOOMED(pi)) { 2746245274Snp rc = ENXIO; 2747245274Snp goto done; 2748245274Snp } 2749245274Snp 2750245274Snp if (!IS_BUSY(sc)) { 2751245274Snp rc = 0; 2752245274Snp break; 2753245274Snp } 2754245274Snp 2755245274Snp if (!(flags & SLEEP_OK)) { 2756245274Snp rc = EBUSY; 2757245274Snp goto done; 2758245274Snp } 2759245274Snp 2760245274Snp if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) { 2761218792Snp rc = EINTR; 2762218792Snp goto done; 2763218792Snp } 2764218792Snp } 2765245274Snp 2766218792Snp KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__)); 2767218792Snp SET_BUSY(sc); 2768245274Snp#ifdef INVARIANTS 2769245274Snp sc->last_op = wmesg; 2770245274Snp sc->last_op_thr = curthread; 2771245274Snp#endif 2772218792Snp 2773245274Snpdone: 2774245274Snp if (!(flags & HOLD_LOCK) || rc) 2775245274Snp ADAPTER_UNLOCK(sc); 2776218792Snp 2777245274Snp return (rc); 2778245274Snp} 2779245274Snp 2780245274Snpvoid 2781245274Snpend_synchronized_op(struct adapter *sc, int flags) 2782245274Snp{ 2783245274Snp 2784245274Snp if (flags & LOCK_HELD) 2785245274Snp ADAPTER_LOCK_ASSERT_OWNED(sc); 2786245274Snp else 2787245274Snp ADAPTER_LOCK(sc); 2788245274Snp 2789218792Snp KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__)); 2790218792Snp CLR_BUSY(sc); 2791245274Snp wakeup(&sc->flags); 2792218792Snp ADAPTER_UNLOCK(sc); 2793218792Snp} 2794218792Snp 2795218792Snpstatic int 2796218792Snpcxgbe_init_synchronized(struct port_info *pi) 2797218792Snp{ 2798218792Snp struct adapter *sc = pi->adapter; 2799218792Snp struct ifnet *ifp = pi->ifp; 2800228561Snp int rc = 0; 2801218792Snp 2802245274Snp ASSERT_SYNCHRONIZED_OP(sc); 2803218792Snp 2804218792Snp if (isset(&sc->open_device_map, pi->port_id)) { 2805218792Snp KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING, 2806218792Snp ("mismatch between open_device_map and if_drv_flags")); 2807218792Snp return (0); /* already running */ 2808218792Snp } 2809218792Snp 2810228561Snp if (!(sc->flags & FULL_INIT_DONE) && 2811228561Snp ((rc = adapter_full_init(sc)) != 0)) 2812218792Snp return (rc); /* error message displayed already */ 2813218792Snp 2814228561Snp if (!(pi->flags & PORT_INIT_DONE) && 2815228561Snp ((rc = port_full_init(pi)) != 0)) 2816228561Snp return (rc); /* error message displayed already */ 2817218792Snp 2818218792Snp rc = update_mac_settings(pi, XGMAC_ALL); 2819218792Snp if (rc) 2820218792Snp goto done; /* error message displayed already */ 2821218792Snp 2822218792Snp rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg); 2823218792Snp if (rc != 0) { 2824218792Snp if_printf(ifp, "start_link failed: %d\n", rc); 2825218792Snp goto done; 2826218792Snp } 2827218792Snp 2828218792Snp rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true); 2829218792Snp if (rc != 0) { 2830218792Snp if_printf(ifp, "enable_vi failed: %d\n", rc); 2831218792Snp goto done; 2832218792Snp } 2833218792Snp 2834218792Snp /* all ok */ 2835218792Snp setbit(&sc->open_device_map, pi->port_id); 2836245274Snp PORT_LOCK(pi); 2837218792Snp ifp->if_drv_flags |= IFF_DRV_RUNNING; 2838245274Snp PORT_UNLOCK(pi); 2839218792Snp 2840218792Snp callout_reset(&pi->tick, hz, cxgbe_tick, pi); 2841218792Snpdone: 2842218792Snp if (rc != 0) 2843218792Snp cxgbe_uninit_synchronized(pi); 2844218792Snp 2845218792Snp return (rc); 2846218792Snp} 2847218792Snp 2848218792Snp/* 2849218792Snp * Idempotent. 2850218792Snp */ 2851218792Snpstatic int 2852218792Snpcxgbe_uninit_synchronized(struct port_info *pi) 2853218792Snp{ 2854218792Snp struct adapter *sc = pi->adapter; 2855218792Snp struct ifnet *ifp = pi->ifp; 2856218792Snp int rc; 2857218792Snp 2858245274Snp ASSERT_SYNCHRONIZED_OP(sc); 2859218792Snp 2860218792Snp /* 2861228561Snp * Disable the VI so that all its data in either direction is discarded 2862228561Snp * by the MPS. Leave everything else (the queues, interrupts, and 1Hz 2863228561Snp * tick) intact as the TP can deliver negative advice or data that it's 2864228561Snp * holding in its RAM (for an offloaded connection) even after the VI is 2865228561Snp * disabled. 2866218792Snp */ 2867228561Snp rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false); 2868228561Snp if (rc) { 2869228561Snp if_printf(ifp, "disable_vi failed: %d\n", rc); 2870228561Snp return (rc); 2871228561Snp } 2872228561Snp 2873218792Snp clrbit(&sc->open_device_map, pi->port_id); 2874245274Snp PORT_LOCK(pi); 2875228561Snp ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2876245274Snp PORT_UNLOCK(pi); 2877218792Snp 2878218792Snp pi->link_cfg.link_ok = 0; 2879218792Snp pi->link_cfg.speed = 0; 2880218792Snp t4_os_link_changed(sc, pi->port_id, 0); 2881218792Snp 2882218792Snp return (0); 2883218792Snp} 2884218792Snp 2885240453Snp/* 2886240453Snp * It is ok for this function to fail midway and return right away. t4_detach 2887240453Snp * will walk the entire sc->irq list and clean up whatever is valid. 2888240453Snp */ 2889218792Snpstatic int 2890240453Snpsetup_intr_handlers(struct adapter *sc) 2891218792Snp{ 2892240453Snp int rc, rid, p, q; 2893222510Snp char s[8]; 2894222510Snp struct irq *irq; 2895228561Snp struct port_info *pi; 2896228561Snp struct sge_rxq *rxq; 2897237263Snp#ifdef TCP_OFFLOAD 2898228561Snp struct sge_ofld_rxq *ofld_rxq; 2899228561Snp#endif 2900218792Snp 2901218792Snp /* 2902218792Snp * Setup interrupts. 2903218792Snp */ 2904222510Snp irq = &sc->irq[0]; 2905222510Snp rid = sc->intr_type == INTR_INTX ? 0 : 1; 2906218792Snp if (sc->intr_count == 1) { 2907228561Snp KASSERT(!(sc->flags & INTR_DIRECT), 2908228561Snp ("%s: single interrupt && INTR_DIRECT?", __func__)); 2909222510Snp 2910240453Snp rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all"); 2911240453Snp if (rc != 0) 2912240453Snp return (rc); 2913218792Snp } else { 2914228561Snp /* Multiple interrupts. */ 2915228561Snp KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports, 2916228561Snp ("%s: too few intr.", __func__)); 2917228561Snp 2918228561Snp /* The first one is always error intr */ 2919240453Snp rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err"); 2920240453Snp if (rc != 0) 2921240453Snp return (rc); 2922222510Snp irq++; 2923222510Snp rid++; 2924218792Snp 2925228561Snp /* The second one is always the firmware event queue */ 2926240453Snp rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, 2927240453Snp "evt"); 2928240453Snp if (rc != 0) 2929240453Snp return (rc); 2930228561Snp irq++; 2931228561Snp rid++; 2932222510Snp 2933228561Snp /* 2934228561Snp * Note that if INTR_DIRECT is not set then either the NIC rx 2935228561Snp * queues or (exclusive or) the TOE rx queueus will be taking 2936228561Snp * direct interrupts. 2937228561Snp * 2938228561Snp * There is no need to check for is_offload(sc) as nofldrxq 2939228561Snp * will be 0 if offload is disabled. 2940228561Snp */ 2941228561Snp for_each_port(sc, p) { 2942228561Snp pi = sc->port[p]; 2943222510Snp 2944237263Snp#ifdef TCP_OFFLOAD 2945228561Snp /* 2946228561Snp * Skip over the NIC queues if they aren't taking direct 2947228561Snp * interrupts. 2948228561Snp */ 2949228561Snp if (!(sc->flags & INTR_DIRECT) && 2950228561Snp pi->nofldrxq > pi->nrxq) 2951228561Snp goto ofld_queues; 2952228561Snp#endif 2953228561Snp rxq = &sc->sge.rxq[pi->first_rxq]; 2954228561Snp for (q = 0; q < pi->nrxq; q++, rxq++) { 2955228561Snp snprintf(s, sizeof(s), "%d.%d", p, q); 2956240453Snp rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq, 2957240453Snp s); 2958240453Snp if (rc != 0) 2959240453Snp return (rc); 2960222510Snp irq++; 2961222510Snp rid++; 2962218792Snp } 2963218792Snp 2964237263Snp#ifdef TCP_OFFLOAD 2965228561Snp /* 2966228561Snp * Skip over the offload queues if they aren't taking 2967228561Snp * direct interrupts. 2968228561Snp */ 2969228561Snp if (!(sc->flags & INTR_DIRECT)) 2970228561Snp continue; 2971228561Snpofld_queues: 2972228561Snp ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq]; 2973228561Snp for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 2974228561Snp snprintf(s, sizeof(s), "%d,%d", p, q); 2975240453Snp rc = t4_alloc_irq(sc, irq, rid, t4_intr, 2976240453Snp ofld_rxq, s); 2977240453Snp if (rc != 0) 2978240453Snp return (rc); 2979228561Snp irq++; 2980228561Snp rid++; 2981218792Snp } 2982228561Snp#endif 2983218792Snp } 2984218792Snp } 2985218792Snp 2986240453Snp return (0); 2987240453Snp} 2988240453Snp 2989240453Snpstatic int 2990240453Snpadapter_full_init(struct adapter *sc) 2991240453Snp{ 2992240453Snp int rc, i; 2993240453Snp 2994240453Snp ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2995240453Snp KASSERT((sc->flags & FULL_INIT_DONE) == 0, 2996240453Snp ("%s: FULL_INIT_DONE already", __func__)); 2997240453Snp 2998240453Snp /* 2999240453Snp * queues that belong to the adapter (not any particular port). 3000240453Snp */ 3001240453Snp rc = t4_setup_adapter_queues(sc); 3002240453Snp if (rc != 0) 3003240453Snp goto done; 3004240453Snp 3005240453Snp for (i = 0; i < nitems(sc->tq); i++) { 3006240453Snp sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT, 3007240453Snp taskqueue_thread_enqueue, &sc->tq[i]); 3008240453Snp if (sc->tq[i] == NULL) { 3009240453Snp device_printf(sc->dev, 3010240453Snp "failed to allocate task queue %d\n", i); 3011240453Snp rc = ENOMEM; 3012240453Snp goto done; 3013240453Snp } 3014240453Snp taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d", 3015240453Snp device_get_nameunit(sc->dev), i); 3016240453Snp } 3017240453Snp 3018218792Snp t4_intr_enable(sc); 3019218792Snp sc->flags |= FULL_INIT_DONE; 3020218792Snpdone: 3021218792Snp if (rc != 0) 3022228561Snp adapter_full_uninit(sc); 3023218792Snp 3024218792Snp return (rc); 3025218792Snp} 3026218792Snp 3027218792Snpstatic int 3028228561Snpadapter_full_uninit(struct adapter *sc) 3029218792Snp{ 3030218792Snp int i; 3031218792Snp 3032218792Snp ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 3033218792Snp 3034220873Snp t4_teardown_adapter_queues(sc); 3035218792Snp 3036240452Snp for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) { 3037228561Snp taskqueue_free(sc->tq[i]); 3038228561Snp sc->tq[i] = NULL; 3039228561Snp } 3040228561Snp 3041218792Snp sc->flags &= ~FULL_INIT_DONE; 3042218792Snp 3043218792Snp return (0); 3044218792Snp} 3045218792Snp 3046218792Snpstatic int 3047228561Snpport_full_init(struct port_info *pi) 3048228561Snp{ 3049228561Snp struct adapter *sc = pi->adapter; 3050228561Snp struct ifnet *ifp = pi->ifp; 3051228561Snp uint16_t *rss; 3052228561Snp struct sge_rxq *rxq; 3053228561Snp int rc, i; 3054228561Snp 3055245274Snp ASSERT_SYNCHRONIZED_OP(sc); 3056228561Snp KASSERT((pi->flags & PORT_INIT_DONE) == 0, 3057228561Snp ("%s: PORT_INIT_DONE already", __func__)); 3058228561Snp 3059228561Snp sysctl_ctx_init(&pi->ctx); 3060228561Snp pi->flags |= PORT_SYSCTL_CTX; 3061228561Snp 3062228561Snp /* 3063228561Snp * Allocate tx/rx/fl queues for this port. 3064228561Snp */ 3065228561Snp rc = t4_setup_port_queues(pi); 3066228561Snp if (rc != 0) 3067228561Snp goto done; /* error message displayed already */ 3068228561Snp 3069228561Snp /* 3070228561Snp * Setup RSS for this port. 3071228561Snp */ 3072228561Snp rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, 3073228561Snp M_ZERO | M_WAITOK); 3074228561Snp for_each_rxq(pi, i, rxq) { 3075228561Snp rss[i] = rxq->iq.abs_id; 3076228561Snp } 3077228561Snp rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 3078228561Snp pi->rss_size, rss, pi->nrxq); 3079228561Snp free(rss, M_CXGBE); 3080228561Snp if (rc != 0) { 3081228561Snp if_printf(ifp, "rss_config failed: %d\n", rc); 3082228561Snp goto done; 3083228561Snp } 3084228561Snp 3085228561Snp pi->flags |= PORT_INIT_DONE; 3086228561Snpdone: 3087228561Snp if (rc != 0) 3088228561Snp port_full_uninit(pi); 3089228561Snp 3090228561Snp return (rc); 3091228561Snp} 3092228561Snp 3093228561Snp/* 3094228561Snp * Idempotent. 3095228561Snp */ 3096228561Snpstatic int 3097228561Snpport_full_uninit(struct port_info *pi) 3098228561Snp{ 3099228561Snp struct adapter *sc = pi->adapter; 3100228561Snp int i; 3101228561Snp struct sge_rxq *rxq; 3102228561Snp struct sge_txq *txq; 3103237263Snp#ifdef TCP_OFFLOAD 3104228561Snp struct sge_ofld_rxq *ofld_rxq; 3105228561Snp struct sge_wrq *ofld_txq; 3106228561Snp#endif 3107228561Snp 3108228561Snp if (pi->flags & PORT_INIT_DONE) { 3109228561Snp 3110228561Snp /* Need to quiesce queues. XXX: ctrl queues? */ 3111228561Snp 3112228561Snp for_each_txq(pi, i, txq) { 3113228561Snp quiesce_eq(sc, &txq->eq); 3114228561Snp } 3115228561Snp 3116237263Snp#ifdef TCP_OFFLOAD 3117228561Snp for_each_ofld_txq(pi, i, ofld_txq) { 3118228561Snp quiesce_eq(sc, &ofld_txq->eq); 3119228561Snp } 3120228561Snp#endif 3121228561Snp 3122228561Snp for_each_rxq(pi, i, rxq) { 3123228561Snp quiesce_iq(sc, &rxq->iq); 3124228561Snp quiesce_fl(sc, &rxq->fl); 3125228561Snp } 3126228561Snp 3127237263Snp#ifdef TCP_OFFLOAD 3128228561Snp for_each_ofld_rxq(pi, i, ofld_rxq) { 3129228561Snp quiesce_iq(sc, &ofld_rxq->iq); 3130228561Snp quiesce_fl(sc, &ofld_rxq->fl); 3131228561Snp } 3132228561Snp#endif 3133228561Snp } 3134228561Snp 3135228561Snp t4_teardown_port_queues(pi); 3136228561Snp pi->flags &= ~PORT_INIT_DONE; 3137228561Snp 3138228561Snp return (0); 3139228561Snp} 3140228561Snp 3141228561Snpstatic void 3142228561Snpquiesce_eq(struct adapter *sc, struct sge_eq *eq) 3143228561Snp{ 3144228561Snp EQ_LOCK(eq); 3145228561Snp eq->flags |= EQ_DOOMED; 3146228561Snp 3147228561Snp /* 3148228561Snp * Wait for the response to a credit flush if one's 3149228561Snp * pending. 3150228561Snp */ 3151228561Snp while (eq->flags & EQ_CRFLUSHED) 3152228561Snp mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0); 3153228561Snp EQ_UNLOCK(eq); 3154228561Snp 3155228561Snp callout_drain(&eq->tx_callout); /* XXX: iffy */ 3156228561Snp pause("callout", 10); /* Still iffy */ 3157228561Snp 3158228561Snp taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task); 3159228561Snp} 3160228561Snp 3161228561Snpstatic void 3162228561Snpquiesce_iq(struct adapter *sc, struct sge_iq *iq) 3163228561Snp{ 3164228561Snp (void) sc; /* unused */ 3165228561Snp 3166228561Snp /* Synchronize with the interrupt handler */ 3167228561Snp while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED)) 3168228561Snp pause("iqfree", 1); 3169228561Snp} 3170228561Snp 3171228561Snpstatic void 3172228561Snpquiesce_fl(struct adapter *sc, struct sge_fl *fl) 3173228561Snp{ 3174228561Snp mtx_lock(&sc->sfl_lock); 3175228561Snp FL_LOCK(fl); 3176228561Snp fl->flags |= FL_DOOMED; 3177228561Snp FL_UNLOCK(fl); 3178228561Snp mtx_unlock(&sc->sfl_lock); 3179228561Snp 3180228561Snp callout_drain(&sc->sfl_callout); 3181228561Snp KASSERT((fl->flags & FL_STARVING) == 0, 3182228561Snp ("%s: still starving", __func__)); 3183228561Snp} 3184228561Snp 3185228561Snpstatic int 3186218792Snpt4_alloc_irq(struct adapter *sc, struct irq *irq, int rid, 3187228561Snp driver_intr_t *handler, void *arg, char *name) 3188218792Snp{ 3189218792Snp int rc; 3190218792Snp 3191218792Snp irq->rid = rid; 3192218792Snp irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid, 3193218792Snp RF_SHAREABLE | RF_ACTIVE); 3194218792Snp if (irq->res == NULL) { 3195218792Snp device_printf(sc->dev, 3196218792Snp "failed to allocate IRQ for rid %d, name %s.\n", rid, name); 3197218792Snp return (ENOMEM); 3198218792Snp } 3199218792Snp 3200218792Snp rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET, 3201218792Snp NULL, handler, arg, &irq->tag); 3202218792Snp if (rc != 0) { 3203218792Snp device_printf(sc->dev, 3204218792Snp "failed to setup interrupt for rid %d, name %s: %d\n", 3205218792Snp rid, name, rc); 3206218792Snp } else if (name) 3207218792Snp bus_describe_intr(sc->dev, irq->res, irq->tag, name); 3208218792Snp 3209218792Snp return (rc); 3210218792Snp} 3211218792Snp 3212218792Snpstatic int 3213218792Snpt4_free_irq(struct adapter *sc, struct irq *irq) 3214218792Snp{ 3215218792Snp if (irq->tag) 3216218792Snp bus_teardown_intr(sc->dev, irq->res, irq->tag); 3217218792Snp if (irq->res) 3218218792Snp bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res); 3219218792Snp 3220218792Snp bzero(irq, sizeof(*irq)); 3221218792Snp 3222218792Snp return (0); 3223218792Snp} 3224218792Snp 3225218792Snpstatic void 3226218792Snpreg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start, 3227218792Snp unsigned int end) 3228218792Snp{ 3229218792Snp uint32_t *p = (uint32_t *)(buf + start); 3230218792Snp 3231218792Snp for ( ; start <= end; start += sizeof(uint32_t)) 3232218792Snp *p++ = t4_read_reg(sc, start); 3233218792Snp} 3234218792Snp 3235218792Snpstatic void 3236218792Snpt4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) 3237218792Snp{ 3238248925Snp int i, n; 3239248925Snp const unsigned int *reg_ranges; 3240248925Snp static const unsigned int t4_reg_ranges[] = { 3241218792Snp 0x1008, 0x1108, 3242218792Snp 0x1180, 0x11b4, 3243218792Snp 0x11fc, 0x123c, 3244218792Snp 0x1300, 0x173c, 3245218792Snp 0x1800, 0x18fc, 3246218792Snp 0x3000, 0x30d8, 3247218792Snp 0x30e0, 0x5924, 3248218792Snp 0x5960, 0x59d4, 3249218792Snp 0x5a00, 0x5af8, 3250218792Snp 0x6000, 0x6098, 3251218792Snp 0x6100, 0x6150, 3252218792Snp 0x6200, 0x6208, 3253218792Snp 0x6240, 0x6248, 3254218792Snp 0x6280, 0x6338, 3255218792Snp 0x6370, 0x638c, 3256218792Snp 0x6400, 0x643c, 3257218792Snp 0x6500, 0x6524, 3258218792Snp 0x6a00, 0x6a38, 3259218792Snp 0x6a60, 0x6a78, 3260218792Snp 0x6b00, 0x6b84, 3261218792Snp 0x6bf0, 0x6c84, 3262218792Snp 0x6cf0, 0x6d84, 3263218792Snp 0x6df0, 0x6e84, 3264218792Snp 0x6ef0, 0x6f84, 3265218792Snp 0x6ff0, 0x7084, 3266218792Snp 0x70f0, 0x7184, 3267218792Snp 0x71f0, 0x7284, 3268218792Snp 0x72f0, 0x7384, 3269218792Snp 0x73f0, 0x7450, 3270218792Snp 0x7500, 0x7530, 3271218792Snp 0x7600, 0x761c, 3272218792Snp 0x7680, 0x76cc, 3273218792Snp 0x7700, 0x7798, 3274218792Snp 0x77c0, 0x77fc, 3275218792Snp 0x7900, 0x79fc, 3276218792Snp 0x7b00, 0x7c38, 3277218792Snp 0x7d00, 0x7efc, 3278218792Snp 0x8dc0, 0x8e1c, 3279218792Snp 0x8e30, 0x8e78, 3280218792Snp 0x8ea0, 0x8f6c, 3281218792Snp 0x8fc0, 0x9074, 3282218792Snp 0x90fc, 0x90fc, 3283218792Snp 0x9400, 0x9458, 3284218792Snp 0x9600, 0x96bc, 3285218792Snp 0x9800, 0x9808, 3286218792Snp 0x9820, 0x983c, 3287218792Snp 0x9850, 0x9864, 3288218792Snp 0x9c00, 0x9c6c, 3289218792Snp 0x9c80, 0x9cec, 3290218792Snp 0x9d00, 0x9d6c, 3291218792Snp 0x9d80, 0x9dec, 3292218792Snp 0x9e00, 0x9e6c, 3293218792Snp 0x9e80, 0x9eec, 3294218792Snp 0x9f00, 0x9f6c, 3295218792Snp 0x9f80, 0x9fec, 3296218792Snp 0xd004, 0xd03c, 3297218792Snp 0xdfc0, 0xdfe0, 3298218792Snp 0xe000, 0xea7c, 3299218792Snp 0xf000, 0x11190, 3300237439Snp 0x19040, 0x1906c, 3301237439Snp 0x19078, 0x19080, 3302237439Snp 0x1908c, 0x19124, 3303218792Snp 0x19150, 0x191b0, 3304218792Snp 0x191d0, 0x191e8, 3305218792Snp 0x19238, 0x1924c, 3306218792Snp 0x193f8, 0x19474, 3307218792Snp 0x19490, 0x194f8, 3308218792Snp 0x19800, 0x19f30, 3309218792Snp 0x1a000, 0x1a06c, 3310218792Snp 0x1a0b0, 0x1a120, 3311218792Snp 0x1a128, 0x1a138, 3312218792Snp 0x1a190, 0x1a1c4, 3313218792Snp 0x1a1fc, 0x1a1fc, 3314218792Snp 0x1e040, 0x1e04c, 3315237439Snp 0x1e284, 0x1e28c, 3316218792Snp 0x1e2c0, 0x1e2c0, 3317218792Snp 0x1e2e0, 0x1e2e0, 3318218792Snp 0x1e300, 0x1e384, 3319218792Snp 0x1e3c0, 0x1e3c8, 3320218792Snp 0x1e440, 0x1e44c, 3321237439Snp 0x1e684, 0x1e68c, 3322218792Snp 0x1e6c0, 0x1e6c0, 3323218792Snp 0x1e6e0, 0x1e6e0, 3324218792Snp 0x1e700, 0x1e784, 3325218792Snp 0x1e7c0, 0x1e7c8, 3326218792Snp 0x1e840, 0x1e84c, 3327237439Snp 0x1ea84, 0x1ea8c, 3328218792Snp 0x1eac0, 0x1eac0, 3329218792Snp 0x1eae0, 0x1eae0, 3330218792Snp 0x1eb00, 0x1eb84, 3331218792Snp 0x1ebc0, 0x1ebc8, 3332218792Snp 0x1ec40, 0x1ec4c, 3333237439Snp 0x1ee84, 0x1ee8c, 3334218792Snp 0x1eec0, 0x1eec0, 3335218792Snp 0x1eee0, 0x1eee0, 3336218792Snp 0x1ef00, 0x1ef84, 3337218792Snp 0x1efc0, 0x1efc8, 3338218792Snp 0x1f040, 0x1f04c, 3339237439Snp 0x1f284, 0x1f28c, 3340218792Snp 0x1f2c0, 0x1f2c0, 3341218792Snp 0x1f2e0, 0x1f2e0, 3342218792Snp 0x1f300, 0x1f384, 3343218792Snp 0x1f3c0, 0x1f3c8, 3344218792Snp 0x1f440, 0x1f44c, 3345237439Snp 0x1f684, 0x1f68c, 3346218792Snp 0x1f6c0, 0x1f6c0, 3347218792Snp 0x1f6e0, 0x1f6e0, 3348218792Snp 0x1f700, 0x1f784, 3349218792Snp 0x1f7c0, 0x1f7c8, 3350218792Snp 0x1f840, 0x1f84c, 3351237439Snp 0x1fa84, 0x1fa8c, 3352218792Snp 0x1fac0, 0x1fac0, 3353218792Snp 0x1fae0, 0x1fae0, 3354218792Snp 0x1fb00, 0x1fb84, 3355218792Snp 0x1fbc0, 0x1fbc8, 3356218792Snp 0x1fc40, 0x1fc4c, 3357237439Snp 0x1fe84, 0x1fe8c, 3358218792Snp 0x1fec0, 0x1fec0, 3359218792Snp 0x1fee0, 0x1fee0, 3360218792Snp 0x1ff00, 0x1ff84, 3361218792Snp 0x1ffc0, 0x1ffc8, 3362218792Snp 0x20000, 0x2002c, 3363218792Snp 0x20100, 0x2013c, 3364218792Snp 0x20190, 0x201c8, 3365218792Snp 0x20200, 0x20318, 3366218792Snp 0x20400, 0x20528, 3367218792Snp 0x20540, 0x20614, 3368218792Snp 0x21000, 0x21040, 3369218792Snp 0x2104c, 0x21060, 3370218792Snp 0x210c0, 0x210ec, 3371218792Snp 0x21200, 0x21268, 3372218792Snp 0x21270, 0x21284, 3373218792Snp 0x212fc, 0x21388, 3374218792Snp 0x21400, 0x21404, 3375218792Snp 0x21500, 0x21518, 3376218792Snp 0x2152c, 0x2153c, 3377218792Snp 0x21550, 0x21554, 3378218792Snp 0x21600, 0x21600, 3379218792Snp 0x21608, 0x21628, 3380218792Snp 0x21630, 0x2163c, 3381218792Snp 0x21700, 0x2171c, 3382218792Snp 0x21780, 0x2178c, 3383218792Snp 0x21800, 0x21c38, 3384218792Snp 0x21c80, 0x21d7c, 3385218792Snp 0x21e00, 0x21e04, 3386218792Snp 0x22000, 0x2202c, 3387218792Snp 0x22100, 0x2213c, 3388218792Snp 0x22190, 0x221c8, 3389218792Snp 0x22200, 0x22318, 3390218792Snp 0x22400, 0x22528, 3391218792Snp 0x22540, 0x22614, 3392218792Snp 0x23000, 0x23040, 3393218792Snp 0x2304c, 0x23060, 3394218792Snp 0x230c0, 0x230ec, 3395218792Snp 0x23200, 0x23268, 3396218792Snp 0x23270, 0x23284, 3397218792Snp 0x232fc, 0x23388, 3398218792Snp 0x23400, 0x23404, 3399218792Snp 0x23500, 0x23518, 3400218792Snp 0x2352c, 0x2353c, 3401218792Snp 0x23550, 0x23554, 3402218792Snp 0x23600, 0x23600, 3403218792Snp 0x23608, 0x23628, 3404218792Snp 0x23630, 0x2363c, 3405218792Snp 0x23700, 0x2371c, 3406218792Snp 0x23780, 0x2378c, 3407218792Snp 0x23800, 0x23c38, 3408218792Snp 0x23c80, 0x23d7c, 3409218792Snp 0x23e00, 0x23e04, 3410218792Snp 0x24000, 0x2402c, 3411218792Snp 0x24100, 0x2413c, 3412218792Snp 0x24190, 0x241c8, 3413218792Snp 0x24200, 0x24318, 3414218792Snp 0x24400, 0x24528, 3415218792Snp 0x24540, 0x24614, 3416218792Snp 0x25000, 0x25040, 3417218792Snp 0x2504c, 0x25060, 3418218792Snp 0x250c0, 0x250ec, 3419218792Snp 0x25200, 0x25268, 3420218792Snp 0x25270, 0x25284, 3421218792Snp 0x252fc, 0x25388, 3422218792Snp 0x25400, 0x25404, 3423218792Snp 0x25500, 0x25518, 3424218792Snp 0x2552c, 0x2553c, 3425218792Snp 0x25550, 0x25554, 3426218792Snp 0x25600, 0x25600, 3427218792Snp 0x25608, 0x25628, 3428218792Snp 0x25630, 0x2563c, 3429218792Snp 0x25700, 0x2571c, 3430218792Snp 0x25780, 0x2578c, 3431218792Snp 0x25800, 0x25c38, 3432218792Snp 0x25c80, 0x25d7c, 3433218792Snp 0x25e00, 0x25e04, 3434218792Snp 0x26000, 0x2602c, 3435218792Snp 0x26100, 0x2613c, 3436218792Snp 0x26190, 0x261c8, 3437218792Snp 0x26200, 0x26318, 3438218792Snp 0x26400, 0x26528, 3439218792Snp 0x26540, 0x26614, 3440218792Snp 0x27000, 0x27040, 3441218792Snp 0x2704c, 0x27060, 3442218792Snp 0x270c0, 0x270ec, 3443218792Snp 0x27200, 0x27268, 3444218792Snp 0x27270, 0x27284, 3445218792Snp 0x272fc, 0x27388, 3446218792Snp 0x27400, 0x27404, 3447218792Snp 0x27500, 0x27518, 3448218792Snp 0x2752c, 0x2753c, 3449218792Snp 0x27550, 0x27554, 3450218792Snp 0x27600, 0x27600, 3451218792Snp 0x27608, 0x27628, 3452218792Snp 0x27630, 0x2763c, 3453218792Snp 0x27700, 0x2771c, 3454218792Snp 0x27780, 0x2778c, 3455218792Snp 0x27800, 0x27c38, 3456218792Snp 0x27c80, 0x27d7c, 3457218792Snp 0x27e00, 0x27e04 3458218792Snp }; 3459248925Snp static const unsigned int t5_reg_ranges[] = { 3460248925Snp 0x1008, 0x1148, 3461248925Snp 0x1180, 0x11b4, 3462248925Snp 0x11fc, 0x123c, 3463248925Snp 0x1280, 0x173c, 3464248925Snp 0x1800, 0x18fc, 3465248925Snp 0x3000, 0x3028, 3466248925Snp 0x3060, 0x30d8, 3467248925Snp 0x30e0, 0x30fc, 3468248925Snp 0x3140, 0x357c, 3469248925Snp 0x35a8, 0x35cc, 3470248925Snp 0x35ec, 0x35ec, 3471248925Snp 0x3600, 0x5624, 3472248925Snp 0x56cc, 0x575c, 3473248925Snp 0x580c, 0x5814, 3474248925Snp 0x5890, 0x58bc, 3475248925Snp 0x5940, 0x59dc, 3476248925Snp 0x59fc, 0x5a18, 3477248925Snp 0x5a60, 0x5a9c, 3478248925Snp 0x5b94, 0x5bfc, 3479248925Snp 0x6000, 0x6040, 3480248925Snp 0x6058, 0x614c, 3481248925Snp 0x7700, 0x7798, 3482248925Snp 0x77c0, 0x78fc, 3483248925Snp 0x7b00, 0x7c54, 3484248925Snp 0x7d00, 0x7efc, 3485248925Snp 0x8dc0, 0x8de0, 3486248925Snp 0x8df8, 0x8e84, 3487248925Snp 0x8ea0, 0x8f84, 3488248925Snp 0x8fc0, 0x90f8, 3489248925Snp 0x9400, 0x9470, 3490248925Snp 0x9600, 0x96f4, 3491248925Snp 0x9800, 0x9808, 3492248925Snp 0x9820, 0x983c, 3493248925Snp 0x9850, 0x9864, 3494248925Snp 0x9c00, 0x9c6c, 3495248925Snp 0x9c80, 0x9cec, 3496248925Snp 0x9d00, 0x9d6c, 3497248925Snp 0x9d80, 0x9dec, 3498248925Snp 0x9e00, 0x9e6c, 3499248925Snp 0x9e80, 0x9eec, 3500248925Snp 0x9f00, 0x9f6c, 3501248925Snp 0x9f80, 0xa020, 3502248925Snp 0xd004, 0xd03c, 3503248925Snp 0xdfc0, 0xdfe0, 3504248925Snp 0xe000, 0x11088, 3505248925Snp 0x1109c, 0x1117c, 3506248925Snp 0x11190, 0x11204, 3507248925Snp 0x19040, 0x1906c, 3508248925Snp 0x19078, 0x19080, 3509248925Snp 0x1908c, 0x19124, 3510248925Snp 0x19150, 0x191b0, 3511248925Snp 0x191d0, 0x191e8, 3512248925Snp 0x19238, 0x19290, 3513248925Snp 0x193f8, 0x19474, 3514248925Snp 0x19490, 0x194cc, 3515248925Snp 0x194f0, 0x194f8, 3516248925Snp 0x19c00, 0x19c60, 3517248925Snp 0x19c94, 0x19e10, 3518248925Snp 0x19e50, 0x19f34, 3519248925Snp 0x19f40, 0x19f50, 3520248925Snp 0x19f90, 0x19fe4, 3521248925Snp 0x1a000, 0x1a06c, 3522248925Snp 0x1a0b0, 0x1a120, 3523248925Snp 0x1a128, 0x1a138, 3524248925Snp 0x1a190, 0x1a1c4, 3525248925Snp 0x1a1fc, 0x1a1fc, 3526248925Snp 0x1e008, 0x1e00c, 3527248925Snp 0x1e040, 0x1e04c, 3528248925Snp 0x1e284, 0x1e290, 3529248925Snp 0x1e2c0, 0x1e2c0, 3530248925Snp 0x1e2e0, 0x1e2e0, 3531248925Snp 0x1e300, 0x1e384, 3532248925Snp 0x1e3c0, 0x1e3c8, 3533248925Snp 0x1e408, 0x1e40c, 3534248925Snp 0x1e440, 0x1e44c, 3535248925Snp 0x1e684, 0x1e690, 3536248925Snp 0x1e6c0, 0x1e6c0, 3537248925Snp 0x1e6e0, 0x1e6e0, 3538248925Snp 0x1e700, 0x1e784, 3539248925Snp 0x1e7c0, 0x1e7c8, 3540248925Snp 0x1e808, 0x1e80c, 3541248925Snp 0x1e840, 0x1e84c, 3542248925Snp 0x1ea84, 0x1ea90, 3543248925Snp 0x1eac0, 0x1eac0, 3544248925Snp 0x1eae0, 0x1eae0, 3545248925Snp 0x1eb00, 0x1eb84, 3546248925Snp 0x1ebc0, 0x1ebc8, 3547248925Snp 0x1ec08, 0x1ec0c, 3548248925Snp 0x1ec40, 0x1ec4c, 3549248925Snp 0x1ee84, 0x1ee90, 3550248925Snp 0x1eec0, 0x1eec0, 3551248925Snp 0x1eee0, 0x1eee0, 3552248925Snp 0x1ef00, 0x1ef84, 3553248925Snp 0x1efc0, 0x1efc8, 3554248925Snp 0x1f008, 0x1f00c, 3555248925Snp 0x1f040, 0x1f04c, 3556248925Snp 0x1f284, 0x1f290, 3557248925Snp 0x1f2c0, 0x1f2c0, 3558248925Snp 0x1f2e0, 0x1f2e0, 3559248925Snp 0x1f300, 0x1f384, 3560248925Snp 0x1f3c0, 0x1f3c8, 3561248925Snp 0x1f408, 0x1f40c, 3562248925Snp 0x1f440, 0x1f44c, 3563248925Snp 0x1f684, 0x1f690, 3564248925Snp 0x1f6c0, 0x1f6c0, 3565248925Snp 0x1f6e0, 0x1f6e0, 3566248925Snp 0x1f700, 0x1f784, 3567248925Snp 0x1f7c0, 0x1f7c8, 3568248925Snp 0x1f808, 0x1f80c, 3569248925Snp 0x1f840, 0x1f84c, 3570248925Snp 0x1fa84, 0x1fa90, 3571248925Snp 0x1fac0, 0x1fac0, 3572248925Snp 0x1fae0, 0x1fae0, 3573248925Snp 0x1fb00, 0x1fb84, 3574248925Snp 0x1fbc0, 0x1fbc8, 3575248925Snp 0x1fc08, 0x1fc0c, 3576248925Snp 0x1fc40, 0x1fc4c, 3577248925Snp 0x1fe84, 0x1fe90, 3578248925Snp 0x1fec0, 0x1fec0, 3579248925Snp 0x1fee0, 0x1fee0, 3580248925Snp 0x1ff00, 0x1ff84, 3581248925Snp 0x1ffc0, 0x1ffc8, 3582248925Snp 0x30000, 0x30040, 3583248925Snp 0x30100, 0x30144, 3584248925Snp 0x30190, 0x301d0, 3585248925Snp 0x30200, 0x30318, 3586248925Snp 0x30400, 0x3052c, 3587248925Snp 0x30540, 0x3061c, 3588248925Snp 0x30800, 0x30834, 3589248925Snp 0x308c0, 0x30908, 3590248925Snp 0x30910, 0x309ac, 3591248925Snp 0x30a00, 0x30a04, 3592248925Snp 0x30a0c, 0x30a2c, 3593248925Snp 0x30a44, 0x30a50, 3594248925Snp 0x30a74, 0x30c24, 3595248925Snp 0x30d08, 0x30d14, 3596248925Snp 0x30d1c, 0x30d20, 3597248925Snp 0x30d3c, 0x30d50, 3598248925Snp 0x31200, 0x3120c, 3599248925Snp 0x31220, 0x31220, 3600248925Snp 0x31240, 0x31240, 3601248925Snp 0x31600, 0x31600, 3602248925Snp 0x31608, 0x3160c, 3603248925Snp 0x31a00, 0x31a1c, 3604248925Snp 0x31e04, 0x31e20, 3605248925Snp 0x31e38, 0x31e3c, 3606248925Snp 0x31e80, 0x31e80, 3607248925Snp 0x31e88, 0x31ea8, 3608248925Snp 0x31eb0, 0x31eb4, 3609248925Snp 0x31ec8, 0x31ed4, 3610248925Snp 0x31fb8, 0x32004, 3611248925Snp 0x32208, 0x3223c, 3612248925Snp 0x32248, 0x3227c, 3613248925Snp 0x32288, 0x322bc, 3614248925Snp 0x322c8, 0x322fc, 3615248925Snp 0x32600, 0x32630, 3616248925Snp 0x32a00, 0x32abc, 3617248925Snp 0x32b00, 0x32b70, 3618248925Snp 0x33000, 0x33048, 3619248925Snp 0x33060, 0x3309c, 3620248925Snp 0x330f0, 0x33148, 3621248925Snp 0x33160, 0x3319c, 3622248925Snp 0x331f0, 0x332e4, 3623248925Snp 0x332f8, 0x333e4, 3624248925Snp 0x333f8, 0x33448, 3625248925Snp 0x33460, 0x3349c, 3626248925Snp 0x334f0, 0x33548, 3627248925Snp 0x33560, 0x3359c, 3628248925Snp 0x335f0, 0x336e4, 3629248925Snp 0x336f8, 0x337e4, 3630248925Snp 0x337f8, 0x337fc, 3631248925Snp 0x33814, 0x33814, 3632248925Snp 0x3382c, 0x3382c, 3633248925Snp 0x33880, 0x3388c, 3634248925Snp 0x338e8, 0x338ec, 3635248925Snp 0x33900, 0x33948, 3636248925Snp 0x33960, 0x3399c, 3637248925Snp 0x339f0, 0x33ae4, 3638248925Snp 0x33af8, 0x33b10, 3639248925Snp 0x33b28, 0x33b28, 3640248925Snp 0x33b3c, 0x33b50, 3641248925Snp 0x33bf0, 0x33c10, 3642248925Snp 0x33c28, 0x33c28, 3643248925Snp 0x33c3c, 0x33c50, 3644248925Snp 0x33cf0, 0x33cfc, 3645248925Snp 0x34000, 0x34040, 3646248925Snp 0x34100, 0x34144, 3647248925Snp 0x34190, 0x341d0, 3648248925Snp 0x34200, 0x34318, 3649248925Snp 0x34400, 0x3452c, 3650248925Snp 0x34540, 0x3461c, 3651248925Snp 0x34800, 0x34834, 3652248925Snp 0x348c0, 0x34908, 3653248925Snp 0x34910, 0x349ac, 3654248925Snp 0x34a00, 0x34a04, 3655248925Snp 0x34a0c, 0x34a2c, 3656248925Snp 0x34a44, 0x34a50, 3657248925Snp 0x34a74, 0x34c24, 3658248925Snp 0x34d08, 0x34d14, 3659248925Snp 0x34d1c, 0x34d20, 3660248925Snp 0x34d3c, 0x34d50, 3661248925Snp 0x35200, 0x3520c, 3662248925Snp 0x35220, 0x35220, 3663248925Snp 0x35240, 0x35240, 3664248925Snp 0x35600, 0x35600, 3665248925Snp 0x35608, 0x3560c, 3666248925Snp 0x35a00, 0x35a1c, 3667248925Snp 0x35e04, 0x35e20, 3668248925Snp 0x35e38, 0x35e3c, 3669248925Snp 0x35e80, 0x35e80, 3670248925Snp 0x35e88, 0x35ea8, 3671248925Snp 0x35eb0, 0x35eb4, 3672248925Snp 0x35ec8, 0x35ed4, 3673248925Snp 0x35fb8, 0x36004, 3674248925Snp 0x36208, 0x3623c, 3675248925Snp 0x36248, 0x3627c, 3676248925Snp 0x36288, 0x362bc, 3677248925Snp 0x362c8, 0x362fc, 3678248925Snp 0x36600, 0x36630, 3679248925Snp 0x36a00, 0x36abc, 3680248925Snp 0x36b00, 0x36b70, 3681248925Snp 0x37000, 0x37048, 3682248925Snp 0x37060, 0x3709c, 3683248925Snp 0x370f0, 0x37148, 3684248925Snp 0x37160, 0x3719c, 3685248925Snp 0x371f0, 0x372e4, 3686248925Snp 0x372f8, 0x373e4, 3687248925Snp 0x373f8, 0x37448, 3688248925Snp 0x37460, 0x3749c, 3689248925Snp 0x374f0, 0x37548, 3690248925Snp 0x37560, 0x3759c, 3691248925Snp 0x375f0, 0x376e4, 3692248925Snp 0x376f8, 0x377e4, 3693248925Snp 0x377f8, 0x377fc, 3694248925Snp 0x37814, 0x37814, 3695248925Snp 0x3782c, 0x3782c, 3696248925Snp 0x37880, 0x3788c, 3697248925Snp 0x378e8, 0x378ec, 3698248925Snp 0x37900, 0x37948, 3699248925Snp 0x37960, 0x3799c, 3700248925Snp 0x379f0, 0x37ae4, 3701248925Snp 0x37af8, 0x37b10, 3702248925Snp 0x37b28, 0x37b28, 3703248925Snp 0x37b3c, 0x37b50, 3704248925Snp 0x37bf0, 0x37c10, 3705248925Snp 0x37c28, 0x37c28, 3706248925Snp 0x37c3c, 0x37c50, 3707248925Snp 0x37cf0, 0x37cfc, 3708248925Snp 0x38000, 0x38040, 3709248925Snp 0x38100, 0x38144, 3710248925Snp 0x38190, 0x381d0, 3711248925Snp 0x38200, 0x38318, 3712248925Snp 0x38400, 0x3852c, 3713248925Snp 0x38540, 0x3861c, 3714248925Snp 0x38800, 0x38834, 3715248925Snp 0x388c0, 0x38908, 3716248925Snp 0x38910, 0x389ac, 3717248925Snp 0x38a00, 0x38a04, 3718248925Snp 0x38a0c, 0x38a2c, 3719248925Snp 0x38a44, 0x38a50, 3720248925Snp 0x38a74, 0x38c24, 3721248925Snp 0x38d08, 0x38d14, 3722248925Snp 0x38d1c, 0x38d20, 3723248925Snp 0x38d3c, 0x38d50, 3724248925Snp 0x39200, 0x3920c, 3725248925Snp 0x39220, 0x39220, 3726248925Snp 0x39240, 0x39240, 3727248925Snp 0x39600, 0x39600, 3728248925Snp 0x39608, 0x3960c, 3729248925Snp 0x39a00, 0x39a1c, 3730248925Snp 0x39e04, 0x39e20, 3731248925Snp 0x39e38, 0x39e3c, 3732248925Snp 0x39e80, 0x39e80, 3733248925Snp 0x39e88, 0x39ea8, 3734248925Snp 0x39eb0, 0x39eb4, 3735248925Snp 0x39ec8, 0x39ed4, 3736248925Snp 0x39fb8, 0x3a004, 3737248925Snp 0x3a208, 0x3a23c, 3738248925Snp 0x3a248, 0x3a27c, 3739248925Snp 0x3a288, 0x3a2bc, 3740248925Snp 0x3a2c8, 0x3a2fc, 3741248925Snp 0x3a600, 0x3a630, 3742248925Snp 0x3aa00, 0x3aabc, 3743248925Snp 0x3ab00, 0x3ab70, 3744248925Snp 0x3b000, 0x3b048, 3745248925Snp 0x3b060, 0x3b09c, 3746248925Snp 0x3b0f0, 0x3b148, 3747248925Snp 0x3b160, 0x3b19c, 3748248925Snp 0x3b1f0, 0x3b2e4, 3749248925Snp 0x3b2f8, 0x3b3e4, 3750248925Snp 0x3b3f8, 0x3b448, 3751248925Snp 0x3b460, 0x3b49c, 3752248925Snp 0x3b4f0, 0x3b548, 3753248925Snp 0x3b560, 0x3b59c, 3754248925Snp 0x3b5f0, 0x3b6e4, 3755248925Snp 0x3b6f8, 0x3b7e4, 3756248925Snp 0x3b7f8, 0x3b7fc, 3757248925Snp 0x3b814, 0x3b814, 3758248925Snp 0x3b82c, 0x3b82c, 3759248925Snp 0x3b880, 0x3b88c, 3760248925Snp 0x3b8e8, 0x3b8ec, 3761248925Snp 0x3b900, 0x3b948, 3762248925Snp 0x3b960, 0x3b99c, 3763248925Snp 0x3b9f0, 0x3bae4, 3764248925Snp 0x3baf8, 0x3bb10, 3765248925Snp 0x3bb28, 0x3bb28, 3766248925Snp 0x3bb3c, 0x3bb50, 3767248925Snp 0x3bbf0, 0x3bc10, 3768248925Snp 0x3bc28, 0x3bc28, 3769248925Snp 0x3bc3c, 0x3bc50, 3770248925Snp 0x3bcf0, 0x3bcfc, 3771248925Snp 0x3c000, 0x3c040, 3772248925Snp 0x3c100, 0x3c144, 3773248925Snp 0x3c190, 0x3c1d0, 3774248925Snp 0x3c200, 0x3c318, 3775248925Snp 0x3c400, 0x3c52c, 3776248925Snp 0x3c540, 0x3c61c, 3777248925Snp 0x3c800, 0x3c834, 3778248925Snp 0x3c8c0, 0x3c908, 3779248925Snp 0x3c910, 0x3c9ac, 3780248925Snp 0x3ca00, 0x3ca04, 3781248925Snp 0x3ca0c, 0x3ca2c, 3782248925Snp 0x3ca44, 0x3ca50, 3783248925Snp 0x3ca74, 0x3cc24, 3784248925Snp 0x3cd08, 0x3cd14, 3785248925Snp 0x3cd1c, 0x3cd20, 3786248925Snp 0x3cd3c, 0x3cd50, 3787248925Snp 0x3d200, 0x3d20c, 3788248925Snp 0x3d220, 0x3d220, 3789248925Snp 0x3d240, 0x3d240, 3790248925Snp 0x3d600, 0x3d600, 3791248925Snp 0x3d608, 0x3d60c, 3792248925Snp 0x3da00, 0x3da1c, 3793248925Snp 0x3de04, 0x3de20, 3794248925Snp 0x3de38, 0x3de3c, 3795248925Snp 0x3de80, 0x3de80, 3796248925Snp 0x3de88, 0x3dea8, 3797248925Snp 0x3deb0, 0x3deb4, 3798248925Snp 0x3dec8, 0x3ded4, 3799248925Snp 0x3dfb8, 0x3e004, 3800248925Snp 0x3e208, 0x3e23c, 3801248925Snp 0x3e248, 0x3e27c, 3802248925Snp 0x3e288, 0x3e2bc, 3803248925Snp 0x3e2c8, 0x3e2fc, 3804248925Snp 0x3e600, 0x3e630, 3805248925Snp 0x3ea00, 0x3eabc, 3806248925Snp 0x3eb00, 0x3eb70, 3807248925Snp 0x3f000, 0x3f048, 3808248925Snp 0x3f060, 0x3f09c, 3809248925Snp 0x3f0f0, 0x3f148, 3810248925Snp 0x3f160, 0x3f19c, 3811248925Snp 0x3f1f0, 0x3f2e4, 3812248925Snp 0x3f2f8, 0x3f3e4, 3813248925Snp 0x3f3f8, 0x3f448, 3814248925Snp 0x3f460, 0x3f49c, 3815248925Snp 0x3f4f0, 0x3f548, 3816248925Snp 0x3f560, 0x3f59c, 3817248925Snp 0x3f5f0, 0x3f6e4, 3818248925Snp 0x3f6f8, 0x3f7e4, 3819248925Snp 0x3f7f8, 0x3f7fc, 3820248925Snp 0x3f814, 0x3f814, 3821248925Snp 0x3f82c, 0x3f82c, 3822248925Snp 0x3f880, 0x3f88c, 3823248925Snp 0x3f8e8, 0x3f8ec, 3824248925Snp 0x3f900, 0x3f948, 3825248925Snp 0x3f960, 0x3f99c, 3826248925Snp 0x3f9f0, 0x3fae4, 3827248925Snp 0x3faf8, 0x3fb10, 3828248925Snp 0x3fb28, 0x3fb28, 3829248925Snp 0x3fb3c, 0x3fb50, 3830248925Snp 0x3fbf0, 0x3fc10, 3831248925Snp 0x3fc28, 0x3fc28, 3832248925Snp 0x3fc3c, 0x3fc50, 3833248925Snp 0x3fcf0, 0x3fcfc, 3834248925Snp 0x40000, 0x4000c, 3835248925Snp 0x40040, 0x40068, 3836248925Snp 0x4007c, 0x40144, 3837248925Snp 0x40180, 0x4018c, 3838248925Snp 0x40200, 0x40298, 3839248925Snp 0x402ac, 0x4033c, 3840248925Snp 0x403f8, 0x403fc, 3841248925Snp 0x41300, 0x413c4, 3842248925Snp 0x41400, 0x4141c, 3843248925Snp 0x41480, 0x414d0, 3844248925Snp 0x44000, 0x44078, 3845248925Snp 0x440c0, 0x44278, 3846248925Snp 0x442c0, 0x44478, 3847248925Snp 0x444c0, 0x44678, 3848248925Snp 0x446c0, 0x44878, 3849248925Snp 0x448c0, 0x449fc, 3850248925Snp 0x45000, 0x45068, 3851248925Snp 0x45080, 0x45084, 3852248925Snp 0x450a0, 0x450b0, 3853248925Snp 0x45200, 0x45268, 3854248925Snp 0x45280, 0x45284, 3855248925Snp 0x452a0, 0x452b0, 3856248925Snp 0x460c0, 0x460e4, 3857248925Snp 0x47000, 0x4708c, 3858248925Snp 0x47200, 0x47250, 3859248925Snp 0x47400, 0x47420, 3860248925Snp 0x47600, 0x47618, 3861248925Snp 0x47800, 0x47814, 3862248925Snp 0x48000, 0x4800c, 3863248925Snp 0x48040, 0x48068, 3864248925Snp 0x4807c, 0x48144, 3865248925Snp 0x48180, 0x4818c, 3866248925Snp 0x48200, 0x48298, 3867248925Snp 0x482ac, 0x4833c, 3868248925Snp 0x483f8, 0x483fc, 3869248925Snp 0x49300, 0x493c4, 3870248925Snp 0x49400, 0x4941c, 3871248925Snp 0x49480, 0x494d0, 3872248925Snp 0x4c000, 0x4c078, 3873248925Snp 0x4c0c0, 0x4c278, 3874248925Snp 0x4c2c0, 0x4c478, 3875248925Snp 0x4c4c0, 0x4c678, 3876248925Snp 0x4c6c0, 0x4c878, 3877248925Snp 0x4c8c0, 0x4c9fc, 3878248925Snp 0x4d000, 0x4d068, 3879248925Snp 0x4d080, 0x4d084, 3880248925Snp 0x4d0a0, 0x4d0b0, 3881248925Snp 0x4d200, 0x4d268, 3882248925Snp 0x4d280, 0x4d284, 3883248925Snp 0x4d2a0, 0x4d2b0, 3884248925Snp 0x4e0c0, 0x4e0e4, 3885248925Snp 0x4f000, 0x4f08c, 3886248925Snp 0x4f200, 0x4f250, 3887248925Snp 0x4f400, 0x4f420, 3888248925Snp 0x4f600, 0x4f618, 3889248925Snp 0x4f800, 0x4f814, 3890248925Snp 0x50000, 0x500cc, 3891248925Snp 0x50400, 0x50400, 3892248925Snp 0x50800, 0x508cc, 3893248925Snp 0x50c00, 0x50c00, 3894248925Snp 0x51000, 0x5101c, 3895248925Snp 0x51300, 0x51308, 3896248925Snp }; 3897218792Snp 3898248925Snp if (is_t4(sc)) { 3899248925Snp reg_ranges = &t4_reg_ranges[0]; 3900248925Snp n = nitems(t4_reg_ranges); 3901248925Snp } else { 3902248925Snp reg_ranges = &t5_reg_ranges[0]; 3903248925Snp n = nitems(t5_reg_ranges); 3904248925Snp } 3905248925Snp 3906248925Snp regs->version = chip_id(sc) | chip_rev(sc) << 10; 3907248925Snp for (i = 0; i < n; i += 2) 3908218792Snp reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]); 3909218792Snp} 3910218792Snp 3911218792Snpstatic void 3912218792Snpcxgbe_tick(void *arg) 3913218792Snp{ 3914218792Snp struct port_info *pi = arg; 3915218792Snp struct ifnet *ifp = pi->ifp; 3916218792Snp struct sge_txq *txq; 3917218792Snp int i, drops; 3918218792Snp struct port_stats *s = &pi->stats; 3919218792Snp 3920218792Snp PORT_LOCK(pi); 3921218792Snp if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3922218792Snp PORT_UNLOCK(pi); 3923218792Snp return; /* without scheduling another callout */ 3924218792Snp } 3925218792Snp 3926218792Snp t4_get_port_stats(pi->adapter, pi->tx_chan, s); 3927218792Snp 3928228561Snp ifp->if_opackets = s->tx_frames - s->tx_pause; 3929228561Snp ifp->if_ipackets = s->rx_frames - s->rx_pause; 3930228561Snp ifp->if_obytes = s->tx_octets - s->tx_pause * 64; 3931228561Snp ifp->if_ibytes = s->rx_octets - s->rx_pause * 64; 3932228561Snp ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause; 3933228561Snp ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause; 3934218792Snp ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 + 3935239259Snp s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 + 3936239259Snp s->rx_trunc3; 3937218792Snp 3938218792Snp drops = s->tx_drop; 3939218792Snp for_each_txq(pi, i, txq) 3940220873Snp drops += txq->br->br_drops; 3941218792Snp ifp->if_snd.ifq_drops = drops; 3942218792Snp 3943218792Snp ifp->if_oerrors = s->tx_error_frames; 3944218792Snp ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long + 3945218792Snp s->rx_fcs_err + s->rx_len_err; 3946218792Snp 3947218792Snp callout_schedule(&pi->tick, hz); 3948218792Snp PORT_UNLOCK(pi); 3949218792Snp} 3950218792Snp 3951237263Snpstatic void 3952237263Snpcxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid) 3953237263Snp{ 3954237263Snp struct ifnet *vlan; 3955237263Snp 3956241494Snp if (arg != ifp || ifp->if_type != IFT_ETHER) 3957237263Snp return; 3958237263Snp 3959237263Snp vlan = VLAN_DEVAT(ifp, vid); 3960237263Snp VLAN_SETCOOKIE(vlan, ifp); 3961237263Snp} 3962237263Snp 3963218792Snpstatic int 3964228561Snpcpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 3965228561Snp{ 3966237263Snp 3967228561Snp#ifdef INVARIANTS 3968237263Snp panic("%s: opcode 0x%02x on iq %p with payload %p", 3969228561Snp __func__, rss->opcode, iq, m); 3970228561Snp#else 3971239336Snp log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n", 3972228561Snp __func__, rss->opcode, iq, m); 3973228561Snp m_freem(m); 3974228561Snp#endif 3975228561Snp return (EDOOFUS); 3976228561Snp} 3977228561Snp 3978228561Snpint 3979228561Snpt4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 3980228561Snp{ 3981228561Snp uintptr_t *loc, new; 3982228561Snp 3983240452Snp if (opcode >= nitems(sc->cpl_handler)) 3984228561Snp return (EINVAL); 3985228561Snp 3986228561Snp new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled; 3987228561Snp loc = (uintptr_t *) &sc->cpl_handler[opcode]; 3988228561Snp atomic_store_rel_ptr(loc, new); 3989228561Snp 3990228561Snp return (0); 3991228561Snp} 3992228561Snp 3993228561Snpstatic int 3994237263Snpan_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl) 3995237263Snp{ 3996237263Snp 3997237263Snp#ifdef INVARIANTS 3998237263Snp panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl); 3999237263Snp#else 4000239336Snp log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n", 4001237263Snp __func__, iq, ctrl); 4002237263Snp#endif 4003237263Snp return (EDOOFUS); 4004237263Snp} 4005237263Snp 4006237263Snpint 4007237263Snpt4_register_an_handler(struct adapter *sc, an_handler_t h) 4008237263Snp{ 4009237263Snp uintptr_t *loc, new; 4010237263Snp 4011237263Snp new = h ? (uintptr_t)h : (uintptr_t)an_not_handled; 4012237263Snp loc = (uintptr_t *) &sc->an_handler; 4013237263Snp atomic_store_rel_ptr(loc, new); 4014237263Snp 4015237263Snp return (0); 4016237263Snp} 4017237263Snp 4018237263Snpstatic int 4019239336Snpfw_msg_not_handled(struct adapter *sc, const __be64 *rpl) 4020239336Snp{ 4021241733Sed const struct cpl_fw6_msg *cpl = 4022241733Sed __containerof(rpl, struct cpl_fw6_msg, data[0]); 4023239336Snp 4024239336Snp#ifdef INVARIANTS 4025239336Snp panic("%s: fw_msg type %d", __func__, cpl->type); 4026239336Snp#else 4027239336Snp log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type); 4028239336Snp#endif 4029239336Snp return (EDOOFUS); 4030239336Snp} 4031239336Snp 4032239336Snpint 4033239336Snpt4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 4034239336Snp{ 4035239336Snp uintptr_t *loc, new; 4036239336Snp 4037240452Snp if (type >= nitems(sc->fw_msg_handler)) 4038239336Snp return (EINVAL); 4039239336Snp 4040247291Snp /* 4041247291Snp * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 4042247291Snp * handler dispatch table. Reject any attempt to install a handler for 4043247291Snp * this subtype. 4044247291Snp */ 4045247291Snp if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 4046247291Snp return (EINVAL); 4047247291Snp 4048239336Snp new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled; 4049239336Snp loc = (uintptr_t *) &sc->fw_msg_handler[type]; 4050239336Snp atomic_store_rel_ptr(loc, new); 4051239336Snp 4052239336Snp return (0); 4053239336Snp} 4054239336Snp 4055239336Snpstatic int 4056218792Snpt4_sysctls(struct adapter *sc) 4057218792Snp{ 4058218792Snp struct sysctl_ctx_list *ctx; 4059218792Snp struct sysctl_oid *oid; 4060228561Snp struct sysctl_oid_list *children, *c0; 4061228561Snp static char *caps[] = { 4062228561Snp "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */ 4063228561Snp "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */ 4064228561Snp "\20\1TOE", /* caps[2] toecaps */ 4065228561Snp "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */ 4066228561Snp "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */ 4067228561Snp "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD" 4068228561Snp "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD", 4069228561Snp "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */ 4070228561Snp }; 4071248925Snp static char *doorbells = {"\20\1UDB\2WRWC\3UDBWC\4KDB"}; 4072218792Snp 4073218792Snp ctx = device_get_sysctl_ctx(sc->dev); 4074228561Snp 4075228561Snp /* 4076228561Snp * dev.t4nex.X. 4077228561Snp */ 4078218792Snp oid = device_get_sysctl_tree(sc->dev); 4079228561Snp c0 = children = SYSCTL_CHILDREN(oid); 4080218792Snp 4081248925Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD, NULL, 4082248925Snp sc->params.nports, "# of ports"); 4083218792Snp 4084218792Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD, 4085248925Snp NULL, chip_rev(sc), "chip hardware revision"); 4086218792Snp 4087218792Snp SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version", 4088218792Snp CTLFLAG_RD, &sc->fw_version, 0, "firmware version"); 4089218792Snp 4090228561Snp SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf", 4091245936Snp CTLFLAG_RD, &sc->cfg_file, 0, "configuration file"); 4092218792Snp 4093248925Snp SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD, NULL, 4094248925Snp sc->cfcsum, "config file checksum"); 4095228561Snp 4096248925Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "doorbells", 4097248925Snp CTLTYPE_STRING | CTLFLAG_RD, doorbells, sc->doorbells, 4098248925Snp sysctl_bitfield, "A", "available doorbells"); 4099248925Snp 4100228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps", 4101228561Snp CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps, 4102228561Snp sysctl_bitfield, "A", "available link capabilities"); 4103228561Snp 4104228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps", 4105228561Snp CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps, 4106228561Snp sysctl_bitfield, "A", "available NIC capabilities"); 4107228561Snp 4108228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps", 4109228561Snp CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps, 4110228561Snp sysctl_bitfield, "A", "available TCP offload capabilities"); 4111228561Snp 4112228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps", 4113228561Snp CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps, 4114228561Snp sysctl_bitfield, "A", "available RDMA capabilities"); 4115228561Snp 4116228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps", 4117228561Snp CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps, 4118228561Snp sysctl_bitfield, "A", "available iSCSI capabilities"); 4119228561Snp 4120228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps", 4121228561Snp CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps, 4122228561Snp sysctl_bitfield, "A", "available FCoE capabilities"); 4123228561Snp 4124248925Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD, NULL, 4125248925Snp sc->params.vpd.cclk, "core clock frequency (in KHz)"); 4126218792Snp 4127219436Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers", 4128228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val, 4129228561Snp sizeof(sc->sge.timer_val), sysctl_int_array, "A", 4130228561Snp "interrupt holdoff timer values (us)"); 4131218792Snp 4132219436Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts", 4133228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val, 4134228561Snp sizeof(sc->sge.counter_val), sysctl_int_array, "A", 4135228561Snp "interrupt holdoff packet counter values"); 4136218792Snp 4137231115Snp#ifdef SBUF_DRAIN 4138228561Snp /* 4139228561Snp * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload. 4140228561Snp */ 4141228561Snp oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc", 4142228561Snp CTLFLAG_RD | CTLFLAG_SKIP, NULL, 4143228561Snp "logs and miscellaneous information"); 4144228561Snp children = SYSCTL_CHILDREN(oid); 4145228561Snp 4146228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl", 4147228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4148228561Snp sysctl_cctrl, "A", "congestion control"); 4149228561Snp 4150247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp0", 4151247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4152247122Snp sysctl_cim_ibq_obq, "A", "CIM IBQ 0 (TP0)"); 4153247122Snp 4154247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_tp1", 4155247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 1, 4156247122Snp sysctl_cim_ibq_obq, "A", "CIM IBQ 1 (TP1)"); 4157247122Snp 4158247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ulp", 4159247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 2, 4160247122Snp sysctl_cim_ibq_obq, "A", "CIM IBQ 2 (ULP)"); 4161247122Snp 4162247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge0", 4163247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 3, 4164247122Snp sysctl_cim_ibq_obq, "A", "CIM IBQ 3 (SGE0)"); 4165247122Snp 4166247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_sge1", 4167247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 4, 4168247122Snp sysctl_cim_ibq_obq, "A", "CIM IBQ 4 (SGE1)"); 4169247122Snp 4170247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_ibq_ncsi", 4171247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 5, 4172247122Snp sysctl_cim_ibq_obq, "A", "CIM IBQ 5 (NCSI)"); 4173247122Snp 4174247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_la", 4175247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4176247122Snp sysctl_cim_la, "A", "CIM logic analyzer"); 4177247122Snp 4178247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp0", 4179247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0 + CIM_NUM_IBQ, 4180247122Snp sysctl_cim_ibq_obq, "A", "CIM OBQ 0 (ULP0)"); 4181247122Snp 4182247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp1", 4183247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 1 + CIM_NUM_IBQ, 4184247122Snp sysctl_cim_ibq_obq, "A", "CIM OBQ 1 (ULP1)"); 4185247122Snp 4186247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp2", 4187247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 2 + CIM_NUM_IBQ, 4188247122Snp sysctl_cim_ibq_obq, "A", "CIM OBQ 2 (ULP2)"); 4189247122Snp 4190247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ulp3", 4191247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 3 + CIM_NUM_IBQ, 4192247122Snp sysctl_cim_ibq_obq, "A", "CIM OBQ 3 (ULP3)"); 4193247122Snp 4194247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge", 4195247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 4 + CIM_NUM_IBQ, 4196247122Snp sysctl_cim_ibq_obq, "A", "CIM OBQ 4 (SGE)"); 4197247122Snp 4198247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_ncsi", 4199247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 5 + CIM_NUM_IBQ, 4200247122Snp sysctl_cim_ibq_obq, "A", "CIM OBQ 5 (NCSI)"); 4201247122Snp 4202248925Snp if (is_t5(sc)) { 4203248925Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge0_rx", 4204248925Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 6 + CIM_NUM_IBQ, 4205248925Snp sysctl_cim_ibq_obq, "A", "CIM OBQ 6 (SGE0-RX)"); 4206248925Snp 4207248925Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_obq_sge1_rx", 4208248925Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 7 + CIM_NUM_IBQ, 4209248925Snp sysctl_cim_ibq_obq, "A", "CIM OBQ 7 (SGE1-RX)"); 4210248925Snp } 4211248925Snp 4212247122Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cim_qcfg", 4213247122Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4214247122Snp sysctl_cim_qcfg, "A", "CIM queue configuration"); 4215247122Snp 4216228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats", 4217228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4218228561Snp sysctl_cpl_stats, "A", "CPL statistics"); 4219228561Snp 4220228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats", 4221228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4222228561Snp sysctl_ddp_stats, "A", "DDP statistics"); 4223228561Snp 4224222551Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog", 4225222551Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4226228561Snp sysctl_devlog, "A", "firmware's device log"); 4227222551Snp 4228228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats", 4229228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4230228561Snp sysctl_fcoe_stats, "A", "FCoE statistics"); 4231228561Snp 4232228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched", 4233228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4234228561Snp sysctl_hw_sched, "A", "hardware scheduler "); 4235228561Snp 4236228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t", 4237228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4238228561Snp sysctl_l2t, "A", "hardware L2 table"); 4239228561Snp 4240228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats", 4241228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4242228561Snp sysctl_lb_stats, "A", "loopback statistics"); 4243228561Snp 4244228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo", 4245228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4246228561Snp sysctl_meminfo, "A", "memory regions"); 4247228561Snp 4248228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus", 4249228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4250228561Snp sysctl_path_mtus, "A", "path MTUs"); 4251228561Snp 4252228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats", 4253228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4254228561Snp sysctl_pm_stats, "A", "PM statistics"); 4255228561Snp 4256228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats", 4257228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4258228561Snp sysctl_rdma_stats, "A", "RDMA statistics"); 4259228561Snp 4260228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats", 4261228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4262228561Snp sysctl_tcp_stats, "A", "TCP statistics"); 4263228561Snp 4264228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids", 4265228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4266228561Snp sysctl_tids, "A", "TID information"); 4267228561Snp 4268228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats", 4269228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4270228561Snp sysctl_tp_err_stats, "A", "TP error statistics"); 4271228561Snp 4272228561Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate", 4273228561Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4274228561Snp sysctl_tx_rate, "A", "Tx rate"); 4275248925Snp 4276248925Snp if (is_t5(sc)) { 4277248925Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "wrwc_stats", 4278248925Snp CTLTYPE_STRING | CTLFLAG_RD, sc, 0, 4279248925Snp sysctl_wrwc_stats, "A", "work request (WC) statistics"); 4280248925Snp } 4281231115Snp#endif 4282228561Snp 4283237263Snp#ifdef TCP_OFFLOAD 4284228561Snp if (is_offload(sc)) { 4285228561Snp /* 4286228561Snp * dev.t4nex.X.toe. 4287228561Snp */ 4288228561Snp oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD, 4289228561Snp NULL, "TOE parameters"); 4290228561Snp children = SYSCTL_CHILDREN(oid); 4291228561Snp 4292228561Snp sc->tt.sndbuf = 256 * 1024; 4293228561Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW, 4294228561Snp &sc->tt.sndbuf, 0, "max hardware send buffer size"); 4295228561Snp 4296228561Snp sc->tt.ddp = 0; 4297228561Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW, 4298228561Snp &sc->tt.ddp, 0, "DDP allowed"); 4299239341Snp 4300239341Snp sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5)); 4301228561Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW, 4302228561Snp &sc->tt.indsz, 0, "DDP max indicate size allowed"); 4303239341Snp 4304239341Snp sc->tt.ddp_thres = 4305239341Snp G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2)); 4306228561Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW, 4307228561Snp &sc->tt.ddp_thres, 0, "DDP threshold"); 4308228561Snp } 4309228561Snp#endif 4310228561Snp 4311228561Snp 4312218792Snp return (0); 4313218792Snp} 4314218792Snp 4315218792Snpstatic int 4316218792Snpcxgbe_sysctls(struct port_info *pi) 4317218792Snp{ 4318218792Snp struct sysctl_ctx_list *ctx; 4319218792Snp struct sysctl_oid *oid; 4320218792Snp struct sysctl_oid_list *children; 4321218792Snp 4322218792Snp ctx = device_get_sysctl_ctx(pi->dev); 4323218792Snp 4324218792Snp /* 4325218792Snp * dev.cxgbe.X. 4326218792Snp */ 4327218792Snp oid = device_get_sysctl_tree(pi->dev); 4328218792Snp children = SYSCTL_CHILDREN(oid); 4329218792Snp 4330218792Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD, 4331218792Snp &pi->nrxq, 0, "# of rx queues"); 4332218792Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD, 4333218792Snp &pi->ntxq, 0, "# of tx queues"); 4334218792Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD, 4335218792Snp &pi->first_rxq, 0, "index of first rx queue"); 4336218792Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD, 4337218792Snp &pi->first_txq, 0, "index of first tx queue"); 4338218792Snp 4339237263Snp#ifdef TCP_OFFLOAD 4340228561Snp if (is_offload(pi->adapter)) { 4341228561Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD, 4342228561Snp &pi->nofldrxq, 0, 4343228561Snp "# of rx queues for offloaded TCP connections"); 4344228561Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD, 4345228561Snp &pi->nofldtxq, 0, 4346228561Snp "# of tx queues for offloaded TCP connections"); 4347228561Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq", 4348228561Snp CTLFLAG_RD, &pi->first_ofld_rxq, 0, 4349228561Snp "index of first TOE rx queue"); 4350228561Snp SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq", 4351228561Snp CTLFLAG_RD, &pi->first_ofld_txq, 0, 4352228561Snp "index of first TOE tx queue"); 4353228561Snp } 4354228561Snp#endif 4355228561Snp 4356218792Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx", 4357218792Snp CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I", 4358218792Snp "holdoff timer index"); 4359218792Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx", 4360218792Snp CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I", 4361218792Snp "holdoff packet counter index"); 4362218792Snp 4363218792Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq", 4364218792Snp CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I", 4365218792Snp "rx queue size"); 4366218792Snp SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq", 4367218792Snp CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I", 4368218792Snp "tx queue size"); 4369218792Snp 4370218792Snp /* 4371218792Snp * dev.cxgbe.X.stats. 4372218792Snp */ 4373218792Snp oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD, 4374218792Snp NULL, "port statistics"); 4375218792Snp children = SYSCTL_CHILDREN(oid); 4376218792Snp 4377218792Snp#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \ 4378218792Snp SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \ 4379218792Snp CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \ 4380218792Snp sysctl_handle_t4_reg64, "QU", desc) 4381218792Snp 4382218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames", 4383218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L)); 4384218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames", 4385218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L)); 4386218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames", 4387218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L)); 4388218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames", 4389218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L)); 4390218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames", 4391218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L)); 4392218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames", 4393218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L)); 4394218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_frames_64", 4395218792Snp "# of tx frames in this range", 4396218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L)); 4397218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127", 4398218792Snp "# of tx frames in this range", 4399218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L)); 4400218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255", 4401218792Snp "# of tx frames in this range", 4402218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L)); 4403218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511", 4404218792Snp "# of tx frames in this range", 4405218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L)); 4406218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023", 4407218792Snp "# of tx frames in this range", 4408218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L)); 4409218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518", 4410218792Snp "# of tx frames in this range", 4411218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L)); 4412218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max", 4413218792Snp "# of tx frames in this range", 4414218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L)); 4415218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames", 4416218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L)); 4417218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted", 4418218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L)); 4419218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted", 4420218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L)); 4421218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted", 4422218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L)); 4423218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted", 4424218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L)); 4425218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted", 4426218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L)); 4427218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted", 4428218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L)); 4429218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted", 4430218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L)); 4431218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted", 4432218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L)); 4433218792Snp SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted", 4434218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L)); 4435218792Snp 4436218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames", 4437218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L)); 4438218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames", 4439218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L)); 4440218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames", 4441218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L)); 4442218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames", 4443218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L)); 4444218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames", 4445218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L)); 4446218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU", 4447218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L)); 4448218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames", 4449218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L)); 4450218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err", 4451218792Snp "# of frames received with bad FCS", 4452218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L)); 4453218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_len_err", 4454218792Snp "# of frames received with length error", 4455218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L)); 4456218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors", 4457218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L)); 4458218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received", 4459218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L)); 4460218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_frames_64", 4461218792Snp "# of rx frames in this range", 4462218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L)); 4463218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127", 4464218792Snp "# of rx frames in this range", 4465218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L)); 4466218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255", 4467218792Snp "# of rx frames in this range", 4468218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L)); 4469218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511", 4470218792Snp "# of rx frames in this range", 4471218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L)); 4472218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023", 4473218792Snp "# of rx frames in this range", 4474218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L)); 4475218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518", 4476218792Snp "# of rx frames in this range", 4477218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L)); 4478218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max", 4479218792Snp "# of rx frames in this range", 4480218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L)); 4481218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received", 4482218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L)); 4483218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received", 4484218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L)); 4485218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received", 4486218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L)); 4487218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received", 4488218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L)); 4489218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received", 4490218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L)); 4491218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received", 4492218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L)); 4493218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received", 4494218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L)); 4495218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received", 4496218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L)); 4497218792Snp SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received", 4498218792Snp PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L)); 4499218792Snp 4500218792Snp#undef SYSCTL_ADD_T4_REG64 4501218792Snp 4502218792Snp#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \ 4503218792Snp SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \ 4504218792Snp &pi->stats.name, desc) 4505218792Snp 4506218792Snp /* We get these from port_stats and they may be stale by upto 1s */ 4507218792Snp SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0, 4508218792Snp "# drops due to buffer-group 0 overflows"); 4509218792Snp SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1, 4510218792Snp "# drops due to buffer-group 1 overflows"); 4511218792Snp SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2, 4512218792Snp "# drops due to buffer-group 2 overflows"); 4513218792Snp SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3, 4514218792Snp "# drops due to buffer-group 3 overflows"); 4515218792Snp SYSCTL_ADD_T4_PORTSTAT(rx_trunc0, 4516218792Snp "# of buffer-group 0 truncated packets"); 4517218792Snp SYSCTL_ADD_T4_PORTSTAT(rx_trunc1, 4518218792Snp "# of buffer-group 1 truncated packets"); 4519218792Snp SYSCTL_ADD_T4_PORTSTAT(rx_trunc2, 4520218792Snp "# of buffer-group 2 truncated packets"); 4521218792Snp SYSCTL_ADD_T4_PORTSTAT(rx_trunc3, 4522218792Snp "# of buffer-group 3 truncated packets"); 4523218792Snp 4524218792Snp#undef SYSCTL_ADD_T4_PORTSTAT 4525218792Snp 4526218792Snp return (0); 4527218792Snp} 4528218792Snp 4529218792Snpstatic int 4530219436Snpsysctl_int_array(SYSCTL_HANDLER_ARGS) 4531219436Snp{ 4532219436Snp int rc, *i; 4533219436Snp struct sbuf sb; 4534219436Snp 4535219436Snp sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND); 4536219436Snp for (i = arg1; arg2; arg2 -= sizeof(int), i++) 4537219436Snp sbuf_printf(&sb, "%d ", *i); 4538219436Snp sbuf_trim(&sb); 4539219436Snp sbuf_finish(&sb); 4540219436Snp rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req); 4541219436Snp sbuf_delete(&sb); 4542219436Snp return (rc); 4543219436Snp} 4544219436Snp 4545219436Snpstatic int 4546228561Snpsysctl_bitfield(SYSCTL_HANDLER_ARGS) 4547228561Snp{ 4548228561Snp int rc; 4549228561Snp struct sbuf *sb; 4550228561Snp 4551228561Snp rc = sysctl_wire_old_buffer(req, 0); 4552228561Snp if (rc != 0) 4553228561Snp return(rc); 4554228561Snp 4555228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 4556228561Snp if (sb == NULL) 4557228561Snp return (ENOMEM); 4558228561Snp 4559228561Snp sbuf_printf(sb, "%b", (int)arg2, (char *)arg1); 4560228561Snp rc = sbuf_finish(sb); 4561228561Snp sbuf_delete(sb); 4562228561Snp 4563228561Snp return (rc); 4564228561Snp} 4565228561Snp 4566228561Snpstatic int 4567218792Snpsysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS) 4568218792Snp{ 4569218792Snp struct port_info *pi = arg1; 4570218792Snp struct adapter *sc = pi->adapter; 4571218792Snp int idx, rc, i; 4572245274Snp struct sge_rxq *rxq; 4573245274Snp uint8_t v; 4574218792Snp 4575218792Snp idx = pi->tmr_idx; 4576218792Snp 4577218792Snp rc = sysctl_handle_int(oidp, &idx, 0, req); 4578218792Snp if (rc != 0 || req->newptr == NULL) 4579218792Snp return (rc); 4580218792Snp 4581218792Snp if (idx < 0 || idx >= SGE_NTIMERS) 4582218792Snp return (EINVAL); 4583218792Snp 4584245274Snp rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4585245274Snp "t4tmr"); 4586245274Snp if (rc) 4587245274Snp return (rc); 4588228561Snp 4589245274Snp v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1); 4590245274Snp for_each_rxq(pi, i, rxq) { 4591228561Snp#ifdef atomic_store_rel_8 4592245274Snp atomic_store_rel_8(&rxq->iq.intr_params, v); 4593228561Snp#else 4594245274Snp rxq->iq.intr_params = v; 4595228561Snp#endif 4596218792Snp } 4597245274Snp pi->tmr_idx = idx; 4598218792Snp 4599245274Snp end_synchronized_op(sc, LOCK_HELD); 4600245274Snp return (0); 4601218792Snp} 4602218792Snp 4603218792Snpstatic int 4604218792Snpsysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS) 4605218792Snp{ 4606218792Snp struct port_info *pi = arg1; 4607218792Snp struct adapter *sc = pi->adapter; 4608218792Snp int idx, rc; 4609218792Snp 4610218792Snp idx = pi->pktc_idx; 4611218792Snp 4612218792Snp rc = sysctl_handle_int(oidp, &idx, 0, req); 4613218792Snp if (rc != 0 || req->newptr == NULL) 4614218792Snp return (rc); 4615218792Snp 4616218792Snp if (idx < -1 || idx >= SGE_NCOUNTERS) 4617218792Snp return (EINVAL); 4618218792Snp 4619245274Snp rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4620245274Snp "t4pktc"); 4621245274Snp if (rc) 4622245274Snp return (rc); 4623245274Snp 4624245274Snp if (pi->flags & PORT_INIT_DONE) 4625228561Snp rc = EBUSY; /* cannot be changed once the queues are created */ 4626245274Snp else 4627218792Snp pi->pktc_idx = idx; 4628218792Snp 4629245274Snp end_synchronized_op(sc, LOCK_HELD); 4630218792Snp return (rc); 4631218792Snp} 4632218792Snp 4633218792Snpstatic int 4634218792Snpsysctl_qsize_rxq(SYSCTL_HANDLER_ARGS) 4635218792Snp{ 4636218792Snp struct port_info *pi = arg1; 4637218792Snp struct adapter *sc = pi->adapter; 4638218792Snp int qsize, rc; 4639218792Snp 4640218792Snp qsize = pi->qsize_rxq; 4641218792Snp 4642218792Snp rc = sysctl_handle_int(oidp, &qsize, 0, req); 4643218792Snp if (rc != 0 || req->newptr == NULL) 4644218792Snp return (rc); 4645218792Snp 4646218792Snp if (qsize < 128 || (qsize & 7)) 4647218792Snp return (EINVAL); 4648218792Snp 4649245274Snp rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4650245274Snp "t4rxqs"); 4651245274Snp if (rc) 4652245274Snp return (rc); 4653245274Snp 4654245274Snp if (pi->flags & PORT_INIT_DONE) 4655228561Snp rc = EBUSY; /* cannot be changed once the queues are created */ 4656245274Snp else 4657218792Snp pi->qsize_rxq = qsize; 4658218792Snp 4659245274Snp end_synchronized_op(sc, LOCK_HELD); 4660218792Snp return (rc); 4661218792Snp} 4662218792Snp 4663218792Snpstatic int 4664218792Snpsysctl_qsize_txq(SYSCTL_HANDLER_ARGS) 4665218792Snp{ 4666218792Snp struct port_info *pi = arg1; 4667218792Snp struct adapter *sc = pi->adapter; 4668218792Snp int qsize, rc; 4669218792Snp 4670218792Snp qsize = pi->qsize_txq; 4671218792Snp 4672218792Snp rc = sysctl_handle_int(oidp, &qsize, 0, req); 4673218792Snp if (rc != 0 || req->newptr == NULL) 4674218792Snp return (rc); 4675218792Snp 4676245274Snp /* bufring size must be powerof2 */ 4677245274Snp if (qsize < 128 || !powerof2(qsize)) 4678218792Snp return (EINVAL); 4679218792Snp 4680245274Snp rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK, 4681245274Snp "t4txqs"); 4682245274Snp if (rc) 4683245274Snp return (rc); 4684245274Snp 4685245274Snp if (pi->flags & PORT_INIT_DONE) 4686228561Snp rc = EBUSY; /* cannot be changed once the queues are created */ 4687245274Snp else 4688218792Snp pi->qsize_txq = qsize; 4689218792Snp 4690245274Snp end_synchronized_op(sc, LOCK_HELD); 4691218792Snp return (rc); 4692218792Snp} 4693218792Snp 4694218792Snpstatic int 4695218792Snpsysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS) 4696218792Snp{ 4697218792Snp struct adapter *sc = arg1; 4698218792Snp int reg = arg2; 4699218792Snp uint64_t val; 4700218792Snp 4701218792Snp val = t4_read_reg64(sc, reg); 4702218792Snp 4703218792Snp return (sysctl_handle_64(oidp, &val, 0, req)); 4704218792Snp} 4705218792Snp 4706231115Snp#ifdef SBUF_DRAIN 4707228561Snpstatic int 4708228561Snpsysctl_cctrl(SYSCTL_HANDLER_ARGS) 4709228561Snp{ 4710228561Snp struct adapter *sc = arg1; 4711228561Snp struct sbuf *sb; 4712228561Snp int rc, i; 4713228561Snp uint16_t incr[NMTUS][NCCTRL_WIN]; 4714228561Snp static const char *dec_fac[] = { 4715228561Snp "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875", 4716228561Snp "0.9375" 4717228561Snp }; 4718228561Snp 4719228561Snp rc = sysctl_wire_old_buffer(req, 0); 4720228561Snp if (rc != 0) 4721228561Snp return (rc); 4722228561Snp 4723228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4724228561Snp if (sb == NULL) 4725228561Snp return (ENOMEM); 4726228561Snp 4727228561Snp t4_read_cong_tbl(sc, incr); 4728228561Snp 4729228561Snp for (i = 0; i < NCCTRL_WIN; ++i) { 4730228561Snp sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i, 4731228561Snp incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i], 4732228561Snp incr[5][i], incr[6][i], incr[7][i]); 4733228561Snp sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n", 4734228561Snp incr[8][i], incr[9][i], incr[10][i], incr[11][i], 4735228561Snp incr[12][i], incr[13][i], incr[14][i], incr[15][i], 4736228561Snp sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]); 4737228561Snp } 4738228561Snp 4739228561Snp rc = sbuf_finish(sb); 4740228561Snp sbuf_delete(sb); 4741228561Snp 4742228561Snp return (rc); 4743228561Snp} 4744228561Snp 4745248925Snpstatic const char *qname[CIM_NUM_IBQ + CIM_NUM_OBQ_T5] = { 4746247122Snp "TP0", "TP1", "ULP", "SGE0", "SGE1", "NC-SI", /* ibq's */ 4747248925Snp "ULP0", "ULP1", "ULP2", "ULP3", "SGE", "NC-SI", /* obq's */ 4748248925Snp "SGE0-RX", "SGE1-RX" /* additional obq's (T5 onwards) */ 4749247122Snp}; 4750247122Snp 4751228561Snpstatic int 4752247122Snpsysctl_cim_ibq_obq(SYSCTL_HANDLER_ARGS) 4753247122Snp{ 4754247122Snp struct adapter *sc = arg1; 4755247122Snp struct sbuf *sb; 4756247122Snp int rc, i, n, qid = arg2; 4757247122Snp uint32_t *buf, *p; 4758247122Snp char *qtype; 4759248925Snp u_int cim_num_obq = is_t4(sc) ? CIM_NUM_OBQ : CIM_NUM_OBQ_T5; 4760247122Snp 4761248925Snp KASSERT(qid >= 0 && qid < CIM_NUM_IBQ + cim_num_obq, 4762247122Snp ("%s: bad qid %d\n", __func__, qid)); 4763247122Snp 4764247122Snp if (qid < CIM_NUM_IBQ) { 4765247122Snp /* inbound queue */ 4766247122Snp qtype = "IBQ"; 4767247122Snp n = 4 * CIM_IBQ_SIZE; 4768247122Snp buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 4769247122Snp rc = t4_read_cim_ibq(sc, qid, buf, n); 4770247122Snp } else { 4771247122Snp /* outbound queue */ 4772247122Snp qtype = "OBQ"; 4773247122Snp qid -= CIM_NUM_IBQ; 4774248925Snp n = 4 * cim_num_obq * CIM_OBQ_SIZE; 4775247122Snp buf = malloc(n * sizeof(uint32_t), M_CXGBE, M_ZERO | M_WAITOK); 4776247122Snp rc = t4_read_cim_obq(sc, qid, buf, n); 4777247122Snp } 4778247122Snp 4779247122Snp if (rc < 0) { 4780247122Snp rc = -rc; 4781247122Snp goto done; 4782247122Snp } 4783247122Snp n = rc * sizeof(uint32_t); /* rc has # of words actually read */ 4784247122Snp 4785247122Snp rc = sysctl_wire_old_buffer(req, 0); 4786247122Snp if (rc != 0) 4787247122Snp goto done; 4788247122Snp 4789248925Snp sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 4790247122Snp if (sb == NULL) { 4791247122Snp rc = ENOMEM; 4792247122Snp goto done; 4793247122Snp } 4794247122Snp 4795247122Snp sbuf_printf(sb, "%s%d %s", qtype , qid, qname[arg2]); 4796247122Snp for (i = 0, p = buf; i < n; i += 16, p += 4) 4797247122Snp sbuf_printf(sb, "\n%#06x: %08x %08x %08x %08x", i, p[0], p[1], 4798247122Snp p[2], p[3]); 4799247122Snp 4800247122Snp rc = sbuf_finish(sb); 4801247122Snp sbuf_delete(sb); 4802247122Snpdone: 4803247122Snp free(buf, M_CXGBE); 4804247122Snp return (rc); 4805247122Snp} 4806247122Snp 4807247122Snpstatic int 4808247122Snpsysctl_cim_la(SYSCTL_HANDLER_ARGS) 4809247122Snp{ 4810247122Snp struct adapter *sc = arg1; 4811247122Snp u_int cfg; 4812247122Snp struct sbuf *sb; 4813247122Snp uint32_t *buf, *p; 4814247122Snp int rc; 4815247122Snp 4816247122Snp rc = -t4_cim_read(sc, A_UP_UP_DBG_LA_CFG, 1, &cfg); 4817247122Snp if (rc != 0) 4818247122Snp return (rc); 4819247122Snp 4820247122Snp rc = sysctl_wire_old_buffer(req, 0); 4821247122Snp if (rc != 0) 4822247122Snp return (rc); 4823247122Snp 4824247122Snp sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 4825247122Snp if (sb == NULL) 4826247122Snp return (ENOMEM); 4827247122Snp 4828247122Snp buf = malloc(sc->params.cim_la_size * sizeof(uint32_t), M_CXGBE, 4829247122Snp M_ZERO | M_WAITOK); 4830247122Snp 4831247122Snp rc = -t4_cim_read_la(sc, buf, NULL); 4832247122Snp if (rc != 0) 4833247122Snp goto done; 4834247122Snp 4835247122Snp sbuf_printf(sb, "Status Data PC%s", 4836247122Snp cfg & F_UPDBGLACAPTPCONLY ? "" : 4837247122Snp " LS0Stat LS0Addr LS0Data"); 4838247122Snp 4839247122Snp KASSERT((sc->params.cim_la_size & 7) == 0, 4840247122Snp ("%s: p will walk off the end of buf", __func__)); 4841247122Snp 4842247122Snp for (p = buf; p < &buf[sc->params.cim_la_size]; p += 8) { 4843247122Snp if (cfg & F_UPDBGLACAPTPCONLY) { 4844247122Snp sbuf_printf(sb, "\n %02x %08x %08x", p[5] & 0xff, 4845247122Snp p[6], p[7]); 4846247122Snp sbuf_printf(sb, "\n %02x %02x%06x %02x%06x", 4847247122Snp (p[3] >> 8) & 0xff, p[3] & 0xff, p[4] >> 8, 4848247122Snp p[4] & 0xff, p[5] >> 8); 4849247122Snp sbuf_printf(sb, "\n %02x %x%07x %x%07x", 4850247122Snp (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 4851247122Snp p[1] & 0xf, p[2] >> 4); 4852247122Snp } else { 4853247122Snp sbuf_printf(sb, 4854247122Snp "\n %02x %x%07x %x%07x %08x %08x " 4855247122Snp "%08x%08x%08x%08x", 4856247122Snp (p[0] >> 4) & 0xff, p[0] & 0xf, p[1] >> 4, 4857247122Snp p[1] & 0xf, p[2] >> 4, p[2] & 0xf, p[3], p[4], p[5], 4858247122Snp p[6], p[7]); 4859247122Snp } 4860247122Snp } 4861247122Snp 4862247122Snp rc = sbuf_finish(sb); 4863247122Snp sbuf_delete(sb); 4864247122Snpdone: 4865247122Snp free(buf, M_CXGBE); 4866247122Snp return (rc); 4867247122Snp} 4868247122Snp 4869247122Snpstatic int 4870247122Snpsysctl_cim_qcfg(SYSCTL_HANDLER_ARGS) 4871247122Snp{ 4872247122Snp struct adapter *sc = arg1; 4873247122Snp struct sbuf *sb; 4874247122Snp int rc, i; 4875248925Snp uint16_t base[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 4876248925Snp uint16_t size[CIM_NUM_IBQ + CIM_NUM_OBQ_T5]; 4877247122Snp uint16_t thres[CIM_NUM_IBQ]; 4878248925Snp uint32_t obq_wr[2 * CIM_NUM_OBQ_T5], *wr = obq_wr; 4879248925Snp uint32_t stat[4 * (CIM_NUM_IBQ + CIM_NUM_OBQ_T5)], *p = stat; 4880248925Snp u_int cim_num_obq, ibq_rdaddr, obq_rdaddr, nq; 4881247122Snp 4882248925Snp if (is_t4(sc)) { 4883248925Snp cim_num_obq = CIM_NUM_OBQ; 4884248925Snp ibq_rdaddr = A_UP_IBQ_0_RDADDR; 4885248925Snp obq_rdaddr = A_UP_OBQ_0_REALADDR; 4886248925Snp } else { 4887248925Snp cim_num_obq = CIM_NUM_OBQ_T5; 4888248925Snp ibq_rdaddr = A_UP_IBQ_0_SHADOW_RDADDR; 4889248925Snp obq_rdaddr = A_UP_OBQ_0_SHADOW_REALADDR; 4890248925Snp } 4891248925Snp nq = CIM_NUM_IBQ + cim_num_obq; 4892248925Snp 4893248925Snp rc = -t4_cim_read(sc, ibq_rdaddr, 4 * nq, stat); 4894247122Snp if (rc == 0) 4895248925Snp rc = -t4_cim_read(sc, obq_rdaddr, 2 * cim_num_obq, obq_wr); 4896247122Snp if (rc != 0) 4897247122Snp return (rc); 4898247122Snp 4899247122Snp t4_read_cimq_cfg(sc, base, size, thres); 4900247122Snp 4901247122Snp rc = sysctl_wire_old_buffer(req, 0); 4902247122Snp if (rc != 0) 4903247122Snp return (rc); 4904247122Snp 4905248925Snp sb = sbuf_new_for_sysctl(NULL, NULL, PAGE_SIZE, req); 4906247122Snp if (sb == NULL) 4907247122Snp return (ENOMEM); 4908247122Snp 4909247122Snp sbuf_printf(sb, "Queue Base Size Thres RdPtr WrPtr SOP EOP Avail"); 4910247122Snp 4911247122Snp for (i = 0; i < CIM_NUM_IBQ; i++, p += 4) 4912248925Snp sbuf_printf(sb, "\n%7s %5x %5u %5u %6x %4x %4u %4u %5u", 4913247122Snp qname[i], base[i], size[i], thres[i], G_IBQRDADDR(p[0]), 4914247122Snp G_IBQWRADDR(p[1]), G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 4915247122Snp G_QUEREMFLITS(p[2]) * 16); 4916248925Snp for ( ; i < nq; i++, p += 4, wr += 2) 4917248925Snp sbuf_printf(sb, "\n%7s %5x %5u %12x %4x %4u %4u %5u", qname[i], 4918247122Snp base[i], size[i], G_QUERDADDR(p[0]) & 0x3fff, 4919247122Snp wr[0] - base[i], G_QUESOPCNT(p[3]), G_QUEEOPCNT(p[3]), 4920247122Snp G_QUEREMFLITS(p[2]) * 16); 4921247122Snp 4922247122Snp rc = sbuf_finish(sb); 4923247122Snp sbuf_delete(sb); 4924247122Snp 4925247122Snp return (rc); 4926247122Snp} 4927247122Snp 4928247122Snpstatic int 4929228561Snpsysctl_cpl_stats(SYSCTL_HANDLER_ARGS) 4930228561Snp{ 4931228561Snp struct adapter *sc = arg1; 4932228561Snp struct sbuf *sb; 4933228561Snp int rc; 4934228561Snp struct tp_cpl_stats stats; 4935228561Snp 4936228561Snp rc = sysctl_wire_old_buffer(req, 0); 4937228561Snp if (rc != 0) 4938228561Snp return (rc); 4939228561Snp 4940228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4941228561Snp if (sb == NULL) 4942228561Snp return (ENOMEM); 4943228561Snp 4944228561Snp t4_tp_get_cpl_stats(sc, &stats); 4945228561Snp 4946228561Snp sbuf_printf(sb, " channel 0 channel 1 channel 2 " 4947228561Snp "channel 3\n"); 4948228561Snp sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n", 4949228561Snp stats.req[0], stats.req[1], stats.req[2], stats.req[3]); 4950228561Snp sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u", 4951228561Snp stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]); 4952228561Snp 4953228561Snp rc = sbuf_finish(sb); 4954228561Snp sbuf_delete(sb); 4955228561Snp 4956228561Snp return (rc); 4957228561Snp} 4958228561Snp 4959228561Snpstatic int 4960228561Snpsysctl_ddp_stats(SYSCTL_HANDLER_ARGS) 4961228561Snp{ 4962228561Snp struct adapter *sc = arg1; 4963228561Snp struct sbuf *sb; 4964228561Snp int rc; 4965228561Snp struct tp_usm_stats stats; 4966228561Snp 4967228561Snp rc = sysctl_wire_old_buffer(req, 0); 4968228561Snp if (rc != 0) 4969228561Snp return(rc); 4970228561Snp 4971228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 4972228561Snp if (sb == NULL) 4973228561Snp return (ENOMEM); 4974228561Snp 4975228561Snp t4_get_usm_stats(sc, &stats); 4976228561Snp 4977228561Snp sbuf_printf(sb, "Frames: %u\n", stats.frames); 4978228561Snp sbuf_printf(sb, "Octets: %ju\n", stats.octets); 4979228561Snp sbuf_printf(sb, "Drops: %u", stats.drops); 4980228561Snp 4981228561Snp rc = sbuf_finish(sb); 4982228561Snp sbuf_delete(sb); 4983228561Snp 4984228561Snp return (rc); 4985228561Snp} 4986228561Snp 4987222551Snpconst char *devlog_level_strings[] = { 4988222551Snp [FW_DEVLOG_LEVEL_EMERG] = "EMERG", 4989222551Snp [FW_DEVLOG_LEVEL_CRIT] = "CRIT", 4990222551Snp [FW_DEVLOG_LEVEL_ERR] = "ERR", 4991222551Snp [FW_DEVLOG_LEVEL_NOTICE] = "NOTICE", 4992222551Snp [FW_DEVLOG_LEVEL_INFO] = "INFO", 4993222551Snp [FW_DEVLOG_LEVEL_DEBUG] = "DEBUG" 4994222551Snp}; 4995222551Snp 4996222551Snpconst char *devlog_facility_strings[] = { 4997222551Snp [FW_DEVLOG_FACILITY_CORE] = "CORE", 4998222551Snp [FW_DEVLOG_FACILITY_SCHED] = "SCHED", 4999222551Snp [FW_DEVLOG_FACILITY_TIMER] = "TIMER", 5000222551Snp [FW_DEVLOG_FACILITY_RES] = "RES", 5001222551Snp [FW_DEVLOG_FACILITY_HW] = "HW", 5002222551Snp [FW_DEVLOG_FACILITY_FLR] = "FLR", 5003222551Snp [FW_DEVLOG_FACILITY_DMAQ] = "DMAQ", 5004222551Snp [FW_DEVLOG_FACILITY_PHY] = "PHY", 5005222551Snp [FW_DEVLOG_FACILITY_MAC] = "MAC", 5006222551Snp [FW_DEVLOG_FACILITY_PORT] = "PORT", 5007222551Snp [FW_DEVLOG_FACILITY_VI] = "VI", 5008222551Snp [FW_DEVLOG_FACILITY_FILTER] = "FILTER", 5009222551Snp [FW_DEVLOG_FACILITY_ACL] = "ACL", 5010222551Snp [FW_DEVLOG_FACILITY_TM] = "TM", 5011222551Snp [FW_DEVLOG_FACILITY_QFC] = "QFC", 5012222551Snp [FW_DEVLOG_FACILITY_DCB] = "DCB", 5013222551Snp [FW_DEVLOG_FACILITY_ETH] = "ETH", 5014222551Snp [FW_DEVLOG_FACILITY_OFLD] = "OFLD", 5015222551Snp [FW_DEVLOG_FACILITY_RI] = "RI", 5016222551Snp [FW_DEVLOG_FACILITY_ISCSI] = "ISCSI", 5017222551Snp [FW_DEVLOG_FACILITY_FCOE] = "FCOE", 5018222551Snp [FW_DEVLOG_FACILITY_FOISCSI] = "FOISCSI", 5019222551Snp [FW_DEVLOG_FACILITY_FOFCOE] = "FOFCOE" 5020222551Snp}; 5021222551Snp 5022222551Snpstatic int 5023222551Snpsysctl_devlog(SYSCTL_HANDLER_ARGS) 5024222551Snp{ 5025222551Snp struct adapter *sc = arg1; 5026222551Snp struct devlog_params *dparams = &sc->params.devlog; 5027222551Snp struct fw_devlog_e *buf, *e; 5028222551Snp int i, j, rc, nentries, first = 0; 5029222551Snp struct sbuf *sb; 5030222551Snp uint64_t ftstamp = UINT64_MAX; 5031222551Snp 5032248925Snp if (dparams->start == 0) { 5033248925Snp dparams->memtype = 0; 5034248925Snp dparams->start = 0x84000; 5035248925Snp dparams->size = 32768; 5036248925Snp } 5037222551Snp 5038222551Snp nentries = dparams->size / sizeof(struct fw_devlog_e); 5039222551Snp 5040222551Snp buf = malloc(dparams->size, M_CXGBE, M_NOWAIT); 5041222551Snp if (buf == NULL) 5042222551Snp return (ENOMEM); 5043222551Snp 5044222551Snp rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size, 5045222551Snp (void *)buf); 5046222551Snp if (rc != 0) 5047222551Snp goto done; 5048222551Snp 5049222551Snp for (i = 0; i < nentries; i++) { 5050222551Snp e = &buf[i]; 5051222551Snp 5052222551Snp if (e->timestamp == 0) 5053222551Snp break; /* end */ 5054222551Snp 5055222551Snp e->timestamp = be64toh(e->timestamp); 5056222551Snp e->seqno = be32toh(e->seqno); 5057222551Snp for (j = 0; j < 8; j++) 5058222551Snp e->params[j] = be32toh(e->params[j]); 5059222551Snp 5060222551Snp if (e->timestamp < ftstamp) { 5061222551Snp ftstamp = e->timestamp; 5062222551Snp first = i; 5063222551Snp } 5064222551Snp } 5065222551Snp 5066222551Snp if (buf[first].timestamp == 0) 5067222551Snp goto done; /* nothing in the log */ 5068222551Snp 5069222551Snp rc = sysctl_wire_old_buffer(req, 0); 5070222551Snp if (rc != 0) 5071222551Snp goto done; 5072222551Snp 5073222551Snp sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5074228561Snp if (sb == NULL) { 5075228561Snp rc = ENOMEM; 5076228561Snp goto done; 5077228561Snp } 5078228561Snp sbuf_printf(sb, "%10s %15s %8s %8s %s\n", 5079222551Snp "Seq#", "Tstamp", "Level", "Facility", "Message"); 5080222551Snp 5081222551Snp i = first; 5082222551Snp do { 5083222551Snp e = &buf[i]; 5084222551Snp if (e->timestamp == 0) 5085222551Snp break; /* end */ 5086222551Snp 5087222551Snp sbuf_printf(sb, "%10d %15ju %8s %8s ", 5088222551Snp e->seqno, e->timestamp, 5089240452Snp (e->level < nitems(devlog_level_strings) ? 5090222551Snp devlog_level_strings[e->level] : "UNKNOWN"), 5091240452Snp (e->facility < nitems(devlog_facility_strings) ? 5092222551Snp devlog_facility_strings[e->facility] : "UNKNOWN")); 5093222551Snp sbuf_printf(sb, e->fmt, e->params[0], e->params[1], 5094222551Snp e->params[2], e->params[3], e->params[4], 5095222551Snp e->params[5], e->params[6], e->params[7]); 5096222551Snp 5097222551Snp if (++i == nentries) 5098222551Snp i = 0; 5099222551Snp } while (i != first); 5100222551Snp 5101222551Snp rc = sbuf_finish(sb); 5102222551Snp sbuf_delete(sb); 5103222551Snpdone: 5104222551Snp free(buf, M_CXGBE); 5105222551Snp return (rc); 5106222551Snp} 5107222551Snp 5108228561Snpstatic int 5109228561Snpsysctl_fcoe_stats(SYSCTL_HANDLER_ARGS) 5110228561Snp{ 5111228561Snp struct adapter *sc = arg1; 5112228561Snp struct sbuf *sb; 5113228561Snp int rc; 5114228561Snp struct tp_fcoe_stats stats[4]; 5115228561Snp 5116228561Snp rc = sysctl_wire_old_buffer(req, 0); 5117228561Snp if (rc != 0) 5118228561Snp return (rc); 5119228561Snp 5120228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5121228561Snp if (sb == NULL) 5122228561Snp return (ENOMEM); 5123228561Snp 5124228561Snp t4_get_fcoe_stats(sc, 0, &stats[0]); 5125228561Snp t4_get_fcoe_stats(sc, 1, &stats[1]); 5126228561Snp t4_get_fcoe_stats(sc, 2, &stats[2]); 5127228561Snp t4_get_fcoe_stats(sc, 3, &stats[3]); 5128228561Snp 5129228561Snp sbuf_printf(sb, " channel 0 channel 1 " 5130228561Snp "channel 2 channel 3\n"); 5131228561Snp sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n", 5132228561Snp stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP, 5133228561Snp stats[3].octetsDDP); 5134228561Snp sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP, 5135228561Snp stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP); 5136228561Snp sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u", 5137228561Snp stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop, 5138228561Snp stats[3].framesDrop); 5139228561Snp 5140228561Snp rc = sbuf_finish(sb); 5141228561Snp sbuf_delete(sb); 5142228561Snp 5143228561Snp return (rc); 5144228561Snp} 5145228561Snp 5146228561Snpstatic int 5147228561Snpsysctl_hw_sched(SYSCTL_HANDLER_ARGS) 5148228561Snp{ 5149228561Snp struct adapter *sc = arg1; 5150228561Snp struct sbuf *sb; 5151228561Snp int rc, i; 5152228561Snp unsigned int map, kbps, ipg, mode; 5153228561Snp unsigned int pace_tab[NTX_SCHED]; 5154228561Snp 5155228561Snp rc = sysctl_wire_old_buffer(req, 0); 5156228561Snp if (rc != 0) 5157228561Snp return (rc); 5158228561Snp 5159228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5160228561Snp if (sb == NULL) 5161228561Snp return (ENOMEM); 5162228561Snp 5163228561Snp map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP); 5164228561Snp mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG)); 5165228561Snp t4_read_pace_tbl(sc, pace_tab); 5166228561Snp 5167228561Snp sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) " 5168228561Snp "Class IPG (0.1 ns) Flow IPG (us)"); 5169228561Snp 5170228561Snp for (i = 0; i < NTX_SCHED; ++i, map >>= 2) { 5171228561Snp t4_get_tx_sched(sc, i, &kbps, &ipg); 5172228561Snp sbuf_printf(sb, "\n %u %-5s %u ", i, 5173228561Snp (mode & (1 << i)) ? "flow" : "class", map & 3); 5174228561Snp if (kbps) 5175228561Snp sbuf_printf(sb, "%9u ", kbps); 5176228561Snp else 5177228561Snp sbuf_printf(sb, " disabled "); 5178228561Snp 5179228561Snp if (ipg) 5180228561Snp sbuf_printf(sb, "%13u ", ipg); 5181228561Snp else 5182228561Snp sbuf_printf(sb, " disabled "); 5183228561Snp 5184228561Snp if (pace_tab[i]) 5185228561Snp sbuf_printf(sb, "%10u", pace_tab[i]); 5186228561Snp else 5187228561Snp sbuf_printf(sb, " disabled"); 5188228561Snp } 5189228561Snp 5190228561Snp rc = sbuf_finish(sb); 5191228561Snp sbuf_delete(sb); 5192228561Snp 5193228561Snp return (rc); 5194228561Snp} 5195228561Snp 5196228561Snpstatic int 5197228561Snpsysctl_lb_stats(SYSCTL_HANDLER_ARGS) 5198228561Snp{ 5199228561Snp struct adapter *sc = arg1; 5200228561Snp struct sbuf *sb; 5201228561Snp int rc, i, j; 5202228561Snp uint64_t *p0, *p1; 5203228561Snp struct lb_port_stats s[2]; 5204228561Snp static const char *stat_name[] = { 5205228561Snp "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:", 5206228561Snp "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:", 5207228561Snp "Frames128To255:", "Frames256To511:", "Frames512To1023:", 5208228561Snp "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:", 5209228561Snp "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:", 5210228561Snp "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:", 5211228561Snp "BG2FramesTrunc:", "BG3FramesTrunc:" 5212228561Snp }; 5213228561Snp 5214228561Snp rc = sysctl_wire_old_buffer(req, 0); 5215228561Snp if (rc != 0) 5216228561Snp return (rc); 5217228561Snp 5218228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5219228561Snp if (sb == NULL) 5220228561Snp return (ENOMEM); 5221228561Snp 5222228561Snp memset(s, 0, sizeof(s)); 5223228561Snp 5224228561Snp for (i = 0; i < 4; i += 2) { 5225228561Snp t4_get_lb_stats(sc, i, &s[0]); 5226228561Snp t4_get_lb_stats(sc, i + 1, &s[1]); 5227228561Snp 5228228561Snp p0 = &s[0].octets; 5229228561Snp p1 = &s[1].octets; 5230228561Snp sbuf_printf(sb, "%s Loopback %u" 5231228561Snp " Loopback %u", i == 0 ? "" : "\n", i, i + 1); 5232228561Snp 5233240452Snp for (j = 0; j < nitems(stat_name); j++) 5234228561Snp sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j], 5235228561Snp *p0++, *p1++); 5236228561Snp } 5237228561Snp 5238228561Snp rc = sbuf_finish(sb); 5239228561Snp sbuf_delete(sb); 5240228561Snp 5241228561Snp return (rc); 5242228561Snp} 5243228561Snp 5244228561Snpstruct mem_desc { 5245228561Snp unsigned int base; 5246228561Snp unsigned int limit; 5247228561Snp unsigned int idx; 5248228561Snp}; 5249228561Snp 5250228561Snpstatic int 5251228561Snpmem_desc_cmp(const void *a, const void *b) 5252228561Snp{ 5253228561Snp return ((const struct mem_desc *)a)->base - 5254228561Snp ((const struct mem_desc *)b)->base; 5255228561Snp} 5256228561Snp 5257228561Snpstatic void 5258228561Snpmem_region_show(struct sbuf *sb, const char *name, unsigned int from, 5259228561Snp unsigned int to) 5260228561Snp{ 5261228561Snp unsigned int size; 5262228561Snp 5263228561Snp size = to - from + 1; 5264228561Snp if (size == 0) 5265228561Snp return; 5266228561Snp 5267228561Snp /* XXX: need humanize_number(3) in libkern for a more readable 'size' */ 5268228561Snp sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size); 5269228561Snp} 5270228561Snp 5271228561Snpstatic int 5272228561Snpsysctl_meminfo(SYSCTL_HANDLER_ARGS) 5273228561Snp{ 5274228561Snp struct adapter *sc = arg1; 5275228561Snp struct sbuf *sb; 5276228561Snp int rc, i, n; 5277248925Snp uint32_t lo, hi, used, alloc; 5278248925Snp static const char *memory[] = {"EDC0:", "EDC1:", "MC:", "MC0:", "MC1:"}; 5279228561Snp static const char *region[] = { 5280228561Snp "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:", 5281228561Snp "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:", 5282228561Snp "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:", 5283228561Snp "TDDP region:", "TPT region:", "STAG region:", "RQ region:", 5284248925Snp "RQUDP region:", "PBL region:", "TXPBL region:", 5285248925Snp "DBVFIFO region:", "ULPRX state:", "ULPTX state:", 5286248925Snp "On-chip queues:" 5287228561Snp }; 5288248925Snp struct mem_desc avail[4]; 5289240452Snp struct mem_desc mem[nitems(region) + 3]; /* up to 3 holes */ 5290228561Snp struct mem_desc *md = mem; 5291228561Snp 5292228561Snp rc = sysctl_wire_old_buffer(req, 0); 5293228561Snp if (rc != 0) 5294228561Snp return (rc); 5295228561Snp 5296228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5297228561Snp if (sb == NULL) 5298228561Snp return (ENOMEM); 5299228561Snp 5300240452Snp for (i = 0; i < nitems(mem); i++) { 5301228561Snp mem[i].limit = 0; 5302228561Snp mem[i].idx = i; 5303228561Snp } 5304228561Snp 5305228561Snp /* Find and sort the populated memory ranges */ 5306228561Snp i = 0; 5307228561Snp lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 5308228561Snp if (lo & F_EDRAM0_ENABLE) { 5309228561Snp hi = t4_read_reg(sc, A_MA_EDRAM0_BAR); 5310228561Snp avail[i].base = G_EDRAM0_BASE(hi) << 20; 5311228561Snp avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20); 5312228561Snp avail[i].idx = 0; 5313228561Snp i++; 5314228561Snp } 5315228561Snp if (lo & F_EDRAM1_ENABLE) { 5316228561Snp hi = t4_read_reg(sc, A_MA_EDRAM1_BAR); 5317228561Snp avail[i].base = G_EDRAM1_BASE(hi) << 20; 5318228561Snp avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20); 5319228561Snp avail[i].idx = 1; 5320228561Snp i++; 5321228561Snp } 5322228561Snp if (lo & F_EXT_MEM_ENABLE) { 5323228561Snp hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 5324228561Snp avail[i].base = G_EXT_MEM_BASE(hi) << 20; 5325248925Snp avail[i].limit = avail[i].base + 5326248925Snp (G_EXT_MEM_SIZE(hi) << 20); 5327248925Snp avail[i].idx = is_t4(sc) ? 2 : 3; /* Call it MC for T4 */ 5328228561Snp i++; 5329228561Snp } 5330248925Snp if (!is_t4(sc) && lo & F_EXT_MEM1_ENABLE) { 5331248925Snp hi = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 5332248925Snp avail[i].base = G_EXT_MEM1_BASE(hi) << 20; 5333248925Snp avail[i].limit = avail[i].base + 5334248925Snp (G_EXT_MEM1_SIZE(hi) << 20); 5335248925Snp avail[i].idx = 4; 5336248925Snp i++; 5337248925Snp } 5338228561Snp if (!i) /* no memory available */ 5339228561Snp return 0; 5340228561Snp qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp); 5341228561Snp 5342228561Snp (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR); 5343228561Snp (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR); 5344228561Snp (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR); 5345228561Snp (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 5346228561Snp (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE); 5347228561Snp (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE); 5348228561Snp (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE); 5349228561Snp (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE); 5350228561Snp (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE); 5351228561Snp 5352228561Snp /* the next few have explicit upper bounds */ 5353228561Snp md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE); 5354228561Snp md->limit = md->base - 1 + 5355228561Snp t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) * 5356228561Snp G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE)); 5357228561Snp md++; 5358228561Snp 5359228561Snp md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE); 5360228561Snp md->limit = md->base - 1 + 5361228561Snp t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) * 5362228561Snp G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE)); 5363228561Snp md++; 5364228561Snp 5365228561Snp if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5366228561Snp hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4; 5367228561Snp md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE); 5368228561Snp md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1; 5369228561Snp } else { 5370228561Snp md->base = 0; 5371240452Snp md->idx = nitems(region); /* hide it */ 5372228561Snp } 5373228561Snp md++; 5374228561Snp 5375228561Snp#define ulp_region(reg) \ 5376228561Snp md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\ 5377228561Snp (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT) 5378228561Snp 5379228561Snp ulp_region(RX_ISCSI); 5380228561Snp ulp_region(RX_TDDP); 5381228561Snp ulp_region(TX_TPT); 5382228561Snp ulp_region(RX_STAG); 5383228561Snp ulp_region(RX_RQ); 5384228561Snp ulp_region(RX_RQUDP); 5385228561Snp ulp_region(RX_PBL); 5386228561Snp ulp_region(TX_PBL); 5387228561Snp#undef ulp_region 5388228561Snp 5389248925Snp md->base = 0; 5390248925Snp md->idx = nitems(region); 5391248925Snp if (!is_t4(sc) && t4_read_reg(sc, A_SGE_CONTROL2) & F_VFIFO_ENABLE) { 5392248925Snp md->base = G_BASEADDR(t4_read_reg(sc, A_SGE_DBVFIFO_BADDR)); 5393248925Snp md->limit = md->base + (G_DBVFIFO_SIZE((t4_read_reg(sc, 5394248925Snp A_SGE_DBVFIFO_SIZE))) << 2) - 1; 5395248925Snp } 5396248925Snp md++; 5397248925Snp 5398228561Snp md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE); 5399228561Snp md->limit = md->base + sc->tids.ntids - 1; 5400228561Snp md++; 5401228561Snp md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE); 5402228561Snp md->limit = md->base + sc->tids.ntids - 1; 5403228561Snp md++; 5404228561Snp 5405228561Snp md->base = sc->vres.ocq.start; 5406228561Snp if (sc->vres.ocq.size) 5407228561Snp md->limit = md->base + sc->vres.ocq.size - 1; 5408228561Snp else 5409240452Snp md->idx = nitems(region); /* hide it */ 5410228561Snp md++; 5411228561Snp 5412228561Snp /* add any address-space holes, there can be up to 3 */ 5413228561Snp for (n = 0; n < i - 1; n++) 5414228561Snp if (avail[n].limit < avail[n + 1].base) 5415228561Snp (md++)->base = avail[n].limit; 5416228561Snp if (avail[n].limit) 5417228561Snp (md++)->base = avail[n].limit; 5418228561Snp 5419228561Snp n = md - mem; 5420228561Snp qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp); 5421228561Snp 5422228561Snp for (lo = 0; lo < i; lo++) 5423228561Snp mem_region_show(sb, memory[avail[lo].idx], avail[lo].base, 5424228561Snp avail[lo].limit - 1); 5425228561Snp 5426228561Snp sbuf_printf(sb, "\n"); 5427228561Snp for (i = 0; i < n; i++) { 5428240452Snp if (mem[i].idx >= nitems(region)) 5429228561Snp continue; /* skip holes */ 5430228561Snp if (!mem[i].limit) 5431228561Snp mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0; 5432228561Snp mem_region_show(sb, region[mem[i].idx], mem[i].base, 5433228561Snp mem[i].limit); 5434228561Snp } 5435228561Snp 5436228561Snp sbuf_printf(sb, "\n"); 5437228561Snp lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR); 5438228561Snp hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1; 5439228561Snp mem_region_show(sb, "uP RAM:", lo, hi); 5440228561Snp 5441228561Snp lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR); 5442228561Snp hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1; 5443228561Snp mem_region_show(sb, "uP Extmem2:", lo, hi); 5444228561Snp 5445228561Snp lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE); 5446228561Snp sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n", 5447228561Snp G_PMRXMAXPAGE(lo), 5448228561Snp t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10, 5449228561Snp (lo & F_PMRXNUMCHN) ? 2 : 1); 5450228561Snp 5451228561Snp lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE); 5452228561Snp hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE); 5453228561Snp sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n", 5454228561Snp G_PMTXMAXPAGE(lo), 5455228561Snp hi >= (1 << 20) ? (hi >> 20) : (hi >> 10), 5456228561Snp hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo)); 5457228561Snp sbuf_printf(sb, "%u p-structs\n", 5458228561Snp t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT)); 5459228561Snp 5460228561Snp for (i = 0; i < 4; i++) { 5461228561Snp lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4); 5462248925Snp if (is_t4(sc)) { 5463248925Snp used = G_USED(lo); 5464248925Snp alloc = G_ALLOC(lo); 5465248925Snp } else { 5466248925Snp used = G_T5_USED(lo); 5467248925Snp alloc = G_T5_ALLOC(lo); 5468248925Snp } 5469228561Snp sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated", 5470248925Snp i, used, alloc); 5471228561Snp } 5472228561Snp for (i = 0; i < 4; i++) { 5473228561Snp lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4); 5474248925Snp if (is_t4(sc)) { 5475248925Snp used = G_USED(lo); 5476248925Snp alloc = G_ALLOC(lo); 5477248925Snp } else { 5478248925Snp used = G_T5_USED(lo); 5479248925Snp alloc = G_T5_ALLOC(lo); 5480248925Snp } 5481228561Snp sbuf_printf(sb, 5482228561Snp "\nLoopback %d using %u pages out of %u allocated", 5483248925Snp i, used, alloc); 5484228561Snp } 5485228561Snp 5486228561Snp rc = sbuf_finish(sb); 5487228561Snp sbuf_delete(sb); 5488228561Snp 5489228561Snp return (rc); 5490228561Snp} 5491228561Snp 5492228561Snpstatic int 5493228561Snpsysctl_path_mtus(SYSCTL_HANDLER_ARGS) 5494228561Snp{ 5495228561Snp struct adapter *sc = arg1; 5496228561Snp struct sbuf *sb; 5497228561Snp int rc; 5498228561Snp uint16_t mtus[NMTUS]; 5499228561Snp 5500228561Snp rc = sysctl_wire_old_buffer(req, 0); 5501228561Snp if (rc != 0) 5502228561Snp return (rc); 5503228561Snp 5504228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5505228561Snp if (sb == NULL) 5506228561Snp return (ENOMEM); 5507228561Snp 5508228561Snp t4_read_mtu_tbl(sc, mtus, NULL); 5509228561Snp 5510228561Snp sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u", 5511228561Snp mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6], 5512228561Snp mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13], 5513228561Snp mtus[14], mtus[15]); 5514228561Snp 5515228561Snp rc = sbuf_finish(sb); 5516228561Snp sbuf_delete(sb); 5517228561Snp 5518228561Snp return (rc); 5519228561Snp} 5520228561Snp 5521228561Snpstatic int 5522228561Snpsysctl_pm_stats(SYSCTL_HANDLER_ARGS) 5523228561Snp{ 5524228561Snp struct adapter *sc = arg1; 5525228561Snp struct sbuf *sb; 5526228561Snp int rc, i; 5527228561Snp uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS]; 5528228561Snp uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS]; 5529228561Snp static const char *pm_stats[] = { 5530228561Snp "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:" 5531228561Snp }; 5532228561Snp 5533228561Snp rc = sysctl_wire_old_buffer(req, 0); 5534228561Snp if (rc != 0) 5535228561Snp return (rc); 5536228561Snp 5537228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5538228561Snp if (sb == NULL) 5539228561Snp return (ENOMEM); 5540228561Snp 5541228561Snp t4_pmtx_get_stats(sc, tx_cnt, tx_cyc); 5542228561Snp t4_pmrx_get_stats(sc, rx_cnt, rx_cyc); 5543228561Snp 5544228561Snp sbuf_printf(sb, " Tx count Tx cycles " 5545228561Snp "Rx count Rx cycles"); 5546228561Snp for (i = 0; i < PM_NSTATS; i++) 5547228561Snp sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju", 5548228561Snp pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]); 5549228561Snp 5550228561Snp rc = sbuf_finish(sb); 5551228561Snp sbuf_delete(sb); 5552228561Snp 5553228561Snp return (rc); 5554228561Snp} 5555228561Snp 5556228561Snpstatic int 5557228561Snpsysctl_rdma_stats(SYSCTL_HANDLER_ARGS) 5558228561Snp{ 5559228561Snp struct adapter *sc = arg1; 5560228561Snp struct sbuf *sb; 5561228561Snp int rc; 5562228561Snp struct tp_rdma_stats stats; 5563228561Snp 5564228561Snp rc = sysctl_wire_old_buffer(req, 0); 5565228561Snp if (rc != 0) 5566228561Snp return (rc); 5567228561Snp 5568228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5569228561Snp if (sb == NULL) 5570228561Snp return (ENOMEM); 5571228561Snp 5572228561Snp t4_tp_get_rdma_stats(sc, &stats); 5573228561Snp sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod); 5574228561Snp sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt); 5575228561Snp 5576228561Snp rc = sbuf_finish(sb); 5577228561Snp sbuf_delete(sb); 5578228561Snp 5579228561Snp return (rc); 5580228561Snp} 5581228561Snp 5582228561Snpstatic int 5583228561Snpsysctl_tcp_stats(SYSCTL_HANDLER_ARGS) 5584228561Snp{ 5585228561Snp struct adapter *sc = arg1; 5586228561Snp struct sbuf *sb; 5587228561Snp int rc; 5588228561Snp struct tp_tcp_stats v4, v6; 5589228561Snp 5590228561Snp rc = sysctl_wire_old_buffer(req, 0); 5591228561Snp if (rc != 0) 5592228561Snp return (rc); 5593228561Snp 5594228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5595228561Snp if (sb == NULL) 5596228561Snp return (ENOMEM); 5597228561Snp 5598228561Snp t4_tp_get_tcp_stats(sc, &v4, &v6); 5599228561Snp sbuf_printf(sb, 5600228561Snp " IP IPv6\n"); 5601228561Snp sbuf_printf(sb, "OutRsts: %20u %20u\n", 5602228561Snp v4.tcpOutRsts, v6.tcpOutRsts); 5603228561Snp sbuf_printf(sb, "InSegs: %20ju %20ju\n", 5604228561Snp v4.tcpInSegs, v6.tcpInSegs); 5605228561Snp sbuf_printf(sb, "OutSegs: %20ju %20ju\n", 5606228561Snp v4.tcpOutSegs, v6.tcpOutSegs); 5607228561Snp sbuf_printf(sb, "RetransSegs: %20ju %20ju", 5608228561Snp v4.tcpRetransSegs, v6.tcpRetransSegs); 5609228561Snp 5610228561Snp rc = sbuf_finish(sb); 5611228561Snp sbuf_delete(sb); 5612228561Snp 5613228561Snp return (rc); 5614228561Snp} 5615228561Snp 5616228561Snpstatic int 5617228561Snpsysctl_tids(SYSCTL_HANDLER_ARGS) 5618228561Snp{ 5619228561Snp struct adapter *sc = arg1; 5620228561Snp struct sbuf *sb; 5621228561Snp int rc; 5622228561Snp struct tid_info *t = &sc->tids; 5623228561Snp 5624228561Snp rc = sysctl_wire_old_buffer(req, 0); 5625228561Snp if (rc != 0) 5626228561Snp return (rc); 5627228561Snp 5628228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5629228561Snp if (sb == NULL) 5630228561Snp return (ENOMEM); 5631228561Snp 5632228561Snp if (t->natids) { 5633228561Snp sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1, 5634228561Snp t->atids_in_use); 5635228561Snp } 5636228561Snp 5637228561Snp if (t->ntids) { 5638228561Snp if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) { 5639228561Snp uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4; 5640228561Snp 5641228561Snp if (b) { 5642228561Snp sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1, 5643228561Snp t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 5644228561Snp t->ntids - 1); 5645228561Snp } else { 5646228561Snp sbuf_printf(sb, "TID range: %u-%u", 5647228561Snp t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4, 5648228561Snp t->ntids - 1); 5649228561Snp } 5650228561Snp } else 5651228561Snp sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1); 5652228561Snp sbuf_printf(sb, ", in use: %u\n", 5653228561Snp atomic_load_acq_int(&t->tids_in_use)); 5654228561Snp } 5655228561Snp 5656228561Snp if (t->nstids) { 5657228561Snp sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base, 5658228561Snp t->stid_base + t->nstids - 1, t->stids_in_use); 5659228561Snp } 5660228561Snp 5661228561Snp if (t->nftids) { 5662228561Snp sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base, 5663228561Snp t->ftid_base + t->nftids - 1); 5664228561Snp } 5665228561Snp 5666228561Snp sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users", 5667228561Snp t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4), 5668228561Snp t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6)); 5669228561Snp 5670228561Snp rc = sbuf_finish(sb); 5671228561Snp sbuf_delete(sb); 5672228561Snp 5673228561Snp return (rc); 5674228561Snp} 5675228561Snp 5676228561Snpstatic int 5677228561Snpsysctl_tp_err_stats(SYSCTL_HANDLER_ARGS) 5678228561Snp{ 5679228561Snp struct adapter *sc = arg1; 5680228561Snp struct sbuf *sb; 5681228561Snp int rc; 5682228561Snp struct tp_err_stats stats; 5683228561Snp 5684228561Snp rc = sysctl_wire_old_buffer(req, 0); 5685228561Snp if (rc != 0) 5686228561Snp return (rc); 5687228561Snp 5688228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5689228561Snp if (sb == NULL) 5690228561Snp return (ENOMEM); 5691228561Snp 5692228561Snp t4_tp_get_err_stats(sc, &stats); 5693228561Snp 5694228561Snp sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5695228561Snp "channel 3\n"); 5696228561Snp sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n", 5697228561Snp stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2], 5698228561Snp stats.macInErrs[3]); 5699228561Snp sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n", 5700228561Snp stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2], 5701228561Snp stats.hdrInErrs[3]); 5702228561Snp sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n", 5703228561Snp stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2], 5704228561Snp stats.tcpInErrs[3]); 5705228561Snp sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n", 5706228561Snp stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2], 5707228561Snp stats.tcp6InErrs[3]); 5708228561Snp sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n", 5709228561Snp stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2], 5710228561Snp stats.tnlCongDrops[3]); 5711228561Snp sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n", 5712228561Snp stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2], 5713228561Snp stats.tnlTxDrops[3]); 5714228561Snp sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n", 5715228561Snp stats.ofldVlanDrops[0], stats.ofldVlanDrops[1], 5716228561Snp stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]); 5717228561Snp sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n", 5718228561Snp stats.ofldChanDrops[0], stats.ofldChanDrops[1], 5719228561Snp stats.ofldChanDrops[2], stats.ofldChanDrops[3]); 5720228561Snp sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u", 5721228561Snp stats.ofldNoNeigh, stats.ofldCongDefer); 5722228561Snp 5723228561Snp rc = sbuf_finish(sb); 5724228561Snp sbuf_delete(sb); 5725228561Snp 5726228561Snp return (rc); 5727228561Snp} 5728228561Snp 5729228561Snpstatic int 5730228561Snpsysctl_tx_rate(SYSCTL_HANDLER_ARGS) 5731228561Snp{ 5732228561Snp struct adapter *sc = arg1; 5733228561Snp struct sbuf *sb; 5734228561Snp int rc; 5735228561Snp u64 nrate[NCHAN], orate[NCHAN]; 5736228561Snp 5737228561Snp rc = sysctl_wire_old_buffer(req, 0); 5738228561Snp if (rc != 0) 5739228561Snp return (rc); 5740228561Snp 5741228561Snp sb = sbuf_new_for_sysctl(NULL, NULL, 256, req); 5742228561Snp if (sb == NULL) 5743228561Snp return (ENOMEM); 5744228561Snp 5745228561Snp t4_get_chan_txrate(sc, nrate, orate); 5746228561Snp sbuf_printf(sb, " channel 0 channel 1 channel 2 " 5747228561Snp "channel 3\n"); 5748228561Snp sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n", 5749228561Snp nrate[0], nrate[1], nrate[2], nrate[3]); 5750228561Snp sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju", 5751228561Snp orate[0], orate[1], orate[2], orate[3]); 5752228561Snp 5753228561Snp rc = sbuf_finish(sb); 5754228561Snp sbuf_delete(sb); 5755228561Snp 5756228561Snp return (rc); 5757228561Snp} 5758248925Snp 5759248925Snpstatic int 5760248925Snpsysctl_wrwc_stats(SYSCTL_HANDLER_ARGS) 5761248925Snp{ 5762248925Snp struct adapter *sc = arg1; 5763248925Snp struct sbuf *sb; 5764248925Snp int rc, v; 5765248925Snp 5766248925Snp rc = sysctl_wire_old_buffer(req, 0); 5767248925Snp if (rc != 0) 5768248925Snp return (rc); 5769248925Snp 5770248925Snp sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req); 5771248925Snp if (sb == NULL) 5772248925Snp return (ENOMEM); 5773248925Snp 5774248925Snp v = t4_read_reg(sc, A_SGE_STAT_CFG); 5775248925Snp if (G_STATSOURCE_T5(v) == 7) { 5776248925Snp if (G_STATMODE(v) == 0) { 5777248925Snp sbuf_printf(sb, "\ntotal %d, incomplete %d", 5778248925Snp t4_read_reg(sc, A_SGE_STAT_TOTAL), 5779248925Snp t4_read_reg(sc, A_SGE_STAT_MATCH)); 5780248925Snp } else if (G_STATMODE(v) == 1) { 5781248925Snp sbuf_printf(sb, "\ntotal %d, data overflow %d", 5782248925Snp t4_read_reg(sc, A_SGE_STAT_TOTAL), 5783248925Snp t4_read_reg(sc, A_SGE_STAT_MATCH)); 5784248925Snp } 5785248925Snp } 5786248925Snp rc = sbuf_finish(sb); 5787248925Snp sbuf_delete(sb); 5788248925Snp 5789248925Snp return (rc); 5790248925Snp} 5791231115Snp#endif 5792228561Snp 5793219286Snpstatic inline void 5794219286Snptxq_start(struct ifnet *ifp, struct sge_txq *txq) 5795219286Snp{ 5796219286Snp struct buf_ring *br; 5797219286Snp struct mbuf *m; 5798219286Snp 5799219286Snp TXQ_LOCK_ASSERT_OWNED(txq); 5800219286Snp 5801220873Snp br = txq->br; 5802219286Snp m = txq->m ? txq->m : drbr_dequeue(ifp, br); 5803219286Snp if (m) 5804219286Snp t4_eth_tx(ifp, txq, m); 5805219286Snp} 5806219286Snp 5807219286Snpvoid 5808228561Snpt4_tx_callout(void *arg) 5809219286Snp{ 5810228561Snp struct sge_eq *eq = arg; 5811228561Snp struct adapter *sc; 5812219286Snp 5813228561Snp if (EQ_TRYLOCK(eq) == 0) 5814228561Snp goto reschedule; 5815228561Snp 5816228561Snp if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) { 5817228561Snp EQ_UNLOCK(eq); 5818228561Snpreschedule: 5819228561Snp if (__predict_true(!(eq->flags && EQ_DOOMED))) 5820228561Snp callout_schedule(&eq->tx_callout, 1); 5821228561Snp return; 5822228561Snp } 5823228561Snp 5824228561Snp EQ_LOCK_ASSERT_OWNED(eq); 5825228561Snp 5826228561Snp if (__predict_true((eq->flags & EQ_DOOMED) == 0)) { 5827228561Snp 5828228561Snp if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 5829228561Snp struct sge_txq *txq = arg; 5830228561Snp struct port_info *pi = txq->ifp->if_softc; 5831228561Snp 5832228561Snp sc = pi->adapter; 5833228561Snp } else { 5834228561Snp struct sge_wrq *wrq = arg; 5835228561Snp 5836228561Snp sc = wrq->adapter; 5837228561Snp } 5838228561Snp 5839228561Snp taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task); 5840228561Snp } 5841228561Snp 5842228561Snp EQ_UNLOCK(eq); 5843228561Snp} 5844228561Snp 5845228561Snpvoid 5846228561Snpt4_tx_task(void *arg, int count) 5847228561Snp{ 5848228561Snp struct sge_eq *eq = arg; 5849228561Snp 5850228561Snp EQ_LOCK(eq); 5851228561Snp if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) { 5852228561Snp struct sge_txq *txq = arg; 5853220649Snp txq_start(txq->ifp, txq); 5854228561Snp } else { 5855228561Snp struct sge_wrq *wrq = arg; 5856228561Snp t4_wrq_tx_locked(wrq->adapter, wrq, NULL); 5857228561Snp } 5858228561Snp EQ_UNLOCK(eq); 5859219286Snp} 5860219286Snp 5861221474Snpstatic uint32_t 5862221474Snpfconf_to_mode(uint32_t fconf) 5863221474Snp{ 5864221474Snp uint32_t mode; 5865221474Snp 5866221474Snp mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR | 5867221474Snp T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT; 5868221474Snp 5869221474Snp if (fconf & F_FRAGMENTATION) 5870221474Snp mode |= T4_FILTER_IP_FRAGMENT; 5871221474Snp 5872221474Snp if (fconf & F_MPSHITTYPE) 5873221474Snp mode |= T4_FILTER_MPS_HIT_TYPE; 5874221474Snp 5875221474Snp if (fconf & F_MACMATCH) 5876221474Snp mode |= T4_FILTER_MAC_IDX; 5877221474Snp 5878221474Snp if (fconf & F_ETHERTYPE) 5879221474Snp mode |= T4_FILTER_ETH_TYPE; 5880221474Snp 5881221474Snp if (fconf & F_PROTOCOL) 5882221474Snp mode |= T4_FILTER_IP_PROTO; 5883221474Snp 5884221474Snp if (fconf & F_TOS) 5885221474Snp mode |= T4_FILTER_IP_TOS; 5886221474Snp 5887221474Snp if (fconf & F_VLAN) 5888228561Snp mode |= T4_FILTER_VLAN; 5889221474Snp 5890221474Snp if (fconf & F_VNIC_ID) 5891228561Snp mode |= T4_FILTER_VNIC; 5892221474Snp 5893221474Snp if (fconf & F_PORT) 5894221474Snp mode |= T4_FILTER_PORT; 5895221474Snp 5896221474Snp if (fconf & F_FCOE) 5897221474Snp mode |= T4_FILTER_FCoE; 5898221474Snp 5899221474Snp return (mode); 5900221474Snp} 5901221474Snp 5902221474Snpstatic uint32_t 5903221474Snpmode_to_fconf(uint32_t mode) 5904221474Snp{ 5905221474Snp uint32_t fconf = 0; 5906221474Snp 5907221474Snp if (mode & T4_FILTER_IP_FRAGMENT) 5908221474Snp fconf |= F_FRAGMENTATION; 5909221474Snp 5910221474Snp if (mode & T4_FILTER_MPS_HIT_TYPE) 5911221474Snp fconf |= F_MPSHITTYPE; 5912221474Snp 5913221474Snp if (mode & T4_FILTER_MAC_IDX) 5914221474Snp fconf |= F_MACMATCH; 5915221474Snp 5916221474Snp if (mode & T4_FILTER_ETH_TYPE) 5917221474Snp fconf |= F_ETHERTYPE; 5918221474Snp 5919221474Snp if (mode & T4_FILTER_IP_PROTO) 5920221474Snp fconf |= F_PROTOCOL; 5921221474Snp 5922221474Snp if (mode & T4_FILTER_IP_TOS) 5923221474Snp fconf |= F_TOS; 5924221474Snp 5925228561Snp if (mode & T4_FILTER_VLAN) 5926221474Snp fconf |= F_VLAN; 5927221474Snp 5928228561Snp if (mode & T4_FILTER_VNIC) 5929221474Snp fconf |= F_VNIC_ID; 5930221474Snp 5931221474Snp if (mode & T4_FILTER_PORT) 5932221474Snp fconf |= F_PORT; 5933221474Snp 5934221474Snp if (mode & T4_FILTER_FCoE) 5935221474Snp fconf |= F_FCOE; 5936221474Snp 5937221474Snp return (fconf); 5938221474Snp} 5939221474Snp 5940221474Snpstatic uint32_t 5941221474Snpfspec_to_fconf(struct t4_filter_specification *fs) 5942221474Snp{ 5943221474Snp uint32_t fconf = 0; 5944221474Snp 5945221474Snp if (fs->val.frag || fs->mask.frag) 5946221474Snp fconf |= F_FRAGMENTATION; 5947221474Snp 5948221474Snp if (fs->val.matchtype || fs->mask.matchtype) 5949221474Snp fconf |= F_MPSHITTYPE; 5950221474Snp 5951221474Snp if (fs->val.macidx || fs->mask.macidx) 5952221474Snp fconf |= F_MACMATCH; 5953221474Snp 5954221474Snp if (fs->val.ethtype || fs->mask.ethtype) 5955221474Snp fconf |= F_ETHERTYPE; 5956221474Snp 5957221474Snp if (fs->val.proto || fs->mask.proto) 5958221474Snp fconf |= F_PROTOCOL; 5959221474Snp 5960221474Snp if (fs->val.tos || fs->mask.tos) 5961221474Snp fconf |= F_TOS; 5962221474Snp 5963228561Snp if (fs->val.vlan_vld || fs->mask.vlan_vld) 5964221474Snp fconf |= F_VLAN; 5965221474Snp 5966228561Snp if (fs->val.vnic_vld || fs->mask.vnic_vld) 5967221474Snp fconf |= F_VNIC_ID; 5968221474Snp 5969221474Snp if (fs->val.iport || fs->mask.iport) 5970221474Snp fconf |= F_PORT; 5971221474Snp 5972221474Snp if (fs->val.fcoe || fs->mask.fcoe) 5973221474Snp fconf |= F_FCOE; 5974221474Snp 5975221474Snp return (fconf); 5976221474Snp} 5977221474Snp 5978221474Snpstatic int 5979221474Snpget_filter_mode(struct adapter *sc, uint32_t *mode) 5980221474Snp{ 5981245274Snp int rc; 5982221474Snp uint32_t fconf; 5983221474Snp 5984245274Snp rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 5985245274Snp "t4getfm"); 5986245274Snp if (rc) 5987245274Snp return (rc); 5988245274Snp 5989221474Snp t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1, 5990221474Snp A_TP_VLAN_PRI_MAP); 5991221474Snp 5992228561Snp if (sc->filter_mode != fconf) { 5993228561Snp log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n", 5994228561Snp device_get_nameunit(sc->dev), sc->filter_mode, fconf); 5995228561Snp sc->filter_mode = fconf; 5996228561Snp } 5997221474Snp 5998228561Snp *mode = fconf_to_mode(sc->filter_mode); 5999228561Snp 6000245274Snp end_synchronized_op(sc, LOCK_HELD); 6001221474Snp return (0); 6002221474Snp} 6003221474Snp 6004221474Snpstatic int 6005221474Snpset_filter_mode(struct adapter *sc, uint32_t mode) 6006221474Snp{ 6007221474Snp uint32_t fconf; 6008221474Snp int rc; 6009221474Snp 6010221474Snp fconf = mode_to_fconf(mode); 6011221474Snp 6012245274Snp rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6013245274Snp "t4setfm"); 6014245274Snp if (rc) 6015245274Snp return (rc); 6016221474Snp 6017221474Snp if (sc->tids.ftids_in_use > 0) { 6018221474Snp rc = EBUSY; 6019221474Snp goto done; 6020221474Snp } 6021221474Snp 6022237263Snp#ifdef TCP_OFFLOAD 6023228561Snp if (sc->offload_map) { 6024228561Snp rc = EBUSY; 6025228561Snp goto done; 6026228561Snp } 6027228561Snp#endif 6028228561Snp 6029228561Snp#ifdef notyet 6030221474Snp rc = -t4_set_filter_mode(sc, fconf); 6031228561Snp if (rc == 0) 6032228561Snp sc->filter_mode = fconf; 6033228561Snp#else 6034228561Snp rc = ENOTSUP; 6035228561Snp#endif 6036228561Snp 6037221474Snpdone: 6038245274Snp end_synchronized_op(sc, LOCK_HELD); 6039221474Snp return (rc); 6040221474Snp} 6041221474Snp 6042222552Snpstatic inline uint64_t 6043222552Snpget_filter_hits(struct adapter *sc, uint32_t fid) 6044222552Snp{ 6045248925Snp uint32_t mw_base, off, tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE); 6046222552Snp uint64_t hits; 6047222552Snp 6048248925Snp memwin_info(sc, 0, &mw_base, NULL); 6049248925Snp off = position_memwin(sc, 0, 6050222552Snp tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE); 6051248925Snp hits = t4_read_reg64(sc, mw_base + off + 16); 6052222552Snp 6053222552Snp return (be64toh(hits)); 6054222552Snp} 6055222552Snp 6056221474Snpstatic int 6057221474Snpget_filter(struct adapter *sc, struct t4_filter *t) 6058221474Snp{ 6059245274Snp int i, rc, nfilters = sc->tids.nftids; 6060221474Snp struct filter_entry *f; 6061221474Snp 6062245274Snp rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK, 6063245274Snp "t4getf"); 6064245274Snp if (rc) 6065245274Snp return (rc); 6066221474Snp 6067221474Snp if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL || 6068221474Snp t->idx >= nfilters) { 6069221474Snp t->idx = 0xffffffff; 6070245274Snp goto done; 6071221474Snp } 6072221474Snp 6073221474Snp f = &sc->tids.ftid_tab[t->idx]; 6074221474Snp for (i = t->idx; i < nfilters; i++, f++) { 6075221474Snp if (f->valid) { 6076221474Snp t->idx = i; 6077222509Snp t->l2tidx = f->l2t ? f->l2t->idx : 0; 6078222509Snp t->smtidx = f->smtidx; 6079222552Snp if (f->fs.hitcnts) 6080222552Snp t->hits = get_filter_hits(sc, t->idx); 6081222552Snp else 6082222552Snp t->hits = UINT64_MAX; 6083221474Snp t->fs = f->fs; 6084221474Snp 6085245274Snp goto done; 6086221474Snp } 6087221474Snp } 6088221474Snp 6089221474Snp t->idx = 0xffffffff; 6090245274Snpdone: 6091245274Snp end_synchronized_op(sc, LOCK_HELD); 6092221474Snp return (0); 6093221474Snp} 6094221474Snp 6095221474Snpstatic int 6096221474Snpset_filter(struct adapter *sc, struct t4_filter *t) 6097221474Snp{ 6098221474Snp unsigned int nfilters, nports; 6099221474Snp struct filter_entry *f; 6100245274Snp int i, rc; 6101221474Snp 6102245274Snp rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf"); 6103245274Snp if (rc) 6104245274Snp return (rc); 6105221474Snp 6106221474Snp nfilters = sc->tids.nftids; 6107221474Snp nports = sc->params.nports; 6108221474Snp 6109245274Snp if (nfilters == 0) { 6110245274Snp rc = ENOTSUP; 6111245274Snp goto done; 6112245274Snp } 6113221474Snp 6114245274Snp if (!(sc->flags & FULL_INIT_DONE)) { 6115245274Snp rc = EAGAIN; 6116245274Snp goto done; 6117245274Snp } 6118221474Snp 6119245274Snp if (t->idx >= nfilters) { 6120245274Snp rc = EINVAL; 6121245274Snp goto done; 6122245274Snp } 6123221474Snp 6124221474Snp /* Validate against the global filter mode */ 6125245274Snp if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) { 6126245274Snp rc = E2BIG; 6127245274Snp goto done; 6128245274Snp } 6129221474Snp 6130245274Snp if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) { 6131245274Snp rc = EINVAL; 6132245274Snp goto done; 6133245274Snp } 6134221474Snp 6135245274Snp if (t->fs.val.iport >= nports) { 6136245274Snp rc = EINVAL; 6137245274Snp goto done; 6138245274Snp } 6139221474Snp 6140221474Snp /* Can't specify an iq if not steering to it */ 6141245274Snp if (!t->fs.dirsteer && t->fs.iq) { 6142245274Snp rc = EINVAL; 6143245274Snp goto done; 6144245274Snp } 6145221474Snp 6146221474Snp /* IPv6 filter idx must be 4 aligned */ 6147221474Snp if (t->fs.type == 1 && 6148245274Snp ((t->idx & 0x3) || t->idx + 4 >= nfilters)) { 6149245274Snp rc = EINVAL; 6150245274Snp goto done; 6151245274Snp } 6152221474Snp 6153221474Snp if (sc->tids.ftid_tab == NULL) { 6154221474Snp KASSERT(sc->tids.ftids_in_use == 0, 6155221474Snp ("%s: no memory allocated but filters_in_use > 0", 6156221474Snp __func__)); 6157221474Snp 6158221474Snp sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) * 6159221474Snp nfilters, M_CXGBE, M_NOWAIT | M_ZERO); 6160245274Snp if (sc->tids.ftid_tab == NULL) { 6161245274Snp rc = ENOMEM; 6162245274Snp goto done; 6163245274Snp } 6164245274Snp mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF); 6165221474Snp } 6166221474Snp 6167221474Snp for (i = 0; i < 4; i++) { 6168221474Snp f = &sc->tids.ftid_tab[t->idx + i]; 6169221474Snp 6170245274Snp if (f->pending || f->valid) { 6171245274Snp rc = EBUSY; 6172245274Snp goto done; 6173245274Snp } 6174245274Snp if (f->locked) { 6175245274Snp rc = EPERM; 6176245274Snp goto done; 6177245274Snp } 6178221474Snp 6179221474Snp if (t->fs.type == 0) 6180221474Snp break; 6181221474Snp } 6182221474Snp 6183221474Snp f = &sc->tids.ftid_tab[t->idx]; 6184221474Snp f->fs = t->fs; 6185221474Snp 6186245274Snp rc = set_filter_wr(sc, t->idx); 6187245274Snpdone: 6188245274Snp end_synchronized_op(sc, 0); 6189245274Snp 6190245274Snp if (rc == 0) { 6191245274Snp mtx_lock(&sc->tids.ftid_lock); 6192245274Snp for (;;) { 6193245274Snp if (f->pending == 0) { 6194245274Snp rc = f->valid ? 0 : EIO; 6195245274Snp break; 6196245274Snp } 6197245274Snp 6198245274Snp if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 6199245274Snp PCATCH, "t4setfw", 0)) { 6200245274Snp rc = EINPROGRESS; 6201245274Snp break; 6202245274Snp } 6203245274Snp } 6204245274Snp mtx_unlock(&sc->tids.ftid_lock); 6205245274Snp } 6206245274Snp return (rc); 6207221474Snp} 6208221474Snp 6209221474Snpstatic int 6210221474Snpdel_filter(struct adapter *sc, struct t4_filter *t) 6211221474Snp{ 6212221474Snp unsigned int nfilters; 6213221474Snp struct filter_entry *f; 6214245274Snp int rc; 6215221474Snp 6216245274Snp rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf"); 6217245274Snp if (rc) 6218245274Snp return (rc); 6219221474Snp 6220221474Snp nfilters = sc->tids.nftids; 6221221474Snp 6222245274Snp if (nfilters == 0) { 6223245274Snp rc = ENOTSUP; 6224245274Snp goto done; 6225245274Snp } 6226221474Snp 6227221474Snp if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 || 6228245274Snp t->idx >= nfilters) { 6229245274Snp rc = EINVAL; 6230245274Snp goto done; 6231245274Snp } 6232221474Snp 6233245274Snp if (!(sc->flags & FULL_INIT_DONE)) { 6234245274Snp rc = EAGAIN; 6235245274Snp goto done; 6236245274Snp } 6237221474Snp 6238221474Snp f = &sc->tids.ftid_tab[t->idx]; 6239221474Snp 6240245274Snp if (f->pending) { 6241245274Snp rc = EBUSY; 6242245274Snp goto done; 6243245274Snp } 6244245274Snp if (f->locked) { 6245245274Snp rc = EPERM; 6246245274Snp goto done; 6247245274Snp } 6248221474Snp 6249221474Snp if (f->valid) { 6250221474Snp t->fs = f->fs; /* extra info for the caller */ 6251245274Snp rc = del_filter_wr(sc, t->idx); 6252221474Snp } 6253221474Snp 6254245274Snpdone: 6255245274Snp end_synchronized_op(sc, 0); 6256245274Snp 6257245274Snp if (rc == 0) { 6258245274Snp mtx_lock(&sc->tids.ftid_lock); 6259245274Snp for (;;) { 6260245274Snp if (f->pending == 0) { 6261245274Snp rc = f->valid ? EIO : 0; 6262245274Snp break; 6263245274Snp } 6264245274Snp 6265245274Snp if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock, 6266245274Snp PCATCH, "t4delfw", 0)) { 6267245274Snp rc = EINPROGRESS; 6268245274Snp break; 6269245274Snp } 6270245274Snp } 6271245274Snp mtx_unlock(&sc->tids.ftid_lock); 6272245274Snp } 6273245274Snp 6274245274Snp return (rc); 6275221474Snp} 6276221474Snp 6277221474Snpstatic void 6278222509Snpclear_filter(struct filter_entry *f) 6279221474Snp{ 6280222509Snp if (f->l2t) 6281222509Snp t4_l2t_release(f->l2t); 6282222509Snp 6283221474Snp bzero(f, sizeof (*f)); 6284221474Snp} 6285221474Snp 6286221474Snpstatic int 6287221474Snpset_filter_wr(struct adapter *sc, int fidx) 6288221474Snp{ 6289221474Snp struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 6290237263Snp struct wrqe *wr; 6291221474Snp struct fw_filter_wr *fwr; 6292221474Snp unsigned int ftid; 6293221474Snp 6294245274Snp ASSERT_SYNCHRONIZED_OP(sc); 6295221474Snp 6296222509Snp if (f->fs.newdmac || f->fs.newvlan) { 6297222509Snp /* This filter needs an L2T entry; allocate one. */ 6298222509Snp f->l2t = t4_l2t_alloc_switching(sc->l2t); 6299222509Snp if (f->l2t == NULL) 6300222509Snp return (EAGAIN); 6301222509Snp if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport, 6302222509Snp f->fs.dmac)) { 6303222509Snp t4_l2t_release(f->l2t); 6304222509Snp f->l2t = NULL; 6305222509Snp return (ENOMEM); 6306222509Snp } 6307222509Snp } 6308221474Snp 6309221474Snp ftid = sc->tids.ftid_base + fidx; 6310221474Snp 6311237263Snp wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 6312237263Snp if (wr == NULL) 6313221474Snp return (ENOMEM); 6314221474Snp 6315237263Snp fwr = wrtod(wr); 6316221474Snp bzero(fwr, sizeof (*fwr)); 6317221474Snp 6318221474Snp fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR)); 6319221474Snp fwr->len16_pkd = htobe32(FW_LEN16(*fwr)); 6320221474Snp fwr->tid_to_iq = 6321221474Snp htobe32(V_FW_FILTER_WR_TID(ftid) | 6322221474Snp V_FW_FILTER_WR_RQTYPE(f->fs.type) | 6323221474Snp V_FW_FILTER_WR_NOREPLY(0) | 6324221474Snp V_FW_FILTER_WR_IQ(f->fs.iq)); 6325221474Snp fwr->del_filter_to_l2tix = 6326221474Snp htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) | 6327221474Snp V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) | 6328221474Snp V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) | 6329221474Snp V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) | 6330221474Snp V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) | 6331221474Snp V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) | 6332221474Snp V_FW_FILTER_WR_DMAC(f->fs.newdmac) | 6333221474Snp V_FW_FILTER_WR_SMAC(f->fs.newsmac) | 6334221474Snp V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT || 6335221474Snp f->fs.newvlan == VLAN_REWRITE) | 6336221474Snp V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE || 6337221474Snp f->fs.newvlan == VLAN_REWRITE) | 6338221474Snp V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) | 6339221474Snp V_FW_FILTER_WR_TXCHAN(f->fs.eport) | 6340221474Snp V_FW_FILTER_WR_PRIO(f->fs.prio) | 6341222509Snp V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0)); 6342221474Snp fwr->ethtype = htobe16(f->fs.val.ethtype); 6343221474Snp fwr->ethtypem = htobe16(f->fs.mask.ethtype); 6344221474Snp fwr->frag_to_ovlan_vldm = 6345221474Snp (V_FW_FILTER_WR_FRAG(f->fs.val.frag) | 6346221474Snp V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) | 6347228561Snp V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) | 6348228561Snp V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) | 6349228561Snp V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) | 6350228561Snp V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld)); 6351221474Snp fwr->smac_sel = 0; 6352221474Snp fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) | 6353228561Snp V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id)); 6354221474Snp fwr->maci_to_matchtypem = 6355221474Snp htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) | 6356221474Snp V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) | 6357221474Snp V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) | 6358221474Snp V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) | 6359221474Snp V_FW_FILTER_WR_PORT(f->fs.val.iport) | 6360221474Snp V_FW_FILTER_WR_PORTM(f->fs.mask.iport) | 6361221474Snp V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) | 6362221474Snp V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype)); 6363221474Snp fwr->ptcl = f->fs.val.proto; 6364221474Snp fwr->ptclm = f->fs.mask.proto; 6365221474Snp fwr->ttyp = f->fs.val.tos; 6366221474Snp fwr->ttypm = f->fs.mask.tos; 6367228561Snp fwr->ivlan = htobe16(f->fs.val.vlan); 6368228561Snp fwr->ivlanm = htobe16(f->fs.mask.vlan); 6369228561Snp fwr->ovlan = htobe16(f->fs.val.vnic); 6370228561Snp fwr->ovlanm = htobe16(f->fs.mask.vnic); 6371221474Snp bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip)); 6372221474Snp bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm)); 6373221474Snp bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip)); 6374221474Snp bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm)); 6375221474Snp fwr->lp = htobe16(f->fs.val.dport); 6376221474Snp fwr->lpm = htobe16(f->fs.mask.dport); 6377221474Snp fwr->fp = htobe16(f->fs.val.sport); 6378221474Snp fwr->fpm = htobe16(f->fs.mask.sport); 6379221474Snp if (f->fs.newsmac) 6380221474Snp bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma)); 6381221474Snp 6382221474Snp f->pending = 1; 6383221474Snp sc->tids.ftids_in_use++; 6384228561Snp 6385237263Snp t4_wrq_tx(sc, wr); 6386228561Snp return (0); 6387221474Snp} 6388221474Snp 6389221474Snpstatic int 6390221474Snpdel_filter_wr(struct adapter *sc, int fidx) 6391221474Snp{ 6392221474Snp struct filter_entry *f = &sc->tids.ftid_tab[fidx]; 6393237263Snp struct wrqe *wr; 6394221474Snp struct fw_filter_wr *fwr; 6395228561Snp unsigned int ftid; 6396221474Snp 6397221474Snp ftid = sc->tids.ftid_base + fidx; 6398221474Snp 6399237263Snp wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq); 6400237263Snp if (wr == NULL) 6401221474Snp return (ENOMEM); 6402237263Snp fwr = wrtod(wr); 6403221474Snp bzero(fwr, sizeof (*fwr)); 6404221474Snp 6405228561Snp t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id); 6406221474Snp 6407221474Snp f->pending = 1; 6408237263Snp t4_wrq_tx(sc, wr); 6409228561Snp return (0); 6410221474Snp} 6411221474Snp 6412239338Snpint 6413239338Snpt4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 6414221474Snp{ 6415228561Snp struct adapter *sc = iq->adapter; 6416228561Snp const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1); 6417221474Snp unsigned int idx = GET_TID(rpl); 6418221474Snp 6419228561Snp KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__, 6420228561Snp rss->opcode)); 6421228561Snp 6422221474Snp if (idx >= sc->tids.ftid_base && 6423221474Snp (idx -= sc->tids.ftid_base) < sc->tids.nftids) { 6424221474Snp unsigned int rc = G_COOKIE(rpl->cookie); 6425221474Snp struct filter_entry *f = &sc->tids.ftid_tab[idx]; 6426221474Snp 6427245274Snp mtx_lock(&sc->tids.ftid_lock); 6428228561Snp if (rc == FW_FILTER_WR_FLT_ADDED) { 6429245274Snp KASSERT(f->pending, ("%s: filter[%u] isn't pending.", 6430245274Snp __func__, idx)); 6431221474Snp f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff; 6432221474Snp f->pending = 0; /* asynchronous setup completed */ 6433221474Snp f->valid = 1; 6434231120Snp } else { 6435231120Snp if (rc != FW_FILTER_WR_FLT_DELETED) { 6436231120Snp /* Add or delete failed, display an error */ 6437231120Snp log(LOG_ERR, 6438231120Snp "filter %u setup failed with error %u\n", 6439231120Snp idx, rc); 6440231120Snp } 6441228561Snp 6442231120Snp clear_filter(f); 6443231120Snp sc->tids.ftids_in_use--; 6444221474Snp } 6445245274Snp wakeup(&sc->tids.ftid_tab); 6446245274Snp mtx_unlock(&sc->tids.ftid_lock); 6447221474Snp } 6448228561Snp 6449228561Snp return (0); 6450221474Snp} 6451221474Snp 6452222973Snpstatic int 6453222973Snpget_sge_context(struct adapter *sc, struct t4_sge_context *cntxt) 6454222973Snp{ 6455245274Snp int rc; 6456222973Snp 6457222973Snp if (cntxt->cid > M_CTXTQID) 6458245274Snp return (EINVAL); 6459222973Snp 6460222973Snp if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS && 6461222973Snp cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM) 6462245274Snp return (EINVAL); 6463222973Snp 6464246575Snp rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ctxt"); 6465246575Snp if (rc) 6466246575Snp return (rc); 6467246575Snp 6468222973Snp if (sc->flags & FW_OK) { 6469246575Snp rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id, 6470246575Snp &cntxt->data[0]); 6471246575Snp if (rc == 0) 6472246575Snp goto done; 6473222973Snp } 6474222973Snp 6475245274Snp /* 6476245274Snp * Read via firmware failed or wasn't even attempted. Read directly via 6477245274Snp * the backdoor. 6478245274Snp */ 6479246575Snp rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id, &cntxt->data[0]); 6480246575Snpdone: 6481246575Snp end_synchronized_op(sc, 0); 6482245274Snp return (rc); 6483245274Snp} 6484222973Snp 6485245274Snpstatic int 6486245274Snpload_fw(struct adapter *sc, struct t4_data *fw) 6487245274Snp{ 6488245274Snp int rc; 6489245274Snp uint8_t *fw_data; 6490245274Snp 6491245274Snp rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw"); 6492245274Snp if (rc) 6493245274Snp return (rc); 6494245274Snp 6495245274Snp if (sc->flags & FULL_INIT_DONE) { 6496245274Snp rc = EBUSY; 6497245274Snp goto done; 6498222973Snp } 6499222973Snp 6500245274Snp fw_data = malloc(fw->len, M_CXGBE, M_WAITOK); 6501245274Snp if (fw_data == NULL) { 6502245274Snp rc = ENOMEM; 6503245274Snp goto done; 6504245274Snp } 6505245274Snp 6506245274Snp rc = copyin(fw->data, fw_data, fw->len); 6507245274Snp if (rc == 0) 6508245274Snp rc = -t4_load_fw(sc, fw_data, fw->len); 6509245274Snp 6510245274Snp free(fw_data, M_CXGBE); 6511245274Snpdone: 6512245274Snp end_synchronized_op(sc, 0); 6513222973Snp return (rc); 6514222973Snp} 6515222973Snp 6516228561Snpstatic int 6517248925Snpread_card_mem(struct adapter *sc, int win, struct t4_mem_range *mr) 6518228561Snp{ 6519248925Snp uint32_t addr, off, remaining, i, n; 6520228561Snp uint32_t *buf, *b; 6521248925Snp uint32_t mw_base, mw_aperture; 6522228561Snp int rc; 6523248925Snp uint8_t *dst; 6524228561Snp 6525248925Snp rc = validate_mem_range(sc, mr->addr, mr->len); 6526248925Snp if (rc != 0) 6527248925Snp return (rc); 6528228561Snp 6529248925Snp memwin_info(sc, win, &mw_base, &mw_aperture); 6530248925Snp buf = b = malloc(min(mr->len, mw_aperture), M_CXGBE, M_WAITOK); 6531248925Snp addr = mr->addr; 6532228561Snp remaining = mr->len; 6533248925Snp dst = (void *)mr->data; 6534228561Snp 6535228561Snp while (remaining) { 6536248925Snp off = position_memwin(sc, win, addr); 6537228561Snp 6538228561Snp /* number of bytes that we'll copy in the inner loop */ 6539248925Snp n = min(remaining, mw_aperture - off); 6540248925Snp for (i = 0; i < n; i += 4) 6541248925Snp *b++ = t4_read_reg(sc, mw_base + off + i); 6542228561Snp 6543248925Snp rc = copyout(buf, dst, n); 6544248925Snp if (rc != 0) 6545248925Snp break; 6546228561Snp 6547248925Snp b = buf; 6548248925Snp dst += n; 6549248925Snp remaining -= n; 6550248925Snp addr += n; 6551228561Snp } 6552228561Snp 6553228561Snp free(buf, M_CXGBE); 6554228561Snp return (rc); 6555228561Snp} 6556228561Snp 6557241399Snpstatic int 6558241399Snpread_i2c(struct adapter *sc, struct t4_i2c_data *i2cd) 6559241399Snp{ 6560241399Snp int rc; 6561241399Snp 6562241399Snp if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports) 6563241399Snp return (EINVAL); 6564241399Snp 6565241399Snp if (i2cd->len > 1) { 6566241399Snp /* XXX: need fw support for longer reads in one go */ 6567241399Snp return (ENOTSUP); 6568241399Snp } 6569241399Snp 6570245274Snp rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd"); 6571245274Snp if (rc) 6572245274Snp return (rc); 6573241399Snp rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr, 6574241399Snp i2cd->offset, &i2cd->data[0]); 6575245274Snp end_synchronized_op(sc, 0); 6576241399Snp 6577241399Snp return (rc); 6578241399Snp} 6579241399Snp 6580218792Snpint 6581218792Snpt4_os_find_pci_capability(struct adapter *sc, int cap) 6582218792Snp{ 6583222102Snp int i; 6584218792Snp 6585222102Snp return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0); 6586218792Snp} 6587218792Snp 6588218792Snpint 6589218792Snpt4_os_pci_save_state(struct adapter *sc) 6590218792Snp{ 6591218792Snp device_t dev; 6592218792Snp struct pci_devinfo *dinfo; 6593218792Snp 6594218792Snp dev = sc->dev; 6595218792Snp dinfo = device_get_ivars(dev); 6596218792Snp 6597218792Snp pci_cfg_save(dev, dinfo, 0); 6598218792Snp return (0); 6599218792Snp} 6600218792Snp 6601218792Snpint 6602218792Snpt4_os_pci_restore_state(struct adapter *sc) 6603218792Snp{ 6604218792Snp device_t dev; 6605218792Snp struct pci_devinfo *dinfo; 6606218792Snp 6607218792Snp dev = sc->dev; 6608218792Snp dinfo = device_get_ivars(dev); 6609218792Snp 6610218792Snp pci_cfg_restore(dev, dinfo); 6611218792Snp return (0); 6612218792Snp} 6613219299Snp 6614218792Snpvoid 6615218792Snpt4_os_portmod_changed(const struct adapter *sc, int idx) 6616218792Snp{ 6617218792Snp struct port_info *pi = sc->port[idx]; 6618218792Snp static const char *mod_str[] = { 6619220232Snp NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 6620218792Snp }; 6621218792Snp 6622218792Snp if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 6623218792Snp if_printf(pi->ifp, "transceiver unplugged.\n"); 6624220232Snp else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 6625220232Snp if_printf(pi->ifp, "unknown transceiver inserted.\n"); 6626220232Snp else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 6627220232Snp if_printf(pi->ifp, "unsupported transceiver inserted.\n"); 6628240452Snp else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) { 6629218792Snp if_printf(pi->ifp, "%s transceiver inserted.\n", 6630218792Snp mod_str[pi->mod_type]); 6631219299Snp } else { 6632219299Snp if_printf(pi->ifp, "transceiver (type %d) inserted.\n", 6633219299Snp pi->mod_type); 6634219299Snp } 6635218792Snp} 6636218792Snp 6637218792Snpvoid 6638218792Snpt4_os_link_changed(struct adapter *sc, int idx, int link_stat) 6639218792Snp{ 6640218792Snp struct port_info *pi = sc->port[idx]; 6641218792Snp struct ifnet *ifp = pi->ifp; 6642218792Snp 6643218792Snp if (link_stat) { 6644218792Snp ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed); 6645218792Snp if_link_state_change(ifp, LINK_STATE_UP); 6646218792Snp } else 6647218792Snp if_link_state_change(ifp, LINK_STATE_DOWN); 6648218792Snp} 6649218792Snp 6650228561Snpvoid 6651228561Snpt4_iterate(void (*func)(struct adapter *, void *), void *arg) 6652228561Snp{ 6653228561Snp struct adapter *sc; 6654228561Snp 6655228561Snp mtx_lock(&t4_list_lock); 6656228561Snp SLIST_FOREACH(sc, &t4_list, link) { 6657228561Snp /* 6658228561Snp * func should not make any assumptions about what state sc is 6659228561Snp * in - the only guarantee is that sc->sc_lock is a valid lock. 6660228561Snp */ 6661228561Snp func(sc, arg); 6662228561Snp } 6663228561Snp mtx_unlock(&t4_list_lock); 6664228561Snp} 6665228561Snp 6666218792Snpstatic int 6667218792Snpt4_open(struct cdev *dev, int flags, int type, struct thread *td) 6668218792Snp{ 6669218792Snp return (0); 6670218792Snp} 6671218792Snp 6672218792Snpstatic int 6673218792Snpt4_close(struct cdev *dev, int flags, int type, struct thread *td) 6674218792Snp{ 6675218792Snp return (0); 6676218792Snp} 6677218792Snp 6678218792Snpstatic int 6679218792Snpt4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, 6680218792Snp struct thread *td) 6681218792Snp{ 6682218792Snp int rc; 6683218792Snp struct adapter *sc = dev->si_drv1; 6684218792Snp 6685218792Snp rc = priv_check(td, PRIV_DRIVER); 6686218792Snp if (rc != 0) 6687218792Snp return (rc); 6688218792Snp 6689218792Snp switch (cmd) { 6690220410Snp case CHELSIO_T4_GETREG: { 6691220410Snp struct t4_reg *edata = (struct t4_reg *)data; 6692220410Snp 6693218792Snp if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 6694218792Snp return (EFAULT); 6695220410Snp 6696220410Snp if (edata->size == 4) 6697220410Snp edata->val = t4_read_reg(sc, edata->addr); 6698220410Snp else if (edata->size == 8) 6699220410Snp edata->val = t4_read_reg64(sc, edata->addr); 6700220410Snp else 6701220410Snp return (EINVAL); 6702220410Snp 6703218792Snp break; 6704218792Snp } 6705220410Snp case CHELSIO_T4_SETREG: { 6706220410Snp struct t4_reg *edata = (struct t4_reg *)data; 6707220410Snp 6708218792Snp if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) 6709218792Snp return (EFAULT); 6710220410Snp 6711220410Snp if (edata->size == 4) { 6712220410Snp if (edata->val & 0xffffffff00000000) 6713220410Snp return (EINVAL); 6714220410Snp t4_write_reg(sc, edata->addr, (uint32_t) edata->val); 6715220410Snp } else if (edata->size == 8) 6716220410Snp t4_write_reg64(sc, edata->addr, edata->val); 6717220410Snp else 6718220410Snp return (EINVAL); 6719218792Snp break; 6720218792Snp } 6721218792Snp case CHELSIO_T4_REGDUMP: { 6722218792Snp struct t4_regdump *regs = (struct t4_regdump *)data; 6723248925Snp int reglen = is_t4(sc) ? T4_REGDUMP_SIZE : T5_REGDUMP_SIZE; 6724218792Snp uint8_t *buf; 6725218792Snp 6726218792Snp if (regs->len < reglen) { 6727218792Snp regs->len = reglen; /* hint to the caller */ 6728218792Snp return (ENOBUFS); 6729218792Snp } 6730218792Snp 6731218792Snp regs->len = reglen; 6732218792Snp buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); 6733218792Snp t4_get_regs(sc, regs, buf); 6734218792Snp rc = copyout(buf, regs->data, reglen); 6735218792Snp free(buf, M_CXGBE); 6736218792Snp break; 6737218792Snp } 6738221474Snp case CHELSIO_T4_GET_FILTER_MODE: 6739221474Snp rc = get_filter_mode(sc, (uint32_t *)data); 6740221474Snp break; 6741221474Snp case CHELSIO_T4_SET_FILTER_MODE: 6742221474Snp rc = set_filter_mode(sc, *(uint32_t *)data); 6743221474Snp break; 6744221474Snp case CHELSIO_T4_GET_FILTER: 6745221474Snp rc = get_filter(sc, (struct t4_filter *)data); 6746221474Snp break; 6747221474Snp case CHELSIO_T4_SET_FILTER: 6748221474Snp rc = set_filter(sc, (struct t4_filter *)data); 6749221474Snp break; 6750221474Snp case CHELSIO_T4_DEL_FILTER: 6751221474Snp rc = del_filter(sc, (struct t4_filter *)data); 6752221474Snp break; 6753222973Snp case CHELSIO_T4_GET_SGE_CONTEXT: 6754222973Snp rc = get_sge_context(sc, (struct t4_sge_context *)data); 6755222973Snp break; 6756245274Snp case CHELSIO_T4_LOAD_FW: 6757245274Snp rc = load_fw(sc, (struct t4_data *)data); 6758228561Snp break; 6759228561Snp case CHELSIO_T4_GET_MEM: 6760248925Snp rc = read_card_mem(sc, 2, (struct t4_mem_range *)data); 6761228561Snp break; 6762241399Snp case CHELSIO_T4_GET_I2C: 6763241399Snp rc = read_i2c(sc, (struct t4_i2c_data *)data); 6764241399Snp break; 6765241409Snp case CHELSIO_T4_CLEAR_STATS: { 6766245518Snp int i; 6767241409Snp u_int port_id = *(uint32_t *)data; 6768245518Snp struct port_info *pi; 6769241409Snp 6770241409Snp if (port_id >= sc->params.nports) 6771241409Snp return (EINVAL); 6772241409Snp 6773245518Snp /* MAC stats */ 6774241409Snp t4_clr_port_stats(sc, port_id); 6775245518Snp 6776245518Snp pi = sc->port[port_id]; 6777245518Snp if (pi->flags & PORT_INIT_DONE) { 6778245518Snp struct sge_rxq *rxq; 6779245518Snp struct sge_txq *txq; 6780245518Snp struct sge_wrq *wrq; 6781245518Snp 6782245518Snp for_each_rxq(pi, i, rxq) { 6783245518Snp#if defined(INET) || defined(INET6) 6784245518Snp rxq->lro.lro_queued = 0; 6785245518Snp rxq->lro.lro_flushed = 0; 6786245518Snp#endif 6787245518Snp rxq->rxcsum = 0; 6788245518Snp rxq->vlan_extraction = 0; 6789245518Snp } 6790245518Snp 6791245518Snp for_each_txq(pi, i, txq) { 6792245518Snp txq->txcsum = 0; 6793245518Snp txq->tso_wrs = 0; 6794245518Snp txq->vlan_insertion = 0; 6795245518Snp txq->imm_wrs = 0; 6796245518Snp txq->sgl_wrs = 0; 6797245518Snp txq->txpkt_wrs = 0; 6798245518Snp txq->txpkts_wrs = 0; 6799245518Snp txq->txpkts_pkts = 0; 6800246093Snp txq->br->br_drops = 0; 6801245518Snp txq->no_dmamap = 0; 6802245518Snp txq->no_desc = 0; 6803245518Snp } 6804245518Snp 6805245518Snp#ifdef TCP_OFFLOAD 6806245518Snp /* nothing to clear for each ofld_rxq */ 6807245518Snp 6808245518Snp for_each_ofld_txq(pi, i, wrq) { 6809245518Snp wrq->tx_wrs = 0; 6810245518Snp wrq->no_desc = 0; 6811245518Snp } 6812245518Snp#endif 6813245518Snp wrq = &sc->sge.ctrlq[pi->port_id]; 6814245518Snp wrq->tx_wrs = 0; 6815245518Snp wrq->no_desc = 0; 6816245518Snp } 6817241409Snp break; 6818241409Snp } 6819218792Snp default: 6820218792Snp rc = EINVAL; 6821218792Snp } 6822218792Snp 6823218792Snp return (rc); 6824218792Snp} 6825218792Snp 6826237263Snp#ifdef TCP_OFFLOAD 6827219392Snpstatic int 6828228561Snptoe_capability(struct port_info *pi, int enable) 6829228561Snp{ 6830228561Snp int rc; 6831228561Snp struct adapter *sc = pi->adapter; 6832228561Snp 6833245274Snp ASSERT_SYNCHRONIZED_OP(sc); 6834228561Snp 6835228561Snp if (!is_offload(sc)) 6836228561Snp return (ENODEV); 6837228561Snp 6838228561Snp if (enable) { 6839237263Snp if (!(sc->flags & FULL_INIT_DONE)) { 6840245274Snp rc = cxgbe_init_synchronized(pi); 6841245274Snp if (rc) 6842245274Snp return (rc); 6843237263Snp } 6844237263Snp 6845228561Snp if (isset(&sc->offload_map, pi->port_id)) 6846228561Snp return (0); 6847228561Snp 6848237263Snp if (!(sc->flags & TOM_INIT_DONE)) { 6849237263Snp rc = t4_activate_uld(sc, ULD_TOM); 6850237263Snp if (rc == EAGAIN) { 6851237263Snp log(LOG_WARNING, 6852237263Snp "You must kldload t4_tom.ko before trying " 6853237263Snp "to enable TOE on a cxgbe interface.\n"); 6854237263Snp } 6855228561Snp if (rc != 0) 6856228561Snp return (rc); 6857237263Snp KASSERT(sc->tom_softc != NULL, 6858237263Snp ("%s: TOM activated but softc NULL", __func__)); 6859237263Snp KASSERT(sc->flags & TOM_INIT_DONE, 6860237263Snp ("%s: TOM activated but flag not set", __func__)); 6861228561Snp } 6862228561Snp 6863228561Snp setbit(&sc->offload_map, pi->port_id); 6864228561Snp } else { 6865228561Snp if (!isset(&sc->offload_map, pi->port_id)) 6866228561Snp return (0); 6867228561Snp 6868237263Snp KASSERT(sc->flags & TOM_INIT_DONE, 6869237263Snp ("%s: TOM never initialized?", __func__)); 6870228561Snp clrbit(&sc->offload_map, pi->port_id); 6871228561Snp } 6872228561Snp 6873228561Snp return (0); 6874228561Snp} 6875228561Snp 6876228561Snp/* 6877228561Snp * Add an upper layer driver to the global list. 6878228561Snp */ 6879228561Snpint 6880228561Snpt4_register_uld(struct uld_info *ui) 6881228561Snp{ 6882228561Snp int rc = 0; 6883228561Snp struct uld_info *u; 6884228561Snp 6885228561Snp mtx_lock(&t4_uld_list_lock); 6886228561Snp SLIST_FOREACH(u, &t4_uld_list, link) { 6887228561Snp if (u->uld_id == ui->uld_id) { 6888228561Snp rc = EEXIST; 6889228561Snp goto done; 6890228561Snp } 6891228561Snp } 6892228561Snp 6893228561Snp SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 6894228561Snp ui->refcount = 0; 6895228561Snpdone: 6896228561Snp mtx_unlock(&t4_uld_list_lock); 6897228561Snp return (rc); 6898228561Snp} 6899228561Snp 6900228561Snpint 6901228561Snpt4_unregister_uld(struct uld_info *ui) 6902228561Snp{ 6903228561Snp int rc = EINVAL; 6904228561Snp struct uld_info *u; 6905228561Snp 6906228561Snp mtx_lock(&t4_uld_list_lock); 6907228561Snp 6908228561Snp SLIST_FOREACH(u, &t4_uld_list, link) { 6909228561Snp if (u == ui) { 6910228561Snp if (ui->refcount > 0) { 6911228561Snp rc = EBUSY; 6912228561Snp goto done; 6913228561Snp } 6914228561Snp 6915228561Snp SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 6916228561Snp rc = 0; 6917228561Snp goto done; 6918228561Snp } 6919228561Snp } 6920228561Snpdone: 6921228561Snp mtx_unlock(&t4_uld_list_lock); 6922228561Snp return (rc); 6923228561Snp} 6924228561Snp 6925237263Snpint 6926237263Snpt4_activate_uld(struct adapter *sc, int id) 6927228561Snp{ 6928228561Snp int rc = EAGAIN; 6929228561Snp struct uld_info *ui; 6930228561Snp 6931245274Snp ASSERT_SYNCHRONIZED_OP(sc); 6932245274Snp 6933228561Snp mtx_lock(&t4_uld_list_lock); 6934228561Snp 6935228561Snp SLIST_FOREACH(ui, &t4_uld_list, link) { 6936228561Snp if (ui->uld_id == id) { 6937237263Snp rc = ui->activate(sc); 6938237263Snp if (rc == 0) 6939228561Snp ui->refcount++; 6940228561Snp goto done; 6941228561Snp } 6942228561Snp } 6943228561Snpdone: 6944228561Snp mtx_unlock(&t4_uld_list_lock); 6945228561Snp 6946228561Snp return (rc); 6947228561Snp} 6948228561Snp 6949237263Snpint 6950237263Snpt4_deactivate_uld(struct adapter *sc, int id) 6951228561Snp{ 6952237263Snp int rc = EINVAL; 6953237263Snp struct uld_info *ui; 6954228561Snp 6955245274Snp ASSERT_SYNCHRONIZED_OP(sc); 6956245274Snp 6957228561Snp mtx_lock(&t4_uld_list_lock); 6958228561Snp 6959237263Snp SLIST_FOREACH(ui, &t4_uld_list, link) { 6960237263Snp if (ui->uld_id == id) { 6961237263Snp rc = ui->deactivate(sc); 6962237263Snp if (rc == 0) 6963237263Snp ui->refcount--; 6964237263Snp goto done; 6965237263Snp } 6966228561Snp } 6967228561Snpdone: 6968228561Snp mtx_unlock(&t4_uld_list_lock); 6969228561Snp 6970228561Snp return (rc); 6971228561Snp} 6972228561Snp#endif 6973228561Snp 6974228561Snp/* 6975228561Snp * Come up with reasonable defaults for some of the tunables, provided they're 6976228561Snp * not set by the user (in which case we'll use the values as is). 6977228561Snp */ 6978228561Snpstatic void 6979228561Snptweak_tunables(void) 6980228561Snp{ 6981228561Snp int nc = mp_ncpus; /* our snapshot of the number of CPUs */ 6982228561Snp 6983228561Snp if (t4_ntxq10g < 1) 6984228561Snp t4_ntxq10g = min(nc, NTXQ_10G); 6985228561Snp 6986228561Snp if (t4_ntxq1g < 1) 6987228561Snp t4_ntxq1g = min(nc, NTXQ_1G); 6988228561Snp 6989228561Snp if (t4_nrxq10g < 1) 6990228561Snp t4_nrxq10g = min(nc, NRXQ_10G); 6991228561Snp 6992228561Snp if (t4_nrxq1g < 1) 6993228561Snp t4_nrxq1g = min(nc, NRXQ_1G); 6994228561Snp 6995237263Snp#ifdef TCP_OFFLOAD 6996228561Snp if (t4_nofldtxq10g < 1) 6997228561Snp t4_nofldtxq10g = min(nc, NOFLDTXQ_10G); 6998228561Snp 6999228561Snp if (t4_nofldtxq1g < 1) 7000228561Snp t4_nofldtxq1g = min(nc, NOFLDTXQ_1G); 7001228561Snp 7002228561Snp if (t4_nofldrxq10g < 1) 7003228561Snp t4_nofldrxq10g = min(nc, NOFLDRXQ_10G); 7004228561Snp 7005228561Snp if (t4_nofldrxq1g < 1) 7006228561Snp t4_nofldrxq1g = min(nc, NOFLDRXQ_1G); 7007238028Snp 7008238028Snp if (t4_toecaps_allowed == -1) 7009238028Snp t4_toecaps_allowed = FW_CAPS_CONFIG_TOE; 7010238028Snp#else 7011238028Snp if (t4_toecaps_allowed == -1) 7012238028Snp t4_toecaps_allowed = 0; 7013228561Snp#endif 7014228561Snp 7015228561Snp if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS) 7016228561Snp t4_tmr_idx_10g = TMR_IDX_10G; 7017228561Snp 7018228561Snp if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS) 7019228561Snp t4_pktc_idx_10g = PKTC_IDX_10G; 7020228561Snp 7021228561Snp if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS) 7022228561Snp t4_tmr_idx_1g = TMR_IDX_1G; 7023228561Snp 7024228561Snp if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS) 7025228561Snp t4_pktc_idx_1g = PKTC_IDX_1G; 7026228561Snp 7027228561Snp if (t4_qsize_txq < 128) 7028228561Snp t4_qsize_txq = 128; 7029228561Snp 7030228561Snp if (t4_qsize_rxq < 128) 7031228561Snp t4_qsize_rxq = 128; 7032228561Snp while (t4_qsize_rxq & 7) 7033228561Snp t4_qsize_rxq++; 7034228561Snp 7035228561Snp t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX; 7036228561Snp} 7037228561Snp 7038228561Snpstatic int 7039249370Snpmod_event(module_t mod, int cmd, void *arg) 7040219392Snp{ 7041228561Snp int rc = 0; 7042249370Snp static int loaded = 0; 7043219392Snp 7044228561Snp switch (cmd) { 7045228561Snp case MOD_LOAD: 7046249370Snp if (atomic_fetchadd_int(&loaded, 1)) 7047249370Snp break; 7048219392Snp t4_sge_modload(); 7049228561Snp mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF); 7050228561Snp SLIST_INIT(&t4_list); 7051237263Snp#ifdef TCP_OFFLOAD 7052228561Snp mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF); 7053228561Snp SLIST_INIT(&t4_uld_list); 7054228561Snp#endif 7055228561Snp tweak_tunables(); 7056228561Snp break; 7057219392Snp 7058228561Snp case MOD_UNLOAD: 7059249370Snp if (atomic_fetchadd_int(&loaded, -1) > 1) 7060249370Snp break; 7061237263Snp#ifdef TCP_OFFLOAD 7062228561Snp mtx_lock(&t4_uld_list_lock); 7063228561Snp if (!SLIST_EMPTY(&t4_uld_list)) { 7064228561Snp rc = EBUSY; 7065228561Snp mtx_unlock(&t4_uld_list_lock); 7066228561Snp break; 7067228561Snp } 7068228561Snp mtx_unlock(&t4_uld_list_lock); 7069228561Snp mtx_destroy(&t4_uld_list_lock); 7070228561Snp#endif 7071228561Snp mtx_lock(&t4_list_lock); 7072228561Snp if (!SLIST_EMPTY(&t4_list)) { 7073228561Snp rc = EBUSY; 7074228561Snp mtx_unlock(&t4_list_lock); 7075228561Snp break; 7076228561Snp } 7077228561Snp mtx_unlock(&t4_list_lock); 7078228561Snp mtx_destroy(&t4_list_lock); 7079228561Snp break; 7080228561Snp } 7081228561Snp 7082228561Snp return (rc); 7083219392Snp} 7084219392Snp 7085248925Snpstatic devclass_t t4_devclass, t5_devclass; 7086248925Snpstatic devclass_t cxgbe_devclass, cxl_devclass; 7087218792Snp 7088249370SnpDRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, mod_event, 0); 7089218792SnpMODULE_VERSION(t4nex, 1); 7090218792Snp 7091249370SnpDRIVER_MODULE(t5nex, pci, t5_driver, t5_devclass, mod_event, 0); 7092248925SnpMODULE_VERSION(t5nex, 1); 7093248925Snp 7094218792SnpDRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0); 7095218792SnpMODULE_VERSION(cxgbe, 1); 7096248925Snp 7097248925SnpDRIVER_MODULE(cxl, t5nex, cxl_driver, cxl_devclass, 0, 0); 7098248925SnpMODULE_VERSION(cxl, 1); 7099