t4_main.c revision 246093
1218792Snp/*-
2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc.
3218792Snp * All rights reserved.
4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org>
5218792Snp *
6218792Snp * Redistribution and use in source and binary forms, with or without
7218792Snp * modification, are permitted provided that the following conditions
8218792Snp * are met:
9218792Snp * 1. Redistributions of source code must retain the above copyright
10218792Snp *    notice, this list of conditions and the following disclaimer.
11218792Snp * 2. Redistributions in binary form must reproduce the above copyright
12218792Snp *    notice, this list of conditions and the following disclaimer in the
13218792Snp *    documentation and/or other materials provided with the distribution.
14218792Snp *
15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18218792Snp * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25218792Snp * SUCH DAMAGE.
26218792Snp */
27218792Snp
28218792Snp#include <sys/cdefs.h>
29218792Snp__FBSDID("$FreeBSD: head/sys/dev/cxgbe/t4_main.c 246093 2013-01-29 20:59:22Z np $");
30218792Snp
31218792Snp#include "opt_inet.h"
32237819Snp#include "opt_inet6.h"
33218792Snp
34218792Snp#include <sys/param.h>
35218792Snp#include <sys/conf.h>
36218792Snp#include <sys/priv.h>
37218792Snp#include <sys/kernel.h>
38218792Snp#include <sys/bus.h>
39218792Snp#include <sys/module.h>
40219286Snp#include <sys/malloc.h>
41219286Snp#include <sys/queue.h>
42219286Snp#include <sys/taskqueue.h>
43218792Snp#include <sys/pciio.h>
44218792Snp#include <dev/pci/pcireg.h>
45218792Snp#include <dev/pci/pcivar.h>
46218792Snp#include <dev/pci/pci_private.h>
47218792Snp#include <sys/firmware.h>
48219436Snp#include <sys/sbuf.h>
49218792Snp#include <sys/smp.h>
50218792Snp#include <sys/socket.h>
51218792Snp#include <sys/sockio.h>
52218792Snp#include <sys/sysctl.h>
53218792Snp#include <net/ethernet.h>
54218792Snp#include <net/if.h>
55218792Snp#include <net/if_types.h>
56218792Snp#include <net/if_dl.h>
57222003Snp#include <net/if_vlan_var.h>
58218792Snp
59218792Snp#include "common/common.h"
60221474Snp#include "common/t4_msg.h"
61218792Snp#include "common/t4_regs.h"
62218792Snp#include "common/t4_regs_values.h"
63218792Snp#include "t4_ioctl.h"
64222509Snp#include "t4_l2t.h"
65218792Snp
66218792Snp/* T4 bus driver interface */
67218792Snpstatic int t4_probe(device_t);
68218792Snpstatic int t4_attach(device_t);
69218792Snpstatic int t4_detach(device_t);
70218792Snpstatic device_method_t t4_methods[] = {
71218792Snp	DEVMETHOD(device_probe,		t4_probe),
72218792Snp	DEVMETHOD(device_attach,	t4_attach),
73218792Snp	DEVMETHOD(device_detach,	t4_detach),
74218792Snp
75227843Smarius	DEVMETHOD_END
76218792Snp};
77218792Snpstatic driver_t t4_driver = {
78218792Snp	"t4nex",
79218792Snp	t4_methods,
80218792Snp	sizeof(struct adapter)
81218792Snp};
82218792Snp
83218792Snp
84218792Snp/* T4 port (cxgbe) interface */
85218792Snpstatic int cxgbe_probe(device_t);
86218792Snpstatic int cxgbe_attach(device_t);
87218792Snpstatic int cxgbe_detach(device_t);
88218792Snpstatic device_method_t cxgbe_methods[] = {
89218792Snp	DEVMETHOD(device_probe,		cxgbe_probe),
90218792Snp	DEVMETHOD(device_attach,	cxgbe_attach),
91218792Snp	DEVMETHOD(device_detach,	cxgbe_detach),
92218792Snp	{ 0, 0 }
93218792Snp};
94218792Snpstatic driver_t cxgbe_driver = {
95218792Snp	"cxgbe",
96218792Snp	cxgbe_methods,
97218792Snp	sizeof(struct port_info)
98218792Snp};
99218792Snp
100218792Snpstatic d_ioctl_t t4_ioctl;
101218792Snpstatic d_open_t t4_open;
102218792Snpstatic d_close_t t4_close;
103218792Snp
104218792Snpstatic struct cdevsw t4_cdevsw = {
105218792Snp       .d_version = D_VERSION,
106218792Snp       .d_flags = 0,
107218792Snp       .d_open = t4_open,
108218792Snp       .d_close = t4_close,
109218792Snp       .d_ioctl = t4_ioctl,
110218792Snp       .d_name = "t4nex",
111218792Snp};
112218792Snp
113218792Snp/* ifnet + media interface */
114218792Snpstatic void cxgbe_init(void *);
115218792Snpstatic int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
116218792Snpstatic int cxgbe_transmit(struct ifnet *, struct mbuf *);
117218792Snpstatic void cxgbe_qflush(struct ifnet *);
118218792Snpstatic int cxgbe_media_change(struct ifnet *);
119218792Snpstatic void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
120218792Snp
121218792SnpMALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services");
122218792Snp
123237263Snp/*
124237263Snp * Correct lock order when you need to acquire multiple locks is t4_list_lock,
125237263Snp * then ADAPTER_LOCK, then t4_uld_list_lock.
126237263Snp */
127228561Snpstatic struct mtx t4_list_lock;
128228561Snpstatic SLIST_HEAD(, adapter) t4_list;
129237263Snp#ifdef TCP_OFFLOAD
130228561Snpstatic struct mtx t4_uld_list_lock;
131228561Snpstatic SLIST_HEAD(, uld_info) t4_uld_list;
132228561Snp#endif
133218792Snp
134218792Snp/*
135228561Snp * Tunables.  See tweak_tunables() too.
136218792Snp */
137218792Snp
138218792Snp/*
139228561Snp * Number of queues for tx and rx, 10G and 1G, NIC and offload.
140218792Snp */
141228561Snp#define NTXQ_10G 16
142228561Snpstatic int t4_ntxq10g = -1;
143228561SnpTUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
144218792Snp
145228561Snp#define NRXQ_10G 8
146228561Snpstatic int t4_nrxq10g = -1;
147228561SnpTUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
148218792Snp
149228561Snp#define NTXQ_1G 4
150228561Snpstatic int t4_ntxq1g = -1;
151228561SnpTUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
152218792Snp
153228561Snp#define NRXQ_1G 2
154228561Snpstatic int t4_nrxq1g = -1;
155228561SnpTUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
156218792Snp
157237263Snp#ifdef TCP_OFFLOAD
158228561Snp#define NOFLDTXQ_10G 8
159228561Snpstatic int t4_nofldtxq10g = -1;
160228561SnpTUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
161228561Snp
162228561Snp#define NOFLDRXQ_10G 2
163228561Snpstatic int t4_nofldrxq10g = -1;
164228561SnpTUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
165228561Snp
166228561Snp#define NOFLDTXQ_1G 2
167228561Snpstatic int t4_nofldtxq1g = -1;
168228561SnpTUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
169228561Snp
170228561Snp#define NOFLDRXQ_1G 1
171228561Snpstatic int t4_nofldrxq1g = -1;
172228561SnpTUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
173228561Snp#endif
174228561Snp
175218792Snp/*
176218792Snp * Holdoff parameters for 10G and 1G ports.
177218792Snp */
178228561Snp#define TMR_IDX_10G 1
179228561Snpstatic int t4_tmr_idx_10g = TMR_IDX_10G;
180228561SnpTUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
181218792Snp
182234833Snp#define PKTC_IDX_10G (-1)
183228561Snpstatic int t4_pktc_idx_10g = PKTC_IDX_10G;
184228561SnpTUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
185218792Snp
186228561Snp#define TMR_IDX_1G 1
187228561Snpstatic int t4_tmr_idx_1g = TMR_IDX_1G;
188228561SnpTUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
189218792Snp
190234833Snp#define PKTC_IDX_1G (-1)
191228561Snpstatic int t4_pktc_idx_1g = PKTC_IDX_1G;
192228561SnpTUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
193218792Snp
194218792Snp/*
195218792Snp * Size (# of entries) of each tx and rx queue.
196218792Snp */
197228561Snpstatic unsigned int t4_qsize_txq = TX_EQ_QSIZE;
198228561SnpTUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
199218792Snp
200228561Snpstatic unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
201228561SnpTUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
202218792Snp
203218792Snp/*
204228561Snp * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
205218792Snp */
206228561Snpstatic int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
207228561SnpTUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
208218792Snp
209218792Snp/*
210228561Snp * Configuration file.
211218792Snp */
212228561Snpstatic char t4_cfg_file[32] = "default";
213228561SnpTUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
214218792Snp
215228561Snp/*
216228561Snp * ASIC features that will be used.  Disable the ones you don't want so that the
217228561Snp * chip resources aren't wasted on features that will not be used.
218228561Snp */
219228561Snpstatic int t4_linkcaps_allowed = 0;	/* No DCBX, PPP, etc. by default */
220228561SnpTUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
221221474Snp
222228561Snpstatic int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
223228561SnpTUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
224228561Snp
225238028Snpstatic int t4_toecaps_allowed = -1;
226228561SnpTUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
227228561Snp
228228561Snpstatic int t4_rdmacaps_allowed = 0;
229228561SnpTUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
230228561Snp
231228561Snpstatic int t4_iscsicaps_allowed = 0;
232228561SnpTUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
233228561Snp
234228561Snpstatic int t4_fcoecaps_allowed = 0;
235228561SnpTUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
236228561Snp
237218792Snpstruct intrs_and_queues {
238219944Snp	int intr_type;		/* INTx, MSI, or MSI-X */
239218792Snp	int nirq;		/* Number of vectors */
240228561Snp	int intr_flags;
241218792Snp	int ntxq10g;		/* # of NIC txq's for each 10G port */
242218792Snp	int nrxq10g;		/* # of NIC rxq's for each 10G port */
243218792Snp	int ntxq1g;		/* # of NIC txq's for each 1G port */
244218792Snp	int nrxq1g;		/* # of NIC rxq's for each 1G port */
245237263Snp#ifdef TCP_OFFLOAD
246228561Snp	int nofldtxq10g;	/* # of TOE txq's for each 10G port */
247228561Snp	int nofldrxq10g;	/* # of TOE rxq's for each 10G port */
248228561Snp	int nofldtxq1g;		/* # of TOE txq's for each 1G port */
249228561Snp	int nofldrxq1g;		/* # of TOE rxq's for each 1G port */
250228561Snp#endif
251218792Snp};
252218792Snp
253221474Snpstruct filter_entry {
254221474Snp        uint32_t valid:1;	/* filter allocated and valid */
255221474Snp        uint32_t locked:1;	/* filter is administratively locked */
256221474Snp        uint32_t pending:1;	/* filter action is pending firmware reply */
257221474Snp	uint32_t smtidx:8;	/* Source MAC Table index for smac */
258222509Snp	struct l2t_entry *l2t;	/* Layer Two Table entry for dmac */
259221474Snp
260221474Snp        struct t4_filter_specification fs;
261221474Snp};
262221474Snp
263218792Snpenum {
264218792Snp	XGMAC_MTU	= (1 << 0),
265218792Snp	XGMAC_PROMISC	= (1 << 1),
266218792Snp	XGMAC_ALLMULTI	= (1 << 2),
267218792Snp	XGMAC_VLANEX	= (1 << 3),
268218792Snp	XGMAC_UCADDR	= (1 << 4),
269218792Snp	XGMAC_MCADDRS	= (1 << 5),
270218792Snp
271218792Snp	XGMAC_ALL	= 0xffff
272218792Snp};
273218792Snp
274218792Snpstatic int map_bars(struct adapter *);
275218792Snpstatic void setup_memwin(struct adapter *);
276218792Snpstatic int cfg_itype_and_nqueues(struct adapter *, int, int,
277218792Snp    struct intrs_and_queues *);
278218792Snpstatic int prep_firmware(struct adapter *);
279228561Snpstatic int upload_config_file(struct adapter *, const struct firmware *,
280228561Snp    uint32_t *, uint32_t *);
281228561Snpstatic int partition_resources(struct adapter *, const struct firmware *);
282228561Snpstatic int get_params__pre_init(struct adapter *);
283228561Snpstatic int get_params__post_init(struct adapter *);
284218792Snpstatic void t4_set_desc(struct adapter *);
285218792Snpstatic void build_medialist(struct port_info *);
286218792Snpstatic int update_mac_settings(struct port_info *, int);
287218792Snpstatic int cxgbe_init_synchronized(struct port_info *);
288218792Snpstatic int cxgbe_uninit_synchronized(struct port_info *);
289240453Snpstatic int setup_intr_handlers(struct adapter *);
290228561Snpstatic int adapter_full_init(struct adapter *);
291228561Snpstatic int adapter_full_uninit(struct adapter *);
292228561Snpstatic int port_full_init(struct port_info *);
293228561Snpstatic int port_full_uninit(struct port_info *);
294228561Snpstatic void quiesce_eq(struct adapter *, struct sge_eq *);
295228561Snpstatic void quiesce_iq(struct adapter *, struct sge_iq *);
296228561Snpstatic void quiesce_fl(struct adapter *, struct sge_fl *);
297218792Snpstatic int t4_alloc_irq(struct adapter *, struct irq *, int rid,
298228561Snp    driver_intr_t *, void *, char *);
299218792Snpstatic int t4_free_irq(struct adapter *, struct irq *);
300218792Snpstatic void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
301218792Snp    unsigned int);
302218792Snpstatic void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
303218792Snpstatic void cxgbe_tick(void *);
304237263Snpstatic void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
305228561Snpstatic int cpl_not_handled(struct sge_iq *, const struct rss_header *,
306228561Snp    struct mbuf *);
307237263Snpstatic int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
308239336Snpstatic int fw_msg_not_handled(struct adapter *, const __be64 *);
309218792Snpstatic int t4_sysctls(struct adapter *);
310218792Snpstatic int cxgbe_sysctls(struct port_info *);
311219436Snpstatic int sysctl_int_array(SYSCTL_HANDLER_ARGS);
312228561Snpstatic int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
313218792Snpstatic int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
314218792Snpstatic int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
315218792Snpstatic int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
316218792Snpstatic int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
317218792Snpstatic int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
318231115Snp#ifdef SBUF_DRAIN
319228561Snpstatic int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
320228561Snpstatic int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
321228561Snpstatic int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
322222551Snpstatic int sysctl_devlog(SYSCTL_HANDLER_ARGS);
323228561Snpstatic int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
324228561Snpstatic int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
325228561Snpstatic int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
326228561Snpstatic int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
327228561Snpstatic int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
328228561Snpstatic int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
329228561Snpstatic int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
330228561Snpstatic int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
331228561Snpstatic int sysctl_tids(SYSCTL_HANDLER_ARGS);
332228561Snpstatic int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
333228561Snpstatic int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
334231115Snp#endif
335219286Snpstatic inline void txq_start(struct ifnet *, struct sge_txq *);
336221474Snpstatic uint32_t fconf_to_mode(uint32_t);
337221474Snpstatic uint32_t mode_to_fconf(uint32_t);
338221474Snpstatic uint32_t fspec_to_fconf(struct t4_filter_specification *);
339221474Snpstatic int get_filter_mode(struct adapter *, uint32_t *);
340221474Snpstatic int set_filter_mode(struct adapter *, uint32_t);
341222552Snpstatic inline uint64_t get_filter_hits(struct adapter *, uint32_t);
342221474Snpstatic int get_filter(struct adapter *, struct t4_filter *);
343221474Snpstatic int set_filter(struct adapter *, struct t4_filter *);
344221474Snpstatic int del_filter(struct adapter *, struct t4_filter *);
345222509Snpstatic void clear_filter(struct filter_entry *);
346221474Snpstatic int set_filter_wr(struct adapter *, int);
347221474Snpstatic int del_filter_wr(struct adapter *, int);
348222973Snpstatic int get_sge_context(struct adapter *, struct t4_sge_context *);
349245274Snpstatic int load_fw(struct adapter *, struct t4_data *);
350228561Snpstatic int read_card_mem(struct adapter *, struct t4_mem_range *);
351241399Snpstatic int read_i2c(struct adapter *, struct t4_i2c_data *);
352237263Snp#ifdef TCP_OFFLOAD
353228561Snpstatic int toe_capability(struct port_info *, int);
354228561Snp#endif
355219392Snpstatic int t4_mod_event(module_t, int, void *);
356218792Snp
357218792Snpstruct t4_pciids {
358218792Snp	uint16_t device;
359218792Snp	char *desc;
360218792Snp} t4_pciids[] = {
361237587Snp	{0xa000, "Chelsio Terminator 4 FPGA"},
362237587Snp	{0x4400, "Chelsio T440-dbg"},
363237587Snp	{0x4401, "Chelsio T420-CR"},
364237587Snp	{0x4402, "Chelsio T422-CR"},
365237587Snp	{0x4403, "Chelsio T440-CR"},
366237587Snp	{0x4404, "Chelsio T420-BCH"},
367237587Snp	{0x4405, "Chelsio T440-BCH"},
368237587Snp	{0x4406, "Chelsio T440-CH"},
369237587Snp	{0x4407, "Chelsio T420-SO"},
370237587Snp	{0x4408, "Chelsio T420-CX"},
371237587Snp	{0x4409, "Chelsio T420-BT"},
372237587Snp	{0x440a, "Chelsio T404-BT"},
373244580Snp	{0x440e, "Chelsio T440-LP-CR"},
374218792Snp};
375218792Snp
376237263Snp#ifdef TCP_OFFLOAD
377237263Snp/*
378237263Snp * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
379237263Snp * exactly the same for both rxq and ofld_rxq.
380237263Snp */
381237263SnpCTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
382228561SnpCTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
383228561Snp#endif
384228561Snp
385239336Snp/* No easy way to include t4_msg.h before adapter.h so we check this way */
386240452SnpCTASSERT(nitems(((struct adapter *)0)->cpl_handler) == NUM_CPL_CMDS);
387240452SnpCTASSERT(nitems(((struct adapter *)0)->fw_msg_handler) == NUM_FW6_TYPES);
388239336Snp
389218792Snpstatic int
390218792Snpt4_probe(device_t dev)
391218792Snp{
392218792Snp	int i;
393218792Snp	uint16_t v = pci_get_vendor(dev);
394218792Snp	uint16_t d = pci_get_device(dev);
395237587Snp	uint8_t f = pci_get_function(dev);
396218792Snp
397218792Snp	if (v != PCI_VENDOR_ID_CHELSIO)
398218792Snp		return (ENXIO);
399218792Snp
400237587Snp	/* Attach only to PF0 of the FPGA */
401237587Snp	if (d == 0xa000 && f != 0)
402237587Snp		return (ENXIO);
403237587Snp
404240452Snp	for (i = 0; i < nitems(t4_pciids); i++) {
405237587Snp		if (d == t4_pciids[i].device) {
406218792Snp			device_set_desc(dev, t4_pciids[i].desc);
407218792Snp			return (BUS_PROBE_DEFAULT);
408218792Snp		}
409218792Snp	}
410218792Snp
411218792Snp	return (ENXIO);
412218792Snp}
413218792Snp
414218792Snpstatic int
415218792Snpt4_attach(device_t dev)
416218792Snp{
417218792Snp	struct adapter *sc;
418218792Snp	int rc = 0, i, n10g, n1g, rqidx, tqidx;
419218792Snp	struct intrs_and_queues iaq;
420218792Snp	struct sge *s;
421237263Snp#ifdef TCP_OFFLOAD
422228561Snp	int ofld_rqidx, ofld_tqidx;
423228561Snp#endif
424218792Snp
425218792Snp	sc = device_get_softc(dev);
426218792Snp	sc->dev = dev;
427218792Snp
428218792Snp	pci_enable_busmaster(dev);
429222085Snp	if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
430228561Snp		uint32_t v;
431228561Snp
432222085Snp		pci_set_max_read_req(dev, 4096);
433240680Sgavin		v = pci_read_config(dev, i + PCIER_DEVICE_CTL, 2);
434240680Sgavin		v |= PCIEM_CTL_RELAXED_ORD_ENABLE;
435240680Sgavin		pci_write_config(dev, i + PCIER_DEVICE_CTL, v, 2);
436222085Snp	}
437222085Snp
438218792Snp	snprintf(sc->lockname, sizeof(sc->lockname), "%s",
439218792Snp	    device_get_nameunit(dev));
440218792Snp	mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
441228561Snp	mtx_lock(&t4_list_lock);
442228561Snp	SLIST_INSERT_HEAD(&t4_list, sc, link);
443228561Snp	mtx_unlock(&t4_list_lock);
444218792Snp
445228561Snp	mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
446228561Snp	TAILQ_INIT(&sc->sfl);
447228561Snp	callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
448228561Snp
449218792Snp	rc = map_bars(sc);
450218792Snp	if (rc != 0)
451218792Snp		goto done; /* error message displayed already */
452218792Snp
453237587Snp	/*
454237587Snp	 * This is the real PF# to which we're attaching.  Works from within PCI
455237587Snp	 * passthrough environments too, where pci_get_function() could return a
456237587Snp	 * different PF# depending on the passthrough configuration.  We need to
457237587Snp	 * use the real PF# in all our communication with the firmware.
458237587Snp	 */
459237587Snp	sc->pf = G_SOURCEPF(t4_read_reg(sc, A_PL_WHOAMI));
460237587Snp	sc->mbox = sc->pf;
461237587Snp
462218792Snp	memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
463237263Snp	sc->an_handler = an_not_handled;
464240452Snp	for (i = 0; i < nitems(sc->cpl_handler); i++)
465228561Snp		sc->cpl_handler[i] = cpl_not_handled;
466240452Snp	for (i = 0; i < nitems(sc->fw_msg_handler); i++)
467239336Snp		sc->fw_msg_handler[i] = fw_msg_not_handled;
468239338Snp	t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, t4_filter_rpl);
469218792Snp
470218792Snp	/* Prepare the adapter for operation */
471218792Snp	rc = -t4_prep_adapter(sc);
472218792Snp	if (rc != 0) {
473218792Snp		device_printf(dev, "failed to prepare adapter: %d.\n", rc);
474218792Snp		goto done;
475218792Snp	}
476218792Snp
477228561Snp	/*
478228561Snp	 * Do this really early, with the memory windows set up even before the
479228561Snp	 * character device.  The userland tool's register i/o and mem read
480228561Snp	 * will work even in "recovery mode".
481228561Snp	 */
482228561Snp	setup_memwin(sc);
483218792Snp	sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT,
484218792Snp	    GID_WHEEL, 0600, "%s", device_get_nameunit(dev));
485218792Snp	sc->cdev->si_drv1 = sc;
486218792Snp
487228561Snp	/* Go no further if recovery mode has been requested. */
488228561Snp	if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
489228561Snp		device_printf(dev, "recovery mode.\n");
490228561Snp		goto done;
491228561Snp	}
492228561Snp
493218792Snp	/* Prepare the firmware for operation */
494218792Snp	rc = prep_firmware(sc);
495218792Snp	if (rc != 0)
496218792Snp		goto done; /* error message displayed already */
497218792Snp
498228561Snp	rc = get_params__pre_init(sc);
499228561Snp	if (rc != 0)
500228561Snp		goto done; /* error message displayed already */
501222551Snp
502228561Snp	rc = t4_sge_init(sc);
503228561Snp	if (rc != 0)
504228561Snp		goto done; /* error message displayed already */
505218792Snp
506228561Snp	if (sc->flags & MASTER_PF) {
507228561Snp		/* get basic stuff going */
508228561Snp		rc = -t4_fw_initialize(sc, sc->mbox);
509228561Snp		if (rc != 0) {
510228561Snp			device_printf(dev, "early init failed: %d.\n", rc);
511228561Snp			goto done;
512228561Snp		}
513218792Snp	}
514218792Snp
515228561Snp	rc = get_params__post_init(sc);
516228561Snp	if (rc != 0)
517228561Snp		goto done; /* error message displayed already */
518218792Snp
519228561Snp	if (sc->flags & MASTER_PF) {
520239341Snp		uint16_t indsz = min(RX_COPY_THRESHOLD - 1, M_INDICATESIZE);
521218792Snp
522228561Snp		/* final tweaks to some settings */
523218792Snp
524228561Snp		t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd,
525228561Snp		    sc->params.b_wnd);
526239341Snp		/* 4K, 16K, 64K, 256K DDP "page sizes" */
527239341Snp		t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(0) | V_HPZ1(2) |
528239341Snp		    V_HPZ2(4) | V_HPZ3(6));
529239341Snp		t4_set_reg_field(sc, A_ULP_RX_CTL, F_TDDPTAGTCB, F_TDDPTAGTCB);
530228561Snp		t4_set_reg_field(sc, A_TP_PARA_REG5,
531228561Snp		    V_INDICATESIZE(M_INDICATESIZE) |
532228561Snp		    F_REARMDDPOFFSET | F_RESETDDPOFFSET,
533239341Snp		    V_INDICATESIZE(indsz) |
534228561Snp		    F_REARMDDPOFFSET | F_RESETDDPOFFSET);
535228561Snp	} else {
536228561Snp		/*
537228561Snp		 * XXX: Verify that we can live with whatever the master driver
538228561Snp		 * has done so far, and hope that it doesn't change any global
539228561Snp		 * setting from underneath us in the future.
540228561Snp		 */
541218792Snp	}
542218792Snp
543228561Snp	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
544228561Snp	    A_TP_VLAN_PRI_MAP);
545218792Snp
546228561Snp	for (i = 0; i < NCHAN; i++)
547228561Snp		sc->params.tp.tx_modq[i] = i;
548218792Snp
549218792Snp	rc = t4_create_dma_tag(sc);
550218792Snp	if (rc != 0)
551218792Snp		goto done; /* error message displayed already */
552218792Snp
553218792Snp	/*
554218792Snp	 * First pass over all the ports - allocate VIs and initialize some
555218792Snp	 * basic parameters like mac address, port type, etc.  We also figure
556218792Snp	 * out whether a port is 10G or 1G and use that information when
557218792Snp	 * calculating how many interrupts to attempt to allocate.
558218792Snp	 */
559218792Snp	n10g = n1g = 0;
560218792Snp	for_each_port(sc, i) {
561218792Snp		struct port_info *pi;
562218792Snp
563218792Snp		pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
564218792Snp		sc->port[i] = pi;
565218792Snp
566218792Snp		/* These must be set before t4_port_init */
567218792Snp		pi->adapter = sc;
568218792Snp		pi->port_id = i;
569218792Snp
570218792Snp		/* Allocate the vi and initialize parameters like mac addr */
571218792Snp		rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
572218792Snp		if (rc != 0) {
573218792Snp			device_printf(dev, "unable to initialize port %d: %d\n",
574218792Snp			    i, rc);
575218792Snp			free(pi, M_CXGBE);
576222510Snp			sc->port[i] = NULL;
577222510Snp			goto done;
578218792Snp		}
579218792Snp
580218792Snp		snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
581218792Snp		    device_get_nameunit(dev), i);
582218792Snp		mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
583218792Snp
584218792Snp		if (is_10G_port(pi)) {
585218792Snp			n10g++;
586228561Snp			pi->tmr_idx = t4_tmr_idx_10g;
587228561Snp			pi->pktc_idx = t4_pktc_idx_10g;
588218792Snp		} else {
589218792Snp			n1g++;
590228561Snp			pi->tmr_idx = t4_tmr_idx_1g;
591228561Snp			pi->pktc_idx = t4_pktc_idx_1g;
592218792Snp		}
593218792Snp
594218792Snp		pi->xact_addr_filt = -1;
595218792Snp
596228561Snp		pi->qsize_rxq = t4_qsize_rxq;
597228561Snp		pi->qsize_txq = t4_qsize_txq;
598218792Snp
599218792Snp		pi->dev = device_add_child(dev, "cxgbe", -1);
600218792Snp		if (pi->dev == NULL) {
601218792Snp			device_printf(dev,
602218792Snp			    "failed to add device for port %d.\n", i);
603218792Snp			rc = ENXIO;
604218792Snp			goto done;
605218792Snp		}
606218792Snp		device_set_softc(pi->dev, pi);
607218792Snp	}
608218792Snp
609218792Snp	/*
610218792Snp	 * Interrupt type, # of interrupts, # of rx/tx queues, etc.
611218792Snp	 */
612218792Snp	rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
613218792Snp	if (rc != 0)
614218792Snp		goto done; /* error message displayed already */
615218792Snp
616218792Snp	sc->intr_type = iaq.intr_type;
617218792Snp	sc->intr_count = iaq.nirq;
618228561Snp	sc->flags |= iaq.intr_flags;
619218792Snp
620218792Snp	s = &sc->sge;
621218792Snp	s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
622218792Snp	s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
623220873Snp	s->neq = s->ntxq + s->nrxq;	/* the free list in an rxq is an eq */
624228561Snp	s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
625218792Snp	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
626222510Snp
627237263Snp#ifdef TCP_OFFLOAD
628228561Snp	if (is_offload(sc)) {
629228561Snp
630228561Snp		s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
631228561Snp		s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
632228561Snp		s->neq += s->nofldtxq + s->nofldrxq;
633228561Snp		s->niq += s->nofldrxq;
634228561Snp
635228561Snp		s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
636228561Snp		    M_CXGBE, M_ZERO | M_WAITOK);
637228561Snp		s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
638228561Snp		    M_CXGBE, M_ZERO | M_WAITOK);
639228561Snp	}
640228561Snp#endif
641228561Snp
642228561Snp	s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
643220873Snp	    M_ZERO | M_WAITOK);
644218792Snp	s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
645218792Snp	    M_ZERO | M_WAITOK);
646218792Snp	s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
647218792Snp	    M_ZERO | M_WAITOK);
648218792Snp	s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
649218792Snp	    M_ZERO | M_WAITOK);
650218792Snp	s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
651218792Snp	    M_ZERO | M_WAITOK);
652218792Snp
653218792Snp	sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
654218792Snp	    M_ZERO | M_WAITOK);
655218792Snp
656228561Snp	t4_init_l2t(sc, M_WAITOK);
657222509Snp
658218792Snp	/*
659218792Snp	 * Second pass over the ports.  This time we know the number of rx and
660218792Snp	 * tx queues that each port should get.
661218792Snp	 */
662218792Snp	rqidx = tqidx = 0;
663237263Snp#ifdef TCP_OFFLOAD
664228561Snp	ofld_rqidx = ofld_tqidx = 0;
665228561Snp#endif
666218792Snp	for_each_port(sc, i) {
667218792Snp		struct port_info *pi = sc->port[i];
668218792Snp
669218792Snp		if (pi == NULL)
670218792Snp			continue;
671218792Snp
672218792Snp		pi->first_rxq = rqidx;
673218792Snp		pi->first_txq = tqidx;
674228561Snp		if (is_10G_port(pi)) {
675228561Snp			pi->nrxq = iaq.nrxq10g;
676228561Snp			pi->ntxq = iaq.ntxq10g;
677228561Snp		} else {
678228561Snp			pi->nrxq = iaq.nrxq1g;
679228561Snp			pi->ntxq = iaq.ntxq1g;
680228561Snp		}
681218792Snp
682218792Snp		rqidx += pi->nrxq;
683218792Snp		tqidx += pi->ntxq;
684228561Snp
685237263Snp#ifdef TCP_OFFLOAD
686228561Snp		if (is_offload(sc)) {
687228561Snp			pi->first_ofld_rxq = ofld_rqidx;
688228561Snp			pi->first_ofld_txq = ofld_tqidx;
689228561Snp			if (is_10G_port(pi)) {
690228561Snp				pi->nofldrxq = iaq.nofldrxq10g;
691228561Snp				pi->nofldtxq = iaq.nofldtxq10g;
692228561Snp			} else {
693228561Snp				pi->nofldrxq = iaq.nofldrxq1g;
694228561Snp				pi->nofldtxq = iaq.nofldtxq1g;
695228561Snp			}
696228561Snp			ofld_rqidx += pi->nofldrxq;
697228561Snp			ofld_tqidx += pi->nofldtxq;
698228561Snp		}
699228561Snp#endif
700218792Snp	}
701218792Snp
702240453Snp	rc = setup_intr_handlers(sc);
703240453Snp	if (rc != 0) {
704240453Snp		device_printf(dev,
705240453Snp		    "failed to setup interrupt handlers: %d\n", rc);
706240453Snp		goto done;
707240453Snp	}
708240453Snp
709218792Snp	rc = bus_generic_attach(dev);
710218792Snp	if (rc != 0) {
711218792Snp		device_printf(dev,
712218792Snp		    "failed to attach all child ports: %d\n", rc);
713218792Snp		goto done;
714218792Snp	}
715218792Snp
716218792Snp	device_printf(dev,
717228561Snp	    "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
718228561Snp	    sc->params.pci.width, sc->params.nports, sc->intr_count,
719228561Snp	    sc->intr_type == INTR_MSIX ? "MSI-X" :
720228561Snp	    (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
721228561Snp	    sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
722228561Snp
723218792Snp	t4_set_desc(sc);
724218792Snp
725218792Snpdone:
726228561Snp	if (rc != 0 && sc->cdev) {
727228561Snp		/* cdev was created and so cxgbetool works; recover that way. */
728228561Snp		device_printf(dev,
729228561Snp		    "error during attach, adapter is now in recovery mode.\n");
730228561Snp		rc = 0;
731228561Snp	}
732228561Snp
733218792Snp	if (rc != 0)
734218792Snp		t4_detach(dev);
735228561Snp	else
736228561Snp		t4_sysctls(sc);
737218792Snp
738218792Snp	return (rc);
739218792Snp}
740218792Snp
741218792Snp/*
742218792Snp * Idempotent
743218792Snp */
744218792Snpstatic int
745218792Snpt4_detach(device_t dev)
746218792Snp{
747218792Snp	struct adapter *sc;
748218792Snp	struct port_info *pi;
749228561Snp	int i, rc;
750218792Snp
751218792Snp	sc = device_get_softc(dev);
752218792Snp
753228561Snp	if (sc->flags & FULL_INIT_DONE)
754228561Snp		t4_intr_disable(sc);
755228561Snp
756228561Snp	if (sc->cdev) {
757218792Snp		destroy_dev(sc->cdev);
758228561Snp		sc->cdev = NULL;
759228561Snp	}
760218792Snp
761228561Snp	rc = bus_generic_detach(dev);
762228561Snp	if (rc) {
763228561Snp		device_printf(dev,
764228561Snp		    "failed to detach child devices: %d\n", rc);
765228561Snp		return (rc);
766228561Snp	}
767228561Snp
768240453Snp	for (i = 0; i < sc->intr_count; i++)
769240453Snp		t4_free_irq(sc, &sc->irq[i]);
770240453Snp
771218792Snp	for (i = 0; i < MAX_NPORTS; i++) {
772218792Snp		pi = sc->port[i];
773218792Snp		if (pi) {
774218792Snp			t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
775218792Snp			if (pi->dev)
776218792Snp				device_delete_child(dev, pi->dev);
777218792Snp
778218792Snp			mtx_destroy(&pi->pi_lock);
779218792Snp			free(pi, M_CXGBE);
780218792Snp		}
781218792Snp	}
782218792Snp
783228561Snp	if (sc->flags & FULL_INIT_DONE)
784228561Snp		adapter_full_uninit(sc);
785228561Snp
786218792Snp	if (sc->flags & FW_OK)
787218792Snp		t4_fw_bye(sc, sc->mbox);
788218792Snp
789219944Snp	if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
790218792Snp		pci_release_msi(dev);
791218792Snp
792218792Snp	if (sc->regs_res)
793218792Snp		bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
794218792Snp		    sc->regs_res);
795218792Snp
796218792Snp	if (sc->msix_res)
797218792Snp		bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
798218792Snp		    sc->msix_res);
799218792Snp
800222509Snp	if (sc->l2t)
801222509Snp		t4_free_l2t(sc->l2t);
802222509Snp
803237263Snp#ifdef TCP_OFFLOAD
804228561Snp	free(sc->sge.ofld_rxq, M_CXGBE);
805228561Snp	free(sc->sge.ofld_txq, M_CXGBE);
806228561Snp#endif
807218792Snp	free(sc->irq, M_CXGBE);
808218792Snp	free(sc->sge.rxq, M_CXGBE);
809218792Snp	free(sc->sge.txq, M_CXGBE);
810220873Snp	free(sc->sge.ctrlq, M_CXGBE);
811218792Snp	free(sc->sge.iqmap, M_CXGBE);
812218792Snp	free(sc->sge.eqmap, M_CXGBE);
813221474Snp	free(sc->tids.ftid_tab, M_CXGBE);
814218792Snp	t4_destroy_dma_tag(sc);
815228561Snp	if (mtx_initialized(&sc->sc_lock)) {
816228561Snp		mtx_lock(&t4_list_lock);
817228561Snp		SLIST_REMOVE(&t4_list, sc, adapter, link);
818228561Snp		mtx_unlock(&t4_list_lock);
819228561Snp		mtx_destroy(&sc->sc_lock);
820228561Snp	}
821218792Snp
822245274Snp	if (mtx_initialized(&sc->tids.ftid_lock))
823245274Snp		mtx_destroy(&sc->tids.ftid_lock);
824228561Snp	if (mtx_initialized(&sc->sfl_lock))
825228561Snp		mtx_destroy(&sc->sfl_lock);
826228561Snp
827218792Snp	bzero(sc, sizeof(*sc));
828218792Snp
829218792Snp	return (0);
830218792Snp}
831218792Snp
832218792Snp
833218792Snpstatic int
834218792Snpcxgbe_probe(device_t dev)
835218792Snp{
836218792Snp	char buf[128];
837218792Snp	struct port_info *pi = device_get_softc(dev);
838218792Snp
839228561Snp	snprintf(buf, sizeof(buf), "port %d", pi->port_id);
840218792Snp	device_set_desc_copy(dev, buf);
841218792Snp
842218792Snp	return (BUS_PROBE_DEFAULT);
843218792Snp}
844218792Snp
845218792Snp#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
846218792Snp    IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
847237831Snp    IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
848237819Snp#define T4_CAP_ENABLE (T4_CAP)
849218792Snp
850218792Snpstatic int
851218792Snpcxgbe_attach(device_t dev)
852218792Snp{
853218792Snp	struct port_info *pi = device_get_softc(dev);
854218792Snp	struct ifnet *ifp;
855218792Snp
856218792Snp	/* Allocate an ifnet and set it up */
857218792Snp	ifp = if_alloc(IFT_ETHER);
858218792Snp	if (ifp == NULL) {
859218792Snp		device_printf(dev, "Cannot allocate ifnet\n");
860218792Snp		return (ENOMEM);
861218792Snp	}
862218792Snp	pi->ifp = ifp;
863218792Snp	ifp->if_softc = pi;
864218792Snp
865218792Snp	callout_init(&pi->tick, CALLOUT_MPSAFE);
866218792Snp
867218792Snp	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
868218792Snp	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
869218792Snp
870218792Snp	ifp->if_init = cxgbe_init;
871218792Snp	ifp->if_ioctl = cxgbe_ioctl;
872218792Snp	ifp->if_transmit = cxgbe_transmit;
873218792Snp	ifp->if_qflush = cxgbe_qflush;
874218792Snp
875218792Snp	ifp->if_capabilities = T4_CAP;
876237263Snp#ifdef TCP_OFFLOAD
877228561Snp	if (is_offload(pi->adapter))
878245933Snp		ifp->if_capabilities |= IFCAP_TOE;
879228561Snp#endif
880218792Snp	ifp->if_capenable = T4_CAP_ENABLE;
881237799Snp	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
882237799Snp	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
883218792Snp
884218792Snp	/* Initialize ifmedia for this port */
885218792Snp	ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
886218792Snp	    cxgbe_media_status);
887218792Snp	build_medialist(pi);
888218792Snp
889237263Snp	pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
890237263Snp	    EVENTHANDLER_PRI_ANY);
891237263Snp
892218792Snp	ether_ifattach(ifp, pi->hw_addr);
893218792Snp
894237263Snp#ifdef TCP_OFFLOAD
895228561Snp	if (is_offload(pi->adapter)) {
896228561Snp		device_printf(dev,
897228561Snp		    "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
898228561Snp		    pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
899228561Snp	} else
900218792Snp#endif
901228561Snp		device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
902218792Snp
903218792Snp	cxgbe_sysctls(pi);
904218792Snp
905218792Snp	return (0);
906218792Snp}
907218792Snp
908218792Snpstatic int
909218792Snpcxgbe_detach(device_t dev)
910218792Snp{
911218792Snp	struct port_info *pi = device_get_softc(dev);
912218792Snp	struct adapter *sc = pi->adapter;
913228561Snp	struct ifnet *ifp = pi->ifp;
914218792Snp
915218792Snp	/* Tell if_ioctl and if_init that the port is going away */
916218792Snp	ADAPTER_LOCK(sc);
917218792Snp	SET_DOOMED(pi);
918218792Snp	wakeup(&sc->flags);
919218792Snp	while (IS_BUSY(sc))
920218792Snp		mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
921218792Snp	SET_BUSY(sc);
922245274Snp#ifdef INVARIANTS
923245274Snp	sc->last_op = "t4detach";
924245274Snp	sc->last_op_thr = curthread;
925245274Snp#endif
926218792Snp	ADAPTER_UNLOCK(sc);
927218792Snp
928237263Snp	if (pi->vlan_c)
929237263Snp		EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
930237263Snp
931228561Snp	PORT_LOCK(pi);
932228561Snp	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
933228561Snp	callout_stop(&pi->tick);
934228561Snp	PORT_UNLOCK(pi);
935228561Snp	callout_drain(&pi->tick);
936218792Snp
937228561Snp	/* Let detach proceed even if these fail. */
938228561Snp	cxgbe_uninit_synchronized(pi);
939228561Snp	port_full_uninit(pi);
940219286Snp
941218792Snp	ifmedia_removeall(&pi->media);
942218792Snp	ether_ifdetach(pi->ifp);
943218792Snp	if_free(pi->ifp);
944218792Snp
945218792Snp	ADAPTER_LOCK(sc);
946218792Snp	CLR_BUSY(sc);
947245274Snp	wakeup(&sc->flags);
948218792Snp	ADAPTER_UNLOCK(sc);
949218792Snp
950218792Snp	return (0);
951218792Snp}
952218792Snp
953218792Snpstatic void
954218792Snpcxgbe_init(void *arg)
955218792Snp{
956218792Snp	struct port_info *pi = arg;
957218792Snp	struct adapter *sc = pi->adapter;
958218792Snp
959245274Snp	if (begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4init") != 0)
960245274Snp		return;
961245274Snp	cxgbe_init_synchronized(pi);
962245274Snp	end_synchronized_op(sc, 0);
963218792Snp}
964218792Snp
965218792Snpstatic int
966218792Snpcxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
967218792Snp{
968218792Snp	int rc = 0, mtu, flags;
969218792Snp	struct port_info *pi = ifp->if_softc;
970218792Snp	struct adapter *sc = pi->adapter;
971218792Snp	struct ifreq *ifr = (struct ifreq *)data;
972218792Snp	uint32_t mask;
973218792Snp
974218792Snp	switch (cmd) {
975218792Snp	case SIOCSIFMTU:
976245274Snp		mtu = ifr->ifr_mtu;
977245274Snp		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO))
978245274Snp			return (EINVAL);
979245274Snp
980245274Snp		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4mtu");
981245274Snp		if (rc)
982218792Snp			return (rc);
983245274Snp		ifp->if_mtu = mtu;
984245274Snp		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
985245274Snp			t4_update_fl_bufsize(ifp);
986245274Snp			rc = update_mac_settings(pi, XGMAC_MTU);
987218792Snp		}
988245274Snp		end_synchronized_op(sc, 0);
989218792Snp		break;
990218792Snp
991218792Snp	case SIOCSIFFLAGS:
992245274Snp		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4flg");
993245274Snp		if (rc)
994245274Snp			return (rc);
995245274Snp
996218792Snp		if (ifp->if_flags & IFF_UP) {
997218792Snp			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
998218792Snp				flags = pi->if_flags;
999218792Snp				if ((ifp->if_flags ^ flags) &
1000218792Snp				    (IFF_PROMISC | IFF_ALLMULTI)) {
1001218792Snp					rc = update_mac_settings(pi,
1002218792Snp					    XGMAC_PROMISC | XGMAC_ALLMULTI);
1003218792Snp				}
1004218792Snp			} else
1005245274Snp				rc = cxgbe_init_synchronized(pi);
1006218792Snp			pi->if_flags = ifp->if_flags;
1007218792Snp		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1008245274Snp			rc = cxgbe_uninit_synchronized(pi);
1009245274Snp		end_synchronized_op(sc, 0);
1010218792Snp		break;
1011218792Snp
1012218792Snp	case SIOCADDMULTI:
1013245274Snp	case SIOCDELMULTI: /* these two are called with a mutex held :-( */
1014245274Snp		rc = begin_synchronized_op(sc, pi, HOLD_LOCK, "t4multi");
1015218792Snp		if (rc)
1016245274Snp			return (rc);
1017245274Snp		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1018218792Snp			rc = update_mac_settings(pi, XGMAC_MCADDRS);
1019245274Snp		end_synchronized_op(sc, LOCK_HELD);
1020218792Snp		break;
1021218792Snp
1022218792Snp	case SIOCSIFCAP:
1023245274Snp		rc = begin_synchronized_op(sc, pi, SLEEP_OK | INTR_OK, "t4cap");
1024218792Snp		if (rc)
1025245274Snp			return (rc);
1026218792Snp
1027218792Snp		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1028218792Snp		if (mask & IFCAP_TXCSUM) {
1029218792Snp			ifp->if_capenable ^= IFCAP_TXCSUM;
1030218792Snp			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1031218792Snp
1032237831Snp			if (IFCAP_TSO4 & ifp->if_capenable &&
1033218792Snp			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1034237799Snp				ifp->if_capenable &= ~IFCAP_TSO4;
1035218792Snp				if_printf(ifp,
1036237831Snp				    "tso4 disabled due to -txcsum.\n");
1037218792Snp			}
1038218792Snp		}
1039237799Snp		if (mask & IFCAP_TXCSUM_IPV6) {
1040237799Snp			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1041237799Snp			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1042237799Snp
1043237799Snp			if (IFCAP_TSO6 & ifp->if_capenable &&
1044237799Snp			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1045237799Snp				ifp->if_capenable &= ~IFCAP_TSO6;
1046237799Snp				if_printf(ifp,
1047237799Snp				    "tso6 disabled due to -txcsum6.\n");
1048237799Snp			}
1049237799Snp		}
1050218792Snp		if (mask & IFCAP_RXCSUM)
1051218792Snp			ifp->if_capenable ^= IFCAP_RXCSUM;
1052237799Snp		if (mask & IFCAP_RXCSUM_IPV6)
1053237799Snp			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1054237799Snp
1055237799Snp		/*
1056237799Snp		 * Note that we leave CSUM_TSO alone (it is always set).  The
1057237799Snp		 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1058237799Snp		 * sending a TSO request our way, so it's sufficient to toggle
1059237799Snp		 * IFCAP_TSOx only.
1060237799Snp		 */
1061218792Snp		if (mask & IFCAP_TSO4) {
1062237799Snp			if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1063237799Snp			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1064237799Snp				if_printf(ifp, "enable txcsum first.\n");
1065237799Snp				rc = EAGAIN;
1066237799Snp				goto fail;
1067237799Snp			}
1068218792Snp			ifp->if_capenable ^= IFCAP_TSO4;
1069218792Snp		}
1070237799Snp		if (mask & IFCAP_TSO6) {
1071237799Snp			if (!(IFCAP_TSO6 & ifp->if_capenable) &&
1072237799Snp			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1073237799Snp				if_printf(ifp, "enable txcsum6 first.\n");
1074237799Snp				rc = EAGAIN;
1075237799Snp				goto fail;
1076237799Snp			}
1077237799Snp			ifp->if_capenable ^= IFCAP_TSO6;
1078237799Snp		}
1079218792Snp		if (mask & IFCAP_LRO) {
1080237819Snp#if defined(INET) || defined(INET6)
1081218792Snp			int i;
1082218792Snp			struct sge_rxq *rxq;
1083218792Snp
1084218792Snp			ifp->if_capenable ^= IFCAP_LRO;
1085218792Snp			for_each_rxq(pi, i, rxq) {
1086218792Snp				if (ifp->if_capenable & IFCAP_LRO)
1087228561Snp					rxq->iq.flags |= IQ_LRO_ENABLED;
1088218792Snp				else
1089228561Snp					rxq->iq.flags &= ~IQ_LRO_ENABLED;
1090218792Snp			}
1091218792Snp#endif
1092218792Snp		}
1093237263Snp#ifdef TCP_OFFLOAD
1094228561Snp		if (mask & IFCAP_TOE) {
1095228561Snp			int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1096228561Snp
1097228561Snp			rc = toe_capability(pi, enable);
1098228561Snp			if (rc != 0)
1099228561Snp				goto fail;
1100228561Snp
1101228561Snp			ifp->if_capenable ^= mask;
1102218792Snp		}
1103218792Snp#endif
1104218792Snp		if (mask & IFCAP_VLAN_HWTAGGING) {
1105218792Snp			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1106245274Snp			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1107218792Snp				rc = update_mac_settings(pi, XGMAC_VLANEX);
1108218792Snp		}
1109218792Snp		if (mask & IFCAP_VLAN_MTU) {
1110218792Snp			ifp->if_capenable ^= IFCAP_VLAN_MTU;
1111218792Snp
1112218792Snp			/* Need to find out how to disable auto-mtu-inflation */
1113218792Snp		}
1114218792Snp		if (mask & IFCAP_VLAN_HWTSO)
1115218792Snp			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1116218792Snp		if (mask & IFCAP_VLAN_HWCSUM)
1117218792Snp			ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1118218792Snp
1119218792Snp#ifdef VLAN_CAPABILITIES
1120218792Snp		VLAN_CAPABILITIES(ifp);
1121218792Snp#endif
1122245274Snpfail:
1123245274Snp		end_synchronized_op(sc, 0);
1124218792Snp		break;
1125218792Snp
1126218792Snp	case SIOCSIFMEDIA:
1127218792Snp	case SIOCGIFMEDIA:
1128218792Snp		ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1129218792Snp		break;
1130218792Snp
1131218792Snp	default:
1132218792Snp		rc = ether_ioctl(ifp, cmd, data);
1133218792Snp	}
1134218792Snp
1135218792Snp	return (rc);
1136218792Snp}
1137218792Snp
1138218792Snpstatic int
1139218792Snpcxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1140218792Snp{
1141218792Snp	struct port_info *pi = ifp->if_softc;
1142218792Snp	struct adapter *sc = pi->adapter;
1143218792Snp	struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1144218792Snp	struct buf_ring *br;
1145218792Snp	int rc;
1146218792Snp
1147218792Snp	M_ASSERTPKTHDR(m);
1148218792Snp
1149228561Snp	if (__predict_false(pi->link_cfg.link_ok == 0)) {
1150218792Snp		m_freem(m);
1151228561Snp		return (ENETDOWN);
1152218792Snp	}
1153218792Snp
1154218792Snp	if (m->m_flags & M_FLOWID)
1155218792Snp		txq += (m->m_pkthdr.flowid % pi->ntxq);
1156220873Snp	br = txq->br;
1157218792Snp
1158218792Snp	if (TXQ_TRYLOCK(txq) == 0) {
1159228561Snp		struct sge_eq *eq = &txq->eq;
1160228561Snp
1161218792Snp		/*
1162228561Snp		 * It is possible that t4_eth_tx finishes up and releases the
1163228561Snp		 * lock between the TRYLOCK above and the drbr_enqueue here.  We
1164228561Snp		 * need to make sure that this mbuf doesn't just sit there in
1165228561Snp		 * the drbr.
1166218792Snp		 */
1167218792Snp
1168228561Snp		rc = drbr_enqueue(ifp, br, m);
1169228561Snp		if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1170228561Snp		    !(eq->flags & EQ_DOOMED))
1171228561Snp			callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1172228561Snp		return (rc);
1173218792Snp	}
1174218792Snp
1175218792Snp	/*
1176218792Snp	 * txq->m is the mbuf that is held up due to a temporary shortage of
1177218792Snp	 * resources and it should be put on the wire first.  Then what's in
1178218792Snp	 * drbr and finally the mbuf that was just passed in to us.
1179218792Snp	 *
1180218792Snp	 * Return code should indicate the fate of the mbuf that was passed in
1181218792Snp	 * this time.
1182218792Snp	 */
1183218792Snp
1184218792Snp	TXQ_LOCK_ASSERT_OWNED(txq);
1185218792Snp	if (drbr_needs_enqueue(ifp, br) || txq->m) {
1186218792Snp
1187218792Snp		/* Queued for transmission. */
1188218792Snp
1189218792Snp		rc = drbr_enqueue(ifp, br, m);
1190218792Snp		m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1191218792Snp		(void) t4_eth_tx(ifp, txq, m);
1192218792Snp		TXQ_UNLOCK(txq);
1193218792Snp		return (rc);
1194218792Snp	}
1195218792Snp
1196218792Snp	/* Direct transmission. */
1197218792Snp	rc = t4_eth_tx(ifp, txq, m);
1198218792Snp	if (rc != 0 && txq->m)
1199218792Snp		rc = 0;	/* held, will be transmitted soon (hopefully) */
1200218792Snp
1201218792Snp	TXQ_UNLOCK(txq);
1202218792Snp	return (rc);
1203218792Snp}
1204218792Snp
1205218792Snpstatic void
1206218792Snpcxgbe_qflush(struct ifnet *ifp)
1207218792Snp{
1208218792Snp	struct port_info *pi = ifp->if_softc;
1209220649Snp	struct sge_txq *txq;
1210220649Snp	int i;
1211220649Snp	struct mbuf *m;
1212218792Snp
1213228561Snp	/* queues do not exist if !PORT_INIT_DONE. */
1214228561Snp	if (pi->flags & PORT_INIT_DONE) {
1215220649Snp		for_each_txq(pi, i, txq) {
1216220649Snp			TXQ_LOCK(txq);
1217220649Snp			m_freem(txq->m);
1218228561Snp			txq->m = NULL;
1219220873Snp			while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1220220649Snp				m_freem(m);
1221220649Snp			TXQ_UNLOCK(txq);
1222220649Snp		}
1223220649Snp	}
1224220649Snp	if_qflush(ifp);
1225218792Snp}
1226218792Snp
1227218792Snpstatic int
1228218792Snpcxgbe_media_change(struct ifnet *ifp)
1229218792Snp{
1230218792Snp	struct port_info *pi = ifp->if_softc;
1231218792Snp
1232218792Snp	device_printf(pi->dev, "%s unimplemented.\n", __func__);
1233218792Snp
1234218792Snp	return (EOPNOTSUPP);
1235218792Snp}
1236218792Snp
1237218792Snpstatic void
1238218792Snpcxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1239218792Snp{
1240218792Snp	struct port_info *pi = ifp->if_softc;
1241218792Snp	struct ifmedia_entry *cur = pi->media.ifm_cur;
1242218792Snp	int speed = pi->link_cfg.speed;
1243218792Snp	int data = (pi->port_type << 8) | pi->mod_type;
1244218792Snp
1245218792Snp	if (cur->ifm_data != data) {
1246218792Snp		build_medialist(pi);
1247218792Snp		cur = pi->media.ifm_cur;
1248218792Snp	}
1249218792Snp
1250218792Snp	ifmr->ifm_status = IFM_AVALID;
1251218792Snp	if (!pi->link_cfg.link_ok)
1252218792Snp		return;
1253218792Snp
1254218792Snp	ifmr->ifm_status |= IFM_ACTIVE;
1255218792Snp
1256218792Snp	/* active and current will differ iff current media is autoselect. */
1257218792Snp	if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1258218792Snp		return;
1259218792Snp
1260218792Snp	ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1261218792Snp	if (speed == SPEED_10000)
1262218792Snp		ifmr->ifm_active |= IFM_10G_T;
1263218792Snp	else if (speed == SPEED_1000)
1264218792Snp		ifmr->ifm_active |= IFM_1000_T;
1265218792Snp	else if (speed == SPEED_100)
1266218792Snp		ifmr->ifm_active |= IFM_100_TX;
1267218792Snp	else if (speed == SPEED_10)
1268218792Snp		ifmr->ifm_active |= IFM_10_T;
1269218792Snp	else
1270218792Snp		KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1271218792Snp			    speed));
1272218792Snp}
1273218792Snp
1274218792Snpvoid
1275218792Snpt4_fatal_err(struct adapter *sc)
1276218792Snp{
1277218792Snp	t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1278218792Snp	t4_intr_disable(sc);
1279218792Snp	log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1280218792Snp	    device_get_nameunit(sc->dev));
1281218792Snp}
1282218792Snp
1283218792Snpstatic int
1284218792Snpmap_bars(struct adapter *sc)
1285218792Snp{
1286218792Snp	sc->regs_rid = PCIR_BAR(0);
1287218792Snp	sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1288218792Snp	    &sc->regs_rid, RF_ACTIVE);
1289218792Snp	if (sc->regs_res == NULL) {
1290218792Snp		device_printf(sc->dev, "cannot map registers.\n");
1291218792Snp		return (ENXIO);
1292218792Snp	}
1293218792Snp	sc->bt = rman_get_bustag(sc->regs_res);
1294218792Snp	sc->bh = rman_get_bushandle(sc->regs_res);
1295218792Snp	sc->mmio_len = rman_get_size(sc->regs_res);
1296218792Snp
1297218792Snp	sc->msix_rid = PCIR_BAR(4);
1298218792Snp	sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1299218792Snp	    &sc->msix_rid, RF_ACTIVE);
1300218792Snp	if (sc->msix_res == NULL) {
1301218792Snp		device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1302218792Snp		return (ENXIO);
1303218792Snp	}
1304218792Snp
1305218792Snp	return (0);
1306218792Snp}
1307218792Snp
1308218792Snpstatic void
1309218792Snpsetup_memwin(struct adapter *sc)
1310218792Snp{
1311237587Snp	uint32_t bar0;
1312218792Snp
1313237587Snp	/*
1314237587Snp	 * Read low 32b of bar0 indirectly via the hardware backdoor mechanism.
1315237587Snp	 * Works from within PCI passthrough environments too, where
1316237587Snp	 * rman_get_start() can return a different value.  We need to program
1317237587Snp	 * the memory window decoders with the actual addresses that will be
1318237587Snp	 * coming across the PCIe link.
1319237587Snp	 */
1320237587Snp	bar0 = t4_hw_pci_read_cfg4(sc, PCIR_BAR(0));
1321237587Snp	bar0 &= (uint32_t) PCIM_BAR_MEM_BASE;
1322218792Snp
1323218792Snp	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0),
1324218792Snp	    	     (bar0 + MEMWIN0_BASE) | V_BIR(0) |
1325218792Snp		     V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
1326218792Snp
1327218792Snp	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1),
1328218792Snp		     (bar0 + MEMWIN1_BASE) | V_BIR(0) |
1329218792Snp		     V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
1330218792Snp
1331218792Snp	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2),
1332218792Snp		     (bar0 + MEMWIN2_BASE) | V_BIR(0) |
1333218792Snp		     V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
1334237587Snp
1335237587Snp	/* flush */
1336237587Snp	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2));
1337218792Snp}
1338218792Snp
1339218792Snpstatic int
1340218792Snpcfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1341218792Snp    struct intrs_and_queues *iaq)
1342218792Snp{
1343228561Snp	int rc, itype, navail, nrxq10g, nrxq1g, n;
1344228561Snp	int nofldrxq10g = 0, nofldrxq1g = 0;
1345218792Snp
1346218792Snp	bzero(iaq, sizeof(*iaq));
1347218792Snp
1348228561Snp	iaq->ntxq10g = t4_ntxq10g;
1349228561Snp	iaq->ntxq1g = t4_ntxq1g;
1350228561Snp	iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1351228561Snp	iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1352237263Snp#ifdef TCP_OFFLOAD
1353237463Snp	if (is_offload(sc)) {
1354237463Snp		iaq->nofldtxq10g = t4_nofldtxq10g;
1355237463Snp		iaq->nofldtxq1g = t4_nofldtxq1g;
1356237463Snp		iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1357237463Snp		iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1358237463Snp	}
1359228561Snp#endif
1360228561Snp
1361219944Snp	for (itype = INTR_MSIX; itype; itype >>= 1) {
1362218792Snp
1363228561Snp		if ((itype & t4_intr_types) == 0)
1364218792Snp			continue;	/* not allowed */
1365218792Snp
1366219944Snp		if (itype == INTR_MSIX)
1367218792Snp			navail = pci_msix_count(sc->dev);
1368219944Snp		else if (itype == INTR_MSI)
1369218792Snp			navail = pci_msi_count(sc->dev);
1370218792Snp		else
1371218792Snp			navail = 1;
1372228561Snprestart:
1373218792Snp		if (navail == 0)
1374218792Snp			continue;
1375218792Snp
1376218792Snp		iaq->intr_type = itype;
1377228561Snp		iaq->intr_flags = 0;
1378218792Snp
1379228561Snp		/*
1380228561Snp		 * Best option: an interrupt vector for errors, one for the
1381228561Snp		 * firmware event queue, and one each for each rxq (NIC as well
1382228561Snp		 * as offload).
1383228561Snp		 */
1384228561Snp		iaq->nirq = T4_EXTRA_INTR;
1385228561Snp		iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
1386228561Snp		iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
1387228561Snp		if (iaq->nirq <= navail &&
1388228561Snp		    (itype != INTR_MSI || powerof2(iaq->nirq))) {
1389228561Snp			iaq->intr_flags |= INTR_DIRECT;
1390228561Snp			goto allocate;
1391228561Snp		}
1392218792Snp
1393228561Snp		/*
1394228561Snp		 * Second best option: an interrupt vector for errors, one for
1395228561Snp		 * the firmware event queue, and one each for either NIC or
1396228561Snp		 * offload rxq's.
1397228561Snp		 */
1398228561Snp		iaq->nirq = T4_EXTRA_INTR;
1399228561Snp		iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
1400228561Snp		iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
1401228561Snp		if (iaq->nirq <= navail &&
1402228561Snp		    (itype != INTR_MSI || powerof2(iaq->nirq)))
1403228561Snp			goto allocate;
1404218792Snp
1405228561Snp		/*
1406228561Snp		 * Next best option: an interrupt vector for errors, one for the
1407228561Snp		 * firmware event queue, and at least one per port.  At this
1408228561Snp		 * point we know we'll have to downsize nrxq or nofldrxq to fit
1409228561Snp		 * what's available to us.
1410228561Snp		 */
1411228561Snp		iaq->nirq = T4_EXTRA_INTR;
1412228561Snp		iaq->nirq += n10g + n1g;
1413228561Snp		if (iaq->nirq <= navail) {
1414228561Snp			int leftover = navail - iaq->nirq;
1415218792Snp
1416228561Snp			if (n10g > 0) {
1417228561Snp				int target = max(nrxq10g, nofldrxq10g);
1418219944Snp
1419228561Snp				n = 1;
1420228561Snp				while (n < target && leftover >= n10g) {
1421228561Snp					leftover -= n10g;
1422228561Snp					iaq->nirq += n10g;
1423228561Snp					n++;
1424228561Snp				}
1425228561Snp				iaq->nrxq10g = min(n, nrxq10g);
1426237263Snp#ifdef TCP_OFFLOAD
1427237463Snp				if (is_offload(sc))
1428237463Snp					iaq->nofldrxq10g = min(n, nofldrxq10g);
1429228561Snp#endif
1430228561Snp			}
1431218792Snp
1432228561Snp			if (n1g > 0) {
1433228561Snp				int target = max(nrxq1g, nofldrxq1g);
1434219944Snp
1435228561Snp				n = 1;
1436228561Snp				while (n < target && leftover >= n1g) {
1437228561Snp					leftover -= n1g;
1438228561Snp					iaq->nirq += n1g;
1439228561Snp					n++;
1440219944Snp				}
1441228561Snp				iaq->nrxq1g = min(n, nrxq1g);
1442237263Snp#ifdef TCP_OFFLOAD
1443237463Snp				if (is_offload(sc))
1444237463Snp					iaq->nofldrxq1g = min(n, nofldrxq1g);
1445228561Snp#endif
1446219944Snp			}
1447219944Snp
1448228561Snp			if (itype != INTR_MSI || powerof2(iaq->nirq))
1449228561Snp				goto allocate;
1450218792Snp		}
1451218792Snp
1452228561Snp		/*
1453228561Snp		 * Least desirable option: one interrupt vector for everything.
1454228561Snp		 */
1455228561Snp		iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1456237263Snp#ifdef TCP_OFFLOAD
1457237463Snp		if (is_offload(sc))
1458237463Snp			iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1459228561Snp#endif
1460228561Snp
1461228561Snpallocate:
1462218792Snp		navail = iaq->nirq;
1463218792Snp		rc = 0;
1464219944Snp		if (itype == INTR_MSIX)
1465218792Snp			rc = pci_alloc_msix(sc->dev, &navail);
1466219944Snp		else if (itype == INTR_MSI)
1467218792Snp			rc = pci_alloc_msi(sc->dev, &navail);
1468218792Snp
1469218792Snp		if (rc == 0) {
1470218792Snp			if (navail == iaq->nirq)
1471218792Snp				return (0);
1472218792Snp
1473218792Snp			/*
1474218792Snp			 * Didn't get the number requested.  Use whatever number
1475218792Snp			 * the kernel is willing to allocate (it's in navail).
1476218792Snp			 */
1477228561Snp			device_printf(sc->dev, "fewer vectors than requested, "
1478228561Snp			    "type=%d, req=%d, rcvd=%d; will downshift req.\n",
1479228561Snp			    itype, iaq->nirq, navail);
1480218792Snp			pci_release_msi(sc->dev);
1481228561Snp			goto restart;
1482218792Snp		}
1483218792Snp
1484218792Snp		device_printf(sc->dev,
1485218792Snp		    "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
1486218792Snp		    itype, rc, iaq->nirq, navail);
1487218792Snp	}
1488218792Snp
1489218792Snp	device_printf(sc->dev,
1490218792Snp	    "failed to find a usable interrupt type.  "
1491228561Snp	    "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
1492218792Snp	    pci_msix_count(sc->dev), pci_msi_count(sc->dev));
1493218792Snp
1494218792Snp	return (ENXIO);
1495218792Snp}
1496218792Snp
1497218792Snp/*
1498228561Snp * Install a compatible firmware (if required), establish contact with it (by
1499228561Snp * saying hello), and reset the device.  If we end up as the master driver,
1500228561Snp * partition adapter resources by providing a configuration file to the
1501228561Snp * firmware.
1502218792Snp */
1503218792Snpstatic int
1504218792Snpprep_firmware(struct adapter *sc)
1505218792Snp{
1506228561Snp	const struct firmware *fw = NULL, *cfg = NULL, *default_cfg;
1507218792Snp	int rc;
1508218792Snp	enum dev_state state;
1509218792Snp
1510228561Snp	default_cfg = firmware_get(T4_CFGNAME);
1511228561Snp
1512218792Snp	/* Check firmware version and install a different one if necessary */
1513218792Snp	rc = t4_check_fw_version(sc);
1514234831Snp	snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
1515234831Snp	    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1516234831Snp	    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1517234831Snp	    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1518234831Snp	    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1519228561Snp	if (rc != 0) {
1520219287Snp		uint32_t v = 0;
1521218792Snp
1522218792Snp		fw = firmware_get(T4_FWNAME);
1523219287Snp		if (fw != NULL) {
1524219287Snp			const struct fw_hdr *hdr = (const void *)fw->data;
1525219287Snp
1526219287Snp			v = ntohl(hdr->fw_ver);
1527219287Snp
1528219287Snp			/*
1529219287Snp			 * The firmware module will not be used if it isn't the
1530219287Snp			 * same major version as what the driver was compiled
1531228561Snp			 * with.
1532219287Snp			 */
1533219287Snp			if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) {
1534219287Snp				device_printf(sc->dev,
1535219287Snp				    "Found firmware image but version %d "
1536219287Snp				    "can not be used with this driver (%d)\n",
1537219287Snp				    G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR);
1538219287Snp
1539219287Snp				firmware_put(fw, FIRMWARE_UNLOAD);
1540219287Snp				fw = NULL;
1541219287Snp			}
1542218792Snp		}
1543218792Snp
1544228561Snp		if (fw == NULL && rc < 0) {
1545219287Snp			device_printf(sc->dev, "No usable firmware. "
1546228561Snp			    "card has %d.%d.%d, driver compiled with %d.%d.%d",
1547219287Snp			    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1548219287Snp			    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1549219287Snp			    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1550219287Snp			    FW_VERSION_MAJOR, FW_VERSION_MINOR,
1551228561Snp			    FW_VERSION_MICRO);
1552228561Snp			rc = EAGAIN;
1553228561Snp			goto done;
1554219287Snp		}
1555219287Snp
1556219287Snp		/*
1557219287Snp		 * Always upgrade, even for minor/micro/build mismatches.
1558219287Snp		 * Downgrade only for a major version mismatch or if
1559219287Snp		 * force_firmware_install was specified.
1560219287Snp		 */
1561228561Snp		if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) {
1562218792Snp			device_printf(sc->dev,
1563219287Snp			    "installing firmware %d.%d.%d.%d on card.\n",
1564219287Snp			    G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v),
1565219287Snp			    G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v));
1566219287Snp
1567219287Snp			rc = -t4_load_fw(sc, fw->data, fw->datasize);
1568219287Snp			if (rc != 0) {
1569219287Snp				device_printf(sc->dev,
1570219287Snp				    "failed to install firmware: %d\n", rc);
1571228561Snp				goto done;
1572219287Snp			} else {
1573219287Snp				/* refresh */
1574219287Snp				(void) t4_check_fw_version(sc);
1575234831Snp				snprintf(sc->fw_version,
1576234831Snp				    sizeof(sc->fw_version), "%u.%u.%u.%u",
1577234831Snp				    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1578234831Snp				    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1579234831Snp				    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1580234831Snp				    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1581219287Snp			}
1582218792Snp		}
1583218792Snp	}
1584218792Snp
1585228561Snp	/* Contact firmware.  */
1586228561Snp	rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
1587218792Snp	if (rc < 0) {
1588218792Snp		rc = -rc;
1589218792Snp		device_printf(sc->dev,
1590218792Snp		    "failed to connect to the firmware: %d.\n", rc);
1591228561Snp		goto done;
1592218792Snp	}
1593228561Snp	if (rc == sc->mbox)
1594228561Snp		sc->flags |= MASTER_PF;
1595218792Snp
1596218792Snp	/* Reset device */
1597218792Snp	rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
1598218792Snp	if (rc != 0) {
1599218792Snp		device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
1600218792Snp		if (rc != ETIMEDOUT && rc != EIO)
1601218792Snp			t4_fw_bye(sc, sc->mbox);
1602228561Snp		goto done;
1603218792Snp	}
1604218792Snp
1605228561Snp	/* Partition adapter resources as specified in the config file. */
1606228561Snp	if (sc->flags & MASTER_PF) {
1607245936Snp		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s",
1608245936Snp		    pci_get_device(sc->dev) == 0x440a ? "uwire" : t4_cfg_file);
1609245936Snp		if (strncmp(sc->cfg_file, "default", sizeof(sc->cfg_file))) {
1610228561Snp			char s[32];
1611228561Snp
1612245936Snp			snprintf(s, sizeof(s), "t4fw_cfg_%s", sc->cfg_file);
1613228561Snp			cfg = firmware_get(s);
1614228561Snp			if (cfg == NULL) {
1615228561Snp				device_printf(sc->dev,
1616228561Snp				    "unable to locate %s module, "
1617228561Snp				    "will use default config file.\n", s);
1618245936Snp				snprintf(sc->cfg_file, sizeof(sc->cfg_file),
1619245936Snp				    "%s", "default");
1620228561Snp			}
1621228561Snp		}
1622228561Snp
1623228561Snp		rc = partition_resources(sc, cfg ? cfg : default_cfg);
1624228561Snp		if (rc != 0)
1625228561Snp			goto done;	/* error message displayed already */
1626245936Snp	} else {
1627245936Snp		snprintf(sc->cfg_file, sizeof(sc->cfg_file), "%s", "notme");
1628245936Snp		sc->cfcsum = (u_int)-1;
1629228561Snp	}
1630228561Snp
1631218792Snp	sc->flags |= FW_OK;
1632218792Snp
1633228561Snpdone:
1634228561Snp	if (fw != NULL)
1635228561Snp		firmware_put(fw, FIRMWARE_UNLOAD);
1636228561Snp	if (cfg != NULL)
1637228561Snp		firmware_put(cfg, FIRMWARE_UNLOAD);
1638228561Snp	if (default_cfg != NULL)
1639228561Snp		firmware_put(default_cfg, FIRMWARE_UNLOAD);
1640228561Snp
1641228561Snp	return (rc);
1642218792Snp}
1643218792Snp
1644228561Snp#define FW_PARAM_DEV(param) \
1645228561Snp	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1646228561Snp	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1647228561Snp#define FW_PARAM_PFVF(param) \
1648228561Snp	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1649228561Snp	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
1650228561Snp
1651228561Snp/*
1652228561Snp * Upload configuration file to card's memory.
1653228561Snp */
1654218792Snpstatic int
1655228561Snpupload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt,
1656228561Snp    uint32_t *ma)
1657222551Snp{
1658228561Snp	int rc, i;
1659228561Snp	uint32_t param, val, mtype, maddr, bar, off, win, remaining;
1660228561Snp	const uint32_t *b;
1661222551Snp
1662228561Snp	/* Figure out where the firmware wants us to upload it. */
1663228561Snp	param = FW_PARAM_DEV(CF);
1664228561Snp	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1665222551Snp	if (rc != 0) {
1666228561Snp		/* Firmwares without config file support will fail this way */
1667222551Snp		device_printf(sc->dev,
1668228561Snp		    "failed to query config file location: %d.\n", rc);
1669222551Snp		return (rc);
1670222551Snp	}
1671228561Snp	*mt = mtype = G_FW_PARAMS_PARAM_Y(val);
1672228561Snp	*ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
1673222551Snp
1674228561Snp	if (maddr & 3) {
1675228561Snp		device_printf(sc->dev,
1676228561Snp		    "cannot upload config file (type %u, addr %x).\n",
1677228561Snp		    mtype, maddr);
1678228561Snp		return (EFAULT);
1679228561Snp	}
1680222551Snp
1681228561Snp	/* Translate mtype/maddr to an address suitable for the PCIe window */
1682228561Snp	val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1683228561Snp	val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE;
1684228561Snp	switch (mtype) {
1685228561Snp	case FW_MEMTYPE_CF_EDC0:
1686228561Snp		if (!(val & F_EDRAM0_ENABLE))
1687228561Snp			goto err;
1688228561Snp		bar = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1689228561Snp		maddr += G_EDRAM0_BASE(bar) << 20;
1690228561Snp		break;
1691228561Snp
1692228561Snp	case FW_MEMTYPE_CF_EDC1:
1693228561Snp		if (!(val & F_EDRAM1_ENABLE))
1694228561Snp			goto err;
1695228561Snp		bar = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1696228561Snp		maddr += G_EDRAM1_BASE(bar) << 20;
1697228561Snp		break;
1698228561Snp
1699228561Snp	case FW_MEMTYPE_CF_EXTMEM:
1700228561Snp		if (!(val & F_EXT_MEM_ENABLE))
1701228561Snp			goto err;
1702228561Snp		bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1703228561Snp		maddr += G_EXT_MEM_BASE(bar) << 20;
1704228561Snp		break;
1705228561Snp
1706228561Snp	default:
1707228561Snperr:
1708228561Snp		device_printf(sc->dev,
1709228561Snp		    "cannot upload config file (type %u, enabled %u).\n",
1710228561Snp		    mtype, val);
1711228561Snp		return (EFAULT);
1712228561Snp	}
1713228561Snp
1714228561Snp	/*
1715228561Snp	 * Position the PCIe window (we use memwin2) to the 16B aligned area
1716228561Snp	 * just at/before the upload location.
1717228561Snp	 */
1718228561Snp	win = maddr & ~0xf;
1719228561Snp	off = maddr - win;  /* offset from the start of the window. */
1720228561Snp	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
1721228561Snp	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
1722228561Snp
1723228561Snp	remaining = fw->datasize;
1724228561Snp	if (remaining > FLASH_CFG_MAX_SIZE ||
1725228561Snp	    remaining > MEMWIN2_APERTURE - off) {
1726228561Snp		device_printf(sc->dev, "cannot upload config file all at once "
1727228561Snp		    "(size %u, max %u, room %u).\n",
1728228561Snp		    remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off);
1729228561Snp		return (EFBIG);
1730228561Snp	}
1731228561Snp
1732228561Snp	/*
1733228561Snp	 * XXX: sheer laziness.  We deliberately added 4 bytes of useless
1734228561Snp	 * stuffing/comments at the end of the config file so it's ok to simply
1735228561Snp	 * throw away the last remaining bytes when the config file is not an
1736228561Snp	 * exact multiple of 4.
1737228561Snp	 */
1738228561Snp	b = fw->data;
1739228561Snp	for (i = 0; remaining >= 4; i += 4, remaining -= 4)
1740228561Snp		t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++);
1741228561Snp
1742228561Snp	return (rc);
1743222551Snp}
1744222551Snp
1745228561Snp/*
1746228561Snp * Partition chip resources for use between various PFs, VFs, etc.  This is done
1747228561Snp * by uploading the firmware configuration file to the adapter and instructing
1748228561Snp * the firmware to process it.
1749228561Snp */
1750222551Snpstatic int
1751228561Snppartition_resources(struct adapter *sc, const struct firmware *cfg)
1752218792Snp{
1753218792Snp	int rc;
1754228561Snp	struct fw_caps_config_cmd caps;
1755228561Snp	uint32_t mtype, maddr, finicsum, cfcsum;
1756218792Snp
1757228561Snp	rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT;
1758228561Snp	if (rc != 0) {
1759228561Snp		mtype = FW_MEMTYPE_CF_FLASH;
1760228561Snp		maddr = t4_flash_cfg_addr(sc);
1761228561Snp	}
1762228561Snp
1763228561Snp	bzero(&caps, sizeof(caps));
1764228561Snp	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1765218792Snp	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1766228561Snp	caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
1767228561Snp	    V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1768228561Snp	    V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
1769228561Snp	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1770228561Snp	if (rc != 0) {
1771228561Snp		device_printf(sc->dev,
1772228561Snp		    "failed to pre-process config file: %d.\n", rc);
1773218792Snp		return (rc);
1774228561Snp	}
1775218792Snp
1776228561Snp	finicsum = be32toh(caps.finicsum);
1777228561Snp	cfcsum = be32toh(caps.cfcsum);
1778228561Snp	if (finicsum != cfcsum) {
1779228561Snp		device_printf(sc->dev,
1780228561Snp		    "WARNING: config file checksum mismatch: %08x %08x\n",
1781228561Snp		    finicsum, cfcsum);
1782228561Snp	}
1783228561Snp	sc->cfcsum = cfcsum;
1784218792Snp
1785228561Snp#define LIMIT_CAPS(x) do { \
1786228561Snp	caps.x &= htobe16(t4_##x##_allowed); \
1787228561Snp	sc->x = htobe16(caps.x); \
1788228561Snp} while (0)
1789228561Snp
1790228561Snp	/*
1791228561Snp	 * Let the firmware know what features will (not) be used so it can tune
1792228561Snp	 * things accordingly.
1793228561Snp	 */
1794228561Snp	LIMIT_CAPS(linkcaps);
1795228561Snp	LIMIT_CAPS(niccaps);
1796228561Snp	LIMIT_CAPS(toecaps);
1797228561Snp	LIMIT_CAPS(rdmacaps);
1798228561Snp	LIMIT_CAPS(iscsicaps);
1799228561Snp	LIMIT_CAPS(fcoecaps);
1800228561Snp#undef LIMIT_CAPS
1801228561Snp
1802228561Snp	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1803218792Snp	    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
1804228561Snp	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1805228561Snp	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
1806228561Snp	if (rc != 0) {
1807228561Snp		device_printf(sc->dev,
1808228561Snp		    "failed to process config file: %d.\n", rc);
1809228561Snp		return (rc);
1810228561Snp	}
1811218792Snp
1812228561Snp	return (0);
1813218792Snp}
1814218792Snp
1815228561Snp/*
1816228561Snp * Retrieve parameters that are needed (or nice to have) prior to calling
1817228561Snp * t4_sge_init and t4_fw_initialize.
1818228561Snp */
1819218792Snpstatic int
1820228561Snpget_params__pre_init(struct adapter *sc)
1821218792Snp{
1822218792Snp	int rc;
1823228561Snp	uint32_t param[2], val[2];
1824228561Snp	struct fw_devlog_cmd cmd;
1825228561Snp	struct devlog_params *dlog = &sc->params.devlog;
1826218792Snp
1827228561Snp	param[0] = FW_PARAM_DEV(PORTVEC);
1828228561Snp	param[1] = FW_PARAM_DEV(CCLK);
1829228561Snp	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1830218792Snp	if (rc != 0) {
1831218792Snp		device_printf(sc->dev,
1832228561Snp		    "failed to query parameters (pre_init): %d.\n", rc);
1833228561Snp		return (rc);
1834218792Snp	}
1835218792Snp
1836218792Snp	sc->params.portvec = val[0];
1837240452Snp	sc->params.nports = bitcount32(val[0]);
1838228561Snp	sc->params.vpd.cclk = val[1];
1839218792Snp
1840228561Snp	/* Read device log parameters. */
1841228561Snp	bzero(&cmd, sizeof(cmd));
1842228561Snp	cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
1843228561Snp	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1844228561Snp	cmd.retval_len16 = htobe32(FW_LEN16(cmd));
1845228561Snp	rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
1846228561Snp	if (rc != 0) {
1847228561Snp		device_printf(sc->dev,
1848228561Snp		    "failed to get devlog parameters: %d.\n", rc);
1849228561Snp		bzero(dlog, sizeof (*dlog));
1850228561Snp		rc = 0;	/* devlog isn't critical for device operation */
1851228561Snp	} else {
1852228561Snp		val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
1853228561Snp		dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
1854228561Snp		dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
1855228561Snp		dlog->size = be32toh(cmd.memsize_devlog);
1856228561Snp	}
1857228561Snp
1858228561Snp	return (rc);
1859228561Snp}
1860228561Snp
1861228561Snp/*
1862228561Snp * Retrieve various parameters that are of interest to the driver.  The device
1863228561Snp * has been initialized by the firmware at this point.
1864228561Snp */
1865228561Snpstatic int
1866228561Snpget_params__post_init(struct adapter *sc)
1867228561Snp{
1868228561Snp	int rc;
1869228561Snp	uint32_t param[7], val[7];
1870228561Snp	struct fw_caps_config_cmd caps;
1871228561Snp
1872228561Snp	param[0] = FW_PARAM_PFVF(IQFLINT_START);
1873228561Snp	param[1] = FW_PARAM_PFVF(EQ_START);
1874228561Snp	param[2] = FW_PARAM_PFVF(FILTER_START);
1875228561Snp	param[3] = FW_PARAM_PFVF(FILTER_END);
1876245434Snp	param[4] = FW_PARAM_PFVF(L2T_START);
1877245434Snp	param[5] = FW_PARAM_PFVF(L2T_END);
1878245434Snp	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1879228561Snp	if (rc != 0) {
1880228561Snp		device_printf(sc->dev,
1881228561Snp		    "failed to query parameters (post_init): %d.\n", rc);
1882228561Snp		return (rc);
1883228561Snp	}
1884228561Snp
1885228561Snp	sc->sge.iq_start = val[0];
1886228561Snp	sc->sge.eq_start = val[1];
1887228561Snp	sc->tids.ftid_base = val[2];
1888228561Snp	sc->tids.nftids = val[3] - val[2] + 1;
1889245434Snp	sc->vres.l2t.start = val[4];
1890245434Snp	sc->vres.l2t.size = val[5] - val[4] + 1;
1891245434Snp	KASSERT(sc->vres.l2t.size <= L2T_SIZE,
1892245434Snp	    ("%s: L2 table size (%u) larger than expected (%u)",
1893245434Snp	    __func__, sc->vres.l2t.size, L2T_SIZE));
1894228561Snp
1895228561Snp	/* get capabilites */
1896228561Snp	bzero(&caps, sizeof(caps));
1897228561Snp	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1898228561Snp	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1899228561Snp	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1900228561Snp	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1901228561Snp	if (rc != 0) {
1902228561Snp		device_printf(sc->dev,
1903228561Snp		    "failed to get card capabilities: %d.\n", rc);
1904228561Snp		return (rc);
1905228561Snp	}
1906228561Snp
1907228561Snp	if (caps.toecaps) {
1908218792Snp		/* query offload-related parameters */
1909228561Snp		param[0] = FW_PARAM_DEV(NTID);
1910228561Snp		param[1] = FW_PARAM_PFVF(SERVER_START);
1911228561Snp		param[2] = FW_PARAM_PFVF(SERVER_END);
1912228561Snp		param[3] = FW_PARAM_PFVF(TDDP_START);
1913228561Snp		param[4] = FW_PARAM_PFVF(TDDP_END);
1914228561Snp		param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
1915228561Snp		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1916218792Snp		if (rc != 0) {
1917218792Snp			device_printf(sc->dev,
1918218792Snp			    "failed to query TOE parameters: %d.\n", rc);
1919228561Snp			return (rc);
1920218792Snp		}
1921218792Snp		sc->tids.ntids = val[0];
1922218792Snp		sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
1923218792Snp		sc->tids.stid_base = val[1];
1924218792Snp		sc->tids.nstids = val[2] - val[1] + 1;
1925218792Snp		sc->vres.ddp.start = val[3];
1926218792Snp		sc->vres.ddp.size = val[4] - val[3] + 1;
1927218792Snp		sc->params.ofldq_wr_cred = val[5];
1928218792Snp		sc->params.offload = 1;
1929218792Snp	}
1930228561Snp	if (caps.rdmacaps) {
1931228561Snp		param[0] = FW_PARAM_PFVF(STAG_START);
1932228561Snp		param[1] = FW_PARAM_PFVF(STAG_END);
1933228561Snp		param[2] = FW_PARAM_PFVF(RQ_START);
1934228561Snp		param[3] = FW_PARAM_PFVF(RQ_END);
1935228561Snp		param[4] = FW_PARAM_PFVF(PBL_START);
1936228561Snp		param[5] = FW_PARAM_PFVF(PBL_END);
1937228561Snp		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1938218792Snp		if (rc != 0) {
1939218792Snp			device_printf(sc->dev,
1940228561Snp			    "failed to query RDMA parameters(1): %d.\n", rc);
1941228561Snp			return (rc);
1942218792Snp		}
1943218792Snp		sc->vres.stag.start = val[0];
1944218792Snp		sc->vres.stag.size = val[1] - val[0] + 1;
1945218792Snp		sc->vres.rq.start = val[2];
1946218792Snp		sc->vres.rq.size = val[3] - val[2] + 1;
1947218792Snp		sc->vres.pbl.start = val[4];
1948218792Snp		sc->vres.pbl.size = val[5] - val[4] + 1;
1949228561Snp
1950228561Snp		param[0] = FW_PARAM_PFVF(SQRQ_START);
1951228561Snp		param[1] = FW_PARAM_PFVF(SQRQ_END);
1952228561Snp		param[2] = FW_PARAM_PFVF(CQ_START);
1953228561Snp		param[3] = FW_PARAM_PFVF(CQ_END);
1954228561Snp		param[4] = FW_PARAM_PFVF(OCQ_START);
1955228561Snp		param[5] = FW_PARAM_PFVF(OCQ_END);
1956228561Snp		rc = -t4_query_params(sc, 0, 0, 0, 6, param, val);
1957228561Snp		if (rc != 0) {
1958228561Snp			device_printf(sc->dev,
1959228561Snp			    "failed to query RDMA parameters(2): %d.\n", rc);
1960228561Snp			return (rc);
1961228561Snp		}
1962228561Snp		sc->vres.qp.start = val[0];
1963228561Snp		sc->vres.qp.size = val[1] - val[0] + 1;
1964228561Snp		sc->vres.cq.start = val[2];
1965228561Snp		sc->vres.cq.size = val[3] - val[2] + 1;
1966228561Snp		sc->vres.ocq.start = val[4];
1967228561Snp		sc->vres.ocq.size = val[5] - val[4] + 1;
1968218792Snp	}
1969228561Snp	if (caps.iscsicaps) {
1970228561Snp		param[0] = FW_PARAM_PFVF(ISCSI_START);
1971228561Snp		param[1] = FW_PARAM_PFVF(ISCSI_END);
1972228561Snp		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1973218792Snp		if (rc != 0) {
1974218792Snp			device_printf(sc->dev,
1975218792Snp			    "failed to query iSCSI parameters: %d.\n", rc);
1976228561Snp			return (rc);
1977218792Snp		}
1978218792Snp		sc->vres.iscsi.start = val[0];
1979218792Snp		sc->vres.iscsi.size = val[1] - val[0] + 1;
1980218792Snp	}
1981218792Snp
1982228561Snp	/* These are finalized by FW initialization, load their values now */
1983228561Snp	val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
1984228561Snp	sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
1985228561Snp	sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
1986228561Snp	t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
1987228561Snp
1988218792Snp	return (rc);
1989218792Snp}
1990218792Snp
1991228561Snp#undef FW_PARAM_PFVF
1992228561Snp#undef FW_PARAM_DEV
1993228561Snp
1994218792Snpstatic void
1995218792Snpt4_set_desc(struct adapter *sc)
1996218792Snp{
1997218792Snp	char buf[128];
1998218792Snp	struct adapter_params *p = &sc->params;
1999218792Snp
2000228561Snp	snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s",
2001228561Snp	    p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec);
2002218792Snp
2003218792Snp	device_set_desc_copy(sc->dev, buf);
2004218792Snp}
2005218792Snp
2006218792Snpstatic void
2007218792Snpbuild_medialist(struct port_info *pi)
2008218792Snp{
2009218792Snp	struct ifmedia *media = &pi->media;
2010218792Snp	int data, m;
2011218792Snp
2012218792Snp	PORT_LOCK(pi);
2013218792Snp
2014218792Snp	ifmedia_removeall(media);
2015218792Snp
2016218792Snp	m = IFM_ETHER | IFM_FDX;
2017218792Snp	data = (pi->port_type << 8) | pi->mod_type;
2018218792Snp
2019218792Snp	switch(pi->port_type) {
2020218792Snp	case FW_PORT_TYPE_BT_XFI:
2021218792Snp		ifmedia_add(media, m | IFM_10G_T, data, NULL);
2022218792Snp		break;
2023218792Snp
2024218792Snp	case FW_PORT_TYPE_BT_XAUI:
2025218792Snp		ifmedia_add(media, m | IFM_10G_T, data, NULL);
2026218792Snp		/* fall through */
2027218792Snp
2028218792Snp	case FW_PORT_TYPE_BT_SGMII:
2029218792Snp		ifmedia_add(media, m | IFM_1000_T, data, NULL);
2030218792Snp		ifmedia_add(media, m | IFM_100_TX, data, NULL);
2031218792Snp		ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
2032218792Snp		ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2033218792Snp		break;
2034218792Snp
2035218792Snp	case FW_PORT_TYPE_CX4:
2036218792Snp		ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
2037218792Snp		ifmedia_set(media, m | IFM_10G_CX4);
2038218792Snp		break;
2039218792Snp
2040218792Snp	case FW_PORT_TYPE_SFP:
2041218792Snp	case FW_PORT_TYPE_FIBER_XFI:
2042218792Snp	case FW_PORT_TYPE_FIBER_XAUI:
2043218792Snp		switch (pi->mod_type) {
2044218792Snp
2045218792Snp		case FW_PORT_MOD_TYPE_LR:
2046218792Snp			ifmedia_add(media, m | IFM_10G_LR, data, NULL);
2047218792Snp			ifmedia_set(media, m | IFM_10G_LR);
2048218792Snp			break;
2049218792Snp
2050218792Snp		case FW_PORT_MOD_TYPE_SR:
2051218792Snp			ifmedia_add(media, m | IFM_10G_SR, data, NULL);
2052218792Snp			ifmedia_set(media, m | IFM_10G_SR);
2053218792Snp			break;
2054218792Snp
2055218792Snp		case FW_PORT_MOD_TYPE_LRM:
2056218792Snp			ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2057218792Snp			ifmedia_set(media, m | IFM_10G_LRM);
2058218792Snp			break;
2059218792Snp
2060218792Snp		case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2061218792Snp		case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2062218792Snp			ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2063218792Snp			ifmedia_set(media, m | IFM_10G_TWINAX);
2064218792Snp			break;
2065218792Snp
2066218792Snp		case FW_PORT_MOD_TYPE_NONE:
2067218792Snp			m &= ~IFM_FDX;
2068218792Snp			ifmedia_add(media, m | IFM_NONE, data, NULL);
2069218792Snp			ifmedia_set(media, m | IFM_NONE);
2070218792Snp			break;
2071218792Snp
2072218792Snp		case FW_PORT_MOD_TYPE_NA:
2073218792Snp		case FW_PORT_MOD_TYPE_ER:
2074218792Snp		default:
2075218792Snp			ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2076218792Snp			ifmedia_set(media, m | IFM_UNKNOWN);
2077218792Snp			break;
2078218792Snp		}
2079218792Snp		break;
2080218792Snp
2081218792Snp	case FW_PORT_TYPE_KX4:
2082218792Snp	case FW_PORT_TYPE_KX:
2083218792Snp	case FW_PORT_TYPE_KR:
2084218792Snp	default:
2085218792Snp		ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2086218792Snp		ifmedia_set(media, m | IFM_UNKNOWN);
2087218792Snp		break;
2088218792Snp	}
2089218792Snp
2090218792Snp	PORT_UNLOCK(pi);
2091218792Snp}
2092218792Snp
2093231172Snp#define FW_MAC_EXACT_CHUNK	7
2094231172Snp
2095218792Snp/*
2096218792Snp * Program the port's XGMAC based on parameters in ifnet.  The caller also
2097218792Snp * indicates which parameters should be programmed (the rest are left alone).
2098218792Snp */
2099218792Snpstatic int
2100218792Snpupdate_mac_settings(struct port_info *pi, int flags)
2101218792Snp{
2102218792Snp	int rc;
2103218792Snp	struct ifnet *ifp = pi->ifp;
2104218792Snp	struct adapter *sc = pi->adapter;
2105218792Snp	int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2106218792Snp
2107245274Snp	ASSERT_SYNCHRONIZED_OP(sc);
2108218792Snp	KASSERT(flags, ("%s: not told what to update.", __func__));
2109218792Snp
2110218792Snp	if (flags & XGMAC_MTU)
2111218792Snp		mtu = ifp->if_mtu;
2112218792Snp
2113218792Snp	if (flags & XGMAC_PROMISC)
2114218792Snp		promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2115218792Snp
2116218792Snp	if (flags & XGMAC_ALLMULTI)
2117218792Snp		allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2118218792Snp
2119218792Snp	if (flags & XGMAC_VLANEX)
2120218792Snp		vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2121218792Snp
2122218792Snp	rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
2123218792Snp	    vlanex, false);
2124218792Snp	if (rc) {
2125218792Snp		if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
2126218792Snp		return (rc);
2127218792Snp	}
2128218792Snp
2129218792Snp	if (flags & XGMAC_UCADDR) {
2130218792Snp		uint8_t ucaddr[ETHER_ADDR_LEN];
2131218792Snp
2132218792Snp		bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2133218792Snp		rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
2134218792Snp		    ucaddr, true, true);
2135218792Snp		if (rc < 0) {
2136218792Snp			rc = -rc;
2137218792Snp			if_printf(ifp, "change_mac failed: %d\n", rc);
2138218792Snp			return (rc);
2139218792Snp		} else {
2140218792Snp			pi->xact_addr_filt = rc;
2141218792Snp			rc = 0;
2142218792Snp		}
2143218792Snp	}
2144218792Snp
2145218792Snp	if (flags & XGMAC_MCADDRS) {
2146231172Snp		const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2147218792Snp		int del = 1;
2148218792Snp		uint64_t hash = 0;
2149218792Snp		struct ifmultiaddr *ifma;
2150231172Snp		int i = 0, j;
2151218792Snp
2152218792Snp		if_maddr_rlock(ifp);
2153218792Snp		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2154238054Snp			if (ifma->ifma_addr->sa_family != AF_LINK)
2155218792Snp				continue;
2156231172Snp			mcaddr[i++] =
2157231172Snp			    LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2158218792Snp
2159231172Snp			if (i == FW_MAC_EXACT_CHUNK) {
2160231172Snp				rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2161231172Snp				    del, i, mcaddr, NULL, &hash, 0);
2162231172Snp				if (rc < 0) {
2163231172Snp					rc = -rc;
2164231172Snp					for (j = 0; j < i; j++) {
2165231172Snp						if_printf(ifp,
2166231172Snp						    "failed to add mc address"
2167231172Snp						    " %02x:%02x:%02x:"
2168231172Snp						    "%02x:%02x:%02x rc=%d\n",
2169231172Snp						    mcaddr[j][0], mcaddr[j][1],
2170231172Snp						    mcaddr[j][2], mcaddr[j][3],
2171231172Snp						    mcaddr[j][4], mcaddr[j][5],
2172231172Snp						    rc);
2173231172Snp					}
2174231172Snp					goto mcfail;
2175231172Snp				}
2176231172Snp				del = 0;
2177231172Snp				i = 0;
2178231172Snp			}
2179231172Snp		}
2180231172Snp		if (i > 0) {
2181231172Snp			rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2182231172Snp			    del, i, mcaddr, NULL, &hash, 0);
2183218792Snp			if (rc < 0) {
2184218792Snp				rc = -rc;
2185231172Snp				for (j = 0; j < i; j++) {
2186231172Snp					if_printf(ifp,
2187231172Snp					    "failed to add mc address"
2188231172Snp					    " %02x:%02x:%02x:"
2189231172Snp					    "%02x:%02x:%02x rc=%d\n",
2190231172Snp					    mcaddr[j][0], mcaddr[j][1],
2191231172Snp					    mcaddr[j][2], mcaddr[j][3],
2192231172Snp					    mcaddr[j][4], mcaddr[j][5],
2193231172Snp					    rc);
2194231172Snp				}
2195218792Snp				goto mcfail;
2196218792Snp			}
2197218792Snp		}
2198218792Snp
2199218792Snp		rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
2200218792Snp		if (rc != 0)
2201218792Snp			if_printf(ifp, "failed to set mc address hash: %d", rc);
2202218792Snpmcfail:
2203218792Snp		if_maddr_runlock(ifp);
2204218792Snp	}
2205218792Snp
2206218792Snp	return (rc);
2207218792Snp}
2208218792Snp
2209245274Snpint
2210245274Snpbegin_synchronized_op(struct adapter *sc, struct port_info *pi, int flags,
2211245274Snp    char *wmesg)
2212218792Snp{
2213245274Snp	int rc, pri;
2214218792Snp
2215245274Snp#ifdef WITNESS
2216245274Snp	/* the caller thinks it's ok to sleep, but is it really? */
2217245274Snp	if (flags & SLEEP_OK)
2218245274Snp		pause("t4slptst", 1);
2219245274Snp#endif
2220218792Snp
2221245274Snp	if (INTR_OK)
2222245274Snp		pri = PCATCH;
2223245274Snp	else
2224245274Snp		pri = 0;
2225245274Snp
2226245274Snp	ADAPTER_LOCK(sc);
2227245274Snp	for (;;) {
2228245274Snp
2229245274Snp		if (pi && IS_DOOMED(pi)) {
2230245274Snp			rc = ENXIO;
2231245274Snp			goto done;
2232245274Snp		}
2233245274Snp
2234245274Snp		if (!IS_BUSY(sc)) {
2235245274Snp			rc = 0;
2236245274Snp			break;
2237245274Snp		}
2238245274Snp
2239245274Snp		if (!(flags & SLEEP_OK)) {
2240245274Snp			rc = EBUSY;
2241245274Snp			goto done;
2242245274Snp		}
2243245274Snp
2244245274Snp		if (mtx_sleep(&sc->flags, &sc->sc_lock, pri, wmesg, 0)) {
2245218792Snp			rc = EINTR;
2246218792Snp			goto done;
2247218792Snp		}
2248218792Snp	}
2249245274Snp
2250218792Snp	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2251218792Snp	SET_BUSY(sc);
2252245274Snp#ifdef INVARIANTS
2253245274Snp	sc->last_op = wmesg;
2254245274Snp	sc->last_op_thr = curthread;
2255245274Snp#endif
2256218792Snp
2257245274Snpdone:
2258245274Snp	if (!(flags & HOLD_LOCK) || rc)
2259245274Snp		ADAPTER_UNLOCK(sc);
2260218792Snp
2261245274Snp	return (rc);
2262245274Snp}
2263245274Snp
2264245274Snpvoid
2265245274Snpend_synchronized_op(struct adapter *sc, int flags)
2266245274Snp{
2267245274Snp
2268245274Snp	if (flags & LOCK_HELD)
2269245274Snp		ADAPTER_LOCK_ASSERT_OWNED(sc);
2270245274Snp	else
2271245274Snp		ADAPTER_LOCK(sc);
2272245274Snp
2273218792Snp	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2274218792Snp	CLR_BUSY(sc);
2275245274Snp	wakeup(&sc->flags);
2276218792Snp	ADAPTER_UNLOCK(sc);
2277218792Snp}
2278218792Snp
2279218792Snpstatic int
2280218792Snpcxgbe_init_synchronized(struct port_info *pi)
2281218792Snp{
2282218792Snp	struct adapter *sc = pi->adapter;
2283218792Snp	struct ifnet *ifp = pi->ifp;
2284228561Snp	int rc = 0;
2285218792Snp
2286245274Snp	ASSERT_SYNCHRONIZED_OP(sc);
2287218792Snp
2288218792Snp	if (isset(&sc->open_device_map, pi->port_id)) {
2289218792Snp		KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2290218792Snp		    ("mismatch between open_device_map and if_drv_flags"));
2291218792Snp		return (0);	/* already running */
2292218792Snp	}
2293218792Snp
2294228561Snp	if (!(sc->flags & FULL_INIT_DONE) &&
2295228561Snp	    ((rc = adapter_full_init(sc)) != 0))
2296218792Snp		return (rc);	/* error message displayed already */
2297218792Snp
2298228561Snp	if (!(pi->flags & PORT_INIT_DONE) &&
2299228561Snp	    ((rc = port_full_init(pi)) != 0))
2300228561Snp		return (rc); /* error message displayed already */
2301218792Snp
2302218792Snp	rc = update_mac_settings(pi, XGMAC_ALL);
2303218792Snp	if (rc)
2304218792Snp		goto done;	/* error message displayed already */
2305218792Snp
2306218792Snp	rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
2307218792Snp	if (rc != 0) {
2308218792Snp		if_printf(ifp, "start_link failed: %d\n", rc);
2309218792Snp		goto done;
2310218792Snp	}
2311218792Snp
2312218792Snp	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
2313218792Snp	if (rc != 0) {
2314218792Snp		if_printf(ifp, "enable_vi failed: %d\n", rc);
2315218792Snp		goto done;
2316218792Snp	}
2317218792Snp
2318218792Snp	/* all ok */
2319218792Snp	setbit(&sc->open_device_map, pi->port_id);
2320245274Snp	PORT_LOCK(pi);
2321218792Snp	ifp->if_drv_flags |= IFF_DRV_RUNNING;
2322245274Snp	PORT_UNLOCK(pi);
2323218792Snp
2324218792Snp	callout_reset(&pi->tick, hz, cxgbe_tick, pi);
2325218792Snpdone:
2326218792Snp	if (rc != 0)
2327218792Snp		cxgbe_uninit_synchronized(pi);
2328218792Snp
2329218792Snp	return (rc);
2330218792Snp}
2331218792Snp
2332218792Snp/*
2333218792Snp * Idempotent.
2334218792Snp */
2335218792Snpstatic int
2336218792Snpcxgbe_uninit_synchronized(struct port_info *pi)
2337218792Snp{
2338218792Snp	struct adapter *sc = pi->adapter;
2339218792Snp	struct ifnet *ifp = pi->ifp;
2340218792Snp	int rc;
2341218792Snp
2342245274Snp	ASSERT_SYNCHRONIZED_OP(sc);
2343218792Snp
2344218792Snp	/*
2345228561Snp	 * Disable the VI so that all its data in either direction is discarded
2346228561Snp	 * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
2347228561Snp	 * tick) intact as the TP can deliver negative advice or data that it's
2348228561Snp	 * holding in its RAM (for an offloaded connection) even after the VI is
2349228561Snp	 * disabled.
2350218792Snp	 */
2351228561Snp	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
2352228561Snp	if (rc) {
2353228561Snp		if_printf(ifp, "disable_vi failed: %d\n", rc);
2354228561Snp		return (rc);
2355228561Snp	}
2356228561Snp
2357218792Snp	clrbit(&sc->open_device_map, pi->port_id);
2358245274Snp	PORT_LOCK(pi);
2359228561Snp	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2360245274Snp	PORT_UNLOCK(pi);
2361218792Snp
2362218792Snp	pi->link_cfg.link_ok = 0;
2363218792Snp	pi->link_cfg.speed = 0;
2364218792Snp	t4_os_link_changed(sc, pi->port_id, 0);
2365218792Snp
2366218792Snp	return (0);
2367218792Snp}
2368218792Snp
2369240453Snp/*
2370240453Snp * It is ok for this function to fail midway and return right away.  t4_detach
2371240453Snp * will walk the entire sc->irq list and clean up whatever is valid.
2372240453Snp */
2373218792Snpstatic int
2374240453Snpsetup_intr_handlers(struct adapter *sc)
2375218792Snp{
2376240453Snp	int rc, rid, p, q;
2377222510Snp	char s[8];
2378222510Snp	struct irq *irq;
2379228561Snp	struct port_info *pi;
2380228561Snp	struct sge_rxq *rxq;
2381237263Snp#ifdef TCP_OFFLOAD
2382228561Snp	struct sge_ofld_rxq *ofld_rxq;
2383228561Snp#endif
2384218792Snp
2385218792Snp	/*
2386218792Snp	 * Setup interrupts.
2387218792Snp	 */
2388222510Snp	irq = &sc->irq[0];
2389222510Snp	rid = sc->intr_type == INTR_INTX ? 0 : 1;
2390218792Snp	if (sc->intr_count == 1) {
2391228561Snp		KASSERT(!(sc->flags & INTR_DIRECT),
2392228561Snp		    ("%s: single interrupt && INTR_DIRECT?", __func__));
2393222510Snp
2394240453Snp		rc = t4_alloc_irq(sc, irq, rid, t4_intr_all, sc, "all");
2395240453Snp		if (rc != 0)
2396240453Snp			return (rc);
2397218792Snp	} else {
2398228561Snp		/* Multiple interrupts. */
2399228561Snp		KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
2400228561Snp		    ("%s: too few intr.", __func__));
2401228561Snp
2402228561Snp		/* The first one is always error intr */
2403240453Snp		rc = t4_alloc_irq(sc, irq, rid, t4_intr_err, sc, "err");
2404240453Snp		if (rc != 0)
2405240453Snp			return (rc);
2406222510Snp		irq++;
2407222510Snp		rid++;
2408218792Snp
2409228561Snp		/* The second one is always the firmware event queue */
2410240453Snp		rc = t4_alloc_irq(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
2411240453Snp		    "evt");
2412240453Snp		if (rc != 0)
2413240453Snp			return (rc);
2414228561Snp		irq++;
2415228561Snp		rid++;
2416222510Snp
2417228561Snp		/*
2418228561Snp		 * Note that if INTR_DIRECT is not set then either the NIC rx
2419228561Snp		 * queues or (exclusive or) the TOE rx queueus will be taking
2420228561Snp		 * direct interrupts.
2421228561Snp		 *
2422228561Snp		 * There is no need to check for is_offload(sc) as nofldrxq
2423228561Snp		 * will be 0 if offload is disabled.
2424228561Snp		 */
2425228561Snp		for_each_port(sc, p) {
2426228561Snp			pi = sc->port[p];
2427222510Snp
2428237263Snp#ifdef TCP_OFFLOAD
2429228561Snp			/*
2430228561Snp			 * Skip over the NIC queues if they aren't taking direct
2431228561Snp			 * interrupts.
2432228561Snp			 */
2433228561Snp			if (!(sc->flags & INTR_DIRECT) &&
2434228561Snp			    pi->nofldrxq > pi->nrxq)
2435228561Snp				goto ofld_queues;
2436228561Snp#endif
2437228561Snp			rxq = &sc->sge.rxq[pi->first_rxq];
2438228561Snp			for (q = 0; q < pi->nrxq; q++, rxq++) {
2439228561Snp				snprintf(s, sizeof(s), "%d.%d", p, q);
2440240453Snp				rc = t4_alloc_irq(sc, irq, rid, t4_intr, rxq,
2441240453Snp				    s);
2442240453Snp				if (rc != 0)
2443240453Snp					return (rc);
2444222510Snp				irq++;
2445222510Snp				rid++;
2446218792Snp			}
2447218792Snp
2448237263Snp#ifdef TCP_OFFLOAD
2449228561Snp			/*
2450228561Snp			 * Skip over the offload queues if they aren't taking
2451228561Snp			 * direct interrupts.
2452228561Snp			 */
2453228561Snp			if (!(sc->flags & INTR_DIRECT))
2454228561Snp				continue;
2455228561Snpofld_queues:
2456228561Snp			ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
2457228561Snp			for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
2458228561Snp				snprintf(s, sizeof(s), "%d,%d", p, q);
2459240453Snp				rc = t4_alloc_irq(sc, irq, rid, t4_intr,
2460240453Snp				    ofld_rxq, s);
2461240453Snp				if (rc != 0)
2462240453Snp					return (rc);
2463228561Snp				irq++;
2464228561Snp				rid++;
2465218792Snp			}
2466228561Snp#endif
2467218792Snp		}
2468218792Snp	}
2469218792Snp
2470240453Snp	return (0);
2471240453Snp}
2472240453Snp
2473240453Snpstatic int
2474240453Snpadapter_full_init(struct adapter *sc)
2475240453Snp{
2476240453Snp	int rc, i;
2477240453Snp
2478240453Snp	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2479240453Snp	KASSERT((sc->flags & FULL_INIT_DONE) == 0,
2480240453Snp	    ("%s: FULL_INIT_DONE already", __func__));
2481240453Snp
2482240453Snp	/*
2483240453Snp	 * queues that belong to the adapter (not any particular port).
2484240453Snp	 */
2485240453Snp	rc = t4_setup_adapter_queues(sc);
2486240453Snp	if (rc != 0)
2487240453Snp		goto done;
2488240453Snp
2489240453Snp	for (i = 0; i < nitems(sc->tq); i++) {
2490240453Snp		sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
2491240453Snp		    taskqueue_thread_enqueue, &sc->tq[i]);
2492240453Snp		if (sc->tq[i] == NULL) {
2493240453Snp			device_printf(sc->dev,
2494240453Snp			    "failed to allocate task queue %d\n", i);
2495240453Snp			rc = ENOMEM;
2496240453Snp			goto done;
2497240453Snp		}
2498240453Snp		taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
2499240453Snp		    device_get_nameunit(sc->dev), i);
2500240453Snp	}
2501240453Snp
2502218792Snp	t4_intr_enable(sc);
2503218792Snp	sc->flags |= FULL_INIT_DONE;
2504218792Snpdone:
2505218792Snp	if (rc != 0)
2506228561Snp		adapter_full_uninit(sc);
2507218792Snp
2508218792Snp	return (rc);
2509218792Snp}
2510218792Snp
2511218792Snpstatic int
2512228561Snpadapter_full_uninit(struct adapter *sc)
2513218792Snp{
2514218792Snp	int i;
2515218792Snp
2516218792Snp	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2517218792Snp
2518220873Snp	t4_teardown_adapter_queues(sc);
2519218792Snp
2520240452Snp	for (i = 0; i < nitems(sc->tq) && sc->tq[i]; i++) {
2521228561Snp		taskqueue_free(sc->tq[i]);
2522228561Snp		sc->tq[i] = NULL;
2523228561Snp	}
2524228561Snp
2525218792Snp	sc->flags &= ~FULL_INIT_DONE;
2526218792Snp
2527218792Snp	return (0);
2528218792Snp}
2529218792Snp
2530218792Snpstatic int
2531228561Snpport_full_init(struct port_info *pi)
2532228561Snp{
2533228561Snp	struct adapter *sc = pi->adapter;
2534228561Snp	struct ifnet *ifp = pi->ifp;
2535228561Snp	uint16_t *rss;
2536228561Snp	struct sge_rxq *rxq;
2537228561Snp	int rc, i;
2538228561Snp
2539245274Snp	ASSERT_SYNCHRONIZED_OP(sc);
2540228561Snp	KASSERT((pi->flags & PORT_INIT_DONE) == 0,
2541228561Snp	    ("%s: PORT_INIT_DONE already", __func__));
2542228561Snp
2543228561Snp	sysctl_ctx_init(&pi->ctx);
2544228561Snp	pi->flags |= PORT_SYSCTL_CTX;
2545228561Snp
2546228561Snp	/*
2547228561Snp	 * Allocate tx/rx/fl queues for this port.
2548228561Snp	 */
2549228561Snp	rc = t4_setup_port_queues(pi);
2550228561Snp	if (rc != 0)
2551228561Snp		goto done;	/* error message displayed already */
2552228561Snp
2553228561Snp	/*
2554228561Snp	 * Setup RSS for this port.
2555228561Snp	 */
2556228561Snp	rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE,
2557228561Snp	    M_ZERO | M_WAITOK);
2558228561Snp	for_each_rxq(pi, i, rxq) {
2559228561Snp		rss[i] = rxq->iq.abs_id;
2560228561Snp	}
2561228561Snp	rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
2562228561Snp	    pi->rss_size, rss, pi->nrxq);
2563228561Snp	free(rss, M_CXGBE);
2564228561Snp	if (rc != 0) {
2565228561Snp		if_printf(ifp, "rss_config failed: %d\n", rc);
2566228561Snp		goto done;
2567228561Snp	}
2568228561Snp
2569228561Snp	pi->flags |= PORT_INIT_DONE;
2570228561Snpdone:
2571228561Snp	if (rc != 0)
2572228561Snp		port_full_uninit(pi);
2573228561Snp
2574228561Snp	return (rc);
2575228561Snp}
2576228561Snp
2577228561Snp/*
2578228561Snp * Idempotent.
2579228561Snp */
2580228561Snpstatic int
2581228561Snpport_full_uninit(struct port_info *pi)
2582228561Snp{
2583228561Snp	struct adapter *sc = pi->adapter;
2584228561Snp	int i;
2585228561Snp	struct sge_rxq *rxq;
2586228561Snp	struct sge_txq *txq;
2587237263Snp#ifdef TCP_OFFLOAD
2588228561Snp	struct sge_ofld_rxq *ofld_rxq;
2589228561Snp	struct sge_wrq *ofld_txq;
2590228561Snp#endif
2591228561Snp
2592228561Snp	if (pi->flags & PORT_INIT_DONE) {
2593228561Snp
2594228561Snp		/* Need to quiesce queues.  XXX: ctrl queues? */
2595228561Snp
2596228561Snp		for_each_txq(pi, i, txq) {
2597228561Snp			quiesce_eq(sc, &txq->eq);
2598228561Snp		}
2599228561Snp
2600237263Snp#ifdef TCP_OFFLOAD
2601228561Snp		for_each_ofld_txq(pi, i, ofld_txq) {
2602228561Snp			quiesce_eq(sc, &ofld_txq->eq);
2603228561Snp		}
2604228561Snp#endif
2605228561Snp
2606228561Snp		for_each_rxq(pi, i, rxq) {
2607228561Snp			quiesce_iq(sc, &rxq->iq);
2608228561Snp			quiesce_fl(sc, &rxq->fl);
2609228561Snp		}
2610228561Snp
2611237263Snp#ifdef TCP_OFFLOAD
2612228561Snp		for_each_ofld_rxq(pi, i, ofld_rxq) {
2613228561Snp			quiesce_iq(sc, &ofld_rxq->iq);
2614228561Snp			quiesce_fl(sc, &ofld_rxq->fl);
2615228561Snp		}
2616228561Snp#endif
2617228561Snp	}
2618228561Snp
2619228561Snp	t4_teardown_port_queues(pi);
2620228561Snp	pi->flags &= ~PORT_INIT_DONE;
2621228561Snp
2622228561Snp	return (0);
2623228561Snp}
2624228561Snp
2625228561Snpstatic void
2626228561Snpquiesce_eq(struct adapter *sc, struct sge_eq *eq)
2627228561Snp{
2628228561Snp	EQ_LOCK(eq);
2629228561Snp	eq->flags |= EQ_DOOMED;
2630228561Snp
2631228561Snp	/*
2632228561Snp	 * Wait for the response to a credit flush if one's
2633228561Snp	 * pending.
2634228561Snp	 */
2635228561Snp	while (eq->flags & EQ_CRFLUSHED)
2636228561Snp		mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
2637228561Snp	EQ_UNLOCK(eq);
2638228561Snp
2639228561Snp	callout_drain(&eq->tx_callout);	/* XXX: iffy */
2640228561Snp	pause("callout", 10);		/* Still iffy */
2641228561Snp
2642228561Snp	taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
2643228561Snp}
2644228561Snp
2645228561Snpstatic void
2646228561Snpquiesce_iq(struct adapter *sc, struct sge_iq *iq)
2647228561Snp{
2648228561Snp	(void) sc;	/* unused */
2649228561Snp
2650228561Snp	/* Synchronize with the interrupt handler */
2651228561Snp	while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
2652228561Snp		pause("iqfree", 1);
2653228561Snp}
2654228561Snp
2655228561Snpstatic void
2656228561Snpquiesce_fl(struct adapter *sc, struct sge_fl *fl)
2657228561Snp{
2658228561Snp	mtx_lock(&sc->sfl_lock);
2659228561Snp	FL_LOCK(fl);
2660228561Snp	fl->flags |= FL_DOOMED;
2661228561Snp	FL_UNLOCK(fl);
2662228561Snp	mtx_unlock(&sc->sfl_lock);
2663228561Snp
2664228561Snp	callout_drain(&sc->sfl_callout);
2665228561Snp	KASSERT((fl->flags & FL_STARVING) == 0,
2666228561Snp	    ("%s: still starving", __func__));
2667228561Snp}
2668228561Snp
2669228561Snpstatic int
2670218792Snpt4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
2671228561Snp    driver_intr_t *handler, void *arg, char *name)
2672218792Snp{
2673218792Snp	int rc;
2674218792Snp
2675218792Snp	irq->rid = rid;
2676218792Snp	irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
2677218792Snp	    RF_SHAREABLE | RF_ACTIVE);
2678218792Snp	if (irq->res == NULL) {
2679218792Snp		device_printf(sc->dev,
2680218792Snp		    "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
2681218792Snp		return (ENOMEM);
2682218792Snp	}
2683218792Snp
2684218792Snp	rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
2685218792Snp	    NULL, handler, arg, &irq->tag);
2686218792Snp	if (rc != 0) {
2687218792Snp		device_printf(sc->dev,
2688218792Snp		    "failed to setup interrupt for rid %d, name %s: %d\n",
2689218792Snp		    rid, name, rc);
2690218792Snp	} else if (name)
2691218792Snp		bus_describe_intr(sc->dev, irq->res, irq->tag, name);
2692218792Snp
2693218792Snp	return (rc);
2694218792Snp}
2695218792Snp
2696218792Snpstatic int
2697218792Snpt4_free_irq(struct adapter *sc, struct irq *irq)
2698218792Snp{
2699218792Snp	if (irq->tag)
2700218792Snp		bus_teardown_intr(sc->dev, irq->res, irq->tag);
2701218792Snp	if (irq->res)
2702218792Snp		bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
2703218792Snp
2704218792Snp	bzero(irq, sizeof(*irq));
2705218792Snp
2706218792Snp	return (0);
2707218792Snp}
2708218792Snp
2709218792Snpstatic void
2710218792Snpreg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
2711218792Snp    unsigned int end)
2712218792Snp{
2713218792Snp	uint32_t *p = (uint32_t *)(buf + start);
2714218792Snp
2715218792Snp	for ( ; start <= end; start += sizeof(uint32_t))
2716218792Snp		*p++ = t4_read_reg(sc, start);
2717218792Snp}
2718218792Snp
2719218792Snpstatic void
2720218792Snpt4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
2721218792Snp{
2722218792Snp	int i;
2723218792Snp	static const unsigned int reg_ranges[] = {
2724218792Snp		0x1008, 0x1108,
2725218792Snp		0x1180, 0x11b4,
2726218792Snp		0x11fc, 0x123c,
2727218792Snp		0x1300, 0x173c,
2728218792Snp		0x1800, 0x18fc,
2729218792Snp		0x3000, 0x30d8,
2730218792Snp		0x30e0, 0x5924,
2731218792Snp		0x5960, 0x59d4,
2732218792Snp		0x5a00, 0x5af8,
2733218792Snp		0x6000, 0x6098,
2734218792Snp		0x6100, 0x6150,
2735218792Snp		0x6200, 0x6208,
2736218792Snp		0x6240, 0x6248,
2737218792Snp		0x6280, 0x6338,
2738218792Snp		0x6370, 0x638c,
2739218792Snp		0x6400, 0x643c,
2740218792Snp		0x6500, 0x6524,
2741218792Snp		0x6a00, 0x6a38,
2742218792Snp		0x6a60, 0x6a78,
2743218792Snp		0x6b00, 0x6b84,
2744218792Snp		0x6bf0, 0x6c84,
2745218792Snp		0x6cf0, 0x6d84,
2746218792Snp		0x6df0, 0x6e84,
2747218792Snp		0x6ef0, 0x6f84,
2748218792Snp		0x6ff0, 0x7084,
2749218792Snp		0x70f0, 0x7184,
2750218792Snp		0x71f0, 0x7284,
2751218792Snp		0x72f0, 0x7384,
2752218792Snp		0x73f0, 0x7450,
2753218792Snp		0x7500, 0x7530,
2754218792Snp		0x7600, 0x761c,
2755218792Snp		0x7680, 0x76cc,
2756218792Snp		0x7700, 0x7798,
2757218792Snp		0x77c0, 0x77fc,
2758218792Snp		0x7900, 0x79fc,
2759218792Snp		0x7b00, 0x7c38,
2760218792Snp		0x7d00, 0x7efc,
2761218792Snp		0x8dc0, 0x8e1c,
2762218792Snp		0x8e30, 0x8e78,
2763218792Snp		0x8ea0, 0x8f6c,
2764218792Snp		0x8fc0, 0x9074,
2765218792Snp		0x90fc, 0x90fc,
2766218792Snp		0x9400, 0x9458,
2767218792Snp		0x9600, 0x96bc,
2768218792Snp		0x9800, 0x9808,
2769218792Snp		0x9820, 0x983c,
2770218792Snp		0x9850, 0x9864,
2771218792Snp		0x9c00, 0x9c6c,
2772218792Snp		0x9c80, 0x9cec,
2773218792Snp		0x9d00, 0x9d6c,
2774218792Snp		0x9d80, 0x9dec,
2775218792Snp		0x9e00, 0x9e6c,
2776218792Snp		0x9e80, 0x9eec,
2777218792Snp		0x9f00, 0x9f6c,
2778218792Snp		0x9f80, 0x9fec,
2779218792Snp		0xd004, 0xd03c,
2780218792Snp		0xdfc0, 0xdfe0,
2781218792Snp		0xe000, 0xea7c,
2782218792Snp		0xf000, 0x11190,
2783237439Snp		0x19040, 0x1906c,
2784237439Snp		0x19078, 0x19080,
2785237439Snp		0x1908c, 0x19124,
2786218792Snp		0x19150, 0x191b0,
2787218792Snp		0x191d0, 0x191e8,
2788218792Snp		0x19238, 0x1924c,
2789218792Snp		0x193f8, 0x19474,
2790218792Snp		0x19490, 0x194f8,
2791218792Snp		0x19800, 0x19f30,
2792218792Snp		0x1a000, 0x1a06c,
2793218792Snp		0x1a0b0, 0x1a120,
2794218792Snp		0x1a128, 0x1a138,
2795218792Snp		0x1a190, 0x1a1c4,
2796218792Snp		0x1a1fc, 0x1a1fc,
2797218792Snp		0x1e040, 0x1e04c,
2798237439Snp		0x1e284, 0x1e28c,
2799218792Snp		0x1e2c0, 0x1e2c0,
2800218792Snp		0x1e2e0, 0x1e2e0,
2801218792Snp		0x1e300, 0x1e384,
2802218792Snp		0x1e3c0, 0x1e3c8,
2803218792Snp		0x1e440, 0x1e44c,
2804237439Snp		0x1e684, 0x1e68c,
2805218792Snp		0x1e6c0, 0x1e6c0,
2806218792Snp		0x1e6e0, 0x1e6e0,
2807218792Snp		0x1e700, 0x1e784,
2808218792Snp		0x1e7c0, 0x1e7c8,
2809218792Snp		0x1e840, 0x1e84c,
2810237439Snp		0x1ea84, 0x1ea8c,
2811218792Snp		0x1eac0, 0x1eac0,
2812218792Snp		0x1eae0, 0x1eae0,
2813218792Snp		0x1eb00, 0x1eb84,
2814218792Snp		0x1ebc0, 0x1ebc8,
2815218792Snp		0x1ec40, 0x1ec4c,
2816237439Snp		0x1ee84, 0x1ee8c,
2817218792Snp		0x1eec0, 0x1eec0,
2818218792Snp		0x1eee0, 0x1eee0,
2819218792Snp		0x1ef00, 0x1ef84,
2820218792Snp		0x1efc0, 0x1efc8,
2821218792Snp		0x1f040, 0x1f04c,
2822237439Snp		0x1f284, 0x1f28c,
2823218792Snp		0x1f2c0, 0x1f2c0,
2824218792Snp		0x1f2e0, 0x1f2e0,
2825218792Snp		0x1f300, 0x1f384,
2826218792Snp		0x1f3c0, 0x1f3c8,
2827218792Snp		0x1f440, 0x1f44c,
2828237439Snp		0x1f684, 0x1f68c,
2829218792Snp		0x1f6c0, 0x1f6c0,
2830218792Snp		0x1f6e0, 0x1f6e0,
2831218792Snp		0x1f700, 0x1f784,
2832218792Snp		0x1f7c0, 0x1f7c8,
2833218792Snp		0x1f840, 0x1f84c,
2834237439Snp		0x1fa84, 0x1fa8c,
2835218792Snp		0x1fac0, 0x1fac0,
2836218792Snp		0x1fae0, 0x1fae0,
2837218792Snp		0x1fb00, 0x1fb84,
2838218792Snp		0x1fbc0, 0x1fbc8,
2839218792Snp		0x1fc40, 0x1fc4c,
2840237439Snp		0x1fe84, 0x1fe8c,
2841218792Snp		0x1fec0, 0x1fec0,
2842218792Snp		0x1fee0, 0x1fee0,
2843218792Snp		0x1ff00, 0x1ff84,
2844218792Snp		0x1ffc0, 0x1ffc8,
2845218792Snp		0x20000, 0x2002c,
2846218792Snp		0x20100, 0x2013c,
2847218792Snp		0x20190, 0x201c8,
2848218792Snp		0x20200, 0x20318,
2849218792Snp		0x20400, 0x20528,
2850218792Snp		0x20540, 0x20614,
2851218792Snp		0x21000, 0x21040,
2852218792Snp		0x2104c, 0x21060,
2853218792Snp		0x210c0, 0x210ec,
2854218792Snp		0x21200, 0x21268,
2855218792Snp		0x21270, 0x21284,
2856218792Snp		0x212fc, 0x21388,
2857218792Snp		0x21400, 0x21404,
2858218792Snp		0x21500, 0x21518,
2859218792Snp		0x2152c, 0x2153c,
2860218792Snp		0x21550, 0x21554,
2861218792Snp		0x21600, 0x21600,
2862218792Snp		0x21608, 0x21628,
2863218792Snp		0x21630, 0x2163c,
2864218792Snp		0x21700, 0x2171c,
2865218792Snp		0x21780, 0x2178c,
2866218792Snp		0x21800, 0x21c38,
2867218792Snp		0x21c80, 0x21d7c,
2868218792Snp		0x21e00, 0x21e04,
2869218792Snp		0x22000, 0x2202c,
2870218792Snp		0x22100, 0x2213c,
2871218792Snp		0x22190, 0x221c8,
2872218792Snp		0x22200, 0x22318,
2873218792Snp		0x22400, 0x22528,
2874218792Snp		0x22540, 0x22614,
2875218792Snp		0x23000, 0x23040,
2876218792Snp		0x2304c, 0x23060,
2877218792Snp		0x230c0, 0x230ec,
2878218792Snp		0x23200, 0x23268,
2879218792Snp		0x23270, 0x23284,
2880218792Snp		0x232fc, 0x23388,
2881218792Snp		0x23400, 0x23404,
2882218792Snp		0x23500, 0x23518,
2883218792Snp		0x2352c, 0x2353c,
2884218792Snp		0x23550, 0x23554,
2885218792Snp		0x23600, 0x23600,
2886218792Snp		0x23608, 0x23628,
2887218792Snp		0x23630, 0x2363c,
2888218792Snp		0x23700, 0x2371c,
2889218792Snp		0x23780, 0x2378c,
2890218792Snp		0x23800, 0x23c38,
2891218792Snp		0x23c80, 0x23d7c,
2892218792Snp		0x23e00, 0x23e04,
2893218792Snp		0x24000, 0x2402c,
2894218792Snp		0x24100, 0x2413c,
2895218792Snp		0x24190, 0x241c8,
2896218792Snp		0x24200, 0x24318,
2897218792Snp		0x24400, 0x24528,
2898218792Snp		0x24540, 0x24614,
2899218792Snp		0x25000, 0x25040,
2900218792Snp		0x2504c, 0x25060,
2901218792Snp		0x250c0, 0x250ec,
2902218792Snp		0x25200, 0x25268,
2903218792Snp		0x25270, 0x25284,
2904218792Snp		0x252fc, 0x25388,
2905218792Snp		0x25400, 0x25404,
2906218792Snp		0x25500, 0x25518,
2907218792Snp		0x2552c, 0x2553c,
2908218792Snp		0x25550, 0x25554,
2909218792Snp		0x25600, 0x25600,
2910218792Snp		0x25608, 0x25628,
2911218792Snp		0x25630, 0x2563c,
2912218792Snp		0x25700, 0x2571c,
2913218792Snp		0x25780, 0x2578c,
2914218792Snp		0x25800, 0x25c38,
2915218792Snp		0x25c80, 0x25d7c,
2916218792Snp		0x25e00, 0x25e04,
2917218792Snp		0x26000, 0x2602c,
2918218792Snp		0x26100, 0x2613c,
2919218792Snp		0x26190, 0x261c8,
2920218792Snp		0x26200, 0x26318,
2921218792Snp		0x26400, 0x26528,
2922218792Snp		0x26540, 0x26614,
2923218792Snp		0x27000, 0x27040,
2924218792Snp		0x2704c, 0x27060,
2925218792Snp		0x270c0, 0x270ec,
2926218792Snp		0x27200, 0x27268,
2927218792Snp		0x27270, 0x27284,
2928218792Snp		0x272fc, 0x27388,
2929218792Snp		0x27400, 0x27404,
2930218792Snp		0x27500, 0x27518,
2931218792Snp		0x2752c, 0x2753c,
2932218792Snp		0x27550, 0x27554,
2933218792Snp		0x27600, 0x27600,
2934218792Snp		0x27608, 0x27628,
2935218792Snp		0x27630, 0x2763c,
2936218792Snp		0x27700, 0x2771c,
2937218792Snp		0x27780, 0x2778c,
2938218792Snp		0x27800, 0x27c38,
2939218792Snp		0x27c80, 0x27d7c,
2940218792Snp		0x27e00, 0x27e04
2941218792Snp	};
2942218792Snp
2943218792Snp	regs->version = 4 | (sc->params.rev << 10);
2944240452Snp	for (i = 0; i < nitems(reg_ranges); i += 2)
2945218792Snp		reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
2946218792Snp}
2947218792Snp
2948218792Snpstatic void
2949218792Snpcxgbe_tick(void *arg)
2950218792Snp{
2951218792Snp	struct port_info *pi = arg;
2952218792Snp	struct ifnet *ifp = pi->ifp;
2953218792Snp	struct sge_txq *txq;
2954218792Snp	int i, drops;
2955218792Snp	struct port_stats *s = &pi->stats;
2956218792Snp
2957218792Snp	PORT_LOCK(pi);
2958218792Snp	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2959218792Snp		PORT_UNLOCK(pi);
2960218792Snp		return;	/* without scheduling another callout */
2961218792Snp	}
2962218792Snp
2963218792Snp	t4_get_port_stats(pi->adapter, pi->tx_chan, s);
2964218792Snp
2965228561Snp	ifp->if_opackets = s->tx_frames - s->tx_pause;
2966228561Snp	ifp->if_ipackets = s->rx_frames - s->rx_pause;
2967228561Snp	ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
2968228561Snp	ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
2969228561Snp	ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
2970228561Snp	ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
2971218792Snp	ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
2972239259Snp	    s->rx_ovflow3 + s->rx_trunc0 + s->rx_trunc1 + s->rx_trunc2 +
2973239259Snp	    s->rx_trunc3;
2974218792Snp
2975218792Snp	drops = s->tx_drop;
2976218792Snp	for_each_txq(pi, i, txq)
2977220873Snp		drops += txq->br->br_drops;
2978218792Snp	ifp->if_snd.ifq_drops = drops;
2979218792Snp
2980218792Snp	ifp->if_oerrors = s->tx_error_frames;
2981218792Snp	ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
2982218792Snp	    s->rx_fcs_err + s->rx_len_err;
2983218792Snp
2984218792Snp	callout_schedule(&pi->tick, hz);
2985218792Snp	PORT_UNLOCK(pi);
2986218792Snp}
2987218792Snp
2988237263Snpstatic void
2989237263Snpcxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
2990237263Snp{
2991237263Snp	struct ifnet *vlan;
2992237263Snp
2993241494Snp	if (arg != ifp || ifp->if_type != IFT_ETHER)
2994237263Snp		return;
2995237263Snp
2996237263Snp	vlan = VLAN_DEVAT(ifp, vid);
2997237263Snp	VLAN_SETCOOKIE(vlan, ifp);
2998237263Snp}
2999237263Snp
3000218792Snpstatic int
3001228561Snpcpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
3002228561Snp{
3003237263Snp
3004228561Snp#ifdef INVARIANTS
3005237263Snp	panic("%s: opcode 0x%02x on iq %p with payload %p",
3006228561Snp	    __func__, rss->opcode, iq, m);
3007228561Snp#else
3008239336Snp	log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p\n",
3009228561Snp	    __func__, rss->opcode, iq, m);
3010228561Snp	m_freem(m);
3011228561Snp#endif
3012228561Snp	return (EDOOFUS);
3013228561Snp}
3014228561Snp
3015228561Snpint
3016228561Snpt4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3017228561Snp{
3018228561Snp	uintptr_t *loc, new;
3019228561Snp
3020240452Snp	if (opcode >= nitems(sc->cpl_handler))
3021228561Snp		return (EINVAL);
3022228561Snp
3023228561Snp	new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3024228561Snp	loc = (uintptr_t *) &sc->cpl_handler[opcode];
3025228561Snp	atomic_store_rel_ptr(loc, new);
3026228561Snp
3027228561Snp	return (0);
3028228561Snp}
3029228561Snp
3030228561Snpstatic int
3031237263Snpan_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
3032237263Snp{
3033237263Snp
3034237263Snp#ifdef INVARIANTS
3035237263Snp	panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
3036237263Snp#else
3037239336Snp	log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)\n",
3038237263Snp	    __func__, iq, ctrl);
3039237263Snp#endif
3040237263Snp	return (EDOOFUS);
3041237263Snp}
3042237263Snp
3043237263Snpint
3044237263Snpt4_register_an_handler(struct adapter *sc, an_handler_t h)
3045237263Snp{
3046237263Snp	uintptr_t *loc, new;
3047237263Snp
3048237263Snp	new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
3049237263Snp	loc = (uintptr_t *) &sc->an_handler;
3050237263Snp	atomic_store_rel_ptr(loc, new);
3051237263Snp
3052237263Snp	return (0);
3053237263Snp}
3054237263Snp
3055237263Snpstatic int
3056239336Snpfw_msg_not_handled(struct adapter *sc, const __be64 *rpl)
3057239336Snp{
3058241733Sed	const struct cpl_fw6_msg *cpl =
3059241733Sed	    __containerof(rpl, struct cpl_fw6_msg, data[0]);
3060239336Snp
3061239336Snp#ifdef INVARIANTS
3062239336Snp	panic("%s: fw_msg type %d", __func__, cpl->type);
3063239336Snp#else
3064239336Snp	log(LOG_ERR, "%s: fw_msg type %d\n", __func__, cpl->type);
3065239336Snp#endif
3066239336Snp	return (EDOOFUS);
3067239336Snp}
3068239336Snp
3069239336Snpint
3070239336Snpt4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h)
3071239336Snp{
3072239336Snp	uintptr_t *loc, new;
3073239336Snp
3074240452Snp	if (type >= nitems(sc->fw_msg_handler))
3075239336Snp		return (EINVAL);
3076239336Snp
3077239336Snp	new = h ? (uintptr_t)h : (uintptr_t)fw_msg_not_handled;
3078239336Snp	loc = (uintptr_t *) &sc->fw_msg_handler[type];
3079239336Snp	atomic_store_rel_ptr(loc, new);
3080239336Snp
3081239336Snp	return (0);
3082239336Snp}
3083239336Snp
3084239336Snpstatic int
3085218792Snpt4_sysctls(struct adapter *sc)
3086218792Snp{
3087218792Snp	struct sysctl_ctx_list *ctx;
3088218792Snp	struct sysctl_oid *oid;
3089228561Snp	struct sysctl_oid_list *children, *c0;
3090228561Snp	static char *caps[] = {
3091228561Snp		"\20\1PPP\2QFC\3DCBX",			/* caps[0] linkcaps */
3092228561Snp		"\20\1NIC\2VM\3IDS\4UM\5UM_ISGL",	/* caps[1] niccaps */
3093228561Snp		"\20\1TOE",				/* caps[2] toecaps */
3094228561Snp		"\20\1RDDP\2RDMAC",			/* caps[3] rdmacaps */
3095228561Snp		"\20\1INITIATOR_PDU\2TARGET_PDU"	/* caps[4] iscsicaps */
3096228561Snp		    "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
3097228561Snp		    "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
3098228561Snp		"\20\1INITIATOR\2TARGET\3CTRL_OFLD"	/* caps[5] fcoecaps */
3099228561Snp	};
3100218792Snp
3101218792Snp	ctx = device_get_sysctl_ctx(sc->dev);
3102228561Snp
3103228561Snp	/*
3104228561Snp	 * dev.t4nex.X.
3105228561Snp	 */
3106218792Snp	oid = device_get_sysctl_tree(sc->dev);
3107228561Snp	c0 = children = SYSCTL_CHILDREN(oid);
3108218792Snp
3109218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD,
3110218792Snp	    &sc->params.nports, 0, "# of ports");
3111218792Snp
3112218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
3113218792Snp	    &sc->params.rev, 0, "chip hardware revision");
3114218792Snp
3115218792Snp	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
3116218792Snp	    CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
3117218792Snp
3118228561Snp	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
3119245936Snp	    CTLFLAG_RD, &sc->cfg_file, 0, "configuration file");
3120218792Snp
3121228561Snp	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD,
3122228561Snp	    &sc->cfcsum, 0, "config file checksum");
3123228561Snp
3124228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
3125228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
3126228561Snp	    sysctl_bitfield, "A", "available link capabilities");
3127228561Snp
3128228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
3129228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
3130228561Snp	    sysctl_bitfield, "A", "available NIC capabilities");
3131228561Snp
3132228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
3133228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
3134228561Snp	    sysctl_bitfield, "A", "available TCP offload capabilities");
3135228561Snp
3136228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
3137228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
3138228561Snp	    sysctl_bitfield, "A", "available RDMA capabilities");
3139228561Snp
3140228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
3141228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
3142228561Snp	    sysctl_bitfield, "A", "available iSCSI capabilities");
3143228561Snp
3144228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
3145228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
3146228561Snp	    sysctl_bitfield, "A", "available FCoE capabilities");
3147228561Snp
3148218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD,
3149218792Snp	    &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)");
3150218792Snp
3151219436Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
3152228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
3153228561Snp	    sizeof(sc->sge.timer_val), sysctl_int_array, "A",
3154228561Snp	    "interrupt holdoff timer values (us)");
3155218792Snp
3156219436Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
3157228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
3158228561Snp	    sizeof(sc->sge.counter_val), sysctl_int_array, "A",
3159228561Snp	    "interrupt holdoff packet counter values");
3160218792Snp
3161231115Snp#ifdef SBUF_DRAIN
3162228561Snp	/*
3163228561Snp	 * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
3164228561Snp	 */
3165228561Snp	oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
3166228561Snp	    CTLFLAG_RD | CTLFLAG_SKIP, NULL,
3167228561Snp	    "logs and miscellaneous information");
3168228561Snp	children = SYSCTL_CHILDREN(oid);
3169228561Snp
3170228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
3171228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3172228561Snp	    sysctl_cctrl, "A", "congestion control");
3173228561Snp
3174228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
3175228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3176228561Snp	    sysctl_cpl_stats, "A", "CPL statistics");
3177228561Snp
3178228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
3179228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3180228561Snp	    sysctl_ddp_stats, "A", "DDP statistics");
3181228561Snp
3182222551Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
3183222551Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3184228561Snp	    sysctl_devlog, "A", "firmware's device log");
3185222551Snp
3186228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
3187228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3188228561Snp	    sysctl_fcoe_stats, "A", "FCoE statistics");
3189228561Snp
3190228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
3191228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3192228561Snp	    sysctl_hw_sched, "A", "hardware scheduler ");
3193228561Snp
3194228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
3195228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3196228561Snp	    sysctl_l2t, "A", "hardware L2 table");
3197228561Snp
3198228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
3199228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3200228561Snp	    sysctl_lb_stats, "A", "loopback statistics");
3201228561Snp
3202228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
3203228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3204228561Snp	    sysctl_meminfo, "A", "memory regions");
3205228561Snp
3206228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
3207228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3208228561Snp	    sysctl_path_mtus, "A", "path MTUs");
3209228561Snp
3210228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
3211228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3212228561Snp	    sysctl_pm_stats, "A", "PM statistics");
3213228561Snp
3214228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
3215228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3216228561Snp	    sysctl_rdma_stats, "A", "RDMA statistics");
3217228561Snp
3218228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
3219228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3220228561Snp	    sysctl_tcp_stats, "A", "TCP statistics");
3221228561Snp
3222228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
3223228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3224228561Snp	    sysctl_tids, "A", "TID information");
3225228561Snp
3226228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
3227228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3228228561Snp	    sysctl_tp_err_stats, "A", "TP error statistics");
3229228561Snp
3230228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
3231228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3232228561Snp	    sysctl_tx_rate, "A", "Tx rate");
3233231115Snp#endif
3234228561Snp
3235237263Snp#ifdef TCP_OFFLOAD
3236228561Snp	if (is_offload(sc)) {
3237228561Snp		/*
3238228561Snp		 * dev.t4nex.X.toe.
3239228561Snp		 */
3240228561Snp		oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
3241228561Snp		    NULL, "TOE parameters");
3242228561Snp		children = SYSCTL_CHILDREN(oid);
3243228561Snp
3244228561Snp		sc->tt.sndbuf = 256 * 1024;
3245228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
3246228561Snp		    &sc->tt.sndbuf, 0, "max hardware send buffer size");
3247228561Snp
3248228561Snp		sc->tt.ddp = 0;
3249228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
3250228561Snp		    &sc->tt.ddp, 0, "DDP allowed");
3251239341Snp
3252239341Snp		sc->tt.indsz = G_INDICATESIZE(t4_read_reg(sc, A_TP_PARA_REG5));
3253228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
3254228561Snp		    &sc->tt.indsz, 0, "DDP max indicate size allowed");
3255239341Snp
3256239341Snp		sc->tt.ddp_thres =
3257239341Snp		    G_RXCOALESCESIZE(t4_read_reg(sc, A_TP_PARA_REG2));
3258228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
3259228561Snp		    &sc->tt.ddp_thres, 0, "DDP threshold");
3260228561Snp	}
3261228561Snp#endif
3262228561Snp
3263228561Snp
3264218792Snp	return (0);
3265218792Snp}
3266218792Snp
3267218792Snpstatic int
3268218792Snpcxgbe_sysctls(struct port_info *pi)
3269218792Snp{
3270218792Snp	struct sysctl_ctx_list *ctx;
3271218792Snp	struct sysctl_oid *oid;
3272218792Snp	struct sysctl_oid_list *children;
3273218792Snp
3274218792Snp	ctx = device_get_sysctl_ctx(pi->dev);
3275218792Snp
3276218792Snp	/*
3277218792Snp	 * dev.cxgbe.X.
3278218792Snp	 */
3279218792Snp	oid = device_get_sysctl_tree(pi->dev);
3280218792Snp	children = SYSCTL_CHILDREN(oid);
3281218792Snp
3282218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
3283218792Snp	    &pi->nrxq, 0, "# of rx queues");
3284218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
3285218792Snp	    &pi->ntxq, 0, "# of tx queues");
3286218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
3287218792Snp	    &pi->first_rxq, 0, "index of first rx queue");
3288218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
3289218792Snp	    &pi->first_txq, 0, "index of first tx queue");
3290218792Snp
3291237263Snp#ifdef TCP_OFFLOAD
3292228561Snp	if (is_offload(pi->adapter)) {
3293228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
3294228561Snp		    &pi->nofldrxq, 0,
3295228561Snp		    "# of rx queues for offloaded TCP connections");
3296228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
3297228561Snp		    &pi->nofldtxq, 0,
3298228561Snp		    "# of tx queues for offloaded TCP connections");
3299228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
3300228561Snp		    CTLFLAG_RD, &pi->first_ofld_rxq, 0,
3301228561Snp		    "index of first TOE rx queue");
3302228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
3303228561Snp		    CTLFLAG_RD, &pi->first_ofld_txq, 0,
3304228561Snp		    "index of first TOE tx queue");
3305228561Snp	}
3306228561Snp#endif
3307228561Snp
3308218792Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
3309218792Snp	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
3310218792Snp	    "holdoff timer index");
3311218792Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
3312218792Snp	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
3313218792Snp	    "holdoff packet counter index");
3314218792Snp
3315218792Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
3316218792Snp	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
3317218792Snp	    "rx queue size");
3318218792Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
3319218792Snp	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
3320218792Snp	    "tx queue size");
3321218792Snp
3322218792Snp	/*
3323218792Snp	 * dev.cxgbe.X.stats.
3324218792Snp	 */
3325218792Snp	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
3326218792Snp	    NULL, "port statistics");
3327218792Snp	children = SYSCTL_CHILDREN(oid);
3328218792Snp
3329218792Snp#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
3330218792Snp	SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
3331218792Snp	    CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \
3332218792Snp	    sysctl_handle_t4_reg64, "QU", desc)
3333218792Snp
3334218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
3335218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
3336218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
3337218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
3338218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
3339218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
3340218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
3341218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
3342218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
3343218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
3344218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
3345218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
3346218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
3347218792Snp	    "# of tx frames in this range",
3348218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
3349218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
3350218792Snp	    "# of tx frames in this range",
3351218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
3352218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
3353218792Snp	    "# of tx frames in this range",
3354218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
3355218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
3356218792Snp	    "# of tx frames in this range",
3357218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
3358218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
3359218792Snp	    "# of tx frames in this range",
3360218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
3361218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
3362218792Snp	    "# of tx frames in this range",
3363218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
3364218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
3365218792Snp	    "# of tx frames in this range",
3366218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
3367218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
3368218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
3369218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
3370218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
3371218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
3372218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
3373218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
3374218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
3375218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
3376218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
3377218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
3378218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
3379218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
3380218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
3381218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
3382218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
3383218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
3384218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
3385218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
3386218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
3387218792Snp
3388218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
3389218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
3390218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
3391218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
3392218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
3393218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
3394218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
3395218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
3396218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
3397218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
3398218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
3399218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
3400218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
3401218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
3402218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
3403218792Snp	    "# of frames received with bad FCS",
3404218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
3405218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
3406218792Snp	    "# of frames received with length error",
3407218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
3408218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
3409218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
3410218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
3411218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
3412218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
3413218792Snp	    "# of rx frames in this range",
3414218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
3415218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
3416218792Snp	    "# of rx frames in this range",
3417218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
3418218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
3419218792Snp	    "# of rx frames in this range",
3420218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
3421218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
3422218792Snp	    "# of rx frames in this range",
3423218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
3424218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
3425218792Snp	    "# of rx frames in this range",
3426218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
3427218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
3428218792Snp	    "# of rx frames in this range",
3429218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
3430218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
3431218792Snp	    "# of rx frames in this range",
3432218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
3433218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
3434218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
3435218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
3436218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
3437218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
3438218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
3439218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
3440218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
3441218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
3442218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
3443218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
3444218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
3445218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
3446218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
3447218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
3448218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
3449218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
3450218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
3451218792Snp
3452218792Snp#undef SYSCTL_ADD_T4_REG64
3453218792Snp
3454218792Snp#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
3455218792Snp	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
3456218792Snp	    &pi->stats.name, desc)
3457218792Snp
3458218792Snp	/* We get these from port_stats and they may be stale by upto 1s */
3459218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
3460218792Snp	    "# drops due to buffer-group 0 overflows");
3461218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
3462218792Snp	    "# drops due to buffer-group 1 overflows");
3463218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
3464218792Snp	    "# drops due to buffer-group 2 overflows");
3465218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
3466218792Snp	    "# drops due to buffer-group 3 overflows");
3467218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
3468218792Snp	    "# of buffer-group 0 truncated packets");
3469218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
3470218792Snp	    "# of buffer-group 1 truncated packets");
3471218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
3472218792Snp	    "# of buffer-group 2 truncated packets");
3473218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
3474218792Snp	    "# of buffer-group 3 truncated packets");
3475218792Snp
3476218792Snp#undef SYSCTL_ADD_T4_PORTSTAT
3477218792Snp
3478218792Snp	return (0);
3479218792Snp}
3480218792Snp
3481218792Snpstatic int
3482219436Snpsysctl_int_array(SYSCTL_HANDLER_ARGS)
3483219436Snp{
3484219436Snp	int rc, *i;
3485219436Snp	struct sbuf sb;
3486219436Snp
3487219436Snp	sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
3488219436Snp	for (i = arg1; arg2; arg2 -= sizeof(int), i++)
3489219436Snp		sbuf_printf(&sb, "%d ", *i);
3490219436Snp	sbuf_trim(&sb);
3491219436Snp	sbuf_finish(&sb);
3492219436Snp	rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
3493219436Snp	sbuf_delete(&sb);
3494219436Snp	return (rc);
3495219436Snp}
3496219436Snp
3497219436Snpstatic int
3498228561Snpsysctl_bitfield(SYSCTL_HANDLER_ARGS)
3499228561Snp{
3500228561Snp	int rc;
3501228561Snp	struct sbuf *sb;
3502228561Snp
3503228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3504228561Snp	if (rc != 0)
3505228561Snp		return(rc);
3506228561Snp
3507228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
3508228561Snp	if (sb == NULL)
3509228561Snp		return (ENOMEM);
3510228561Snp
3511228561Snp	sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
3512228561Snp	rc = sbuf_finish(sb);
3513228561Snp	sbuf_delete(sb);
3514228561Snp
3515228561Snp	return (rc);
3516228561Snp}
3517228561Snp
3518228561Snpstatic int
3519218792Snpsysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
3520218792Snp{
3521218792Snp	struct port_info *pi = arg1;
3522218792Snp	struct adapter *sc = pi->adapter;
3523218792Snp	int idx, rc, i;
3524245274Snp	struct sge_rxq *rxq;
3525245274Snp	uint8_t v;
3526218792Snp
3527218792Snp	idx = pi->tmr_idx;
3528218792Snp
3529218792Snp	rc = sysctl_handle_int(oidp, &idx, 0, req);
3530218792Snp	if (rc != 0 || req->newptr == NULL)
3531218792Snp		return (rc);
3532218792Snp
3533218792Snp	if (idx < 0 || idx >= SGE_NTIMERS)
3534218792Snp		return (EINVAL);
3535218792Snp
3536245274Snp	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3537245274Snp	    "t4tmr");
3538245274Snp	if (rc)
3539245274Snp		return (rc);
3540228561Snp
3541245274Snp	v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
3542245274Snp	for_each_rxq(pi, i, rxq) {
3543228561Snp#ifdef atomic_store_rel_8
3544245274Snp		atomic_store_rel_8(&rxq->iq.intr_params, v);
3545228561Snp#else
3546245274Snp		rxq->iq.intr_params = v;
3547228561Snp#endif
3548218792Snp	}
3549245274Snp	pi->tmr_idx = idx;
3550218792Snp
3551245274Snp	end_synchronized_op(sc, LOCK_HELD);
3552245274Snp	return (0);
3553218792Snp}
3554218792Snp
3555218792Snpstatic int
3556218792Snpsysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
3557218792Snp{
3558218792Snp	struct port_info *pi = arg1;
3559218792Snp	struct adapter *sc = pi->adapter;
3560218792Snp	int idx, rc;
3561218792Snp
3562218792Snp	idx = pi->pktc_idx;
3563218792Snp
3564218792Snp	rc = sysctl_handle_int(oidp, &idx, 0, req);
3565218792Snp	if (rc != 0 || req->newptr == NULL)
3566218792Snp		return (rc);
3567218792Snp
3568218792Snp	if (idx < -1 || idx >= SGE_NCOUNTERS)
3569218792Snp		return (EINVAL);
3570218792Snp
3571245274Snp	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3572245274Snp	    "t4pktc");
3573245274Snp	if (rc)
3574245274Snp		return (rc);
3575245274Snp
3576245274Snp	if (pi->flags & PORT_INIT_DONE)
3577228561Snp		rc = EBUSY; /* cannot be changed once the queues are created */
3578245274Snp	else
3579218792Snp		pi->pktc_idx = idx;
3580218792Snp
3581245274Snp	end_synchronized_op(sc, LOCK_HELD);
3582218792Snp	return (rc);
3583218792Snp}
3584218792Snp
3585218792Snpstatic int
3586218792Snpsysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
3587218792Snp{
3588218792Snp	struct port_info *pi = arg1;
3589218792Snp	struct adapter *sc = pi->adapter;
3590218792Snp	int qsize, rc;
3591218792Snp
3592218792Snp	qsize = pi->qsize_rxq;
3593218792Snp
3594218792Snp	rc = sysctl_handle_int(oidp, &qsize, 0, req);
3595218792Snp	if (rc != 0 || req->newptr == NULL)
3596218792Snp		return (rc);
3597218792Snp
3598218792Snp	if (qsize < 128 || (qsize & 7))
3599218792Snp		return (EINVAL);
3600218792Snp
3601245274Snp	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3602245274Snp	    "t4rxqs");
3603245274Snp	if (rc)
3604245274Snp		return (rc);
3605245274Snp
3606245274Snp	if (pi->flags & PORT_INIT_DONE)
3607228561Snp		rc = EBUSY; /* cannot be changed once the queues are created */
3608245274Snp	else
3609218792Snp		pi->qsize_rxq = qsize;
3610218792Snp
3611245274Snp	end_synchronized_op(sc, LOCK_HELD);
3612218792Snp	return (rc);
3613218792Snp}
3614218792Snp
3615218792Snpstatic int
3616218792Snpsysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
3617218792Snp{
3618218792Snp	struct port_info *pi = arg1;
3619218792Snp	struct adapter *sc = pi->adapter;
3620218792Snp	int qsize, rc;
3621218792Snp
3622218792Snp	qsize = pi->qsize_txq;
3623218792Snp
3624218792Snp	rc = sysctl_handle_int(oidp, &qsize, 0, req);
3625218792Snp	if (rc != 0 || req->newptr == NULL)
3626218792Snp		return (rc);
3627218792Snp
3628245274Snp	/* bufring size must be powerof2 */
3629245274Snp	if (qsize < 128 || !powerof2(qsize))
3630218792Snp		return (EINVAL);
3631218792Snp
3632245274Snp	rc = begin_synchronized_op(sc, pi, HOLD_LOCK | SLEEP_OK | INTR_OK,
3633245274Snp	    "t4txqs");
3634245274Snp	if (rc)
3635245274Snp		return (rc);
3636245274Snp
3637245274Snp	if (pi->flags & PORT_INIT_DONE)
3638228561Snp		rc = EBUSY; /* cannot be changed once the queues are created */
3639245274Snp	else
3640218792Snp		pi->qsize_txq = qsize;
3641218792Snp
3642245274Snp	end_synchronized_op(sc, LOCK_HELD);
3643218792Snp	return (rc);
3644218792Snp}
3645218792Snp
3646218792Snpstatic int
3647218792Snpsysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
3648218792Snp{
3649218792Snp	struct adapter *sc = arg1;
3650218792Snp	int reg = arg2;
3651218792Snp	uint64_t val;
3652218792Snp
3653218792Snp	val = t4_read_reg64(sc, reg);
3654218792Snp
3655218792Snp	return (sysctl_handle_64(oidp, &val, 0, req));
3656218792Snp}
3657218792Snp
3658231115Snp#ifdef SBUF_DRAIN
3659228561Snpstatic int
3660228561Snpsysctl_cctrl(SYSCTL_HANDLER_ARGS)
3661228561Snp{
3662228561Snp	struct adapter *sc = arg1;
3663228561Snp	struct sbuf *sb;
3664228561Snp	int rc, i;
3665228561Snp	uint16_t incr[NMTUS][NCCTRL_WIN];
3666228561Snp	static const char *dec_fac[] = {
3667228561Snp		"0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
3668228561Snp		"0.9375"
3669228561Snp	};
3670228561Snp
3671228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3672228561Snp	if (rc != 0)
3673228561Snp		return (rc);
3674228561Snp
3675228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3676228561Snp	if (sb == NULL)
3677228561Snp		return (ENOMEM);
3678228561Snp
3679228561Snp	t4_read_cong_tbl(sc, incr);
3680228561Snp
3681228561Snp	for (i = 0; i < NCCTRL_WIN; ++i) {
3682228561Snp		sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
3683228561Snp		    incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
3684228561Snp		    incr[5][i], incr[6][i], incr[7][i]);
3685228561Snp		sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
3686228561Snp		    incr[8][i], incr[9][i], incr[10][i], incr[11][i],
3687228561Snp		    incr[12][i], incr[13][i], incr[14][i], incr[15][i],
3688228561Snp		    sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
3689228561Snp	}
3690228561Snp
3691228561Snp	rc = sbuf_finish(sb);
3692228561Snp	sbuf_delete(sb);
3693228561Snp
3694228561Snp	return (rc);
3695228561Snp}
3696228561Snp
3697228561Snpstatic int
3698228561Snpsysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
3699228561Snp{
3700228561Snp	struct adapter *sc = arg1;
3701228561Snp	struct sbuf *sb;
3702228561Snp	int rc;
3703228561Snp	struct tp_cpl_stats stats;
3704228561Snp
3705228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3706228561Snp	if (rc != 0)
3707228561Snp		return (rc);
3708228561Snp
3709228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3710228561Snp	if (sb == NULL)
3711228561Snp		return (ENOMEM);
3712228561Snp
3713228561Snp	t4_tp_get_cpl_stats(sc, &stats);
3714228561Snp
3715228561Snp	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
3716228561Snp	    "channel 3\n");
3717228561Snp	sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
3718228561Snp		   stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
3719228561Snp	sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
3720228561Snp		   stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
3721228561Snp
3722228561Snp	rc = sbuf_finish(sb);
3723228561Snp	sbuf_delete(sb);
3724228561Snp
3725228561Snp	return (rc);
3726228561Snp}
3727228561Snp
3728228561Snpstatic int
3729228561Snpsysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
3730228561Snp{
3731228561Snp	struct adapter *sc = arg1;
3732228561Snp	struct sbuf *sb;
3733228561Snp	int rc;
3734228561Snp	struct tp_usm_stats stats;
3735228561Snp
3736228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3737228561Snp	if (rc != 0)
3738228561Snp		return(rc);
3739228561Snp
3740228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3741228561Snp	if (sb == NULL)
3742228561Snp		return (ENOMEM);
3743228561Snp
3744228561Snp	t4_get_usm_stats(sc, &stats);
3745228561Snp
3746228561Snp	sbuf_printf(sb, "Frames: %u\n", stats.frames);
3747228561Snp	sbuf_printf(sb, "Octets: %ju\n", stats.octets);
3748228561Snp	sbuf_printf(sb, "Drops:  %u", stats.drops);
3749228561Snp
3750228561Snp	rc = sbuf_finish(sb);
3751228561Snp	sbuf_delete(sb);
3752228561Snp
3753228561Snp	return (rc);
3754228561Snp}
3755228561Snp
3756222551Snpconst char *devlog_level_strings[] = {
3757222551Snp	[FW_DEVLOG_LEVEL_EMERG]		= "EMERG",
3758222551Snp	[FW_DEVLOG_LEVEL_CRIT]		= "CRIT",
3759222551Snp	[FW_DEVLOG_LEVEL_ERR]		= "ERR",
3760222551Snp	[FW_DEVLOG_LEVEL_NOTICE]	= "NOTICE",
3761222551Snp	[FW_DEVLOG_LEVEL_INFO]		= "INFO",
3762222551Snp	[FW_DEVLOG_LEVEL_DEBUG]		= "DEBUG"
3763222551Snp};
3764222551Snp
3765222551Snpconst char *devlog_facility_strings[] = {
3766222551Snp	[FW_DEVLOG_FACILITY_CORE]	= "CORE",
3767222551Snp	[FW_DEVLOG_FACILITY_SCHED]	= "SCHED",
3768222551Snp	[FW_DEVLOG_FACILITY_TIMER]	= "TIMER",
3769222551Snp	[FW_DEVLOG_FACILITY_RES]	= "RES",
3770222551Snp	[FW_DEVLOG_FACILITY_HW]		= "HW",
3771222551Snp	[FW_DEVLOG_FACILITY_FLR]	= "FLR",
3772222551Snp	[FW_DEVLOG_FACILITY_DMAQ]	= "DMAQ",
3773222551Snp	[FW_DEVLOG_FACILITY_PHY]	= "PHY",
3774222551Snp	[FW_DEVLOG_FACILITY_MAC]	= "MAC",
3775222551Snp	[FW_DEVLOG_FACILITY_PORT]	= "PORT",
3776222551Snp	[FW_DEVLOG_FACILITY_VI]		= "VI",
3777222551Snp	[FW_DEVLOG_FACILITY_FILTER]	= "FILTER",
3778222551Snp	[FW_DEVLOG_FACILITY_ACL]	= "ACL",
3779222551Snp	[FW_DEVLOG_FACILITY_TM]		= "TM",
3780222551Snp	[FW_DEVLOG_FACILITY_QFC]	= "QFC",
3781222551Snp	[FW_DEVLOG_FACILITY_DCB]	= "DCB",
3782222551Snp	[FW_DEVLOG_FACILITY_ETH]	= "ETH",
3783222551Snp	[FW_DEVLOG_FACILITY_OFLD]	= "OFLD",
3784222551Snp	[FW_DEVLOG_FACILITY_RI]		= "RI",
3785222551Snp	[FW_DEVLOG_FACILITY_ISCSI]	= "ISCSI",
3786222551Snp	[FW_DEVLOG_FACILITY_FCOE]	= "FCOE",
3787222551Snp	[FW_DEVLOG_FACILITY_FOISCSI]	= "FOISCSI",
3788222551Snp	[FW_DEVLOG_FACILITY_FOFCOE]	= "FOFCOE"
3789222551Snp};
3790222551Snp
3791222551Snpstatic int
3792222551Snpsysctl_devlog(SYSCTL_HANDLER_ARGS)
3793222551Snp{
3794222551Snp	struct adapter *sc = arg1;
3795222551Snp	struct devlog_params *dparams = &sc->params.devlog;
3796222551Snp	struct fw_devlog_e *buf, *e;
3797222551Snp	int i, j, rc, nentries, first = 0;
3798222551Snp	struct sbuf *sb;
3799222551Snp	uint64_t ftstamp = UINT64_MAX;
3800222551Snp
3801222551Snp	if (dparams->start == 0)
3802222551Snp		return (ENXIO);
3803222551Snp
3804222551Snp	nentries = dparams->size / sizeof(struct fw_devlog_e);
3805222551Snp
3806222551Snp	buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
3807222551Snp	if (buf == NULL)
3808222551Snp		return (ENOMEM);
3809222551Snp
3810222551Snp	rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size,
3811222551Snp	    (void *)buf);
3812222551Snp	if (rc != 0)
3813222551Snp		goto done;
3814222551Snp
3815222551Snp	for (i = 0; i < nentries; i++) {
3816222551Snp		e = &buf[i];
3817222551Snp
3818222551Snp		if (e->timestamp == 0)
3819222551Snp			break;	/* end */
3820222551Snp
3821222551Snp		e->timestamp = be64toh(e->timestamp);
3822222551Snp		e->seqno = be32toh(e->seqno);
3823222551Snp		for (j = 0; j < 8; j++)
3824222551Snp			e->params[j] = be32toh(e->params[j]);
3825222551Snp
3826222551Snp		if (e->timestamp < ftstamp) {
3827222551Snp			ftstamp = e->timestamp;
3828222551Snp			first = i;
3829222551Snp		}
3830222551Snp	}
3831222551Snp
3832222551Snp	if (buf[first].timestamp == 0)
3833222551Snp		goto done;	/* nothing in the log */
3834222551Snp
3835222551Snp	rc = sysctl_wire_old_buffer(req, 0);
3836222551Snp	if (rc != 0)
3837222551Snp		goto done;
3838222551Snp
3839222551Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3840228561Snp	if (sb == NULL) {
3841228561Snp		rc = ENOMEM;
3842228561Snp		goto done;
3843228561Snp	}
3844228561Snp	sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
3845222551Snp	    "Seq#", "Tstamp", "Level", "Facility", "Message");
3846222551Snp
3847222551Snp	i = first;
3848222551Snp	do {
3849222551Snp		e = &buf[i];
3850222551Snp		if (e->timestamp == 0)
3851222551Snp			break;	/* end */
3852222551Snp
3853222551Snp		sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
3854222551Snp		    e->seqno, e->timestamp,
3855240452Snp		    (e->level < nitems(devlog_level_strings) ?
3856222551Snp			devlog_level_strings[e->level] : "UNKNOWN"),
3857240452Snp		    (e->facility < nitems(devlog_facility_strings) ?
3858222551Snp			devlog_facility_strings[e->facility] : "UNKNOWN"));
3859222551Snp		sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
3860222551Snp		    e->params[2], e->params[3], e->params[4],
3861222551Snp		    e->params[5], e->params[6], e->params[7]);
3862222551Snp
3863222551Snp		if (++i == nentries)
3864222551Snp			i = 0;
3865222551Snp	} while (i != first);
3866222551Snp
3867222551Snp	rc = sbuf_finish(sb);
3868222551Snp	sbuf_delete(sb);
3869222551Snpdone:
3870222551Snp	free(buf, M_CXGBE);
3871222551Snp	return (rc);
3872222551Snp}
3873222551Snp
3874228561Snpstatic int
3875228561Snpsysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
3876228561Snp{
3877228561Snp	struct adapter *sc = arg1;
3878228561Snp	struct sbuf *sb;
3879228561Snp	int rc;
3880228561Snp	struct tp_fcoe_stats stats[4];
3881228561Snp
3882228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3883228561Snp	if (rc != 0)
3884228561Snp		return (rc);
3885228561Snp
3886228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3887228561Snp	if (sb == NULL)
3888228561Snp		return (ENOMEM);
3889228561Snp
3890228561Snp	t4_get_fcoe_stats(sc, 0, &stats[0]);
3891228561Snp	t4_get_fcoe_stats(sc, 1, &stats[1]);
3892228561Snp	t4_get_fcoe_stats(sc, 2, &stats[2]);
3893228561Snp	t4_get_fcoe_stats(sc, 3, &stats[3]);
3894228561Snp
3895228561Snp	sbuf_printf(sb, "                   channel 0        channel 1        "
3896228561Snp	    "channel 2        channel 3\n");
3897228561Snp	sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
3898228561Snp	    stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
3899228561Snp	    stats[3].octetsDDP);
3900228561Snp	sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
3901228561Snp	    stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
3902228561Snp	sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
3903228561Snp	    stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
3904228561Snp	    stats[3].framesDrop);
3905228561Snp
3906228561Snp	rc = sbuf_finish(sb);
3907228561Snp	sbuf_delete(sb);
3908228561Snp
3909228561Snp	return (rc);
3910228561Snp}
3911228561Snp
3912228561Snpstatic int
3913228561Snpsysctl_hw_sched(SYSCTL_HANDLER_ARGS)
3914228561Snp{
3915228561Snp	struct adapter *sc = arg1;
3916228561Snp	struct sbuf *sb;
3917228561Snp	int rc, i;
3918228561Snp	unsigned int map, kbps, ipg, mode;
3919228561Snp	unsigned int pace_tab[NTX_SCHED];
3920228561Snp
3921228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3922228561Snp	if (rc != 0)
3923228561Snp		return (rc);
3924228561Snp
3925228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3926228561Snp	if (sb == NULL)
3927228561Snp		return (ENOMEM);
3928228561Snp
3929228561Snp	map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
3930228561Snp	mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
3931228561Snp	t4_read_pace_tbl(sc, pace_tab);
3932228561Snp
3933228561Snp	sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
3934228561Snp	    "Class IPG (0.1 ns)   Flow IPG (us)");
3935228561Snp
3936228561Snp	for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
3937228561Snp		t4_get_tx_sched(sc, i, &kbps, &ipg);
3938228561Snp		sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
3939228561Snp		    (mode & (1 << i)) ? "flow" : "class", map & 3);
3940228561Snp		if (kbps)
3941228561Snp			sbuf_printf(sb, "%9u     ", kbps);
3942228561Snp		else
3943228561Snp			sbuf_printf(sb, " disabled     ");
3944228561Snp
3945228561Snp		if (ipg)
3946228561Snp			sbuf_printf(sb, "%13u        ", ipg);
3947228561Snp		else
3948228561Snp			sbuf_printf(sb, "     disabled        ");
3949228561Snp
3950228561Snp		if (pace_tab[i])
3951228561Snp			sbuf_printf(sb, "%10u", pace_tab[i]);
3952228561Snp		else
3953228561Snp			sbuf_printf(sb, "  disabled");
3954228561Snp	}
3955228561Snp
3956228561Snp	rc = sbuf_finish(sb);
3957228561Snp	sbuf_delete(sb);
3958228561Snp
3959228561Snp	return (rc);
3960228561Snp}
3961228561Snp
3962228561Snpstatic int
3963228561Snpsysctl_lb_stats(SYSCTL_HANDLER_ARGS)
3964228561Snp{
3965228561Snp	struct adapter *sc = arg1;
3966228561Snp	struct sbuf *sb;
3967228561Snp	int rc, i, j;
3968228561Snp	uint64_t *p0, *p1;
3969228561Snp	struct lb_port_stats s[2];
3970228561Snp	static const char *stat_name[] = {
3971228561Snp		"OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
3972228561Snp		"UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
3973228561Snp		"Frames128To255:", "Frames256To511:", "Frames512To1023:",
3974228561Snp		"Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
3975228561Snp		"BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
3976228561Snp		"BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
3977228561Snp		"BG2FramesTrunc:", "BG3FramesTrunc:"
3978228561Snp	};
3979228561Snp
3980228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3981228561Snp	if (rc != 0)
3982228561Snp		return (rc);
3983228561Snp
3984228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3985228561Snp	if (sb == NULL)
3986228561Snp		return (ENOMEM);
3987228561Snp
3988228561Snp	memset(s, 0, sizeof(s));
3989228561Snp
3990228561Snp	for (i = 0; i < 4; i += 2) {
3991228561Snp		t4_get_lb_stats(sc, i, &s[0]);
3992228561Snp		t4_get_lb_stats(sc, i + 1, &s[1]);
3993228561Snp
3994228561Snp		p0 = &s[0].octets;
3995228561Snp		p1 = &s[1].octets;
3996228561Snp		sbuf_printf(sb, "%s                       Loopback %u"
3997228561Snp		    "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
3998228561Snp
3999240452Snp		for (j = 0; j < nitems(stat_name); j++)
4000228561Snp			sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
4001228561Snp				   *p0++, *p1++);
4002228561Snp	}
4003228561Snp
4004228561Snp	rc = sbuf_finish(sb);
4005228561Snp	sbuf_delete(sb);
4006228561Snp
4007228561Snp	return (rc);
4008228561Snp}
4009228561Snp
4010228561Snpstruct mem_desc {
4011228561Snp	unsigned int base;
4012228561Snp	unsigned int limit;
4013228561Snp	unsigned int idx;
4014228561Snp};
4015228561Snp
4016228561Snpstatic int
4017228561Snpmem_desc_cmp(const void *a, const void *b)
4018228561Snp{
4019228561Snp	return ((const struct mem_desc *)a)->base -
4020228561Snp	       ((const struct mem_desc *)b)->base;
4021228561Snp}
4022228561Snp
4023228561Snpstatic void
4024228561Snpmem_region_show(struct sbuf *sb, const char *name, unsigned int from,
4025228561Snp    unsigned int to)
4026228561Snp{
4027228561Snp	unsigned int size;
4028228561Snp
4029228561Snp	size = to - from + 1;
4030228561Snp	if (size == 0)
4031228561Snp		return;
4032228561Snp
4033228561Snp	/* XXX: need humanize_number(3) in libkern for a more readable 'size' */
4034228561Snp	sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
4035228561Snp}
4036228561Snp
4037228561Snpstatic int
4038228561Snpsysctl_meminfo(SYSCTL_HANDLER_ARGS)
4039228561Snp{
4040228561Snp	struct adapter *sc = arg1;
4041228561Snp	struct sbuf *sb;
4042228561Snp	int rc, i, n;
4043228561Snp	uint32_t lo, hi;
4044228561Snp	static const char *memory[] = { "EDC0:", "EDC1:", "MC:" };
4045228561Snp	static const char *region[] = {
4046228561Snp		"DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
4047228561Snp		"Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
4048228561Snp		"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
4049228561Snp		"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
4050228561Snp		"RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:",
4051228561Snp		"ULPTX state:", "On-chip queues:"
4052228561Snp	};
4053228561Snp	struct mem_desc avail[3];
4054240452Snp	struct mem_desc mem[nitems(region) + 3];	/* up to 3 holes */
4055228561Snp	struct mem_desc *md = mem;
4056228561Snp
4057228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4058228561Snp	if (rc != 0)
4059228561Snp		return (rc);
4060228561Snp
4061228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
4062228561Snp	if (sb == NULL)
4063228561Snp		return (ENOMEM);
4064228561Snp
4065240452Snp	for (i = 0; i < nitems(mem); i++) {
4066228561Snp		mem[i].limit = 0;
4067228561Snp		mem[i].idx = i;
4068228561Snp	}
4069228561Snp
4070228561Snp	/* Find and sort the populated memory ranges */
4071228561Snp	i = 0;
4072228561Snp	lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
4073228561Snp	if (lo & F_EDRAM0_ENABLE) {
4074228561Snp		hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
4075228561Snp		avail[i].base = G_EDRAM0_BASE(hi) << 20;
4076228561Snp		avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
4077228561Snp		avail[i].idx = 0;
4078228561Snp		i++;
4079228561Snp	}
4080228561Snp	if (lo & F_EDRAM1_ENABLE) {
4081228561Snp		hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
4082228561Snp		avail[i].base = G_EDRAM1_BASE(hi) << 20;
4083228561Snp		avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
4084228561Snp		avail[i].idx = 1;
4085228561Snp		i++;
4086228561Snp	}
4087228561Snp	if (lo & F_EXT_MEM_ENABLE) {
4088228561Snp		hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
4089228561Snp		avail[i].base = G_EXT_MEM_BASE(hi) << 20;
4090228561Snp		avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20);
4091228561Snp		avail[i].idx = 2;
4092228561Snp		i++;
4093228561Snp	}
4094228561Snp	if (!i)                                    /* no memory available */
4095228561Snp		return 0;
4096228561Snp	qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
4097228561Snp
4098228561Snp	(md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
4099228561Snp	(md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
4100228561Snp	(md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
4101228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
4102228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
4103228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
4104228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
4105228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
4106228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
4107228561Snp
4108228561Snp	/* the next few have explicit upper bounds */
4109228561Snp	md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
4110228561Snp	md->limit = md->base - 1 +
4111228561Snp		    t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
4112228561Snp		    G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
4113228561Snp	md++;
4114228561Snp
4115228561Snp	md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
4116228561Snp	md->limit = md->base - 1 +
4117228561Snp		    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
4118228561Snp		    G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
4119228561Snp	md++;
4120228561Snp
4121228561Snp	if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4122228561Snp		hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
4123228561Snp		md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
4124228561Snp		md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
4125228561Snp	} else {
4126228561Snp		md->base = 0;
4127240452Snp		md->idx = nitems(region);  /* hide it */
4128228561Snp	}
4129228561Snp	md++;
4130228561Snp
4131228561Snp#define ulp_region(reg) \
4132228561Snp	md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
4133228561Snp	(md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
4134228561Snp
4135228561Snp	ulp_region(RX_ISCSI);
4136228561Snp	ulp_region(RX_TDDP);
4137228561Snp	ulp_region(TX_TPT);
4138228561Snp	ulp_region(RX_STAG);
4139228561Snp	ulp_region(RX_RQ);
4140228561Snp	ulp_region(RX_RQUDP);
4141228561Snp	ulp_region(RX_PBL);
4142228561Snp	ulp_region(TX_PBL);
4143228561Snp#undef ulp_region
4144228561Snp
4145228561Snp	md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
4146228561Snp	md->limit = md->base + sc->tids.ntids - 1;
4147228561Snp	md++;
4148228561Snp	md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
4149228561Snp	md->limit = md->base + sc->tids.ntids - 1;
4150228561Snp	md++;
4151228561Snp
4152228561Snp	md->base = sc->vres.ocq.start;
4153228561Snp	if (sc->vres.ocq.size)
4154228561Snp		md->limit = md->base + sc->vres.ocq.size - 1;
4155228561Snp	else
4156240452Snp		md->idx = nitems(region);  /* hide it */
4157228561Snp	md++;
4158228561Snp
4159228561Snp	/* add any address-space holes, there can be up to 3 */
4160228561Snp	for (n = 0; n < i - 1; n++)
4161228561Snp		if (avail[n].limit < avail[n + 1].base)
4162228561Snp			(md++)->base = avail[n].limit;
4163228561Snp	if (avail[n].limit)
4164228561Snp		(md++)->base = avail[n].limit;
4165228561Snp
4166228561Snp	n = md - mem;
4167228561Snp	qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
4168228561Snp
4169228561Snp	for (lo = 0; lo < i; lo++)
4170228561Snp		mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
4171228561Snp				avail[lo].limit - 1);
4172228561Snp
4173228561Snp	sbuf_printf(sb, "\n");
4174228561Snp	for (i = 0; i < n; i++) {
4175240452Snp		if (mem[i].idx >= nitems(region))
4176228561Snp			continue;                        /* skip holes */
4177228561Snp		if (!mem[i].limit)
4178228561Snp			mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
4179228561Snp		mem_region_show(sb, region[mem[i].idx], mem[i].base,
4180228561Snp				mem[i].limit);
4181228561Snp	}
4182228561Snp
4183228561Snp	sbuf_printf(sb, "\n");
4184228561Snp	lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
4185228561Snp	hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
4186228561Snp	mem_region_show(sb, "uP RAM:", lo, hi);
4187228561Snp
4188228561Snp	lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
4189228561Snp	hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
4190228561Snp	mem_region_show(sb, "uP Extmem2:", lo, hi);
4191228561Snp
4192228561Snp	lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
4193228561Snp	sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
4194228561Snp		   G_PMRXMAXPAGE(lo),
4195228561Snp		   t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
4196228561Snp		   (lo & F_PMRXNUMCHN) ? 2 : 1);
4197228561Snp
4198228561Snp	lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
4199228561Snp	hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
4200228561Snp	sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
4201228561Snp		   G_PMTXMAXPAGE(lo),
4202228561Snp		   hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
4203228561Snp		   hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
4204228561Snp	sbuf_printf(sb, "%u p-structs\n",
4205228561Snp		   t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
4206228561Snp
4207228561Snp	for (i = 0; i < 4; i++) {
4208228561Snp		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
4209228561Snp		sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
4210228561Snp			   i, G_USED(lo), G_ALLOC(lo));
4211228561Snp	}
4212228561Snp	for (i = 0; i < 4; i++) {
4213228561Snp		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
4214228561Snp		sbuf_printf(sb,
4215228561Snp			   "\nLoopback %d using %u pages out of %u allocated",
4216228561Snp			   i, G_USED(lo), G_ALLOC(lo));
4217228561Snp	}
4218228561Snp
4219228561Snp	rc = sbuf_finish(sb);
4220228561Snp	sbuf_delete(sb);
4221228561Snp
4222228561Snp	return (rc);
4223228561Snp}
4224228561Snp
4225228561Snpstatic int
4226228561Snpsysctl_path_mtus(SYSCTL_HANDLER_ARGS)
4227228561Snp{
4228228561Snp	struct adapter *sc = arg1;
4229228561Snp	struct sbuf *sb;
4230228561Snp	int rc;
4231228561Snp	uint16_t mtus[NMTUS];
4232228561Snp
4233228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4234228561Snp	if (rc != 0)
4235228561Snp		return (rc);
4236228561Snp
4237228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4238228561Snp	if (sb == NULL)
4239228561Snp		return (ENOMEM);
4240228561Snp
4241228561Snp	t4_read_mtu_tbl(sc, mtus, NULL);
4242228561Snp
4243228561Snp	sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
4244228561Snp	    mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
4245228561Snp	    mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
4246228561Snp	    mtus[14], mtus[15]);
4247228561Snp
4248228561Snp	rc = sbuf_finish(sb);
4249228561Snp	sbuf_delete(sb);
4250228561Snp
4251228561Snp	return (rc);
4252228561Snp}
4253228561Snp
4254228561Snpstatic int
4255228561Snpsysctl_pm_stats(SYSCTL_HANDLER_ARGS)
4256228561Snp{
4257228561Snp	struct adapter *sc = arg1;
4258228561Snp	struct sbuf *sb;
4259228561Snp	int rc, i;
4260228561Snp	uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS];
4261228561Snp	uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS];
4262228561Snp	static const char *pm_stats[] = {
4263228561Snp		"Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:"
4264228561Snp	};
4265228561Snp
4266228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4267228561Snp	if (rc != 0)
4268228561Snp		return (rc);
4269228561Snp
4270228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4271228561Snp	if (sb == NULL)
4272228561Snp		return (ENOMEM);
4273228561Snp
4274228561Snp	t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
4275228561Snp	t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
4276228561Snp
4277228561Snp	sbuf_printf(sb, "                Tx count            Tx cycles    "
4278228561Snp	    "Rx count            Rx cycles");
4279228561Snp	for (i = 0; i < PM_NSTATS; i++)
4280228561Snp		sbuf_printf(sb, "\n%-13s %10u %20ju  %10u %20ju",
4281228561Snp		    pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]);
4282228561Snp
4283228561Snp	rc = sbuf_finish(sb);
4284228561Snp	sbuf_delete(sb);
4285228561Snp
4286228561Snp	return (rc);
4287228561Snp}
4288228561Snp
4289228561Snpstatic int
4290228561Snpsysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
4291228561Snp{
4292228561Snp	struct adapter *sc = arg1;
4293228561Snp	struct sbuf *sb;
4294228561Snp	int rc;
4295228561Snp	struct tp_rdma_stats stats;
4296228561Snp
4297228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4298228561Snp	if (rc != 0)
4299228561Snp		return (rc);
4300228561Snp
4301228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4302228561Snp	if (sb == NULL)
4303228561Snp		return (ENOMEM);
4304228561Snp
4305228561Snp	t4_tp_get_rdma_stats(sc, &stats);
4306228561Snp	sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
4307228561Snp	sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
4308228561Snp
4309228561Snp	rc = sbuf_finish(sb);
4310228561Snp	sbuf_delete(sb);
4311228561Snp
4312228561Snp	return (rc);
4313228561Snp}
4314228561Snp
4315228561Snpstatic int
4316228561Snpsysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
4317228561Snp{
4318228561Snp	struct adapter *sc = arg1;
4319228561Snp	struct sbuf *sb;
4320228561Snp	int rc;
4321228561Snp	struct tp_tcp_stats v4, v6;
4322228561Snp
4323228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4324228561Snp	if (rc != 0)
4325228561Snp		return (rc);
4326228561Snp
4327228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4328228561Snp	if (sb == NULL)
4329228561Snp		return (ENOMEM);
4330228561Snp
4331228561Snp	t4_tp_get_tcp_stats(sc, &v4, &v6);
4332228561Snp	sbuf_printf(sb,
4333228561Snp	    "                                IP                 IPv6\n");
4334228561Snp	sbuf_printf(sb, "OutRsts:      %20u %20u\n",
4335228561Snp	    v4.tcpOutRsts, v6.tcpOutRsts);
4336228561Snp	sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
4337228561Snp	    v4.tcpInSegs, v6.tcpInSegs);
4338228561Snp	sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
4339228561Snp	    v4.tcpOutSegs, v6.tcpOutSegs);
4340228561Snp	sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
4341228561Snp	    v4.tcpRetransSegs, v6.tcpRetransSegs);
4342228561Snp
4343228561Snp	rc = sbuf_finish(sb);
4344228561Snp	sbuf_delete(sb);
4345228561Snp
4346228561Snp	return (rc);
4347228561Snp}
4348228561Snp
4349228561Snpstatic int
4350228561Snpsysctl_tids(SYSCTL_HANDLER_ARGS)
4351228561Snp{
4352228561Snp	struct adapter *sc = arg1;
4353228561Snp	struct sbuf *sb;
4354228561Snp	int rc;
4355228561Snp	struct tid_info *t = &sc->tids;
4356228561Snp
4357228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4358228561Snp	if (rc != 0)
4359228561Snp		return (rc);
4360228561Snp
4361228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4362228561Snp	if (sb == NULL)
4363228561Snp		return (ENOMEM);
4364228561Snp
4365228561Snp	if (t->natids) {
4366228561Snp		sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
4367228561Snp		    t->atids_in_use);
4368228561Snp	}
4369228561Snp
4370228561Snp	if (t->ntids) {
4371228561Snp		if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4372228561Snp			uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
4373228561Snp
4374228561Snp			if (b) {
4375228561Snp				sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
4376228561Snp				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4377228561Snp				    t->ntids - 1);
4378228561Snp			} else {
4379228561Snp				sbuf_printf(sb, "TID range: %u-%u",
4380228561Snp				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4381228561Snp				    t->ntids - 1);
4382228561Snp			}
4383228561Snp		} else
4384228561Snp			sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
4385228561Snp		sbuf_printf(sb, ", in use: %u\n",
4386228561Snp		    atomic_load_acq_int(&t->tids_in_use));
4387228561Snp	}
4388228561Snp
4389228561Snp	if (t->nstids) {
4390228561Snp		sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
4391228561Snp		    t->stid_base + t->nstids - 1, t->stids_in_use);
4392228561Snp	}
4393228561Snp
4394228561Snp	if (t->nftids) {
4395228561Snp		sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
4396228561Snp		    t->ftid_base + t->nftids - 1);
4397228561Snp	}
4398228561Snp
4399228561Snp	sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
4400228561Snp	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
4401228561Snp	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
4402228561Snp
4403228561Snp	rc = sbuf_finish(sb);
4404228561Snp	sbuf_delete(sb);
4405228561Snp
4406228561Snp	return (rc);
4407228561Snp}
4408228561Snp
4409228561Snpstatic int
4410228561Snpsysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
4411228561Snp{
4412228561Snp	struct adapter *sc = arg1;
4413228561Snp	struct sbuf *sb;
4414228561Snp	int rc;
4415228561Snp	struct tp_err_stats stats;
4416228561Snp
4417228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4418228561Snp	if (rc != 0)
4419228561Snp		return (rc);
4420228561Snp
4421228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4422228561Snp	if (sb == NULL)
4423228561Snp		return (ENOMEM);
4424228561Snp
4425228561Snp	t4_tp_get_err_stats(sc, &stats);
4426228561Snp
4427228561Snp	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
4428228561Snp		      "channel 3\n");
4429228561Snp	sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
4430228561Snp	    stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
4431228561Snp	    stats.macInErrs[3]);
4432228561Snp	sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
4433228561Snp	    stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
4434228561Snp	    stats.hdrInErrs[3]);
4435228561Snp	sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
4436228561Snp	    stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
4437228561Snp	    stats.tcpInErrs[3]);
4438228561Snp	sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
4439228561Snp	    stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
4440228561Snp	    stats.tcp6InErrs[3]);
4441228561Snp	sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
4442228561Snp	    stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
4443228561Snp	    stats.tnlCongDrops[3]);
4444228561Snp	sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
4445228561Snp	    stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
4446228561Snp	    stats.tnlTxDrops[3]);
4447228561Snp	sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
4448228561Snp	    stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
4449228561Snp	    stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
4450228561Snp	sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
4451228561Snp	    stats.ofldChanDrops[0], stats.ofldChanDrops[1],
4452228561Snp	    stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
4453228561Snp	sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
4454228561Snp	    stats.ofldNoNeigh, stats.ofldCongDefer);
4455228561Snp
4456228561Snp	rc = sbuf_finish(sb);
4457228561Snp	sbuf_delete(sb);
4458228561Snp
4459228561Snp	return (rc);
4460228561Snp}
4461228561Snp
4462228561Snpstatic int
4463228561Snpsysctl_tx_rate(SYSCTL_HANDLER_ARGS)
4464228561Snp{
4465228561Snp	struct adapter *sc = arg1;
4466228561Snp	struct sbuf *sb;
4467228561Snp	int rc;
4468228561Snp	u64 nrate[NCHAN], orate[NCHAN];
4469228561Snp
4470228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4471228561Snp	if (rc != 0)
4472228561Snp		return (rc);
4473228561Snp
4474228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4475228561Snp	if (sb == NULL)
4476228561Snp		return (ENOMEM);
4477228561Snp
4478228561Snp	t4_get_chan_txrate(sc, nrate, orate);
4479228561Snp	sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
4480228561Snp		 "channel 3\n");
4481228561Snp	sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
4482228561Snp	    nrate[0], nrate[1], nrate[2], nrate[3]);
4483228561Snp	sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
4484228561Snp	    orate[0], orate[1], orate[2], orate[3]);
4485228561Snp
4486228561Snp	rc = sbuf_finish(sb);
4487228561Snp	sbuf_delete(sb);
4488228561Snp
4489228561Snp	return (rc);
4490228561Snp}
4491231115Snp#endif
4492228561Snp
4493219286Snpstatic inline void
4494219286Snptxq_start(struct ifnet *ifp, struct sge_txq *txq)
4495219286Snp{
4496219286Snp	struct buf_ring *br;
4497219286Snp	struct mbuf *m;
4498219286Snp
4499219286Snp	TXQ_LOCK_ASSERT_OWNED(txq);
4500219286Snp
4501220873Snp	br = txq->br;
4502219286Snp	m = txq->m ? txq->m : drbr_dequeue(ifp, br);
4503219286Snp	if (m)
4504219286Snp		t4_eth_tx(ifp, txq, m);
4505219286Snp}
4506219286Snp
4507219286Snpvoid
4508228561Snpt4_tx_callout(void *arg)
4509219286Snp{
4510228561Snp	struct sge_eq *eq = arg;
4511228561Snp	struct adapter *sc;
4512219286Snp
4513228561Snp	if (EQ_TRYLOCK(eq) == 0)
4514228561Snp		goto reschedule;
4515228561Snp
4516228561Snp	if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
4517228561Snp		EQ_UNLOCK(eq);
4518228561Snpreschedule:
4519228561Snp		if (__predict_true(!(eq->flags && EQ_DOOMED)))
4520228561Snp			callout_schedule(&eq->tx_callout, 1);
4521228561Snp		return;
4522228561Snp	}
4523228561Snp
4524228561Snp	EQ_LOCK_ASSERT_OWNED(eq);
4525228561Snp
4526228561Snp	if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
4527228561Snp
4528228561Snp		if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4529228561Snp			struct sge_txq *txq = arg;
4530228561Snp			struct port_info *pi = txq->ifp->if_softc;
4531228561Snp
4532228561Snp			sc = pi->adapter;
4533228561Snp		} else {
4534228561Snp			struct sge_wrq *wrq = arg;
4535228561Snp
4536228561Snp			sc = wrq->adapter;
4537228561Snp		}
4538228561Snp
4539228561Snp		taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
4540228561Snp	}
4541228561Snp
4542228561Snp	EQ_UNLOCK(eq);
4543228561Snp}
4544228561Snp
4545228561Snpvoid
4546228561Snpt4_tx_task(void *arg, int count)
4547228561Snp{
4548228561Snp	struct sge_eq *eq = arg;
4549228561Snp
4550228561Snp	EQ_LOCK(eq);
4551228561Snp	if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4552228561Snp		struct sge_txq *txq = arg;
4553220649Snp		txq_start(txq->ifp, txq);
4554228561Snp	} else {
4555228561Snp		struct sge_wrq *wrq = arg;
4556228561Snp		t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
4557228561Snp	}
4558228561Snp	EQ_UNLOCK(eq);
4559219286Snp}
4560219286Snp
4561221474Snpstatic uint32_t
4562221474Snpfconf_to_mode(uint32_t fconf)
4563221474Snp{
4564221474Snp	uint32_t mode;
4565221474Snp
4566221474Snp	mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
4567221474Snp	    T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
4568221474Snp
4569221474Snp	if (fconf & F_FRAGMENTATION)
4570221474Snp		mode |= T4_FILTER_IP_FRAGMENT;
4571221474Snp
4572221474Snp	if (fconf & F_MPSHITTYPE)
4573221474Snp		mode |= T4_FILTER_MPS_HIT_TYPE;
4574221474Snp
4575221474Snp	if (fconf & F_MACMATCH)
4576221474Snp		mode |= T4_FILTER_MAC_IDX;
4577221474Snp
4578221474Snp	if (fconf & F_ETHERTYPE)
4579221474Snp		mode |= T4_FILTER_ETH_TYPE;
4580221474Snp
4581221474Snp	if (fconf & F_PROTOCOL)
4582221474Snp		mode |= T4_FILTER_IP_PROTO;
4583221474Snp
4584221474Snp	if (fconf & F_TOS)
4585221474Snp		mode |= T4_FILTER_IP_TOS;
4586221474Snp
4587221474Snp	if (fconf & F_VLAN)
4588228561Snp		mode |= T4_FILTER_VLAN;
4589221474Snp
4590221474Snp	if (fconf & F_VNIC_ID)
4591228561Snp		mode |= T4_FILTER_VNIC;
4592221474Snp
4593221474Snp	if (fconf & F_PORT)
4594221474Snp		mode |= T4_FILTER_PORT;
4595221474Snp
4596221474Snp	if (fconf & F_FCOE)
4597221474Snp		mode |= T4_FILTER_FCoE;
4598221474Snp
4599221474Snp	return (mode);
4600221474Snp}
4601221474Snp
4602221474Snpstatic uint32_t
4603221474Snpmode_to_fconf(uint32_t mode)
4604221474Snp{
4605221474Snp	uint32_t fconf = 0;
4606221474Snp
4607221474Snp	if (mode & T4_FILTER_IP_FRAGMENT)
4608221474Snp		fconf |= F_FRAGMENTATION;
4609221474Snp
4610221474Snp	if (mode & T4_FILTER_MPS_HIT_TYPE)
4611221474Snp		fconf |= F_MPSHITTYPE;
4612221474Snp
4613221474Snp	if (mode & T4_FILTER_MAC_IDX)
4614221474Snp		fconf |= F_MACMATCH;
4615221474Snp
4616221474Snp	if (mode & T4_FILTER_ETH_TYPE)
4617221474Snp		fconf |= F_ETHERTYPE;
4618221474Snp
4619221474Snp	if (mode & T4_FILTER_IP_PROTO)
4620221474Snp		fconf |= F_PROTOCOL;
4621221474Snp
4622221474Snp	if (mode & T4_FILTER_IP_TOS)
4623221474Snp		fconf |= F_TOS;
4624221474Snp
4625228561Snp	if (mode & T4_FILTER_VLAN)
4626221474Snp		fconf |= F_VLAN;
4627221474Snp
4628228561Snp	if (mode & T4_FILTER_VNIC)
4629221474Snp		fconf |= F_VNIC_ID;
4630221474Snp
4631221474Snp	if (mode & T4_FILTER_PORT)
4632221474Snp		fconf |= F_PORT;
4633221474Snp
4634221474Snp	if (mode & T4_FILTER_FCoE)
4635221474Snp		fconf |= F_FCOE;
4636221474Snp
4637221474Snp	return (fconf);
4638221474Snp}
4639221474Snp
4640221474Snpstatic uint32_t
4641221474Snpfspec_to_fconf(struct t4_filter_specification *fs)
4642221474Snp{
4643221474Snp	uint32_t fconf = 0;
4644221474Snp
4645221474Snp	if (fs->val.frag || fs->mask.frag)
4646221474Snp		fconf |= F_FRAGMENTATION;
4647221474Snp
4648221474Snp	if (fs->val.matchtype || fs->mask.matchtype)
4649221474Snp		fconf |= F_MPSHITTYPE;
4650221474Snp
4651221474Snp	if (fs->val.macidx || fs->mask.macidx)
4652221474Snp		fconf |= F_MACMATCH;
4653221474Snp
4654221474Snp	if (fs->val.ethtype || fs->mask.ethtype)
4655221474Snp		fconf |= F_ETHERTYPE;
4656221474Snp
4657221474Snp	if (fs->val.proto || fs->mask.proto)
4658221474Snp		fconf |= F_PROTOCOL;
4659221474Snp
4660221474Snp	if (fs->val.tos || fs->mask.tos)
4661221474Snp		fconf |= F_TOS;
4662221474Snp
4663228561Snp	if (fs->val.vlan_vld || fs->mask.vlan_vld)
4664221474Snp		fconf |= F_VLAN;
4665221474Snp
4666228561Snp	if (fs->val.vnic_vld || fs->mask.vnic_vld)
4667221474Snp		fconf |= F_VNIC_ID;
4668221474Snp
4669221474Snp	if (fs->val.iport || fs->mask.iport)
4670221474Snp		fconf |= F_PORT;
4671221474Snp
4672221474Snp	if (fs->val.fcoe || fs->mask.fcoe)
4673221474Snp		fconf |= F_FCOE;
4674221474Snp
4675221474Snp	return (fconf);
4676221474Snp}
4677221474Snp
4678221474Snpstatic int
4679221474Snpget_filter_mode(struct adapter *sc, uint32_t *mode)
4680221474Snp{
4681245274Snp	int rc;
4682221474Snp	uint32_t fconf;
4683221474Snp
4684245274Snp	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
4685245274Snp	    "t4getfm");
4686245274Snp	if (rc)
4687245274Snp		return (rc);
4688245274Snp
4689221474Snp	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
4690221474Snp	    A_TP_VLAN_PRI_MAP);
4691221474Snp
4692228561Snp	if (sc->filter_mode != fconf) {
4693228561Snp		log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
4694228561Snp		    device_get_nameunit(sc->dev), sc->filter_mode, fconf);
4695228561Snp		sc->filter_mode = fconf;
4696228561Snp	}
4697221474Snp
4698228561Snp	*mode = fconf_to_mode(sc->filter_mode);
4699228561Snp
4700245274Snp	end_synchronized_op(sc, LOCK_HELD);
4701221474Snp	return (0);
4702221474Snp}
4703221474Snp
4704221474Snpstatic int
4705221474Snpset_filter_mode(struct adapter *sc, uint32_t mode)
4706221474Snp{
4707221474Snp	uint32_t fconf;
4708221474Snp	int rc;
4709221474Snp
4710221474Snp	fconf = mode_to_fconf(mode);
4711221474Snp
4712245274Snp	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
4713245274Snp	    "t4setfm");
4714245274Snp	if (rc)
4715245274Snp		return (rc);
4716221474Snp
4717221474Snp	if (sc->tids.ftids_in_use > 0) {
4718221474Snp		rc = EBUSY;
4719221474Snp		goto done;
4720221474Snp	}
4721221474Snp
4722237263Snp#ifdef TCP_OFFLOAD
4723228561Snp	if (sc->offload_map) {
4724228561Snp		rc = EBUSY;
4725228561Snp		goto done;
4726228561Snp	}
4727228561Snp#endif
4728228561Snp
4729228561Snp#ifdef notyet
4730221474Snp	rc = -t4_set_filter_mode(sc, fconf);
4731228561Snp	if (rc == 0)
4732228561Snp		sc->filter_mode = fconf;
4733228561Snp#else
4734228561Snp	rc = ENOTSUP;
4735228561Snp#endif
4736228561Snp
4737221474Snpdone:
4738245274Snp	end_synchronized_op(sc, LOCK_HELD);
4739221474Snp	return (rc);
4740221474Snp}
4741221474Snp
4742222552Snpstatic inline uint64_t
4743222552Snpget_filter_hits(struct adapter *sc, uint32_t fid)
4744222552Snp{
4745222552Snp	uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
4746222552Snp	uint64_t hits;
4747222552Snp
4748222552Snp	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0),
4749222552Snp	    tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
4750222552Snp	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0));
4751222552Snp	hits = t4_read_reg64(sc, MEMWIN0_BASE + 16);
4752222552Snp
4753222552Snp	return (be64toh(hits));
4754222552Snp}
4755222552Snp
4756221474Snpstatic int
4757221474Snpget_filter(struct adapter *sc, struct t4_filter *t)
4758221474Snp{
4759245274Snp	int i, rc, nfilters = sc->tids.nftids;
4760221474Snp	struct filter_entry *f;
4761221474Snp
4762245274Snp	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
4763245274Snp	    "t4getf");
4764245274Snp	if (rc)
4765245274Snp		return (rc);
4766221474Snp
4767221474Snp	if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
4768221474Snp	    t->idx >= nfilters) {
4769221474Snp		t->idx = 0xffffffff;
4770245274Snp		goto done;
4771221474Snp	}
4772221474Snp
4773221474Snp	f = &sc->tids.ftid_tab[t->idx];
4774221474Snp	for (i = t->idx; i < nfilters; i++, f++) {
4775221474Snp		if (f->valid) {
4776221474Snp			t->idx = i;
4777222509Snp			t->l2tidx = f->l2t ? f->l2t->idx : 0;
4778222509Snp			t->smtidx = f->smtidx;
4779222552Snp			if (f->fs.hitcnts)
4780222552Snp				t->hits = get_filter_hits(sc, t->idx);
4781222552Snp			else
4782222552Snp				t->hits = UINT64_MAX;
4783221474Snp			t->fs = f->fs;
4784221474Snp
4785245274Snp			goto done;
4786221474Snp		}
4787221474Snp	}
4788221474Snp
4789221474Snp	t->idx = 0xffffffff;
4790245274Snpdone:
4791245274Snp	end_synchronized_op(sc, LOCK_HELD);
4792221474Snp	return (0);
4793221474Snp}
4794221474Snp
4795221474Snpstatic int
4796221474Snpset_filter(struct adapter *sc, struct t4_filter *t)
4797221474Snp{
4798221474Snp	unsigned int nfilters, nports;
4799221474Snp	struct filter_entry *f;
4800245274Snp	int i, rc;
4801221474Snp
4802245274Snp	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4setf");
4803245274Snp	if (rc)
4804245274Snp		return (rc);
4805221474Snp
4806221474Snp	nfilters = sc->tids.nftids;
4807221474Snp	nports = sc->params.nports;
4808221474Snp
4809245274Snp	if (nfilters == 0) {
4810245274Snp		rc = ENOTSUP;
4811245274Snp		goto done;
4812245274Snp	}
4813221474Snp
4814245274Snp	if (!(sc->flags & FULL_INIT_DONE)) {
4815245274Snp		rc = EAGAIN;
4816245274Snp		goto done;
4817245274Snp	}
4818221474Snp
4819245274Snp	if (t->idx >= nfilters) {
4820245274Snp		rc = EINVAL;
4821245274Snp		goto done;
4822245274Snp	}
4823221474Snp
4824221474Snp	/* Validate against the global filter mode */
4825245274Snp	if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode) {
4826245274Snp		rc = E2BIG;
4827245274Snp		goto done;
4828245274Snp	}
4829221474Snp
4830245274Snp	if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
4831245274Snp		rc = EINVAL;
4832245274Snp		goto done;
4833245274Snp	}
4834221474Snp
4835245274Snp	if (t->fs.val.iport >= nports) {
4836245274Snp		rc = EINVAL;
4837245274Snp		goto done;
4838245274Snp	}
4839221474Snp
4840221474Snp	/* Can't specify an iq if not steering to it */
4841245274Snp	if (!t->fs.dirsteer && t->fs.iq) {
4842245274Snp		rc = EINVAL;
4843245274Snp		goto done;
4844245274Snp	}
4845221474Snp
4846221474Snp	/* IPv6 filter idx must be 4 aligned */
4847221474Snp	if (t->fs.type == 1 &&
4848245274Snp	    ((t->idx & 0x3) || t->idx + 4 >= nfilters)) {
4849245274Snp		rc = EINVAL;
4850245274Snp		goto done;
4851245274Snp	}
4852221474Snp
4853221474Snp	if (sc->tids.ftid_tab == NULL) {
4854221474Snp		KASSERT(sc->tids.ftids_in_use == 0,
4855221474Snp		    ("%s: no memory allocated but filters_in_use > 0",
4856221474Snp		    __func__));
4857221474Snp
4858221474Snp		sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
4859221474Snp		    nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
4860245274Snp		if (sc->tids.ftid_tab == NULL) {
4861245274Snp			rc = ENOMEM;
4862245274Snp			goto done;
4863245274Snp		}
4864245274Snp		mtx_init(&sc->tids.ftid_lock, "T4 filters", 0, MTX_DEF);
4865221474Snp	}
4866221474Snp
4867221474Snp	for (i = 0; i < 4; i++) {
4868221474Snp		f = &sc->tids.ftid_tab[t->idx + i];
4869221474Snp
4870245274Snp		if (f->pending || f->valid) {
4871245274Snp			rc = EBUSY;
4872245274Snp			goto done;
4873245274Snp		}
4874245274Snp		if (f->locked) {
4875245274Snp			rc = EPERM;
4876245274Snp			goto done;
4877245274Snp		}
4878221474Snp
4879221474Snp		if (t->fs.type == 0)
4880221474Snp			break;
4881221474Snp	}
4882221474Snp
4883221474Snp	f = &sc->tids.ftid_tab[t->idx];
4884221474Snp	f->fs = t->fs;
4885221474Snp
4886245274Snp	rc = set_filter_wr(sc, t->idx);
4887245274Snpdone:
4888245274Snp	end_synchronized_op(sc, 0);
4889245274Snp
4890245274Snp	if (rc == 0) {
4891245274Snp		mtx_lock(&sc->tids.ftid_lock);
4892245274Snp		for (;;) {
4893245274Snp			if (f->pending == 0) {
4894245274Snp				rc = f->valid ? 0 : EIO;
4895245274Snp				break;
4896245274Snp			}
4897245274Snp
4898245274Snp			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
4899245274Snp			    PCATCH, "t4setfw", 0)) {
4900245274Snp				rc = EINPROGRESS;
4901245274Snp				break;
4902245274Snp			}
4903245274Snp		}
4904245274Snp		mtx_unlock(&sc->tids.ftid_lock);
4905245274Snp	}
4906245274Snp	return (rc);
4907221474Snp}
4908221474Snp
4909221474Snpstatic int
4910221474Snpdel_filter(struct adapter *sc, struct t4_filter *t)
4911221474Snp{
4912221474Snp	unsigned int nfilters;
4913221474Snp	struct filter_entry *f;
4914245274Snp	int rc;
4915221474Snp
4916245274Snp	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4delf");
4917245274Snp	if (rc)
4918245274Snp		return (rc);
4919221474Snp
4920221474Snp	nfilters = sc->tids.nftids;
4921221474Snp
4922245274Snp	if (nfilters == 0) {
4923245274Snp		rc = ENOTSUP;
4924245274Snp		goto done;
4925245274Snp	}
4926221474Snp
4927221474Snp	if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
4928245274Snp	    t->idx >= nfilters) {
4929245274Snp		rc = EINVAL;
4930245274Snp		goto done;
4931245274Snp	}
4932221474Snp
4933245274Snp	if (!(sc->flags & FULL_INIT_DONE)) {
4934245274Snp		rc = EAGAIN;
4935245274Snp		goto done;
4936245274Snp	}
4937221474Snp
4938221474Snp	f = &sc->tids.ftid_tab[t->idx];
4939221474Snp
4940245274Snp	if (f->pending) {
4941245274Snp		rc = EBUSY;
4942245274Snp		goto done;
4943245274Snp	}
4944245274Snp	if (f->locked) {
4945245274Snp		rc = EPERM;
4946245274Snp		goto done;
4947245274Snp	}
4948221474Snp
4949221474Snp	if (f->valid) {
4950221474Snp		t->fs = f->fs;	/* extra info for the caller */
4951245274Snp		rc = del_filter_wr(sc, t->idx);
4952221474Snp	}
4953221474Snp
4954245274Snpdone:
4955245274Snp	end_synchronized_op(sc, 0);
4956245274Snp
4957245274Snp	if (rc == 0) {
4958245274Snp		mtx_lock(&sc->tids.ftid_lock);
4959245274Snp		for (;;) {
4960245274Snp			if (f->pending == 0) {
4961245274Snp				rc = f->valid ? EIO : 0;
4962245274Snp				break;
4963245274Snp			}
4964245274Snp
4965245274Snp			if (mtx_sleep(&sc->tids.ftid_tab, &sc->tids.ftid_lock,
4966245274Snp			    PCATCH, "t4delfw", 0)) {
4967245274Snp				rc = EINPROGRESS;
4968245274Snp				break;
4969245274Snp			}
4970245274Snp		}
4971245274Snp		mtx_unlock(&sc->tids.ftid_lock);
4972245274Snp	}
4973245274Snp
4974245274Snp	return (rc);
4975221474Snp}
4976221474Snp
4977221474Snpstatic void
4978222509Snpclear_filter(struct filter_entry *f)
4979221474Snp{
4980222509Snp	if (f->l2t)
4981222509Snp		t4_l2t_release(f->l2t);
4982222509Snp
4983221474Snp	bzero(f, sizeof (*f));
4984221474Snp}
4985221474Snp
4986221474Snpstatic int
4987221474Snpset_filter_wr(struct adapter *sc, int fidx)
4988221474Snp{
4989221474Snp	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
4990237263Snp	struct wrqe *wr;
4991221474Snp	struct fw_filter_wr *fwr;
4992221474Snp	unsigned int ftid;
4993221474Snp
4994245274Snp	ASSERT_SYNCHRONIZED_OP(sc);
4995221474Snp
4996222509Snp	if (f->fs.newdmac || f->fs.newvlan) {
4997222509Snp		/* This filter needs an L2T entry; allocate one. */
4998222509Snp		f->l2t = t4_l2t_alloc_switching(sc->l2t);
4999222509Snp		if (f->l2t == NULL)
5000222509Snp			return (EAGAIN);
5001222509Snp		if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
5002222509Snp		    f->fs.dmac)) {
5003222509Snp			t4_l2t_release(f->l2t);
5004222509Snp			f->l2t = NULL;
5005222509Snp			return (ENOMEM);
5006222509Snp		}
5007222509Snp	}
5008221474Snp
5009221474Snp	ftid = sc->tids.ftid_base + fidx;
5010221474Snp
5011237263Snp	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
5012237263Snp	if (wr == NULL)
5013221474Snp		return (ENOMEM);
5014221474Snp
5015237263Snp	fwr = wrtod(wr);
5016221474Snp	bzero(fwr, sizeof (*fwr));
5017221474Snp
5018221474Snp	fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
5019221474Snp	fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
5020221474Snp	fwr->tid_to_iq =
5021221474Snp	    htobe32(V_FW_FILTER_WR_TID(ftid) |
5022221474Snp		V_FW_FILTER_WR_RQTYPE(f->fs.type) |
5023221474Snp		V_FW_FILTER_WR_NOREPLY(0) |
5024221474Snp		V_FW_FILTER_WR_IQ(f->fs.iq));
5025221474Snp	fwr->del_filter_to_l2tix =
5026221474Snp	    htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
5027221474Snp		V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
5028221474Snp		V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
5029221474Snp		V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
5030221474Snp		V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
5031221474Snp		V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
5032221474Snp		V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
5033221474Snp		V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
5034221474Snp		V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
5035221474Snp		    f->fs.newvlan == VLAN_REWRITE) |
5036221474Snp		V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
5037221474Snp		    f->fs.newvlan == VLAN_REWRITE) |
5038221474Snp		V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
5039221474Snp		V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
5040221474Snp		V_FW_FILTER_WR_PRIO(f->fs.prio) |
5041222509Snp		V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
5042221474Snp	fwr->ethtype = htobe16(f->fs.val.ethtype);
5043221474Snp	fwr->ethtypem = htobe16(f->fs.mask.ethtype);
5044221474Snp	fwr->frag_to_ovlan_vldm =
5045221474Snp	    (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
5046221474Snp		V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
5047228561Snp		V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
5048228561Snp		V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
5049228561Snp		V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
5050228561Snp		V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
5051221474Snp	fwr->smac_sel = 0;
5052221474Snp	fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
5053228561Snp	    V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
5054221474Snp	fwr->maci_to_matchtypem =
5055221474Snp	    htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
5056221474Snp		V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
5057221474Snp		V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
5058221474Snp		V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
5059221474Snp		V_FW_FILTER_WR_PORT(f->fs.val.iport) |
5060221474Snp		V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
5061221474Snp		V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
5062221474Snp		V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
5063221474Snp	fwr->ptcl = f->fs.val.proto;
5064221474Snp	fwr->ptclm = f->fs.mask.proto;
5065221474Snp	fwr->ttyp = f->fs.val.tos;
5066221474Snp	fwr->ttypm = f->fs.mask.tos;
5067228561Snp	fwr->ivlan = htobe16(f->fs.val.vlan);
5068228561Snp	fwr->ivlanm = htobe16(f->fs.mask.vlan);
5069228561Snp	fwr->ovlan = htobe16(f->fs.val.vnic);
5070228561Snp	fwr->ovlanm = htobe16(f->fs.mask.vnic);
5071221474Snp	bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
5072221474Snp	bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
5073221474Snp	bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
5074221474Snp	bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
5075221474Snp	fwr->lp = htobe16(f->fs.val.dport);
5076221474Snp	fwr->lpm = htobe16(f->fs.mask.dport);
5077221474Snp	fwr->fp = htobe16(f->fs.val.sport);
5078221474Snp	fwr->fpm = htobe16(f->fs.mask.sport);
5079221474Snp	if (f->fs.newsmac)
5080221474Snp		bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
5081221474Snp
5082221474Snp	f->pending = 1;
5083221474Snp	sc->tids.ftids_in_use++;
5084228561Snp
5085237263Snp	t4_wrq_tx(sc, wr);
5086228561Snp	return (0);
5087221474Snp}
5088221474Snp
5089221474Snpstatic int
5090221474Snpdel_filter_wr(struct adapter *sc, int fidx)
5091221474Snp{
5092221474Snp	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
5093237263Snp	struct wrqe *wr;
5094221474Snp	struct fw_filter_wr *fwr;
5095228561Snp	unsigned int ftid;
5096221474Snp
5097221474Snp	ftid = sc->tids.ftid_base + fidx;
5098221474Snp
5099237263Snp	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
5100237263Snp	if (wr == NULL)
5101221474Snp		return (ENOMEM);
5102237263Snp	fwr = wrtod(wr);
5103221474Snp	bzero(fwr, sizeof (*fwr));
5104221474Snp
5105228561Snp	t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
5106221474Snp
5107221474Snp	f->pending = 1;
5108237263Snp	t4_wrq_tx(sc, wr);
5109228561Snp	return (0);
5110221474Snp}
5111221474Snp
5112239338Snpint
5113239338Snpt4_filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
5114221474Snp{
5115228561Snp	struct adapter *sc = iq->adapter;
5116228561Snp	const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
5117221474Snp	unsigned int idx = GET_TID(rpl);
5118221474Snp
5119228561Snp	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
5120228561Snp	    rss->opcode));
5121228561Snp
5122221474Snp	if (idx >= sc->tids.ftid_base &&
5123221474Snp	    (idx -= sc->tids.ftid_base) < sc->tids.nftids) {
5124221474Snp		unsigned int rc = G_COOKIE(rpl->cookie);
5125221474Snp		struct filter_entry *f = &sc->tids.ftid_tab[idx];
5126221474Snp
5127245274Snp		mtx_lock(&sc->tids.ftid_lock);
5128228561Snp		if (rc == FW_FILTER_WR_FLT_ADDED) {
5129245274Snp			KASSERT(f->pending, ("%s: filter[%u] isn't pending.",
5130245274Snp			    __func__, idx));
5131221474Snp			f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
5132221474Snp			f->pending = 0;  /* asynchronous setup completed */
5133221474Snp			f->valid = 1;
5134231120Snp		} else {
5135231120Snp			if (rc != FW_FILTER_WR_FLT_DELETED) {
5136231120Snp				/* Add or delete failed, display an error */
5137231120Snp				log(LOG_ERR,
5138231120Snp				    "filter %u setup failed with error %u\n",
5139231120Snp				    idx, rc);
5140231120Snp			}
5141228561Snp
5142231120Snp			clear_filter(f);
5143231120Snp			sc->tids.ftids_in_use--;
5144221474Snp		}
5145245274Snp		wakeup(&sc->tids.ftid_tab);
5146245274Snp		mtx_unlock(&sc->tids.ftid_lock);
5147221474Snp	}
5148228561Snp
5149228561Snp	return (0);
5150221474Snp}
5151221474Snp
5152222973Snpstatic int
5153222973Snpget_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
5154222973Snp{
5155245274Snp	int rc;
5156222973Snp
5157222973Snp	if (cntxt->cid > M_CTXTQID)
5158245274Snp		return (EINVAL);
5159222973Snp
5160222973Snp	if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
5161222973Snp	    cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
5162245274Snp		return (EINVAL);
5163222973Snp
5164222973Snp	if (sc->flags & FW_OK) {
5165245274Snp		rc = begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4ctxt");
5166245274Snp		if (rc == 0) {
5167245274Snp			rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid,
5168245274Snp			    cntxt->mem_id, &cntxt->data[0]);
5169245274Snp			end_synchronized_op(sc, LOCK_HELD);
5170245274Snp			if (rc == 0)
5171245274Snp				return (0);
5172245274Snp		}
5173222973Snp	}
5174222973Snp
5175245274Snp	/*
5176245274Snp	 * Read via firmware failed or wasn't even attempted.  Read directly via
5177245274Snp	 * the backdoor.
5178245274Snp	 */
5179245274Snp	rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id,
5180245274Snp	    &cntxt->data[0]);
5181245274Snp	return (rc);
5182245274Snp}
5183222973Snp
5184245274Snpstatic int
5185245274Snpload_fw(struct adapter *sc, struct t4_data *fw)
5186245274Snp{
5187245274Snp	int rc;
5188245274Snp	uint8_t *fw_data;
5189245274Snp
5190245274Snp	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4ldfw");
5191245274Snp	if (rc)
5192245274Snp		return (rc);
5193245274Snp
5194245274Snp	if (sc->flags & FULL_INIT_DONE) {
5195245274Snp		rc = EBUSY;
5196245274Snp		goto done;
5197222973Snp	}
5198222973Snp
5199245274Snp	fw_data = malloc(fw->len, M_CXGBE, M_WAITOK);
5200245274Snp	if (fw_data == NULL) {
5201245274Snp		rc = ENOMEM;
5202245274Snp		goto done;
5203245274Snp	}
5204245274Snp
5205245274Snp	rc = copyin(fw->data, fw_data, fw->len);
5206245274Snp	if (rc == 0)
5207245274Snp		rc = -t4_load_fw(sc, fw_data, fw->len);
5208245274Snp
5209245274Snp	free(fw_data, M_CXGBE);
5210245274Snpdone:
5211245274Snp	end_synchronized_op(sc, 0);
5212222973Snp	return (rc);
5213222973Snp}
5214222973Snp
5215228561Snpstatic int
5216228561Snpread_card_mem(struct adapter *sc, struct t4_mem_range *mr)
5217228561Snp{
5218228561Snp	uint32_t base, size, lo, hi, win, off, remaining, i, n;
5219228561Snp	uint32_t *buf, *b;
5220228561Snp	int rc;
5221228561Snp
5222228561Snp	/* reads are in multiples of 32 bits */
5223228561Snp	if (mr->addr & 3 || mr->len & 3 || mr->len == 0)
5224228561Snp		return (EINVAL);
5225228561Snp
5226228561Snp	/*
5227228561Snp	 * We don't want to deal with potential holes so we mandate that the
5228228561Snp	 * requested region must lie entirely within one of the 3 memories.
5229228561Snp	 */
5230228561Snp	lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
5231228561Snp	if (lo & F_EDRAM0_ENABLE) {
5232228561Snp		hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5233228561Snp		base = G_EDRAM0_BASE(hi) << 20;
5234228561Snp		size = G_EDRAM0_SIZE(hi) << 20;
5235228561Snp		if (size > 0 &&
5236228561Snp		    mr->addr >= base && mr->addr < base + size &&
5237228561Snp		    mr->addr + mr->len <= base + size)
5238228561Snp			goto proceed;
5239228561Snp	}
5240228561Snp	if (lo & F_EDRAM1_ENABLE) {
5241228561Snp		hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5242228561Snp		base = G_EDRAM1_BASE(hi) << 20;
5243228561Snp		size = G_EDRAM1_SIZE(hi) << 20;
5244228561Snp		if (size > 0 &&
5245228561Snp		    mr->addr >= base && mr->addr < base + size &&
5246228561Snp		    mr->addr + mr->len <= base + size)
5247228561Snp			goto proceed;
5248228561Snp	}
5249228561Snp	if (lo & F_EXT_MEM_ENABLE) {
5250228561Snp		hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5251228561Snp		base = G_EXT_MEM_BASE(hi) << 20;
5252228561Snp		size = G_EXT_MEM_SIZE(hi) << 20;
5253228561Snp		if (size > 0 &&
5254228561Snp		    mr->addr >= base && mr->addr < base + size &&
5255228561Snp		    mr->addr + mr->len <= base + size)
5256228561Snp			goto proceed;
5257228561Snp	}
5258228561Snp	return (ENXIO);
5259228561Snp
5260228561Snpproceed:
5261228561Snp	buf = b = malloc(mr->len, M_CXGBE, M_WAITOK);
5262228561Snp
5263228561Snp	/*
5264228561Snp	 * Position the PCIe window (we use memwin2) to the 16B aligned area
5265228561Snp	 * just at/before the requested region.
5266228561Snp	 */
5267228561Snp	win = mr->addr & ~0xf;
5268228561Snp	off = mr->addr - win;  /* offset of the requested region in the win */
5269228561Snp	remaining = mr->len;
5270228561Snp
5271228561Snp	while (remaining) {
5272228561Snp		t4_write_reg(sc,
5273228561Snp		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
5274228561Snp		t4_read_reg(sc,
5275228561Snp		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
5276228561Snp
5277228561Snp		/* number of bytes that we'll copy in the inner loop */
5278228561Snp		n = min(remaining, MEMWIN2_APERTURE - off);
5279228561Snp
5280228561Snp		for (i = 0; i < n; i += 4, remaining -= 4)
5281228561Snp			*b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i);
5282228561Snp
5283228561Snp		win += MEMWIN2_APERTURE;
5284228561Snp		off = 0;
5285228561Snp	}
5286228561Snp
5287228561Snp	rc = copyout(buf, mr->data, mr->len);
5288228561Snp	free(buf, M_CXGBE);
5289228561Snp
5290228561Snp	return (rc);
5291228561Snp}
5292228561Snp
5293241399Snpstatic int
5294241399Snpread_i2c(struct adapter *sc, struct t4_i2c_data *i2cd)
5295241399Snp{
5296241399Snp	int rc;
5297241399Snp
5298241399Snp	if (i2cd->len == 0 || i2cd->port_id >= sc->params.nports)
5299241399Snp		return (EINVAL);
5300241399Snp
5301241399Snp	if (i2cd->len > 1) {
5302241399Snp		/* XXX: need fw support for longer reads in one go */
5303241399Snp		return (ENOTSUP);
5304241399Snp	}
5305241399Snp
5306245274Snp	rc = begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4i2crd");
5307245274Snp	if (rc)
5308245274Snp		return (rc);
5309241399Snp	rc = -t4_i2c_rd(sc, sc->mbox, i2cd->port_id, i2cd->dev_addr,
5310241399Snp	    i2cd->offset, &i2cd->data[0]);
5311245274Snp	end_synchronized_op(sc, 0);
5312241399Snp
5313241399Snp	return (rc);
5314241399Snp}
5315241399Snp
5316218792Snpint
5317218792Snpt4_os_find_pci_capability(struct adapter *sc, int cap)
5318218792Snp{
5319222102Snp	int i;
5320218792Snp
5321222102Snp	return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
5322218792Snp}
5323218792Snp
5324218792Snpint
5325218792Snpt4_os_pci_save_state(struct adapter *sc)
5326218792Snp{
5327218792Snp	device_t dev;
5328218792Snp	struct pci_devinfo *dinfo;
5329218792Snp
5330218792Snp	dev = sc->dev;
5331218792Snp	dinfo = device_get_ivars(dev);
5332218792Snp
5333218792Snp	pci_cfg_save(dev, dinfo, 0);
5334218792Snp	return (0);
5335218792Snp}
5336218792Snp
5337218792Snpint
5338218792Snpt4_os_pci_restore_state(struct adapter *sc)
5339218792Snp{
5340218792Snp	device_t dev;
5341218792Snp	struct pci_devinfo *dinfo;
5342218792Snp
5343218792Snp	dev = sc->dev;
5344218792Snp	dinfo = device_get_ivars(dev);
5345218792Snp
5346218792Snp	pci_cfg_restore(dev, dinfo);
5347218792Snp	return (0);
5348218792Snp}
5349219299Snp
5350218792Snpvoid
5351218792Snpt4_os_portmod_changed(const struct adapter *sc, int idx)
5352218792Snp{
5353218792Snp	struct port_info *pi = sc->port[idx];
5354218792Snp	static const char *mod_str[] = {
5355220232Snp		NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
5356218792Snp	};
5357218792Snp
5358218792Snp	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
5359218792Snp		if_printf(pi->ifp, "transceiver unplugged.\n");
5360220232Snp	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
5361220232Snp		if_printf(pi->ifp, "unknown transceiver inserted.\n");
5362220232Snp	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
5363220232Snp		if_printf(pi->ifp, "unsupported transceiver inserted.\n");
5364240452Snp	else if (pi->mod_type > 0 && pi->mod_type < nitems(mod_str)) {
5365218792Snp		if_printf(pi->ifp, "%s transceiver inserted.\n",
5366218792Snp		    mod_str[pi->mod_type]);
5367219299Snp	} else {
5368219299Snp		if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
5369219299Snp		    pi->mod_type);
5370219299Snp	}
5371218792Snp}
5372218792Snp
5373218792Snpvoid
5374218792Snpt4_os_link_changed(struct adapter *sc, int idx, int link_stat)
5375218792Snp{
5376218792Snp	struct port_info *pi = sc->port[idx];
5377218792Snp	struct ifnet *ifp = pi->ifp;
5378218792Snp
5379218792Snp	if (link_stat) {
5380218792Snp		ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
5381218792Snp		if_link_state_change(ifp, LINK_STATE_UP);
5382218792Snp	} else
5383218792Snp		if_link_state_change(ifp, LINK_STATE_DOWN);
5384218792Snp}
5385218792Snp
5386228561Snpvoid
5387228561Snpt4_iterate(void (*func)(struct adapter *, void *), void *arg)
5388228561Snp{
5389228561Snp	struct adapter *sc;
5390228561Snp
5391228561Snp	mtx_lock(&t4_list_lock);
5392228561Snp	SLIST_FOREACH(sc, &t4_list, link) {
5393228561Snp		/*
5394228561Snp		 * func should not make any assumptions about what state sc is
5395228561Snp		 * in - the only guarantee is that sc->sc_lock is a valid lock.
5396228561Snp		 */
5397228561Snp		func(sc, arg);
5398228561Snp	}
5399228561Snp	mtx_unlock(&t4_list_lock);
5400228561Snp}
5401228561Snp
5402218792Snpstatic int
5403218792Snpt4_open(struct cdev *dev, int flags, int type, struct thread *td)
5404218792Snp{
5405218792Snp       return (0);
5406218792Snp}
5407218792Snp
5408218792Snpstatic int
5409218792Snpt4_close(struct cdev *dev, int flags, int type, struct thread *td)
5410218792Snp{
5411218792Snp       return (0);
5412218792Snp}
5413218792Snp
5414218792Snpstatic int
5415218792Snpt4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
5416218792Snp    struct thread *td)
5417218792Snp{
5418218792Snp	int rc;
5419218792Snp	struct adapter *sc = dev->si_drv1;
5420218792Snp
5421218792Snp	rc = priv_check(td, PRIV_DRIVER);
5422218792Snp	if (rc != 0)
5423218792Snp		return (rc);
5424218792Snp
5425218792Snp	switch (cmd) {
5426220410Snp	case CHELSIO_T4_GETREG: {
5427220410Snp		struct t4_reg *edata = (struct t4_reg *)data;
5428220410Snp
5429218792Snp		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5430218792Snp			return (EFAULT);
5431220410Snp
5432220410Snp		if (edata->size == 4)
5433220410Snp			edata->val = t4_read_reg(sc, edata->addr);
5434220410Snp		else if (edata->size == 8)
5435220410Snp			edata->val = t4_read_reg64(sc, edata->addr);
5436220410Snp		else
5437220410Snp			return (EINVAL);
5438220410Snp
5439218792Snp		break;
5440218792Snp	}
5441220410Snp	case CHELSIO_T4_SETREG: {
5442220410Snp		struct t4_reg *edata = (struct t4_reg *)data;
5443220410Snp
5444218792Snp		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5445218792Snp			return (EFAULT);
5446220410Snp
5447220410Snp		if (edata->size == 4) {
5448220410Snp			if (edata->val & 0xffffffff00000000)
5449220410Snp				return (EINVAL);
5450220410Snp			t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
5451220410Snp		} else if (edata->size == 8)
5452220410Snp			t4_write_reg64(sc, edata->addr, edata->val);
5453220410Snp		else
5454220410Snp			return (EINVAL);
5455218792Snp		break;
5456218792Snp	}
5457218792Snp	case CHELSIO_T4_REGDUMP: {
5458218792Snp		struct t4_regdump *regs = (struct t4_regdump *)data;
5459218792Snp		int reglen = T4_REGDUMP_SIZE;
5460218792Snp		uint8_t *buf;
5461218792Snp
5462218792Snp		if (regs->len < reglen) {
5463218792Snp			regs->len = reglen; /* hint to the caller */
5464218792Snp			return (ENOBUFS);
5465218792Snp		}
5466218792Snp
5467218792Snp		regs->len = reglen;
5468218792Snp		buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
5469218792Snp		t4_get_regs(sc, regs, buf);
5470218792Snp		rc = copyout(buf, regs->data, reglen);
5471218792Snp		free(buf, M_CXGBE);
5472218792Snp		break;
5473218792Snp	}
5474221474Snp	case CHELSIO_T4_GET_FILTER_MODE:
5475221474Snp		rc = get_filter_mode(sc, (uint32_t *)data);
5476221474Snp		break;
5477221474Snp	case CHELSIO_T4_SET_FILTER_MODE:
5478221474Snp		rc = set_filter_mode(sc, *(uint32_t *)data);
5479221474Snp		break;
5480221474Snp	case CHELSIO_T4_GET_FILTER:
5481221474Snp		rc = get_filter(sc, (struct t4_filter *)data);
5482221474Snp		break;
5483221474Snp	case CHELSIO_T4_SET_FILTER:
5484221474Snp		rc = set_filter(sc, (struct t4_filter *)data);
5485221474Snp		break;
5486221474Snp	case CHELSIO_T4_DEL_FILTER:
5487221474Snp		rc = del_filter(sc, (struct t4_filter *)data);
5488221474Snp		break;
5489222973Snp	case CHELSIO_T4_GET_SGE_CONTEXT:
5490222973Snp		rc = get_sge_context(sc, (struct t4_sge_context *)data);
5491222973Snp		break;
5492245274Snp	case CHELSIO_T4_LOAD_FW:
5493245274Snp		rc = load_fw(sc, (struct t4_data *)data);
5494228561Snp		break;
5495228561Snp	case CHELSIO_T4_GET_MEM:
5496228561Snp		rc = read_card_mem(sc, (struct t4_mem_range *)data);
5497228561Snp		break;
5498241399Snp	case CHELSIO_T4_GET_I2C:
5499241399Snp		rc = read_i2c(sc, (struct t4_i2c_data *)data);
5500241399Snp		break;
5501241409Snp	case CHELSIO_T4_CLEAR_STATS: {
5502245518Snp		int i;
5503241409Snp		u_int port_id = *(uint32_t *)data;
5504245518Snp		struct port_info *pi;
5505241409Snp
5506241409Snp		if (port_id >= sc->params.nports)
5507241409Snp			return (EINVAL);
5508241409Snp
5509245518Snp		/* MAC stats */
5510241409Snp		t4_clr_port_stats(sc, port_id);
5511245518Snp
5512245518Snp		pi = sc->port[port_id];
5513245518Snp		if (pi->flags & PORT_INIT_DONE) {
5514245518Snp			struct sge_rxq *rxq;
5515245518Snp			struct sge_txq *txq;
5516245518Snp			struct sge_wrq *wrq;
5517245518Snp
5518245518Snp			for_each_rxq(pi, i, rxq) {
5519245518Snp#if defined(INET) || defined(INET6)
5520245518Snp				rxq->lro.lro_queued = 0;
5521245518Snp				rxq->lro.lro_flushed = 0;
5522245518Snp#endif
5523245518Snp				rxq->rxcsum = 0;
5524245518Snp				rxq->vlan_extraction = 0;
5525245518Snp			}
5526245518Snp
5527245518Snp			for_each_txq(pi, i, txq) {
5528245518Snp				txq->txcsum = 0;
5529245518Snp				txq->tso_wrs = 0;
5530245518Snp				txq->vlan_insertion = 0;
5531245518Snp				txq->imm_wrs = 0;
5532245518Snp				txq->sgl_wrs = 0;
5533245518Snp				txq->txpkt_wrs = 0;
5534245518Snp				txq->txpkts_wrs = 0;
5535245518Snp				txq->txpkts_pkts = 0;
5536246093Snp				txq->br->br_drops = 0;
5537245518Snp				txq->no_dmamap = 0;
5538245518Snp				txq->no_desc = 0;
5539245518Snp			}
5540245518Snp
5541245518Snp#ifdef TCP_OFFLOAD
5542245518Snp			/* nothing to clear for each ofld_rxq */
5543245518Snp
5544245518Snp			for_each_ofld_txq(pi, i, wrq) {
5545245518Snp				wrq->tx_wrs = 0;
5546245518Snp				wrq->no_desc = 0;
5547245518Snp			}
5548245518Snp#endif
5549245518Snp			wrq = &sc->sge.ctrlq[pi->port_id];
5550245518Snp			wrq->tx_wrs = 0;
5551245518Snp			wrq->no_desc = 0;
5552245518Snp		}
5553241409Snp		break;
5554241409Snp	}
5555218792Snp	default:
5556218792Snp		rc = EINVAL;
5557218792Snp	}
5558218792Snp
5559218792Snp	return (rc);
5560218792Snp}
5561218792Snp
5562237263Snp#ifdef TCP_OFFLOAD
5563219392Snpstatic int
5564228561Snptoe_capability(struct port_info *pi, int enable)
5565228561Snp{
5566228561Snp	int rc;
5567228561Snp	struct adapter *sc = pi->adapter;
5568228561Snp
5569245274Snp	ASSERT_SYNCHRONIZED_OP(sc);
5570228561Snp
5571228561Snp	if (!is_offload(sc))
5572228561Snp		return (ENODEV);
5573228561Snp
5574228561Snp	if (enable) {
5575237263Snp		if (!(sc->flags & FULL_INIT_DONE)) {
5576245274Snp			rc = cxgbe_init_synchronized(pi);
5577245274Snp			if (rc)
5578245274Snp				return (rc);
5579237263Snp		}
5580237263Snp
5581228561Snp		if (isset(&sc->offload_map, pi->port_id))
5582228561Snp			return (0);
5583228561Snp
5584237263Snp		if (!(sc->flags & TOM_INIT_DONE)) {
5585237263Snp			rc = t4_activate_uld(sc, ULD_TOM);
5586237263Snp			if (rc == EAGAIN) {
5587237263Snp				log(LOG_WARNING,
5588237263Snp				    "You must kldload t4_tom.ko before trying "
5589237263Snp				    "to enable TOE on a cxgbe interface.\n");
5590237263Snp			}
5591228561Snp			if (rc != 0)
5592228561Snp				return (rc);
5593237263Snp			KASSERT(sc->tom_softc != NULL,
5594237263Snp			    ("%s: TOM activated but softc NULL", __func__));
5595237263Snp			KASSERT(sc->flags & TOM_INIT_DONE,
5596237263Snp			    ("%s: TOM activated but flag not set", __func__));
5597228561Snp		}
5598228561Snp
5599228561Snp		setbit(&sc->offload_map, pi->port_id);
5600228561Snp	} else {
5601228561Snp		if (!isset(&sc->offload_map, pi->port_id))
5602228561Snp			return (0);
5603228561Snp
5604237263Snp		KASSERT(sc->flags & TOM_INIT_DONE,
5605237263Snp		    ("%s: TOM never initialized?", __func__));
5606228561Snp		clrbit(&sc->offload_map, pi->port_id);
5607228561Snp	}
5608228561Snp
5609228561Snp	return (0);
5610228561Snp}
5611228561Snp
5612228561Snp/*
5613228561Snp * Add an upper layer driver to the global list.
5614228561Snp */
5615228561Snpint
5616228561Snpt4_register_uld(struct uld_info *ui)
5617228561Snp{
5618228561Snp	int rc = 0;
5619228561Snp	struct uld_info *u;
5620228561Snp
5621228561Snp	mtx_lock(&t4_uld_list_lock);
5622228561Snp	SLIST_FOREACH(u, &t4_uld_list, link) {
5623228561Snp	    if (u->uld_id == ui->uld_id) {
5624228561Snp		    rc = EEXIST;
5625228561Snp		    goto done;
5626228561Snp	    }
5627228561Snp	}
5628228561Snp
5629228561Snp	SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
5630228561Snp	ui->refcount = 0;
5631228561Snpdone:
5632228561Snp	mtx_unlock(&t4_uld_list_lock);
5633228561Snp	return (rc);
5634228561Snp}
5635228561Snp
5636228561Snpint
5637228561Snpt4_unregister_uld(struct uld_info *ui)
5638228561Snp{
5639228561Snp	int rc = EINVAL;
5640228561Snp	struct uld_info *u;
5641228561Snp
5642228561Snp	mtx_lock(&t4_uld_list_lock);
5643228561Snp
5644228561Snp	SLIST_FOREACH(u, &t4_uld_list, link) {
5645228561Snp	    if (u == ui) {
5646228561Snp		    if (ui->refcount > 0) {
5647228561Snp			    rc = EBUSY;
5648228561Snp			    goto done;
5649228561Snp		    }
5650228561Snp
5651228561Snp		    SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
5652228561Snp		    rc = 0;
5653228561Snp		    goto done;
5654228561Snp	    }
5655228561Snp	}
5656228561Snpdone:
5657228561Snp	mtx_unlock(&t4_uld_list_lock);
5658228561Snp	return (rc);
5659228561Snp}
5660228561Snp
5661237263Snpint
5662237263Snpt4_activate_uld(struct adapter *sc, int id)
5663228561Snp{
5664228561Snp	int rc = EAGAIN;
5665228561Snp	struct uld_info *ui;
5666228561Snp
5667245274Snp	ASSERT_SYNCHRONIZED_OP(sc);
5668245274Snp
5669228561Snp	mtx_lock(&t4_uld_list_lock);
5670228561Snp
5671228561Snp	SLIST_FOREACH(ui, &t4_uld_list, link) {
5672228561Snp		if (ui->uld_id == id) {
5673237263Snp			rc = ui->activate(sc);
5674237263Snp			if (rc == 0)
5675228561Snp				ui->refcount++;
5676228561Snp			goto done;
5677228561Snp		}
5678228561Snp	}
5679228561Snpdone:
5680228561Snp	mtx_unlock(&t4_uld_list_lock);
5681228561Snp
5682228561Snp	return (rc);
5683228561Snp}
5684228561Snp
5685237263Snpint
5686237263Snpt4_deactivate_uld(struct adapter *sc, int id)
5687228561Snp{
5688237263Snp	int rc = EINVAL;
5689237263Snp	struct uld_info *ui;
5690228561Snp
5691245274Snp	ASSERT_SYNCHRONIZED_OP(sc);
5692245274Snp
5693228561Snp	mtx_lock(&t4_uld_list_lock);
5694228561Snp
5695237263Snp	SLIST_FOREACH(ui, &t4_uld_list, link) {
5696237263Snp		if (ui->uld_id == id) {
5697237263Snp			rc = ui->deactivate(sc);
5698237263Snp			if (rc == 0)
5699237263Snp				ui->refcount--;
5700237263Snp			goto done;
5701237263Snp		}
5702228561Snp	}
5703228561Snpdone:
5704228561Snp	mtx_unlock(&t4_uld_list_lock);
5705228561Snp
5706228561Snp	return (rc);
5707228561Snp}
5708228561Snp#endif
5709228561Snp
5710228561Snp/*
5711228561Snp * Come up with reasonable defaults for some of the tunables, provided they're
5712228561Snp * not set by the user (in which case we'll use the values as is).
5713228561Snp */
5714228561Snpstatic void
5715228561Snptweak_tunables(void)
5716228561Snp{
5717228561Snp	int nc = mp_ncpus;	/* our snapshot of the number of CPUs */
5718228561Snp
5719228561Snp	if (t4_ntxq10g < 1)
5720228561Snp		t4_ntxq10g = min(nc, NTXQ_10G);
5721228561Snp
5722228561Snp	if (t4_ntxq1g < 1)
5723228561Snp		t4_ntxq1g = min(nc, NTXQ_1G);
5724228561Snp
5725228561Snp	if (t4_nrxq10g < 1)
5726228561Snp		t4_nrxq10g = min(nc, NRXQ_10G);
5727228561Snp
5728228561Snp	if (t4_nrxq1g < 1)
5729228561Snp		t4_nrxq1g = min(nc, NRXQ_1G);
5730228561Snp
5731237263Snp#ifdef TCP_OFFLOAD
5732228561Snp	if (t4_nofldtxq10g < 1)
5733228561Snp		t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
5734228561Snp
5735228561Snp	if (t4_nofldtxq1g < 1)
5736228561Snp		t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
5737228561Snp
5738228561Snp	if (t4_nofldrxq10g < 1)
5739228561Snp		t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
5740228561Snp
5741228561Snp	if (t4_nofldrxq1g < 1)
5742228561Snp		t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
5743238028Snp
5744238028Snp	if (t4_toecaps_allowed == -1)
5745238028Snp		t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
5746238028Snp#else
5747238028Snp	if (t4_toecaps_allowed == -1)
5748238028Snp		t4_toecaps_allowed = 0;
5749228561Snp#endif
5750228561Snp
5751228561Snp	if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
5752228561Snp		t4_tmr_idx_10g = TMR_IDX_10G;
5753228561Snp
5754228561Snp	if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
5755228561Snp		t4_pktc_idx_10g = PKTC_IDX_10G;
5756228561Snp
5757228561Snp	if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
5758228561Snp		t4_tmr_idx_1g = TMR_IDX_1G;
5759228561Snp
5760228561Snp	if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
5761228561Snp		t4_pktc_idx_1g = PKTC_IDX_1G;
5762228561Snp
5763228561Snp	if (t4_qsize_txq < 128)
5764228561Snp		t4_qsize_txq = 128;
5765228561Snp
5766228561Snp	if (t4_qsize_rxq < 128)
5767228561Snp		t4_qsize_rxq = 128;
5768228561Snp	while (t4_qsize_rxq & 7)
5769228561Snp		t4_qsize_rxq++;
5770228561Snp
5771228561Snp	t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
5772228561Snp}
5773228561Snp
5774228561Snpstatic int
5775219392Snpt4_mod_event(module_t mod, int cmd, void *arg)
5776219392Snp{
5777228561Snp	int rc = 0;
5778219392Snp
5779228561Snp	switch (cmd) {
5780228561Snp	case MOD_LOAD:
5781219392Snp		t4_sge_modload();
5782228561Snp		mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF);
5783228561Snp		SLIST_INIT(&t4_list);
5784237263Snp#ifdef TCP_OFFLOAD
5785228561Snp		mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF);
5786228561Snp		SLIST_INIT(&t4_uld_list);
5787228561Snp#endif
5788228561Snp		tweak_tunables();
5789228561Snp		break;
5790219392Snp
5791228561Snp	case MOD_UNLOAD:
5792237263Snp#ifdef TCP_OFFLOAD
5793228561Snp		mtx_lock(&t4_uld_list_lock);
5794228561Snp		if (!SLIST_EMPTY(&t4_uld_list)) {
5795228561Snp			rc = EBUSY;
5796228561Snp			mtx_unlock(&t4_uld_list_lock);
5797228561Snp			break;
5798228561Snp		}
5799228561Snp		mtx_unlock(&t4_uld_list_lock);
5800228561Snp		mtx_destroy(&t4_uld_list_lock);
5801228561Snp#endif
5802228561Snp		mtx_lock(&t4_list_lock);
5803228561Snp		if (!SLIST_EMPTY(&t4_list)) {
5804228561Snp			rc = EBUSY;
5805228561Snp			mtx_unlock(&t4_list_lock);
5806228561Snp			break;
5807228561Snp		}
5808228561Snp		mtx_unlock(&t4_list_lock);
5809228561Snp		mtx_destroy(&t4_list_lock);
5810228561Snp		break;
5811228561Snp	}
5812228561Snp
5813228561Snp	return (rc);
5814219392Snp}
5815219392Snp
5816218792Snpstatic devclass_t t4_devclass;
5817218792Snpstatic devclass_t cxgbe_devclass;
5818218792Snp
5819219392SnpDRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0);
5820218792SnpMODULE_VERSION(t4nex, 1);
5821218792Snp
5822218792SnpDRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
5823218792SnpMODULE_VERSION(cxgbe, 1);
5824