t4_main.c revision 237439
1218792Snp/*-
2218792Snp * Copyright (c) 2011 Chelsio Communications, Inc.
3218792Snp * All rights reserved.
4218792Snp * Written by: Navdeep Parhar <np@FreeBSD.org>
5218792Snp *
6218792Snp * Redistribution and use in source and binary forms, with or without
7218792Snp * modification, are permitted provided that the following conditions
8218792Snp * are met:
9218792Snp * 1. Redistributions of source code must retain the above copyright
10218792Snp *    notice, this list of conditions and the following disclaimer.
11218792Snp * 2. Redistributions in binary form must reproduce the above copyright
12218792Snp *    notice, this list of conditions and the following disclaimer in the
13218792Snp *    documentation and/or other materials provided with the distribution.
14218792Snp *
15218792Snp * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16218792Snp * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17218792Snp * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18218792Snp * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19218792Snp * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20218792Snp * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21218792Snp * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22218792Snp * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23218792Snp * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24218792Snp * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25218792Snp * SUCH DAMAGE.
26218792Snp */
27218792Snp
28218792Snp#include <sys/cdefs.h>
29218792Snp__FBSDID("$FreeBSD: head/sys/dev/cxgbe/t4_main.c 237439 2012-06-22 08:37:33Z np $");
30218792Snp
31218792Snp#include "opt_inet.h"
32218792Snp
33218792Snp#include <sys/param.h>
34218792Snp#include <sys/conf.h>
35218792Snp#include <sys/priv.h>
36218792Snp#include <sys/kernel.h>
37218792Snp#include <sys/bus.h>
38218792Snp#include <sys/module.h>
39219286Snp#include <sys/malloc.h>
40219286Snp#include <sys/queue.h>
41219286Snp#include <sys/taskqueue.h>
42218792Snp#include <sys/pciio.h>
43218792Snp#include <dev/pci/pcireg.h>
44218792Snp#include <dev/pci/pcivar.h>
45218792Snp#include <dev/pci/pci_private.h>
46218792Snp#include <sys/firmware.h>
47219436Snp#include <sys/sbuf.h>
48218792Snp#include <sys/smp.h>
49218792Snp#include <sys/socket.h>
50218792Snp#include <sys/sockio.h>
51218792Snp#include <sys/sysctl.h>
52218792Snp#include <net/ethernet.h>
53218792Snp#include <net/if.h>
54218792Snp#include <net/if_types.h>
55218792Snp#include <net/if_dl.h>
56222003Snp#include <net/if_vlan_var.h>
57218792Snp
58218792Snp#include "common/common.h"
59221474Snp#include "common/t4_msg.h"
60218792Snp#include "common/t4_regs.h"
61218792Snp#include "common/t4_regs_values.h"
62218792Snp#include "t4_ioctl.h"
63222509Snp#include "t4_l2t.h"
64218792Snp
65218792Snp/* T4 bus driver interface */
66218792Snpstatic int t4_probe(device_t);
67218792Snpstatic int t4_attach(device_t);
68218792Snpstatic int t4_detach(device_t);
69218792Snpstatic device_method_t t4_methods[] = {
70218792Snp	DEVMETHOD(device_probe,		t4_probe),
71218792Snp	DEVMETHOD(device_attach,	t4_attach),
72218792Snp	DEVMETHOD(device_detach,	t4_detach),
73218792Snp
74227843Smarius	DEVMETHOD_END
75218792Snp};
76218792Snpstatic driver_t t4_driver = {
77218792Snp	"t4nex",
78218792Snp	t4_methods,
79218792Snp	sizeof(struct adapter)
80218792Snp};
81218792Snp
82218792Snp
83218792Snp/* T4 port (cxgbe) interface */
84218792Snpstatic int cxgbe_probe(device_t);
85218792Snpstatic int cxgbe_attach(device_t);
86218792Snpstatic int cxgbe_detach(device_t);
87218792Snpstatic device_method_t cxgbe_methods[] = {
88218792Snp	DEVMETHOD(device_probe,		cxgbe_probe),
89218792Snp	DEVMETHOD(device_attach,	cxgbe_attach),
90218792Snp	DEVMETHOD(device_detach,	cxgbe_detach),
91218792Snp	{ 0, 0 }
92218792Snp};
93218792Snpstatic driver_t cxgbe_driver = {
94218792Snp	"cxgbe",
95218792Snp	cxgbe_methods,
96218792Snp	sizeof(struct port_info)
97218792Snp};
98218792Snp
99218792Snpstatic d_ioctl_t t4_ioctl;
100218792Snpstatic d_open_t t4_open;
101218792Snpstatic d_close_t t4_close;
102218792Snp
103218792Snpstatic struct cdevsw t4_cdevsw = {
104218792Snp       .d_version = D_VERSION,
105218792Snp       .d_flags = 0,
106218792Snp       .d_open = t4_open,
107218792Snp       .d_close = t4_close,
108218792Snp       .d_ioctl = t4_ioctl,
109218792Snp       .d_name = "t4nex",
110218792Snp};
111218792Snp
112218792Snp/* ifnet + media interface */
113218792Snpstatic void cxgbe_init(void *);
114218792Snpstatic int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
115218792Snpstatic int cxgbe_transmit(struct ifnet *, struct mbuf *);
116218792Snpstatic void cxgbe_qflush(struct ifnet *);
117218792Snpstatic int cxgbe_media_change(struct ifnet *);
118218792Snpstatic void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
119218792Snp
120218792SnpMALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services");
121218792Snp
122237263Snp/*
123237263Snp * Correct lock order when you need to acquire multiple locks is t4_list_lock,
124237263Snp * then ADAPTER_LOCK, then t4_uld_list_lock.
125237263Snp */
126228561Snpstatic struct mtx t4_list_lock;
127228561Snpstatic SLIST_HEAD(, adapter) t4_list;
128237263Snp#ifdef TCP_OFFLOAD
129228561Snpstatic struct mtx t4_uld_list_lock;
130228561Snpstatic SLIST_HEAD(, uld_info) t4_uld_list;
131228561Snp#endif
132218792Snp
133218792Snp/*
134228561Snp * Tunables.  See tweak_tunables() too.
135218792Snp */
136218792Snp
137218792Snp/*
138228561Snp * Number of queues for tx and rx, 10G and 1G, NIC and offload.
139218792Snp */
140228561Snp#define NTXQ_10G 16
141228561Snpstatic int t4_ntxq10g = -1;
142228561SnpTUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
143218792Snp
144228561Snp#define NRXQ_10G 8
145228561Snpstatic int t4_nrxq10g = -1;
146228561SnpTUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
147218792Snp
148228561Snp#define NTXQ_1G 4
149228561Snpstatic int t4_ntxq1g = -1;
150228561SnpTUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
151218792Snp
152228561Snp#define NRXQ_1G 2
153228561Snpstatic int t4_nrxq1g = -1;
154228561SnpTUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
155218792Snp
156237263Snp#ifdef TCP_OFFLOAD
157228561Snp#define NOFLDTXQ_10G 8
158228561Snpstatic int t4_nofldtxq10g = -1;
159228561SnpTUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
160228561Snp
161228561Snp#define NOFLDRXQ_10G 2
162228561Snpstatic int t4_nofldrxq10g = -1;
163228561SnpTUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
164228561Snp
165228561Snp#define NOFLDTXQ_1G 2
166228561Snpstatic int t4_nofldtxq1g = -1;
167228561SnpTUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
168228561Snp
169228561Snp#define NOFLDRXQ_1G 1
170228561Snpstatic int t4_nofldrxq1g = -1;
171228561SnpTUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
172228561Snp#endif
173228561Snp
174218792Snp/*
175218792Snp * Holdoff parameters for 10G and 1G ports.
176218792Snp */
177228561Snp#define TMR_IDX_10G 1
178228561Snpstatic int t4_tmr_idx_10g = TMR_IDX_10G;
179228561SnpTUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
180218792Snp
181234833Snp#define PKTC_IDX_10G (-1)
182228561Snpstatic int t4_pktc_idx_10g = PKTC_IDX_10G;
183228561SnpTUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
184218792Snp
185228561Snp#define TMR_IDX_1G 1
186228561Snpstatic int t4_tmr_idx_1g = TMR_IDX_1G;
187228561SnpTUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
188218792Snp
189234833Snp#define PKTC_IDX_1G (-1)
190228561Snpstatic int t4_pktc_idx_1g = PKTC_IDX_1G;
191228561SnpTUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
192218792Snp
193218792Snp/*
194218792Snp * Size (# of entries) of each tx and rx queue.
195218792Snp */
196228561Snpstatic unsigned int t4_qsize_txq = TX_EQ_QSIZE;
197228561SnpTUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
198218792Snp
199228561Snpstatic unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
200228561SnpTUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
201218792Snp
202218792Snp/*
203228561Snp * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
204218792Snp */
205228561Snpstatic int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
206228561SnpTUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
207218792Snp
208218792Snp/*
209228561Snp * Configuration file.
210218792Snp */
211228561Snpstatic char t4_cfg_file[32] = "default";
212228561SnpTUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
213218792Snp
214228561Snp/*
215228561Snp * ASIC features that will be used.  Disable the ones you don't want so that the
216228561Snp * chip resources aren't wasted on features that will not be used.
217228561Snp */
218228561Snpstatic int t4_linkcaps_allowed = 0;	/* No DCBX, PPP, etc. by default */
219228561SnpTUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
220221474Snp
221228561Snpstatic int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
222228561SnpTUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
223228561Snp
224228561Snpstatic int t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
225228561SnpTUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
226228561Snp
227228561Snpstatic int t4_rdmacaps_allowed = 0;
228228561SnpTUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
229228561Snp
230228561Snpstatic int t4_iscsicaps_allowed = 0;
231228561SnpTUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
232228561Snp
233228561Snpstatic int t4_fcoecaps_allowed = 0;
234228561SnpTUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
235228561Snp
236218792Snpstruct intrs_and_queues {
237219944Snp	int intr_type;		/* INTx, MSI, or MSI-X */
238218792Snp	int nirq;		/* Number of vectors */
239228561Snp	int intr_flags;
240218792Snp	int ntxq10g;		/* # of NIC txq's for each 10G port */
241218792Snp	int nrxq10g;		/* # of NIC rxq's for each 10G port */
242218792Snp	int ntxq1g;		/* # of NIC txq's for each 1G port */
243218792Snp	int nrxq1g;		/* # of NIC rxq's for each 1G port */
244237263Snp#ifdef TCP_OFFLOAD
245228561Snp	int nofldtxq10g;	/* # of TOE txq's for each 10G port */
246228561Snp	int nofldrxq10g;	/* # of TOE rxq's for each 10G port */
247228561Snp	int nofldtxq1g;		/* # of TOE txq's for each 1G port */
248228561Snp	int nofldrxq1g;		/* # of TOE rxq's for each 1G port */
249228561Snp#endif
250218792Snp};
251218792Snp
252221474Snpstruct filter_entry {
253221474Snp        uint32_t valid:1;	/* filter allocated and valid */
254221474Snp        uint32_t locked:1;	/* filter is administratively locked */
255221474Snp        uint32_t pending:1;	/* filter action is pending firmware reply */
256221474Snp	uint32_t smtidx:8;	/* Source MAC Table index for smac */
257222509Snp	struct l2t_entry *l2t;	/* Layer Two Table entry for dmac */
258221474Snp
259221474Snp        struct t4_filter_specification fs;
260221474Snp};
261221474Snp
262218792Snpenum {
263218792Snp	XGMAC_MTU	= (1 << 0),
264218792Snp	XGMAC_PROMISC	= (1 << 1),
265218792Snp	XGMAC_ALLMULTI	= (1 << 2),
266218792Snp	XGMAC_VLANEX	= (1 << 3),
267218792Snp	XGMAC_UCADDR	= (1 << 4),
268218792Snp	XGMAC_MCADDRS	= (1 << 5),
269218792Snp
270218792Snp	XGMAC_ALL	= 0xffff
271218792Snp};
272218792Snp
273218792Snpstatic int map_bars(struct adapter *);
274218792Snpstatic void setup_memwin(struct adapter *);
275218792Snpstatic int cfg_itype_and_nqueues(struct adapter *, int, int,
276218792Snp    struct intrs_and_queues *);
277218792Snpstatic int prep_firmware(struct adapter *);
278228561Snpstatic int upload_config_file(struct adapter *, const struct firmware *,
279228561Snp    uint32_t *, uint32_t *);
280228561Snpstatic int partition_resources(struct adapter *, const struct firmware *);
281228561Snpstatic int get_params__pre_init(struct adapter *);
282228561Snpstatic int get_params__post_init(struct adapter *);
283218792Snpstatic void t4_set_desc(struct adapter *);
284218792Snpstatic void build_medialist(struct port_info *);
285218792Snpstatic int update_mac_settings(struct port_info *, int);
286218792Snpstatic int cxgbe_init_locked(struct port_info *);
287218792Snpstatic int cxgbe_init_synchronized(struct port_info *);
288218792Snpstatic int cxgbe_uninit_locked(struct port_info *);
289218792Snpstatic int cxgbe_uninit_synchronized(struct port_info *);
290228561Snpstatic int adapter_full_init(struct adapter *);
291228561Snpstatic int adapter_full_uninit(struct adapter *);
292228561Snpstatic int port_full_init(struct port_info *);
293228561Snpstatic int port_full_uninit(struct port_info *);
294228561Snpstatic void quiesce_eq(struct adapter *, struct sge_eq *);
295228561Snpstatic void quiesce_iq(struct adapter *, struct sge_iq *);
296228561Snpstatic void quiesce_fl(struct adapter *, struct sge_fl *);
297218792Snpstatic int t4_alloc_irq(struct adapter *, struct irq *, int rid,
298228561Snp    driver_intr_t *, void *, char *);
299218792Snpstatic int t4_free_irq(struct adapter *, struct irq *);
300218792Snpstatic void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
301218792Snp    unsigned int);
302218792Snpstatic void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
303218792Snpstatic void cxgbe_tick(void *);
304237263Snpstatic void cxgbe_vlan_config(void *, struct ifnet *, uint16_t);
305228561Snpstatic int cpl_not_handled(struct sge_iq *, const struct rss_header *,
306228561Snp    struct mbuf *);
307237263Snpstatic int an_not_handled(struct sge_iq *, const struct rsp_ctrl *);
308218792Snpstatic int t4_sysctls(struct adapter *);
309218792Snpstatic int cxgbe_sysctls(struct port_info *);
310219436Snpstatic int sysctl_int_array(SYSCTL_HANDLER_ARGS);
311228561Snpstatic int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
312218792Snpstatic int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
313218792Snpstatic int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
314218792Snpstatic int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
315218792Snpstatic int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
316218792Snpstatic int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
317231115Snp#ifdef SBUF_DRAIN
318228561Snpstatic int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
319228561Snpstatic int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
320228561Snpstatic int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
321222551Snpstatic int sysctl_devlog(SYSCTL_HANDLER_ARGS);
322228561Snpstatic int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
323228561Snpstatic int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
324228561Snpstatic int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
325228561Snpstatic int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
326228561Snpstatic int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
327228561Snpstatic int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
328228561Snpstatic int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
329228561Snpstatic int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
330228561Snpstatic int sysctl_tids(SYSCTL_HANDLER_ARGS);
331228561Snpstatic int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
332228561Snpstatic int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
333231115Snp#endif
334219286Snpstatic inline void txq_start(struct ifnet *, struct sge_txq *);
335221474Snpstatic uint32_t fconf_to_mode(uint32_t);
336221474Snpstatic uint32_t mode_to_fconf(uint32_t);
337221474Snpstatic uint32_t fspec_to_fconf(struct t4_filter_specification *);
338221474Snpstatic int get_filter_mode(struct adapter *, uint32_t *);
339221474Snpstatic int set_filter_mode(struct adapter *, uint32_t);
340222552Snpstatic inline uint64_t get_filter_hits(struct adapter *, uint32_t);
341221474Snpstatic int get_filter(struct adapter *, struct t4_filter *);
342221474Snpstatic int set_filter(struct adapter *, struct t4_filter *);
343221474Snpstatic int del_filter(struct adapter *, struct t4_filter *);
344222509Snpstatic void clear_filter(struct filter_entry *);
345221474Snpstatic int set_filter_wr(struct adapter *, int);
346221474Snpstatic int del_filter_wr(struct adapter *, int);
347228561Snpstatic int filter_rpl(struct sge_iq *, const struct rss_header *,
348228561Snp    struct mbuf *);
349222973Snpstatic int get_sge_context(struct adapter *, struct t4_sge_context *);
350228561Snpstatic int read_card_mem(struct adapter *, struct t4_mem_range *);
351237263Snp#ifdef TCP_OFFLOAD
352228561Snpstatic int toe_capability(struct port_info *, int);
353228561Snp#endif
354219392Snpstatic int t4_mod_event(module_t, int, void *);
355218792Snp
356218792Snpstruct t4_pciids {
357218792Snp	uint16_t device;
358218792Snp	uint8_t mpf;
359218792Snp	char *desc;
360218792Snp} t4_pciids[] = {
361218792Snp	{0xa000, 0, "Chelsio Terminator 4 FPGA"},
362218792Snp	{0x4400, 4, "Chelsio T440-dbg"},
363218792Snp	{0x4401, 4, "Chelsio T420-CR"},
364218792Snp	{0x4402, 4, "Chelsio T422-CR"},
365218792Snp	{0x4403, 4, "Chelsio T440-CR"},
366218792Snp	{0x4404, 4, "Chelsio T420-BCH"},
367218792Snp	{0x4405, 4, "Chelsio T440-BCH"},
368218792Snp	{0x4406, 4, "Chelsio T440-CH"},
369218792Snp	{0x4407, 4, "Chelsio T420-SO"},
370218792Snp	{0x4408, 4, "Chelsio T420-CX"},
371218792Snp	{0x4409, 4, "Chelsio T420-BT"},
372218792Snp	{0x440a, 4, "Chelsio T404-BT"},
373218792Snp};
374218792Snp
375237263Snp#ifdef TCP_OFFLOAD
376237263Snp/*
377237263Snp * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
378237263Snp * exactly the same for both rxq and ofld_rxq.
379237263Snp */
380237263SnpCTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
381228561SnpCTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
382228561Snp#endif
383228561Snp
384218792Snpstatic int
385218792Snpt4_probe(device_t dev)
386218792Snp{
387218792Snp	int i;
388218792Snp	uint16_t v = pci_get_vendor(dev);
389218792Snp	uint16_t d = pci_get_device(dev);
390218792Snp
391218792Snp	if (v != PCI_VENDOR_ID_CHELSIO)
392218792Snp		return (ENXIO);
393218792Snp
394218792Snp	for (i = 0; i < ARRAY_SIZE(t4_pciids); i++) {
395218792Snp		if (d == t4_pciids[i].device &&
396218792Snp		    pci_get_function(dev) == t4_pciids[i].mpf) {
397218792Snp			device_set_desc(dev, t4_pciids[i].desc);
398218792Snp			return (BUS_PROBE_DEFAULT);
399218792Snp		}
400218792Snp	}
401218792Snp
402218792Snp	return (ENXIO);
403218792Snp}
404218792Snp
405218792Snpstatic int
406218792Snpt4_attach(device_t dev)
407218792Snp{
408218792Snp	struct adapter *sc;
409218792Snp	int rc = 0, i, n10g, n1g, rqidx, tqidx;
410218792Snp	struct intrs_and_queues iaq;
411218792Snp	struct sge *s;
412237263Snp#ifdef TCP_OFFLOAD
413228561Snp	int ofld_rqidx, ofld_tqidx;
414228561Snp#endif
415218792Snp
416218792Snp	sc = device_get_softc(dev);
417218792Snp	sc->dev = dev;
418218792Snp	sc->pf = pci_get_function(dev);
419218792Snp	sc->mbox = sc->pf;
420218792Snp
421218792Snp	pci_enable_busmaster(dev);
422222085Snp	if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
423228561Snp		uint32_t v;
424228561Snp
425222085Snp		pci_set_max_read_req(dev, 4096);
426222085Snp		v = pci_read_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, 2);
427222085Snp		v |= PCIM_EXP_CTL_RELAXED_ORD_ENABLE;
428222085Snp		pci_write_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, v, 2);
429222085Snp	}
430222085Snp
431218792Snp	snprintf(sc->lockname, sizeof(sc->lockname), "%s",
432218792Snp	    device_get_nameunit(dev));
433218792Snp	mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
434228561Snp	mtx_lock(&t4_list_lock);
435228561Snp	SLIST_INSERT_HEAD(&t4_list, sc, link);
436228561Snp	mtx_unlock(&t4_list_lock);
437218792Snp
438228561Snp	mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
439228561Snp	TAILQ_INIT(&sc->sfl);
440228561Snp	callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
441228561Snp
442218792Snp	rc = map_bars(sc);
443218792Snp	if (rc != 0)
444218792Snp		goto done; /* error message displayed already */
445218792Snp
446218792Snp	memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
447237263Snp	sc->an_handler = an_not_handled;
448228561Snp	for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++)
449228561Snp		sc->cpl_handler[i] = cpl_not_handled;
450228561Snp	t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, filter_rpl);
451218792Snp
452218792Snp	/* Prepare the adapter for operation */
453218792Snp	rc = -t4_prep_adapter(sc);
454218792Snp	if (rc != 0) {
455218792Snp		device_printf(dev, "failed to prepare adapter: %d.\n", rc);
456218792Snp		goto done;
457218792Snp	}
458218792Snp
459228561Snp	/*
460228561Snp	 * Do this really early, with the memory windows set up even before the
461228561Snp	 * character device.  The userland tool's register i/o and mem read
462228561Snp	 * will work even in "recovery mode".
463228561Snp	 */
464228561Snp	setup_memwin(sc);
465218792Snp	sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT,
466218792Snp	    GID_WHEEL, 0600, "%s", device_get_nameunit(dev));
467218792Snp	sc->cdev->si_drv1 = sc;
468218792Snp
469228561Snp	/* Go no further if recovery mode has been requested. */
470228561Snp	if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
471228561Snp		device_printf(dev, "recovery mode.\n");
472228561Snp		goto done;
473228561Snp	}
474228561Snp
475218792Snp	/* Prepare the firmware for operation */
476218792Snp	rc = prep_firmware(sc);
477218792Snp	if (rc != 0)
478218792Snp		goto done; /* error message displayed already */
479218792Snp
480228561Snp	rc = get_params__pre_init(sc);
481228561Snp	if (rc != 0)
482228561Snp		goto done; /* error message displayed already */
483222551Snp
484228561Snp	rc = t4_sge_init(sc);
485228561Snp	if (rc != 0)
486228561Snp		goto done; /* error message displayed already */
487218792Snp
488228561Snp	if (sc->flags & MASTER_PF) {
489228561Snp		/* get basic stuff going */
490228561Snp		rc = -t4_fw_initialize(sc, sc->mbox);
491228561Snp		if (rc != 0) {
492228561Snp			device_printf(dev, "early init failed: %d.\n", rc);
493228561Snp			goto done;
494228561Snp		}
495218792Snp	}
496218792Snp
497228561Snp	rc = get_params__post_init(sc);
498228561Snp	if (rc != 0)
499228561Snp		goto done; /* error message displayed already */
500218792Snp
501228561Snp	if (sc->flags & MASTER_PF) {
502218792Snp
503228561Snp		/* final tweaks to some settings */
504218792Snp
505228561Snp		t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd,
506228561Snp		    sc->params.b_wnd);
507228561Snp		t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
508228561Snp		t4_set_reg_field(sc, A_TP_PARA_REG3, F_TUNNELCNGDROP0 |
509228561Snp		    F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 | F_TUNNELCNGDROP3, 0);
510228561Snp		t4_set_reg_field(sc, A_TP_PARA_REG5,
511228561Snp		    V_INDICATESIZE(M_INDICATESIZE) |
512228561Snp		    F_REARMDDPOFFSET | F_RESETDDPOFFSET,
513228561Snp		    V_INDICATESIZE(M_INDICATESIZE) |
514228561Snp		    F_REARMDDPOFFSET | F_RESETDDPOFFSET);
515228561Snp	} else {
516228561Snp		/*
517228561Snp		 * XXX: Verify that we can live with whatever the master driver
518228561Snp		 * has done so far, and hope that it doesn't change any global
519228561Snp		 * setting from underneath us in the future.
520228561Snp		 */
521218792Snp	}
522218792Snp
523228561Snp	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
524228561Snp	    A_TP_VLAN_PRI_MAP);
525218792Snp
526228561Snp	for (i = 0; i < NCHAN; i++)
527228561Snp		sc->params.tp.tx_modq[i] = i;
528218792Snp
529218792Snp	rc = t4_create_dma_tag(sc);
530218792Snp	if (rc != 0)
531218792Snp		goto done; /* error message displayed already */
532218792Snp
533218792Snp	/*
534218792Snp	 * First pass over all the ports - allocate VIs and initialize some
535218792Snp	 * basic parameters like mac address, port type, etc.  We also figure
536218792Snp	 * out whether a port is 10G or 1G and use that information when
537218792Snp	 * calculating how many interrupts to attempt to allocate.
538218792Snp	 */
539218792Snp	n10g = n1g = 0;
540218792Snp	for_each_port(sc, i) {
541218792Snp		struct port_info *pi;
542218792Snp
543218792Snp		pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
544218792Snp		sc->port[i] = pi;
545218792Snp
546218792Snp		/* These must be set before t4_port_init */
547218792Snp		pi->adapter = sc;
548218792Snp		pi->port_id = i;
549218792Snp
550218792Snp		/* Allocate the vi and initialize parameters like mac addr */
551218792Snp		rc = -t4_port_init(pi, sc->mbox, sc->pf, 0);
552218792Snp		if (rc != 0) {
553218792Snp			device_printf(dev, "unable to initialize port %d: %d\n",
554218792Snp			    i, rc);
555218792Snp			free(pi, M_CXGBE);
556222510Snp			sc->port[i] = NULL;
557222510Snp			goto done;
558218792Snp		}
559218792Snp
560218792Snp		snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
561218792Snp		    device_get_nameunit(dev), i);
562218792Snp		mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
563218792Snp
564218792Snp		if (is_10G_port(pi)) {
565218792Snp			n10g++;
566228561Snp			pi->tmr_idx = t4_tmr_idx_10g;
567228561Snp			pi->pktc_idx = t4_pktc_idx_10g;
568218792Snp		} else {
569218792Snp			n1g++;
570228561Snp			pi->tmr_idx = t4_tmr_idx_1g;
571228561Snp			pi->pktc_idx = t4_pktc_idx_1g;
572218792Snp		}
573218792Snp
574218792Snp		pi->xact_addr_filt = -1;
575218792Snp
576228561Snp		pi->qsize_rxq = t4_qsize_rxq;
577228561Snp		pi->qsize_txq = t4_qsize_txq;
578218792Snp
579218792Snp		pi->dev = device_add_child(dev, "cxgbe", -1);
580218792Snp		if (pi->dev == NULL) {
581218792Snp			device_printf(dev,
582218792Snp			    "failed to add device for port %d.\n", i);
583218792Snp			rc = ENXIO;
584218792Snp			goto done;
585218792Snp		}
586218792Snp		device_set_softc(pi->dev, pi);
587218792Snp	}
588218792Snp
589218792Snp	/*
590218792Snp	 * Interrupt type, # of interrupts, # of rx/tx queues, etc.
591218792Snp	 */
592218792Snp	rc = cfg_itype_and_nqueues(sc, n10g, n1g, &iaq);
593218792Snp	if (rc != 0)
594218792Snp		goto done; /* error message displayed already */
595218792Snp
596218792Snp	sc->intr_type = iaq.intr_type;
597218792Snp	sc->intr_count = iaq.nirq;
598228561Snp	sc->flags |= iaq.intr_flags;
599218792Snp
600218792Snp	s = &sc->sge;
601218792Snp	s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
602218792Snp	s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
603220873Snp	s->neq = s->ntxq + s->nrxq;	/* the free list in an rxq is an eq */
604228561Snp	s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
605218792Snp	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
606222510Snp
607237263Snp#ifdef TCP_OFFLOAD
608228561Snp	if (is_offload(sc)) {
609228561Snp
610228561Snp		s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
611228561Snp		s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
612228561Snp		s->neq += s->nofldtxq + s->nofldrxq;
613228561Snp		s->niq += s->nofldrxq;
614228561Snp
615228561Snp		s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
616228561Snp		    M_CXGBE, M_ZERO | M_WAITOK);
617228561Snp		s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
618228561Snp		    M_CXGBE, M_ZERO | M_WAITOK);
619228561Snp	}
620228561Snp#endif
621228561Snp
622228561Snp	s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
623220873Snp	    M_ZERO | M_WAITOK);
624218792Snp	s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
625218792Snp	    M_ZERO | M_WAITOK);
626218792Snp	s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
627218792Snp	    M_ZERO | M_WAITOK);
628218792Snp	s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
629218792Snp	    M_ZERO | M_WAITOK);
630218792Snp	s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
631218792Snp	    M_ZERO | M_WAITOK);
632218792Snp
633218792Snp	sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
634218792Snp	    M_ZERO | M_WAITOK);
635218792Snp
636228561Snp	t4_init_l2t(sc, M_WAITOK);
637222509Snp
638218792Snp	/*
639218792Snp	 * Second pass over the ports.  This time we know the number of rx and
640218792Snp	 * tx queues that each port should get.
641218792Snp	 */
642218792Snp	rqidx = tqidx = 0;
643237263Snp#ifdef TCP_OFFLOAD
644228561Snp	ofld_rqidx = ofld_tqidx = 0;
645228561Snp#endif
646218792Snp	for_each_port(sc, i) {
647218792Snp		struct port_info *pi = sc->port[i];
648218792Snp
649218792Snp		if (pi == NULL)
650218792Snp			continue;
651218792Snp
652218792Snp		pi->first_rxq = rqidx;
653218792Snp		pi->first_txq = tqidx;
654228561Snp		if (is_10G_port(pi)) {
655228561Snp			pi->nrxq = iaq.nrxq10g;
656228561Snp			pi->ntxq = iaq.ntxq10g;
657228561Snp		} else {
658228561Snp			pi->nrxq = iaq.nrxq1g;
659228561Snp			pi->ntxq = iaq.ntxq1g;
660228561Snp		}
661218792Snp
662218792Snp		rqidx += pi->nrxq;
663218792Snp		tqidx += pi->ntxq;
664228561Snp
665237263Snp#ifdef TCP_OFFLOAD
666228561Snp		if (is_offload(sc)) {
667228561Snp			pi->first_ofld_rxq = ofld_rqidx;
668228561Snp			pi->first_ofld_txq = ofld_tqidx;
669228561Snp			if (is_10G_port(pi)) {
670228561Snp				pi->nofldrxq = iaq.nofldrxq10g;
671228561Snp				pi->nofldtxq = iaq.nofldtxq10g;
672228561Snp			} else {
673228561Snp				pi->nofldrxq = iaq.nofldrxq1g;
674228561Snp				pi->nofldtxq = iaq.nofldtxq1g;
675228561Snp			}
676228561Snp			ofld_rqidx += pi->nofldrxq;
677228561Snp			ofld_tqidx += pi->nofldtxq;
678228561Snp		}
679228561Snp#endif
680218792Snp	}
681218792Snp
682218792Snp	rc = bus_generic_attach(dev);
683218792Snp	if (rc != 0) {
684218792Snp		device_printf(dev,
685218792Snp		    "failed to attach all child ports: %d\n", rc);
686218792Snp		goto done;
687218792Snp	}
688218792Snp
689218792Snp	device_printf(dev,
690228561Snp	    "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
691228561Snp	    sc->params.pci.width, sc->params.nports, sc->intr_count,
692228561Snp	    sc->intr_type == INTR_MSIX ? "MSI-X" :
693228561Snp	    (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
694228561Snp	    sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
695228561Snp
696218792Snp	t4_set_desc(sc);
697218792Snp
698218792Snpdone:
699228561Snp	if (rc != 0 && sc->cdev) {
700228561Snp		/* cdev was created and so cxgbetool works; recover that way. */
701228561Snp		device_printf(dev,
702228561Snp		    "error during attach, adapter is now in recovery mode.\n");
703228561Snp		rc = 0;
704228561Snp	}
705228561Snp
706218792Snp	if (rc != 0)
707218792Snp		t4_detach(dev);
708228561Snp	else
709228561Snp		t4_sysctls(sc);
710218792Snp
711218792Snp	return (rc);
712218792Snp}
713218792Snp
714218792Snp/*
715218792Snp * Idempotent
716218792Snp */
717218792Snpstatic int
718218792Snpt4_detach(device_t dev)
719218792Snp{
720218792Snp	struct adapter *sc;
721218792Snp	struct port_info *pi;
722228561Snp	int i, rc;
723218792Snp
724218792Snp	sc = device_get_softc(dev);
725218792Snp
726228561Snp	if (sc->flags & FULL_INIT_DONE)
727228561Snp		t4_intr_disable(sc);
728228561Snp
729228561Snp	if (sc->cdev) {
730218792Snp		destroy_dev(sc->cdev);
731228561Snp		sc->cdev = NULL;
732228561Snp	}
733218792Snp
734228561Snp	rc = bus_generic_detach(dev);
735228561Snp	if (rc) {
736228561Snp		device_printf(dev,
737228561Snp		    "failed to detach child devices: %d\n", rc);
738228561Snp		return (rc);
739228561Snp	}
740228561Snp
741218792Snp	for (i = 0; i < MAX_NPORTS; i++) {
742218792Snp		pi = sc->port[i];
743218792Snp		if (pi) {
744218792Snp			t4_free_vi(pi->adapter, sc->mbox, sc->pf, 0, pi->viid);
745218792Snp			if (pi->dev)
746218792Snp				device_delete_child(dev, pi->dev);
747218792Snp
748218792Snp			mtx_destroy(&pi->pi_lock);
749218792Snp			free(pi, M_CXGBE);
750218792Snp		}
751218792Snp	}
752218792Snp
753228561Snp	if (sc->flags & FULL_INIT_DONE)
754228561Snp		adapter_full_uninit(sc);
755228561Snp
756218792Snp	if (sc->flags & FW_OK)
757218792Snp		t4_fw_bye(sc, sc->mbox);
758218792Snp
759219944Snp	if (sc->intr_type == INTR_MSI || sc->intr_type == INTR_MSIX)
760218792Snp		pci_release_msi(dev);
761218792Snp
762218792Snp	if (sc->regs_res)
763218792Snp		bus_release_resource(dev, SYS_RES_MEMORY, sc->regs_rid,
764218792Snp		    sc->regs_res);
765218792Snp
766218792Snp	if (sc->msix_res)
767218792Snp		bus_release_resource(dev, SYS_RES_MEMORY, sc->msix_rid,
768218792Snp		    sc->msix_res);
769218792Snp
770222509Snp	if (sc->l2t)
771222509Snp		t4_free_l2t(sc->l2t);
772222509Snp
773237263Snp#ifdef TCP_OFFLOAD
774228561Snp	free(sc->sge.ofld_rxq, M_CXGBE);
775228561Snp	free(sc->sge.ofld_txq, M_CXGBE);
776228561Snp#endif
777218792Snp	free(sc->irq, M_CXGBE);
778218792Snp	free(sc->sge.rxq, M_CXGBE);
779218792Snp	free(sc->sge.txq, M_CXGBE);
780220873Snp	free(sc->sge.ctrlq, M_CXGBE);
781218792Snp	free(sc->sge.iqmap, M_CXGBE);
782218792Snp	free(sc->sge.eqmap, M_CXGBE);
783221474Snp	free(sc->tids.ftid_tab, M_CXGBE);
784218792Snp	t4_destroy_dma_tag(sc);
785228561Snp	if (mtx_initialized(&sc->sc_lock)) {
786228561Snp		mtx_lock(&t4_list_lock);
787228561Snp		SLIST_REMOVE(&t4_list, sc, adapter, link);
788228561Snp		mtx_unlock(&t4_list_lock);
789228561Snp		mtx_destroy(&sc->sc_lock);
790228561Snp	}
791218792Snp
792228561Snp	if (mtx_initialized(&sc->sfl_lock))
793228561Snp		mtx_destroy(&sc->sfl_lock);
794228561Snp
795218792Snp	bzero(sc, sizeof(*sc));
796218792Snp
797218792Snp	return (0);
798218792Snp}
799218792Snp
800218792Snp
801218792Snpstatic int
802218792Snpcxgbe_probe(device_t dev)
803218792Snp{
804218792Snp	char buf[128];
805218792Snp	struct port_info *pi = device_get_softc(dev);
806218792Snp
807228561Snp	snprintf(buf, sizeof(buf), "port %d", pi->port_id);
808218792Snp	device_set_desc_copy(dev, buf);
809218792Snp
810218792Snp	return (BUS_PROBE_DEFAULT);
811218792Snp}
812218792Snp
813218792Snp#define T4_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
814218792Snp    IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
815218792Snp    IFCAP_VLAN_HWTSO)
816218792Snp#define T4_CAP_ENABLE (T4_CAP & ~IFCAP_TSO6)
817218792Snp
818218792Snpstatic int
819218792Snpcxgbe_attach(device_t dev)
820218792Snp{
821218792Snp	struct port_info *pi = device_get_softc(dev);
822218792Snp	struct ifnet *ifp;
823218792Snp
824218792Snp	/* Allocate an ifnet and set it up */
825218792Snp	ifp = if_alloc(IFT_ETHER);
826218792Snp	if (ifp == NULL) {
827218792Snp		device_printf(dev, "Cannot allocate ifnet\n");
828218792Snp		return (ENOMEM);
829218792Snp	}
830218792Snp	pi->ifp = ifp;
831218792Snp	ifp->if_softc = pi;
832218792Snp
833218792Snp	callout_init(&pi->tick, CALLOUT_MPSAFE);
834218792Snp
835218792Snp	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
836218792Snp	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
837218792Snp
838218792Snp	ifp->if_init = cxgbe_init;
839218792Snp	ifp->if_ioctl = cxgbe_ioctl;
840218792Snp	ifp->if_transmit = cxgbe_transmit;
841218792Snp	ifp->if_qflush = cxgbe_qflush;
842218792Snp
843218792Snp	ifp->if_capabilities = T4_CAP;
844237263Snp#ifdef TCP_OFFLOAD
845228561Snp	if (is_offload(pi->adapter))
846228561Snp		ifp->if_capabilities |= IFCAP_TOE4;
847228561Snp#endif
848218792Snp	ifp->if_capenable = T4_CAP_ENABLE;
849218792Snp	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO;
850218792Snp
851218792Snp	/* Initialize ifmedia for this port */
852218792Snp	ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
853218792Snp	    cxgbe_media_status);
854218792Snp	build_medialist(pi);
855218792Snp
856237263Snp	pi->vlan_c = EVENTHANDLER_REGISTER(vlan_config, cxgbe_vlan_config, ifp,
857237263Snp	    EVENTHANDLER_PRI_ANY);
858237263Snp
859218792Snp	ether_ifattach(ifp, pi->hw_addr);
860218792Snp
861237263Snp#ifdef TCP_OFFLOAD
862228561Snp	if (is_offload(pi->adapter)) {
863228561Snp		device_printf(dev,
864228561Snp		    "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
865228561Snp		    pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
866228561Snp	} else
867218792Snp#endif
868228561Snp		device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
869218792Snp
870218792Snp	cxgbe_sysctls(pi);
871218792Snp
872218792Snp	return (0);
873218792Snp}
874218792Snp
875218792Snpstatic int
876218792Snpcxgbe_detach(device_t dev)
877218792Snp{
878218792Snp	struct port_info *pi = device_get_softc(dev);
879218792Snp	struct adapter *sc = pi->adapter;
880228561Snp	struct ifnet *ifp = pi->ifp;
881218792Snp
882218792Snp	/* Tell if_ioctl and if_init that the port is going away */
883218792Snp	ADAPTER_LOCK(sc);
884218792Snp	SET_DOOMED(pi);
885218792Snp	wakeup(&sc->flags);
886218792Snp	while (IS_BUSY(sc))
887218792Snp		mtx_sleep(&sc->flags, &sc->sc_lock, 0, "t4detach", 0);
888218792Snp	SET_BUSY(sc);
889218792Snp	ADAPTER_UNLOCK(sc);
890218792Snp
891237263Snp	if (pi->vlan_c)
892237263Snp		EVENTHANDLER_DEREGISTER(vlan_config, pi->vlan_c);
893237263Snp
894228561Snp	PORT_LOCK(pi);
895228561Snp	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
896228561Snp	callout_stop(&pi->tick);
897228561Snp	PORT_UNLOCK(pi);
898228561Snp	callout_drain(&pi->tick);
899218792Snp
900228561Snp	/* Let detach proceed even if these fail. */
901228561Snp	cxgbe_uninit_synchronized(pi);
902228561Snp	port_full_uninit(pi);
903219286Snp
904218792Snp	ifmedia_removeall(&pi->media);
905218792Snp	ether_ifdetach(pi->ifp);
906218792Snp	if_free(pi->ifp);
907218792Snp
908218792Snp	ADAPTER_LOCK(sc);
909218792Snp	CLR_BUSY(sc);
910218792Snp	wakeup_one(&sc->flags);
911218792Snp	ADAPTER_UNLOCK(sc);
912218792Snp
913218792Snp	return (0);
914218792Snp}
915218792Snp
916218792Snpstatic void
917218792Snpcxgbe_init(void *arg)
918218792Snp{
919218792Snp	struct port_info *pi = arg;
920218792Snp	struct adapter *sc = pi->adapter;
921218792Snp
922218792Snp	ADAPTER_LOCK(sc);
923218792Snp	cxgbe_init_locked(pi); /* releases adapter lock */
924218792Snp	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
925218792Snp}
926218792Snp
927218792Snpstatic int
928218792Snpcxgbe_ioctl(struct ifnet *ifp, unsigned long cmd, caddr_t data)
929218792Snp{
930218792Snp	int rc = 0, mtu, flags;
931218792Snp	struct port_info *pi = ifp->if_softc;
932218792Snp	struct adapter *sc = pi->adapter;
933218792Snp	struct ifreq *ifr = (struct ifreq *)data;
934218792Snp	uint32_t mask;
935218792Snp
936218792Snp	switch (cmd) {
937218792Snp	case SIOCSIFMTU:
938218792Snp		ADAPTER_LOCK(sc);
939218792Snp		rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
940218792Snp		if (rc) {
941218792Snpfail:
942218792Snp			ADAPTER_UNLOCK(sc);
943218792Snp			return (rc);
944218792Snp		}
945218792Snp
946218792Snp		mtu = ifr->ifr_mtu;
947218792Snp		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
948218792Snp			rc = EINVAL;
949218792Snp		} else {
950218792Snp			ifp->if_mtu = mtu;
951218792Snp			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
952218792Snp				t4_update_fl_bufsize(ifp);
953218792Snp				PORT_LOCK(pi);
954218792Snp				rc = update_mac_settings(pi, XGMAC_MTU);
955218792Snp				PORT_UNLOCK(pi);
956218792Snp			}
957218792Snp		}
958218792Snp		ADAPTER_UNLOCK(sc);
959218792Snp		break;
960218792Snp
961218792Snp	case SIOCSIFFLAGS:
962218792Snp		ADAPTER_LOCK(sc);
963218792Snp		if (IS_DOOMED(pi)) {
964218792Snp			rc = ENXIO;
965218792Snp			goto fail;
966218792Snp		}
967218792Snp		if (ifp->if_flags & IFF_UP) {
968218792Snp			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
969218792Snp				flags = pi->if_flags;
970218792Snp				if ((ifp->if_flags ^ flags) &
971218792Snp				    (IFF_PROMISC | IFF_ALLMULTI)) {
972218792Snp					if (IS_BUSY(sc)) {
973218792Snp						rc = EBUSY;
974218792Snp						goto fail;
975218792Snp					}
976218792Snp					PORT_LOCK(pi);
977218792Snp					rc = update_mac_settings(pi,
978218792Snp					    XGMAC_PROMISC | XGMAC_ALLMULTI);
979218792Snp					PORT_UNLOCK(pi);
980218792Snp				}
981218792Snp				ADAPTER_UNLOCK(sc);
982218792Snp			} else
983218792Snp				rc = cxgbe_init_locked(pi);
984218792Snp			pi->if_flags = ifp->if_flags;
985218792Snp		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
986218792Snp			rc = cxgbe_uninit_locked(pi);
987218792Snp		else
988218792Snp			ADAPTER_UNLOCK(sc);
989218792Snp
990218792Snp		ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
991218792Snp		break;
992218792Snp
993218792Snp	case SIOCADDMULTI:
994218792Snp	case SIOCDELMULTI: /* these two can be called with a mutex held :-( */
995218792Snp		ADAPTER_LOCK(sc);
996218792Snp		rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
997218792Snp		if (rc)
998218792Snp			goto fail;
999218792Snp
1000218792Snp		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1001218792Snp			PORT_LOCK(pi);
1002218792Snp			rc = update_mac_settings(pi, XGMAC_MCADDRS);
1003218792Snp			PORT_UNLOCK(pi);
1004218792Snp		}
1005218792Snp		ADAPTER_UNLOCK(sc);
1006218792Snp		break;
1007218792Snp
1008218792Snp	case SIOCSIFCAP:
1009218792Snp		ADAPTER_LOCK(sc);
1010218792Snp		rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1011218792Snp		if (rc)
1012218792Snp			goto fail;
1013218792Snp
1014218792Snp		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1015218792Snp		if (mask & IFCAP_TXCSUM) {
1016218792Snp			ifp->if_capenable ^= IFCAP_TXCSUM;
1017218792Snp			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1018218792Snp
1019218792Snp			if (IFCAP_TSO & ifp->if_capenable &&
1020218792Snp			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1021218792Snp				ifp->if_capenable &= ~IFCAP_TSO;
1022218792Snp				ifp->if_hwassist &= ~CSUM_TSO;
1023218792Snp				if_printf(ifp,
1024218792Snp				    "tso disabled due to -txcsum.\n");
1025218792Snp			}
1026218792Snp		}
1027218792Snp		if (mask & IFCAP_RXCSUM)
1028218792Snp			ifp->if_capenable ^= IFCAP_RXCSUM;
1029218792Snp		if (mask & IFCAP_TSO4) {
1030218792Snp			ifp->if_capenable ^= IFCAP_TSO4;
1031218792Snp
1032218792Snp			if (IFCAP_TSO & ifp->if_capenable) {
1033218792Snp				if (IFCAP_TXCSUM & ifp->if_capenable)
1034218792Snp					ifp->if_hwassist |= CSUM_TSO;
1035218792Snp				else {
1036218792Snp					ifp->if_capenable &= ~IFCAP_TSO;
1037218792Snp					ifp->if_hwassist &= ~CSUM_TSO;
1038218792Snp					if_printf(ifp,
1039218792Snp					    "enable txcsum first.\n");
1040218792Snp					rc = EAGAIN;
1041228561Snp					goto fail;
1042218792Snp				}
1043218792Snp			} else
1044218792Snp				ifp->if_hwassist &= ~CSUM_TSO;
1045218792Snp		}
1046218792Snp		if (mask & IFCAP_LRO) {
1047218792Snp#ifdef INET
1048218792Snp			int i;
1049218792Snp			struct sge_rxq *rxq;
1050218792Snp
1051218792Snp			ifp->if_capenable ^= IFCAP_LRO;
1052218792Snp			for_each_rxq(pi, i, rxq) {
1053218792Snp				if (ifp->if_capenable & IFCAP_LRO)
1054228561Snp					rxq->iq.flags |= IQ_LRO_ENABLED;
1055218792Snp				else
1056228561Snp					rxq->iq.flags &= ~IQ_LRO_ENABLED;
1057218792Snp			}
1058218792Snp#endif
1059218792Snp		}
1060237263Snp#ifdef TCP_OFFLOAD
1061228561Snp		if (mask & IFCAP_TOE) {
1062228561Snp			int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
1063228561Snp
1064228561Snp			rc = toe_capability(pi, enable);
1065228561Snp			if (rc != 0)
1066228561Snp				goto fail;
1067228561Snp
1068228561Snp			ifp->if_capenable ^= mask;
1069218792Snp		}
1070218792Snp#endif
1071218792Snp		if (mask & IFCAP_VLAN_HWTAGGING) {
1072218792Snp			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1073218792Snp			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1074218792Snp				PORT_LOCK(pi);
1075218792Snp				rc = update_mac_settings(pi, XGMAC_VLANEX);
1076218792Snp				PORT_UNLOCK(pi);
1077218792Snp			}
1078218792Snp		}
1079218792Snp		if (mask & IFCAP_VLAN_MTU) {
1080218792Snp			ifp->if_capenable ^= IFCAP_VLAN_MTU;
1081218792Snp
1082218792Snp			/* Need to find out how to disable auto-mtu-inflation */
1083218792Snp		}
1084218792Snp		if (mask & IFCAP_VLAN_HWTSO)
1085218792Snp			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1086218792Snp		if (mask & IFCAP_VLAN_HWCSUM)
1087218792Snp			ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
1088218792Snp
1089218792Snp#ifdef VLAN_CAPABILITIES
1090218792Snp		VLAN_CAPABILITIES(ifp);
1091218792Snp#endif
1092218792Snp		ADAPTER_UNLOCK(sc);
1093218792Snp		break;
1094218792Snp
1095218792Snp	case SIOCSIFMEDIA:
1096218792Snp	case SIOCGIFMEDIA:
1097218792Snp		ifmedia_ioctl(ifp, ifr, &pi->media, cmd);
1098218792Snp		break;
1099218792Snp
1100218792Snp	default:
1101218792Snp		rc = ether_ioctl(ifp, cmd, data);
1102218792Snp	}
1103218792Snp
1104218792Snp	return (rc);
1105218792Snp}
1106218792Snp
1107218792Snpstatic int
1108218792Snpcxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
1109218792Snp{
1110218792Snp	struct port_info *pi = ifp->if_softc;
1111218792Snp	struct adapter *sc = pi->adapter;
1112218792Snp	struct sge_txq *txq = &sc->sge.txq[pi->first_txq];
1113218792Snp	struct buf_ring *br;
1114218792Snp	int rc;
1115218792Snp
1116218792Snp	M_ASSERTPKTHDR(m);
1117218792Snp
1118228561Snp	if (__predict_false(pi->link_cfg.link_ok == 0)) {
1119218792Snp		m_freem(m);
1120228561Snp		return (ENETDOWN);
1121218792Snp	}
1122218792Snp
1123218792Snp	if (m->m_flags & M_FLOWID)
1124218792Snp		txq += (m->m_pkthdr.flowid % pi->ntxq);
1125220873Snp	br = txq->br;
1126218792Snp
1127218792Snp	if (TXQ_TRYLOCK(txq) == 0) {
1128228561Snp		struct sge_eq *eq = &txq->eq;
1129228561Snp
1130218792Snp		/*
1131228561Snp		 * It is possible that t4_eth_tx finishes up and releases the
1132228561Snp		 * lock between the TRYLOCK above and the drbr_enqueue here.  We
1133228561Snp		 * need to make sure that this mbuf doesn't just sit there in
1134228561Snp		 * the drbr.
1135218792Snp		 */
1136218792Snp
1137228561Snp		rc = drbr_enqueue(ifp, br, m);
1138228561Snp		if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
1139228561Snp		    !(eq->flags & EQ_DOOMED))
1140228561Snp			callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
1141228561Snp		return (rc);
1142218792Snp	}
1143218792Snp
1144218792Snp	/*
1145218792Snp	 * txq->m is the mbuf that is held up due to a temporary shortage of
1146218792Snp	 * resources and it should be put on the wire first.  Then what's in
1147218792Snp	 * drbr and finally the mbuf that was just passed in to us.
1148218792Snp	 *
1149218792Snp	 * Return code should indicate the fate of the mbuf that was passed in
1150218792Snp	 * this time.
1151218792Snp	 */
1152218792Snp
1153218792Snp	TXQ_LOCK_ASSERT_OWNED(txq);
1154218792Snp	if (drbr_needs_enqueue(ifp, br) || txq->m) {
1155218792Snp
1156218792Snp		/* Queued for transmission. */
1157218792Snp
1158218792Snp		rc = drbr_enqueue(ifp, br, m);
1159218792Snp		m = txq->m ? txq->m : drbr_dequeue(ifp, br);
1160218792Snp		(void) t4_eth_tx(ifp, txq, m);
1161218792Snp		TXQ_UNLOCK(txq);
1162218792Snp		return (rc);
1163218792Snp	}
1164218792Snp
1165218792Snp	/* Direct transmission. */
1166218792Snp	rc = t4_eth_tx(ifp, txq, m);
1167218792Snp	if (rc != 0 && txq->m)
1168218792Snp		rc = 0;	/* held, will be transmitted soon (hopefully) */
1169218792Snp
1170218792Snp	TXQ_UNLOCK(txq);
1171218792Snp	return (rc);
1172218792Snp}
1173218792Snp
1174218792Snpstatic void
1175218792Snpcxgbe_qflush(struct ifnet *ifp)
1176218792Snp{
1177218792Snp	struct port_info *pi = ifp->if_softc;
1178220649Snp	struct sge_txq *txq;
1179220649Snp	int i;
1180220649Snp	struct mbuf *m;
1181218792Snp
1182228561Snp	/* queues do not exist if !PORT_INIT_DONE. */
1183228561Snp	if (pi->flags & PORT_INIT_DONE) {
1184220649Snp		for_each_txq(pi, i, txq) {
1185220649Snp			TXQ_LOCK(txq);
1186220649Snp			m_freem(txq->m);
1187228561Snp			txq->m = NULL;
1188220873Snp			while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
1189220649Snp				m_freem(m);
1190220649Snp			TXQ_UNLOCK(txq);
1191220649Snp		}
1192220649Snp	}
1193220649Snp	if_qflush(ifp);
1194218792Snp}
1195218792Snp
1196218792Snpstatic int
1197218792Snpcxgbe_media_change(struct ifnet *ifp)
1198218792Snp{
1199218792Snp	struct port_info *pi = ifp->if_softc;
1200218792Snp
1201218792Snp	device_printf(pi->dev, "%s unimplemented.\n", __func__);
1202218792Snp
1203218792Snp	return (EOPNOTSUPP);
1204218792Snp}
1205218792Snp
1206218792Snpstatic void
1207218792Snpcxgbe_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1208218792Snp{
1209218792Snp	struct port_info *pi = ifp->if_softc;
1210218792Snp	struct ifmedia_entry *cur = pi->media.ifm_cur;
1211218792Snp	int speed = pi->link_cfg.speed;
1212218792Snp	int data = (pi->port_type << 8) | pi->mod_type;
1213218792Snp
1214218792Snp	if (cur->ifm_data != data) {
1215218792Snp		build_medialist(pi);
1216218792Snp		cur = pi->media.ifm_cur;
1217218792Snp	}
1218218792Snp
1219218792Snp	ifmr->ifm_status = IFM_AVALID;
1220218792Snp	if (!pi->link_cfg.link_ok)
1221218792Snp		return;
1222218792Snp
1223218792Snp	ifmr->ifm_status |= IFM_ACTIVE;
1224218792Snp
1225218792Snp	/* active and current will differ iff current media is autoselect. */
1226218792Snp	if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
1227218792Snp		return;
1228218792Snp
1229218792Snp	ifmr->ifm_active = IFM_ETHER | IFM_FDX;
1230218792Snp	if (speed == SPEED_10000)
1231218792Snp		ifmr->ifm_active |= IFM_10G_T;
1232218792Snp	else if (speed == SPEED_1000)
1233218792Snp		ifmr->ifm_active |= IFM_1000_T;
1234218792Snp	else if (speed == SPEED_100)
1235218792Snp		ifmr->ifm_active |= IFM_100_TX;
1236218792Snp	else if (speed == SPEED_10)
1237218792Snp		ifmr->ifm_active |= IFM_10_T;
1238218792Snp	else
1239218792Snp		KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
1240218792Snp			    speed));
1241218792Snp}
1242218792Snp
1243218792Snpvoid
1244218792Snpt4_fatal_err(struct adapter *sc)
1245218792Snp{
1246218792Snp	t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0);
1247218792Snp	t4_intr_disable(sc);
1248218792Snp	log(LOG_EMERG, "%s: encountered fatal error, adapter stopped.\n",
1249218792Snp	    device_get_nameunit(sc->dev));
1250218792Snp}
1251218792Snp
1252218792Snpstatic int
1253218792Snpmap_bars(struct adapter *sc)
1254218792Snp{
1255218792Snp	sc->regs_rid = PCIR_BAR(0);
1256218792Snp	sc->regs_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1257218792Snp	    &sc->regs_rid, RF_ACTIVE);
1258218792Snp	if (sc->regs_res == NULL) {
1259218792Snp		device_printf(sc->dev, "cannot map registers.\n");
1260218792Snp		return (ENXIO);
1261218792Snp	}
1262218792Snp	sc->bt = rman_get_bustag(sc->regs_res);
1263218792Snp	sc->bh = rman_get_bushandle(sc->regs_res);
1264218792Snp	sc->mmio_len = rman_get_size(sc->regs_res);
1265218792Snp
1266218792Snp	sc->msix_rid = PCIR_BAR(4);
1267218792Snp	sc->msix_res = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY,
1268218792Snp	    &sc->msix_rid, RF_ACTIVE);
1269218792Snp	if (sc->msix_res == NULL) {
1270218792Snp		device_printf(sc->dev, "cannot map MSI-X BAR.\n");
1271218792Snp		return (ENXIO);
1272218792Snp	}
1273218792Snp
1274218792Snp	return (0);
1275218792Snp}
1276218792Snp
1277218792Snpstatic void
1278218792Snpsetup_memwin(struct adapter *sc)
1279218792Snp{
1280218792Snp	u_long bar0;
1281218792Snp
1282218792Snp	bar0 = rman_get_start(sc->regs_res);
1283218792Snp
1284218792Snp	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0),
1285218792Snp	    	     (bar0 + MEMWIN0_BASE) | V_BIR(0) |
1286218792Snp		     V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
1287218792Snp
1288218792Snp	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1),
1289218792Snp		     (bar0 + MEMWIN1_BASE) | V_BIR(0) |
1290218792Snp		     V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
1291218792Snp
1292218792Snp	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2),
1293218792Snp		     (bar0 + MEMWIN2_BASE) | V_BIR(0) |
1294218792Snp		     V_WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
1295218792Snp}
1296218792Snp
1297218792Snpstatic int
1298218792Snpcfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
1299218792Snp    struct intrs_and_queues *iaq)
1300218792Snp{
1301228561Snp	int rc, itype, navail, nrxq10g, nrxq1g, n;
1302228561Snp	int nofldrxq10g = 0, nofldrxq1g = 0;
1303218792Snp
1304218792Snp	bzero(iaq, sizeof(*iaq));
1305218792Snp
1306228561Snp	iaq->ntxq10g = t4_ntxq10g;
1307228561Snp	iaq->ntxq1g = t4_ntxq1g;
1308228561Snp	iaq->nrxq10g = nrxq10g = t4_nrxq10g;
1309228561Snp	iaq->nrxq1g = nrxq1g = t4_nrxq1g;
1310237263Snp#ifdef TCP_OFFLOAD
1311228561Snp	iaq->nofldtxq10g = t4_nofldtxq10g;
1312228561Snp	iaq->nofldtxq1g = t4_nofldtxq1g;
1313228561Snp	iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
1314228561Snp	iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
1315228561Snp#endif
1316228561Snp
1317219944Snp	for (itype = INTR_MSIX; itype; itype >>= 1) {
1318218792Snp
1319228561Snp		if ((itype & t4_intr_types) == 0)
1320218792Snp			continue;	/* not allowed */
1321218792Snp
1322219944Snp		if (itype == INTR_MSIX)
1323218792Snp			navail = pci_msix_count(sc->dev);
1324219944Snp		else if (itype == INTR_MSI)
1325218792Snp			navail = pci_msi_count(sc->dev);
1326218792Snp		else
1327218792Snp			navail = 1;
1328228561Snprestart:
1329218792Snp		if (navail == 0)
1330218792Snp			continue;
1331218792Snp
1332218792Snp		iaq->intr_type = itype;
1333228561Snp		iaq->intr_flags = 0;
1334218792Snp
1335228561Snp		/*
1336228561Snp		 * Best option: an interrupt vector for errors, one for the
1337228561Snp		 * firmware event queue, and one each for each rxq (NIC as well
1338228561Snp		 * as offload).
1339228561Snp		 */
1340228561Snp		iaq->nirq = T4_EXTRA_INTR;
1341228561Snp		iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
1342228561Snp		iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
1343228561Snp		if (iaq->nirq <= navail &&
1344228561Snp		    (itype != INTR_MSI || powerof2(iaq->nirq))) {
1345228561Snp			iaq->intr_flags |= INTR_DIRECT;
1346228561Snp			goto allocate;
1347228561Snp		}
1348218792Snp
1349228561Snp		/*
1350228561Snp		 * Second best option: an interrupt vector for errors, one for
1351228561Snp		 * the firmware event queue, and one each for either NIC or
1352228561Snp		 * offload rxq's.
1353228561Snp		 */
1354228561Snp		iaq->nirq = T4_EXTRA_INTR;
1355228561Snp		iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
1356228561Snp		iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
1357228561Snp		if (iaq->nirq <= navail &&
1358228561Snp		    (itype != INTR_MSI || powerof2(iaq->nirq)))
1359228561Snp			goto allocate;
1360218792Snp
1361228561Snp		/*
1362228561Snp		 * Next best option: an interrupt vector for errors, one for the
1363228561Snp		 * firmware event queue, and at least one per port.  At this
1364228561Snp		 * point we know we'll have to downsize nrxq or nofldrxq to fit
1365228561Snp		 * what's available to us.
1366228561Snp		 */
1367228561Snp		iaq->nirq = T4_EXTRA_INTR;
1368228561Snp		iaq->nirq += n10g + n1g;
1369228561Snp		if (iaq->nirq <= navail) {
1370228561Snp			int leftover = navail - iaq->nirq;
1371218792Snp
1372228561Snp			if (n10g > 0) {
1373228561Snp				int target = max(nrxq10g, nofldrxq10g);
1374219944Snp
1375228561Snp				n = 1;
1376228561Snp				while (n < target && leftover >= n10g) {
1377228561Snp					leftover -= n10g;
1378228561Snp					iaq->nirq += n10g;
1379228561Snp					n++;
1380228561Snp				}
1381228561Snp				iaq->nrxq10g = min(n, nrxq10g);
1382237263Snp#ifdef TCP_OFFLOAD
1383228561Snp				iaq->nofldrxq10g = min(n, nofldrxq10g);
1384228561Snp#endif
1385228561Snp			}
1386218792Snp
1387228561Snp			if (n1g > 0) {
1388228561Snp				int target = max(nrxq1g, nofldrxq1g);
1389219944Snp
1390228561Snp				n = 1;
1391228561Snp				while (n < target && leftover >= n1g) {
1392228561Snp					leftover -= n1g;
1393228561Snp					iaq->nirq += n1g;
1394228561Snp					n++;
1395219944Snp				}
1396228561Snp				iaq->nrxq1g = min(n, nrxq1g);
1397237263Snp#ifdef TCP_OFFLOAD
1398228561Snp				iaq->nofldrxq1g = min(n, nofldrxq1g);
1399228561Snp#endif
1400219944Snp			}
1401219944Snp
1402228561Snp			if (itype != INTR_MSI || powerof2(iaq->nirq))
1403228561Snp				goto allocate;
1404218792Snp		}
1405218792Snp
1406228561Snp		/*
1407228561Snp		 * Least desirable option: one interrupt vector for everything.
1408228561Snp		 */
1409228561Snp		iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
1410237263Snp#ifdef TCP_OFFLOAD
1411228561Snp		iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
1412228561Snp#endif
1413228561Snp
1414228561Snpallocate:
1415218792Snp		navail = iaq->nirq;
1416218792Snp		rc = 0;
1417219944Snp		if (itype == INTR_MSIX)
1418218792Snp			rc = pci_alloc_msix(sc->dev, &navail);
1419219944Snp		else if (itype == INTR_MSI)
1420218792Snp			rc = pci_alloc_msi(sc->dev, &navail);
1421218792Snp
1422218792Snp		if (rc == 0) {
1423218792Snp			if (navail == iaq->nirq)
1424218792Snp				return (0);
1425218792Snp
1426218792Snp			/*
1427218792Snp			 * Didn't get the number requested.  Use whatever number
1428218792Snp			 * the kernel is willing to allocate (it's in navail).
1429218792Snp			 */
1430228561Snp			device_printf(sc->dev, "fewer vectors than requested, "
1431228561Snp			    "type=%d, req=%d, rcvd=%d; will downshift req.\n",
1432228561Snp			    itype, iaq->nirq, navail);
1433218792Snp			pci_release_msi(sc->dev);
1434228561Snp			goto restart;
1435218792Snp		}
1436218792Snp
1437218792Snp		device_printf(sc->dev,
1438218792Snp		    "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
1439218792Snp		    itype, rc, iaq->nirq, navail);
1440218792Snp	}
1441218792Snp
1442218792Snp	device_printf(sc->dev,
1443218792Snp	    "failed to find a usable interrupt type.  "
1444228561Snp	    "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
1445218792Snp	    pci_msix_count(sc->dev), pci_msi_count(sc->dev));
1446218792Snp
1447218792Snp	return (ENXIO);
1448218792Snp}
1449218792Snp
1450218792Snp/*
1451228561Snp * Install a compatible firmware (if required), establish contact with it (by
1452228561Snp * saying hello), and reset the device.  If we end up as the master driver,
1453228561Snp * partition adapter resources by providing a configuration file to the
1454228561Snp * firmware.
1455218792Snp */
1456218792Snpstatic int
1457218792Snpprep_firmware(struct adapter *sc)
1458218792Snp{
1459228561Snp	const struct firmware *fw = NULL, *cfg = NULL, *default_cfg;
1460218792Snp	int rc;
1461218792Snp	enum dev_state state;
1462218792Snp
1463228561Snp	default_cfg = firmware_get(T4_CFGNAME);
1464228561Snp
1465218792Snp	/* Check firmware version and install a different one if necessary */
1466218792Snp	rc = t4_check_fw_version(sc);
1467234831Snp	snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
1468234831Snp	    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1469234831Snp	    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1470234831Snp	    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1471234831Snp	    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1472228561Snp	if (rc != 0) {
1473219287Snp		uint32_t v = 0;
1474218792Snp
1475218792Snp		fw = firmware_get(T4_FWNAME);
1476219287Snp		if (fw != NULL) {
1477219287Snp			const struct fw_hdr *hdr = (const void *)fw->data;
1478219287Snp
1479219287Snp			v = ntohl(hdr->fw_ver);
1480219287Snp
1481219287Snp			/*
1482219287Snp			 * The firmware module will not be used if it isn't the
1483219287Snp			 * same major version as what the driver was compiled
1484228561Snp			 * with.
1485219287Snp			 */
1486219287Snp			if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) {
1487219287Snp				device_printf(sc->dev,
1488219287Snp				    "Found firmware image but version %d "
1489219287Snp				    "can not be used with this driver (%d)\n",
1490219287Snp				    G_FW_HDR_FW_VER_MAJOR(v), FW_VERSION_MAJOR);
1491219287Snp
1492219287Snp				firmware_put(fw, FIRMWARE_UNLOAD);
1493219287Snp				fw = NULL;
1494219287Snp			}
1495218792Snp		}
1496218792Snp
1497228561Snp		if (fw == NULL && rc < 0) {
1498219287Snp			device_printf(sc->dev, "No usable firmware. "
1499228561Snp			    "card has %d.%d.%d, driver compiled with %d.%d.%d",
1500219287Snp			    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1501219287Snp			    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1502219287Snp			    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1503219287Snp			    FW_VERSION_MAJOR, FW_VERSION_MINOR,
1504228561Snp			    FW_VERSION_MICRO);
1505228561Snp			rc = EAGAIN;
1506228561Snp			goto done;
1507219287Snp		}
1508219287Snp
1509219287Snp		/*
1510219287Snp		 * Always upgrade, even for minor/micro/build mismatches.
1511219287Snp		 * Downgrade only for a major version mismatch or if
1512219287Snp		 * force_firmware_install was specified.
1513219287Snp		 */
1514228561Snp		if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) {
1515218792Snp			device_printf(sc->dev,
1516219287Snp			    "installing firmware %d.%d.%d.%d on card.\n",
1517219287Snp			    G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v),
1518219287Snp			    G_FW_HDR_FW_VER_MICRO(v), G_FW_HDR_FW_VER_BUILD(v));
1519219287Snp
1520219287Snp			rc = -t4_load_fw(sc, fw->data, fw->datasize);
1521219287Snp			if (rc != 0) {
1522219287Snp				device_printf(sc->dev,
1523219287Snp				    "failed to install firmware: %d\n", rc);
1524228561Snp				goto done;
1525219287Snp			} else {
1526219287Snp				/* refresh */
1527219287Snp				(void) t4_check_fw_version(sc);
1528234831Snp				snprintf(sc->fw_version,
1529234831Snp				    sizeof(sc->fw_version), "%u.%u.%u.%u",
1530234831Snp				    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
1531234831Snp				    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
1532234831Snp				    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
1533234831Snp				    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
1534219287Snp			}
1535218792Snp		}
1536218792Snp	}
1537218792Snp
1538228561Snp	/* Contact firmware.  */
1539228561Snp	rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
1540218792Snp	if (rc < 0) {
1541218792Snp		rc = -rc;
1542218792Snp		device_printf(sc->dev,
1543218792Snp		    "failed to connect to the firmware: %d.\n", rc);
1544228561Snp		goto done;
1545218792Snp	}
1546228561Snp	if (rc == sc->mbox)
1547228561Snp		sc->flags |= MASTER_PF;
1548218792Snp
1549218792Snp	/* Reset device */
1550218792Snp	rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
1551218792Snp	if (rc != 0) {
1552218792Snp		device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
1553218792Snp		if (rc != ETIMEDOUT && rc != EIO)
1554218792Snp			t4_fw_bye(sc, sc->mbox);
1555228561Snp		goto done;
1556218792Snp	}
1557218792Snp
1558228561Snp	/* Partition adapter resources as specified in the config file. */
1559228561Snp	if (sc->flags & MASTER_PF) {
1560228561Snp		if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) {
1561228561Snp			char s[32];
1562228561Snp
1563228561Snp			snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file);
1564228561Snp			cfg = firmware_get(s);
1565228561Snp			if (cfg == NULL) {
1566228561Snp				device_printf(sc->dev,
1567228561Snp				    "unable to locate %s module, "
1568228561Snp				    "will use default config file.\n", s);
1569228561Snp			}
1570228561Snp		}
1571228561Snp
1572228561Snp		rc = partition_resources(sc, cfg ? cfg : default_cfg);
1573228561Snp		if (rc != 0)
1574228561Snp			goto done;	/* error message displayed already */
1575228561Snp	}
1576228561Snp
1577218792Snp	sc->flags |= FW_OK;
1578218792Snp
1579228561Snpdone:
1580228561Snp	if (fw != NULL)
1581228561Snp		firmware_put(fw, FIRMWARE_UNLOAD);
1582228561Snp	if (cfg != NULL)
1583228561Snp		firmware_put(cfg, FIRMWARE_UNLOAD);
1584228561Snp	if (default_cfg != NULL)
1585228561Snp		firmware_put(default_cfg, FIRMWARE_UNLOAD);
1586228561Snp
1587228561Snp	return (rc);
1588218792Snp}
1589218792Snp
1590228561Snp#define FW_PARAM_DEV(param) \
1591228561Snp	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
1592228561Snp	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
1593228561Snp#define FW_PARAM_PFVF(param) \
1594228561Snp	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
1595228561Snp	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
1596228561Snp
1597228561Snp/*
1598228561Snp * Upload configuration file to card's memory.
1599228561Snp */
1600218792Snpstatic int
1601228561Snpupload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt,
1602228561Snp    uint32_t *ma)
1603222551Snp{
1604228561Snp	int rc, i;
1605228561Snp	uint32_t param, val, mtype, maddr, bar, off, win, remaining;
1606228561Snp	const uint32_t *b;
1607222551Snp
1608228561Snp	/* Figure out where the firmware wants us to upload it. */
1609228561Snp	param = FW_PARAM_DEV(CF);
1610228561Snp	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1611222551Snp	if (rc != 0) {
1612228561Snp		/* Firmwares without config file support will fail this way */
1613222551Snp		device_printf(sc->dev,
1614228561Snp		    "failed to query config file location: %d.\n", rc);
1615222551Snp		return (rc);
1616222551Snp	}
1617228561Snp	*mt = mtype = G_FW_PARAMS_PARAM_Y(val);
1618228561Snp	*ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
1619222551Snp
1620228561Snp	if (maddr & 3) {
1621228561Snp		device_printf(sc->dev,
1622228561Snp		    "cannot upload config file (type %u, addr %x).\n",
1623228561Snp		    mtype, maddr);
1624228561Snp		return (EFAULT);
1625228561Snp	}
1626222551Snp
1627228561Snp	/* Translate mtype/maddr to an address suitable for the PCIe window */
1628228561Snp	val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
1629228561Snp	val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE;
1630228561Snp	switch (mtype) {
1631228561Snp	case FW_MEMTYPE_CF_EDC0:
1632228561Snp		if (!(val & F_EDRAM0_ENABLE))
1633228561Snp			goto err;
1634228561Snp		bar = t4_read_reg(sc, A_MA_EDRAM0_BAR);
1635228561Snp		maddr += G_EDRAM0_BASE(bar) << 20;
1636228561Snp		break;
1637228561Snp
1638228561Snp	case FW_MEMTYPE_CF_EDC1:
1639228561Snp		if (!(val & F_EDRAM1_ENABLE))
1640228561Snp			goto err;
1641228561Snp		bar = t4_read_reg(sc, A_MA_EDRAM1_BAR);
1642228561Snp		maddr += G_EDRAM1_BASE(bar) << 20;
1643228561Snp		break;
1644228561Snp
1645228561Snp	case FW_MEMTYPE_CF_EXTMEM:
1646228561Snp		if (!(val & F_EXT_MEM_ENABLE))
1647228561Snp			goto err;
1648228561Snp		bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
1649228561Snp		maddr += G_EXT_MEM_BASE(bar) << 20;
1650228561Snp		break;
1651228561Snp
1652228561Snp	default:
1653228561Snperr:
1654228561Snp		device_printf(sc->dev,
1655228561Snp		    "cannot upload config file (type %u, enabled %u).\n",
1656228561Snp		    mtype, val);
1657228561Snp		return (EFAULT);
1658228561Snp	}
1659228561Snp
1660228561Snp	/*
1661228561Snp	 * Position the PCIe window (we use memwin2) to the 16B aligned area
1662228561Snp	 * just at/before the upload location.
1663228561Snp	 */
1664228561Snp	win = maddr & ~0xf;
1665228561Snp	off = maddr - win;  /* offset from the start of the window. */
1666228561Snp	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
1667228561Snp	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
1668228561Snp
1669228561Snp	remaining = fw->datasize;
1670228561Snp	if (remaining > FLASH_CFG_MAX_SIZE ||
1671228561Snp	    remaining > MEMWIN2_APERTURE - off) {
1672228561Snp		device_printf(sc->dev, "cannot upload config file all at once "
1673228561Snp		    "(size %u, max %u, room %u).\n",
1674228561Snp		    remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off);
1675228561Snp		return (EFBIG);
1676228561Snp	}
1677228561Snp
1678228561Snp	/*
1679228561Snp	 * XXX: sheer laziness.  We deliberately added 4 bytes of useless
1680228561Snp	 * stuffing/comments at the end of the config file so it's ok to simply
1681228561Snp	 * throw away the last remaining bytes when the config file is not an
1682228561Snp	 * exact multiple of 4.
1683228561Snp	 */
1684228561Snp	b = fw->data;
1685228561Snp	for (i = 0; remaining >= 4; i += 4, remaining -= 4)
1686228561Snp		t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++);
1687228561Snp
1688228561Snp	return (rc);
1689222551Snp}
1690222551Snp
1691228561Snp/*
1692228561Snp * Partition chip resources for use between various PFs, VFs, etc.  This is done
1693228561Snp * by uploading the firmware configuration file to the adapter and instructing
1694228561Snp * the firmware to process it.
1695228561Snp */
1696222551Snpstatic int
1697228561Snppartition_resources(struct adapter *sc, const struct firmware *cfg)
1698218792Snp{
1699218792Snp	int rc;
1700228561Snp	struct fw_caps_config_cmd caps;
1701228561Snp	uint32_t mtype, maddr, finicsum, cfcsum;
1702218792Snp
1703228561Snp	rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT;
1704228561Snp	if (rc != 0) {
1705228561Snp		mtype = FW_MEMTYPE_CF_FLASH;
1706228561Snp		maddr = t4_flash_cfg_addr(sc);
1707228561Snp	}
1708228561Snp
1709228561Snp	bzero(&caps, sizeof(caps));
1710228561Snp	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1711218792Snp	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1712228561Snp	caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
1713228561Snp	    V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
1714228561Snp	    V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
1715228561Snp	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1716228561Snp	if (rc != 0) {
1717228561Snp		device_printf(sc->dev,
1718228561Snp		    "failed to pre-process config file: %d.\n", rc);
1719218792Snp		return (rc);
1720228561Snp	}
1721218792Snp
1722228561Snp	finicsum = be32toh(caps.finicsum);
1723228561Snp	cfcsum = be32toh(caps.cfcsum);
1724228561Snp	if (finicsum != cfcsum) {
1725228561Snp		device_printf(sc->dev,
1726228561Snp		    "WARNING: config file checksum mismatch: %08x %08x\n",
1727228561Snp		    finicsum, cfcsum);
1728228561Snp	}
1729228561Snp	sc->cfcsum = cfcsum;
1730218792Snp
1731228561Snp#define LIMIT_CAPS(x) do { \
1732228561Snp	caps.x &= htobe16(t4_##x##_allowed); \
1733228561Snp	sc->x = htobe16(caps.x); \
1734228561Snp} while (0)
1735228561Snp
1736228561Snp	/*
1737228561Snp	 * Let the firmware know what features will (not) be used so it can tune
1738228561Snp	 * things accordingly.
1739228561Snp	 */
1740228561Snp	LIMIT_CAPS(linkcaps);
1741228561Snp	LIMIT_CAPS(niccaps);
1742228561Snp	LIMIT_CAPS(toecaps);
1743228561Snp	LIMIT_CAPS(rdmacaps);
1744228561Snp	LIMIT_CAPS(iscsicaps);
1745228561Snp	LIMIT_CAPS(fcoecaps);
1746228561Snp#undef LIMIT_CAPS
1747228561Snp
1748228561Snp	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1749218792Snp	    F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
1750228561Snp	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1751228561Snp	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
1752228561Snp	if (rc != 0) {
1753228561Snp		device_printf(sc->dev,
1754228561Snp		    "failed to process config file: %d.\n", rc);
1755228561Snp		return (rc);
1756228561Snp	}
1757218792Snp
1758228561Snp	return (0);
1759218792Snp}
1760218792Snp
1761228561Snp/*
1762228561Snp * Retrieve parameters that are needed (or nice to have) prior to calling
1763228561Snp * t4_sge_init and t4_fw_initialize.
1764228561Snp */
1765218792Snpstatic int
1766228561Snpget_params__pre_init(struct adapter *sc)
1767218792Snp{
1768218792Snp	int rc;
1769228561Snp	uint32_t param[2], val[2];
1770228561Snp	struct fw_devlog_cmd cmd;
1771228561Snp	struct devlog_params *dlog = &sc->params.devlog;
1772218792Snp
1773228561Snp	param[0] = FW_PARAM_DEV(PORTVEC);
1774228561Snp	param[1] = FW_PARAM_DEV(CCLK);
1775228561Snp	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1776218792Snp	if (rc != 0) {
1777218792Snp		device_printf(sc->dev,
1778228561Snp		    "failed to query parameters (pre_init): %d.\n", rc);
1779228561Snp		return (rc);
1780218792Snp	}
1781218792Snp
1782218792Snp	sc->params.portvec = val[0];
1783218792Snp	sc->params.nports = 0;
1784218792Snp	while (val[0]) {
1785218792Snp		sc->params.nports++;
1786218792Snp		val[0] &= val[0] - 1;
1787218792Snp	}
1788218792Snp
1789228561Snp	sc->params.vpd.cclk = val[1];
1790218792Snp
1791228561Snp	/* Read device log parameters. */
1792228561Snp	bzero(&cmd, sizeof(cmd));
1793228561Snp	cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
1794228561Snp	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1795228561Snp	cmd.retval_len16 = htobe32(FW_LEN16(cmd));
1796228561Snp	rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
1797228561Snp	if (rc != 0) {
1798228561Snp		device_printf(sc->dev,
1799228561Snp		    "failed to get devlog parameters: %d.\n", rc);
1800228561Snp		bzero(dlog, sizeof (*dlog));
1801228561Snp		rc = 0;	/* devlog isn't critical for device operation */
1802228561Snp	} else {
1803228561Snp		val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
1804228561Snp		dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
1805228561Snp		dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
1806228561Snp		dlog->size = be32toh(cmd.memsize_devlog);
1807228561Snp	}
1808228561Snp
1809228561Snp	return (rc);
1810228561Snp}
1811228561Snp
1812228561Snp/*
1813228561Snp * Retrieve various parameters that are of interest to the driver.  The device
1814228561Snp * has been initialized by the firmware at this point.
1815228561Snp */
1816228561Snpstatic int
1817228561Snpget_params__post_init(struct adapter *sc)
1818228561Snp{
1819228561Snp	int rc;
1820228561Snp	uint32_t param[7], val[7];
1821228561Snp	struct fw_caps_config_cmd caps;
1822228561Snp
1823228561Snp	param[0] = FW_PARAM_PFVF(IQFLINT_START);
1824228561Snp	param[1] = FW_PARAM_PFVF(EQ_START);
1825228561Snp	param[2] = FW_PARAM_PFVF(FILTER_START);
1826228561Snp	param[3] = FW_PARAM_PFVF(FILTER_END);
1827228561Snp	rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
1828228561Snp	if (rc != 0) {
1829228561Snp		device_printf(sc->dev,
1830228561Snp		    "failed to query parameters (post_init): %d.\n", rc);
1831228561Snp		return (rc);
1832228561Snp	}
1833228561Snp
1834228561Snp	sc->sge.iq_start = val[0];
1835228561Snp	sc->sge.eq_start = val[1];
1836228561Snp	sc->tids.ftid_base = val[2];
1837228561Snp	sc->tids.nftids = val[3] - val[2] + 1;
1838228561Snp
1839228561Snp	/* get capabilites */
1840228561Snp	bzero(&caps, sizeof(caps));
1841228561Snp	caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
1842228561Snp	    F_FW_CMD_REQUEST | F_FW_CMD_READ);
1843228561Snp	caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
1844228561Snp	rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
1845228561Snp	if (rc != 0) {
1846228561Snp		device_printf(sc->dev,
1847228561Snp		    "failed to get card capabilities: %d.\n", rc);
1848228561Snp		return (rc);
1849228561Snp	}
1850228561Snp
1851228561Snp	if (caps.toecaps) {
1852218792Snp		/* query offload-related parameters */
1853228561Snp		param[0] = FW_PARAM_DEV(NTID);
1854228561Snp		param[1] = FW_PARAM_PFVF(SERVER_START);
1855228561Snp		param[2] = FW_PARAM_PFVF(SERVER_END);
1856228561Snp		param[3] = FW_PARAM_PFVF(TDDP_START);
1857228561Snp		param[4] = FW_PARAM_PFVF(TDDP_END);
1858228561Snp		param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
1859228561Snp		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1860218792Snp		if (rc != 0) {
1861218792Snp			device_printf(sc->dev,
1862218792Snp			    "failed to query TOE parameters: %d.\n", rc);
1863228561Snp			return (rc);
1864218792Snp		}
1865218792Snp		sc->tids.ntids = val[0];
1866218792Snp		sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
1867218792Snp		sc->tids.stid_base = val[1];
1868218792Snp		sc->tids.nstids = val[2] - val[1] + 1;
1869218792Snp		sc->vres.ddp.start = val[3];
1870218792Snp		sc->vres.ddp.size = val[4] - val[3] + 1;
1871218792Snp		sc->params.ofldq_wr_cred = val[5];
1872218792Snp		sc->params.offload = 1;
1873218792Snp	}
1874228561Snp	if (caps.rdmacaps) {
1875228561Snp		param[0] = FW_PARAM_PFVF(STAG_START);
1876228561Snp		param[1] = FW_PARAM_PFVF(STAG_END);
1877228561Snp		param[2] = FW_PARAM_PFVF(RQ_START);
1878228561Snp		param[3] = FW_PARAM_PFVF(RQ_END);
1879228561Snp		param[4] = FW_PARAM_PFVF(PBL_START);
1880228561Snp		param[5] = FW_PARAM_PFVF(PBL_END);
1881228561Snp		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
1882218792Snp		if (rc != 0) {
1883218792Snp			device_printf(sc->dev,
1884228561Snp			    "failed to query RDMA parameters(1): %d.\n", rc);
1885228561Snp			return (rc);
1886218792Snp		}
1887218792Snp		sc->vres.stag.start = val[0];
1888218792Snp		sc->vres.stag.size = val[1] - val[0] + 1;
1889218792Snp		sc->vres.rq.start = val[2];
1890218792Snp		sc->vres.rq.size = val[3] - val[2] + 1;
1891218792Snp		sc->vres.pbl.start = val[4];
1892218792Snp		sc->vres.pbl.size = val[5] - val[4] + 1;
1893228561Snp
1894228561Snp		param[0] = FW_PARAM_PFVF(SQRQ_START);
1895228561Snp		param[1] = FW_PARAM_PFVF(SQRQ_END);
1896228561Snp		param[2] = FW_PARAM_PFVF(CQ_START);
1897228561Snp		param[3] = FW_PARAM_PFVF(CQ_END);
1898228561Snp		param[4] = FW_PARAM_PFVF(OCQ_START);
1899228561Snp		param[5] = FW_PARAM_PFVF(OCQ_END);
1900228561Snp		rc = -t4_query_params(sc, 0, 0, 0, 6, param, val);
1901228561Snp		if (rc != 0) {
1902228561Snp			device_printf(sc->dev,
1903228561Snp			    "failed to query RDMA parameters(2): %d.\n", rc);
1904228561Snp			return (rc);
1905228561Snp		}
1906228561Snp		sc->vres.qp.start = val[0];
1907228561Snp		sc->vres.qp.size = val[1] - val[0] + 1;
1908228561Snp		sc->vres.cq.start = val[2];
1909228561Snp		sc->vres.cq.size = val[3] - val[2] + 1;
1910228561Snp		sc->vres.ocq.start = val[4];
1911228561Snp		sc->vres.ocq.size = val[5] - val[4] + 1;
1912218792Snp	}
1913228561Snp	if (caps.iscsicaps) {
1914228561Snp		param[0] = FW_PARAM_PFVF(ISCSI_START);
1915228561Snp		param[1] = FW_PARAM_PFVF(ISCSI_END);
1916228561Snp		rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
1917218792Snp		if (rc != 0) {
1918218792Snp			device_printf(sc->dev,
1919218792Snp			    "failed to query iSCSI parameters: %d.\n", rc);
1920228561Snp			return (rc);
1921218792Snp		}
1922218792Snp		sc->vres.iscsi.start = val[0];
1923218792Snp		sc->vres.iscsi.size = val[1] - val[0] + 1;
1924218792Snp	}
1925218792Snp
1926228561Snp	/* These are finalized by FW initialization, load their values now */
1927228561Snp	val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
1928228561Snp	sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
1929228561Snp	sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
1930228561Snp	t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
1931228561Snp
1932218792Snp	return (rc);
1933218792Snp}
1934218792Snp
1935228561Snp#undef FW_PARAM_PFVF
1936228561Snp#undef FW_PARAM_DEV
1937228561Snp
1938218792Snpstatic void
1939218792Snpt4_set_desc(struct adapter *sc)
1940218792Snp{
1941218792Snp	char buf[128];
1942218792Snp	struct adapter_params *p = &sc->params;
1943218792Snp
1944228561Snp	snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s",
1945228561Snp	    p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec);
1946218792Snp
1947218792Snp	device_set_desc_copy(sc->dev, buf);
1948218792Snp}
1949218792Snp
1950218792Snpstatic void
1951218792Snpbuild_medialist(struct port_info *pi)
1952218792Snp{
1953218792Snp	struct ifmedia *media = &pi->media;
1954218792Snp	int data, m;
1955218792Snp
1956218792Snp	PORT_LOCK(pi);
1957218792Snp
1958218792Snp	ifmedia_removeall(media);
1959218792Snp
1960218792Snp	m = IFM_ETHER | IFM_FDX;
1961218792Snp	data = (pi->port_type << 8) | pi->mod_type;
1962218792Snp
1963218792Snp	switch(pi->port_type) {
1964218792Snp	case FW_PORT_TYPE_BT_XFI:
1965218792Snp		ifmedia_add(media, m | IFM_10G_T, data, NULL);
1966218792Snp		break;
1967218792Snp
1968218792Snp	case FW_PORT_TYPE_BT_XAUI:
1969218792Snp		ifmedia_add(media, m | IFM_10G_T, data, NULL);
1970218792Snp		/* fall through */
1971218792Snp
1972218792Snp	case FW_PORT_TYPE_BT_SGMII:
1973218792Snp		ifmedia_add(media, m | IFM_1000_T, data, NULL);
1974218792Snp		ifmedia_add(media, m | IFM_100_TX, data, NULL);
1975218792Snp		ifmedia_add(media, IFM_ETHER | IFM_AUTO, data, NULL);
1976218792Snp		ifmedia_set(media, IFM_ETHER | IFM_AUTO);
1977218792Snp		break;
1978218792Snp
1979218792Snp	case FW_PORT_TYPE_CX4:
1980218792Snp		ifmedia_add(media, m | IFM_10G_CX4, data, NULL);
1981218792Snp		ifmedia_set(media, m | IFM_10G_CX4);
1982218792Snp		break;
1983218792Snp
1984218792Snp	case FW_PORT_TYPE_SFP:
1985218792Snp	case FW_PORT_TYPE_FIBER_XFI:
1986218792Snp	case FW_PORT_TYPE_FIBER_XAUI:
1987218792Snp		switch (pi->mod_type) {
1988218792Snp
1989218792Snp		case FW_PORT_MOD_TYPE_LR:
1990218792Snp			ifmedia_add(media, m | IFM_10G_LR, data, NULL);
1991218792Snp			ifmedia_set(media, m | IFM_10G_LR);
1992218792Snp			break;
1993218792Snp
1994218792Snp		case FW_PORT_MOD_TYPE_SR:
1995218792Snp			ifmedia_add(media, m | IFM_10G_SR, data, NULL);
1996218792Snp			ifmedia_set(media, m | IFM_10G_SR);
1997218792Snp			break;
1998218792Snp
1999218792Snp		case FW_PORT_MOD_TYPE_LRM:
2000218792Snp			ifmedia_add(media, m | IFM_10G_LRM, data, NULL);
2001218792Snp			ifmedia_set(media, m | IFM_10G_LRM);
2002218792Snp			break;
2003218792Snp
2004218792Snp		case FW_PORT_MOD_TYPE_TWINAX_PASSIVE:
2005218792Snp		case FW_PORT_MOD_TYPE_TWINAX_ACTIVE:
2006218792Snp			ifmedia_add(media, m | IFM_10G_TWINAX, data, NULL);
2007218792Snp			ifmedia_set(media, m | IFM_10G_TWINAX);
2008218792Snp			break;
2009218792Snp
2010218792Snp		case FW_PORT_MOD_TYPE_NONE:
2011218792Snp			m &= ~IFM_FDX;
2012218792Snp			ifmedia_add(media, m | IFM_NONE, data, NULL);
2013218792Snp			ifmedia_set(media, m | IFM_NONE);
2014218792Snp			break;
2015218792Snp
2016218792Snp		case FW_PORT_MOD_TYPE_NA:
2017218792Snp		case FW_PORT_MOD_TYPE_ER:
2018218792Snp		default:
2019218792Snp			ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2020218792Snp			ifmedia_set(media, m | IFM_UNKNOWN);
2021218792Snp			break;
2022218792Snp		}
2023218792Snp		break;
2024218792Snp
2025218792Snp	case FW_PORT_TYPE_KX4:
2026218792Snp	case FW_PORT_TYPE_KX:
2027218792Snp	case FW_PORT_TYPE_KR:
2028218792Snp	default:
2029218792Snp		ifmedia_add(media, m | IFM_UNKNOWN, data, NULL);
2030218792Snp		ifmedia_set(media, m | IFM_UNKNOWN);
2031218792Snp		break;
2032218792Snp	}
2033218792Snp
2034218792Snp	PORT_UNLOCK(pi);
2035218792Snp}
2036218792Snp
2037231172Snp#define FW_MAC_EXACT_CHUNK	7
2038231172Snp
2039218792Snp/*
2040218792Snp * Program the port's XGMAC based on parameters in ifnet.  The caller also
2041218792Snp * indicates which parameters should be programmed (the rest are left alone).
2042218792Snp */
2043218792Snpstatic int
2044218792Snpupdate_mac_settings(struct port_info *pi, int flags)
2045218792Snp{
2046218792Snp	int rc;
2047218792Snp	struct ifnet *ifp = pi->ifp;
2048218792Snp	struct adapter *sc = pi->adapter;
2049218792Snp	int mtu = -1, promisc = -1, allmulti = -1, vlanex = -1;
2050218792Snp
2051218792Snp	PORT_LOCK_ASSERT_OWNED(pi);
2052218792Snp	KASSERT(flags, ("%s: not told what to update.", __func__));
2053218792Snp
2054218792Snp	if (flags & XGMAC_MTU)
2055218792Snp		mtu = ifp->if_mtu;
2056218792Snp
2057218792Snp	if (flags & XGMAC_PROMISC)
2058218792Snp		promisc = ifp->if_flags & IFF_PROMISC ? 1 : 0;
2059218792Snp
2060218792Snp	if (flags & XGMAC_ALLMULTI)
2061218792Snp		allmulti = ifp->if_flags & IFF_ALLMULTI ? 1 : 0;
2062218792Snp
2063218792Snp	if (flags & XGMAC_VLANEX)
2064218792Snp		vlanex = ifp->if_capenable & IFCAP_VLAN_HWTAGGING ? 1 : 0;
2065218792Snp
2066218792Snp	rc = -t4_set_rxmode(sc, sc->mbox, pi->viid, mtu, promisc, allmulti, 1,
2067218792Snp	    vlanex, false);
2068218792Snp	if (rc) {
2069218792Snp		if_printf(ifp, "set_rxmode (%x) failed: %d\n", flags, rc);
2070218792Snp		return (rc);
2071218792Snp	}
2072218792Snp
2073218792Snp	if (flags & XGMAC_UCADDR) {
2074218792Snp		uint8_t ucaddr[ETHER_ADDR_LEN];
2075218792Snp
2076218792Snp		bcopy(IF_LLADDR(ifp), ucaddr, sizeof(ucaddr));
2077218792Snp		rc = t4_change_mac(sc, sc->mbox, pi->viid, pi->xact_addr_filt,
2078218792Snp		    ucaddr, true, true);
2079218792Snp		if (rc < 0) {
2080218792Snp			rc = -rc;
2081218792Snp			if_printf(ifp, "change_mac failed: %d\n", rc);
2082218792Snp			return (rc);
2083218792Snp		} else {
2084218792Snp			pi->xact_addr_filt = rc;
2085218792Snp			rc = 0;
2086218792Snp		}
2087218792Snp	}
2088218792Snp
2089218792Snp	if (flags & XGMAC_MCADDRS) {
2090231172Snp		const uint8_t *mcaddr[FW_MAC_EXACT_CHUNK];
2091218792Snp		int del = 1;
2092218792Snp		uint64_t hash = 0;
2093218792Snp		struct ifmultiaddr *ifma;
2094231172Snp		int i = 0, j;
2095218792Snp
2096218792Snp		if_maddr_rlock(ifp);
2097218792Snp		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2098231172Snp			if (ifma->ifma_addr->sa_family == AF_LINK)
2099218792Snp				continue;
2100231172Snp			mcaddr[i++] =
2101231172Snp			    LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
2102218792Snp
2103231172Snp			if (i == FW_MAC_EXACT_CHUNK) {
2104231172Snp				rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2105231172Snp				    del, i, mcaddr, NULL, &hash, 0);
2106231172Snp				if (rc < 0) {
2107231172Snp					rc = -rc;
2108231172Snp					for (j = 0; j < i; j++) {
2109231172Snp						if_printf(ifp,
2110231172Snp						    "failed to add mc address"
2111231172Snp						    " %02x:%02x:%02x:"
2112231172Snp						    "%02x:%02x:%02x rc=%d\n",
2113231172Snp						    mcaddr[j][0], mcaddr[j][1],
2114231172Snp						    mcaddr[j][2], mcaddr[j][3],
2115231172Snp						    mcaddr[j][4], mcaddr[j][5],
2116231172Snp						    rc);
2117231172Snp					}
2118231172Snp					goto mcfail;
2119231172Snp				}
2120231172Snp				del = 0;
2121231172Snp				i = 0;
2122231172Snp			}
2123231172Snp		}
2124231172Snp		if (i > 0) {
2125231172Snp			rc = t4_alloc_mac_filt(sc, sc->mbox, pi->viid,
2126231172Snp			    del, i, mcaddr, NULL, &hash, 0);
2127218792Snp			if (rc < 0) {
2128218792Snp				rc = -rc;
2129231172Snp				for (j = 0; j < i; j++) {
2130231172Snp					if_printf(ifp,
2131231172Snp					    "failed to add mc address"
2132231172Snp					    " %02x:%02x:%02x:"
2133231172Snp					    "%02x:%02x:%02x rc=%d\n",
2134231172Snp					    mcaddr[j][0], mcaddr[j][1],
2135231172Snp					    mcaddr[j][2], mcaddr[j][3],
2136231172Snp					    mcaddr[j][4], mcaddr[j][5],
2137231172Snp					    rc);
2138231172Snp				}
2139218792Snp				goto mcfail;
2140218792Snp			}
2141218792Snp		}
2142218792Snp
2143218792Snp		rc = -t4_set_addr_hash(sc, sc->mbox, pi->viid, 0, hash, 0);
2144218792Snp		if (rc != 0)
2145218792Snp			if_printf(ifp, "failed to set mc address hash: %d", rc);
2146218792Snpmcfail:
2147218792Snp		if_maddr_runlock(ifp);
2148218792Snp	}
2149218792Snp
2150218792Snp	return (rc);
2151218792Snp}
2152218792Snp
2153218792Snpstatic int
2154218792Snpcxgbe_init_locked(struct port_info *pi)
2155218792Snp{
2156218792Snp	struct adapter *sc = pi->adapter;
2157218792Snp	int rc = 0;
2158218792Snp
2159218792Snp	ADAPTER_LOCK_ASSERT_OWNED(sc);
2160218792Snp
2161218792Snp	while (!IS_DOOMED(pi) && IS_BUSY(sc)) {
2162218792Snp		if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4init", 0)) {
2163218792Snp			rc = EINTR;
2164218792Snp			goto done;
2165218792Snp		}
2166218792Snp	}
2167218792Snp	if (IS_DOOMED(pi)) {
2168218792Snp		rc = ENXIO;
2169218792Snp		goto done;
2170218792Snp	}
2171218792Snp	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2172218792Snp
2173218792Snp	/* Give up the adapter lock, port init code can sleep. */
2174218792Snp	SET_BUSY(sc);
2175218792Snp	ADAPTER_UNLOCK(sc);
2176218792Snp
2177218792Snp	rc = cxgbe_init_synchronized(pi);
2178218792Snp
2179218792Snpdone:
2180218792Snp	ADAPTER_LOCK(sc);
2181218792Snp	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2182218792Snp	CLR_BUSY(sc);
2183218792Snp	wakeup_one(&sc->flags);
2184218792Snp	ADAPTER_UNLOCK(sc);
2185218792Snp	return (rc);
2186218792Snp}
2187218792Snp
2188218792Snpstatic int
2189218792Snpcxgbe_init_synchronized(struct port_info *pi)
2190218792Snp{
2191218792Snp	struct adapter *sc = pi->adapter;
2192218792Snp	struct ifnet *ifp = pi->ifp;
2193228561Snp	int rc = 0;
2194218792Snp
2195218792Snp	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2196218792Snp
2197218792Snp	if (isset(&sc->open_device_map, pi->port_id)) {
2198218792Snp		KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2199218792Snp		    ("mismatch between open_device_map and if_drv_flags"));
2200218792Snp		return (0);	/* already running */
2201218792Snp	}
2202218792Snp
2203228561Snp	if (!(sc->flags & FULL_INIT_DONE) &&
2204228561Snp	    ((rc = adapter_full_init(sc)) != 0))
2205218792Snp		return (rc);	/* error message displayed already */
2206218792Snp
2207228561Snp	if (!(pi->flags & PORT_INIT_DONE) &&
2208228561Snp	    ((rc = port_full_init(pi)) != 0))
2209228561Snp		return (rc); /* error message displayed already */
2210218792Snp
2211218792Snp	PORT_LOCK(pi);
2212218792Snp	rc = update_mac_settings(pi, XGMAC_ALL);
2213218792Snp	PORT_UNLOCK(pi);
2214218792Snp	if (rc)
2215218792Snp		goto done;	/* error message displayed already */
2216218792Snp
2217218792Snp	rc = -t4_link_start(sc, sc->mbox, pi->tx_chan, &pi->link_cfg);
2218218792Snp	if (rc != 0) {
2219218792Snp		if_printf(ifp, "start_link failed: %d\n", rc);
2220218792Snp		goto done;
2221218792Snp	}
2222218792Snp
2223218792Snp	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, true, true);
2224218792Snp	if (rc != 0) {
2225218792Snp		if_printf(ifp, "enable_vi failed: %d\n", rc);
2226218792Snp		goto done;
2227218792Snp	}
2228218792Snp
2229218792Snp	/* all ok */
2230218792Snp	setbit(&sc->open_device_map, pi->port_id);
2231218792Snp	ifp->if_drv_flags |= IFF_DRV_RUNNING;
2232218792Snp
2233218792Snp	callout_reset(&pi->tick, hz, cxgbe_tick, pi);
2234218792Snpdone:
2235218792Snp	if (rc != 0)
2236218792Snp		cxgbe_uninit_synchronized(pi);
2237218792Snp
2238218792Snp	return (rc);
2239218792Snp}
2240218792Snp
2241218792Snpstatic int
2242218792Snpcxgbe_uninit_locked(struct port_info *pi)
2243218792Snp{
2244218792Snp	struct adapter *sc = pi->adapter;
2245218792Snp	int rc;
2246218792Snp
2247218792Snp	ADAPTER_LOCK_ASSERT_OWNED(sc);
2248218792Snp
2249218792Snp	while (!IS_DOOMED(pi) && IS_BUSY(sc)) {
2250218792Snp		if (mtx_sleep(&sc->flags, &sc->sc_lock, PCATCH, "t4uninit", 0)) {
2251218792Snp			rc = EINTR;
2252218792Snp			goto done;
2253218792Snp		}
2254218792Snp	}
2255218792Snp	if (IS_DOOMED(pi)) {
2256218792Snp		rc = ENXIO;
2257218792Snp		goto done;
2258218792Snp	}
2259218792Snp	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
2260218792Snp	SET_BUSY(sc);
2261218792Snp	ADAPTER_UNLOCK(sc);
2262218792Snp
2263218792Snp	rc = cxgbe_uninit_synchronized(pi);
2264218792Snp
2265218792Snp	ADAPTER_LOCK(sc);
2266218792Snp	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
2267218792Snp	CLR_BUSY(sc);
2268218792Snp	wakeup_one(&sc->flags);
2269218792Snpdone:
2270218792Snp	ADAPTER_UNLOCK(sc);
2271218792Snp	return (rc);
2272218792Snp}
2273218792Snp
2274218792Snp/*
2275218792Snp * Idempotent.
2276218792Snp */
2277218792Snpstatic int
2278218792Snpcxgbe_uninit_synchronized(struct port_info *pi)
2279218792Snp{
2280218792Snp	struct adapter *sc = pi->adapter;
2281218792Snp	struct ifnet *ifp = pi->ifp;
2282218792Snp	int rc;
2283218792Snp
2284218792Snp	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2285218792Snp
2286218792Snp	/*
2287228561Snp	 * Disable the VI so that all its data in either direction is discarded
2288228561Snp	 * by the MPS.  Leave everything else (the queues, interrupts, and 1Hz
2289228561Snp	 * tick) intact as the TP can deliver negative advice or data that it's
2290228561Snp	 * holding in its RAM (for an offloaded connection) even after the VI is
2291228561Snp	 * disabled.
2292218792Snp	 */
2293228561Snp	rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
2294228561Snp	if (rc) {
2295228561Snp		if_printf(ifp, "disable_vi failed: %d\n", rc);
2296228561Snp		return (rc);
2297228561Snp	}
2298228561Snp
2299218792Snp	clrbit(&sc->open_device_map, pi->port_id);
2300228561Snp	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
2301218792Snp
2302218792Snp	pi->link_cfg.link_ok = 0;
2303218792Snp	pi->link_cfg.speed = 0;
2304218792Snp	t4_os_link_changed(sc, pi->port_id, 0);
2305218792Snp
2306218792Snp	return (0);
2307218792Snp}
2308218792Snp
2309222510Snp#define T4_ALLOC_IRQ(sc, irq, rid, handler, arg, name) do { \
2310222510Snp	rc = t4_alloc_irq(sc, irq, rid, handler, arg, name); \
2311218792Snp	if (rc != 0) \
2312218792Snp		goto done; \
2313218792Snp} while (0)
2314228561Snp
2315218792Snpstatic int
2316228561Snpadapter_full_init(struct adapter *sc)
2317218792Snp{
2318222510Snp	int rc, i, rid, p, q;
2319222510Snp	char s[8];
2320222510Snp	struct irq *irq;
2321228561Snp	struct port_info *pi;
2322228561Snp	struct sge_rxq *rxq;
2323237263Snp#ifdef TCP_OFFLOAD
2324228561Snp	struct sge_ofld_rxq *ofld_rxq;
2325228561Snp#endif
2326218792Snp
2327218792Snp	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2328228561Snp	KASSERT((sc->flags & FULL_INIT_DONE) == 0,
2329228561Snp	    ("%s: FULL_INIT_DONE already", __func__));
2330218792Snp
2331218792Snp	/*
2332220873Snp	 * queues that belong to the adapter (not any particular port).
2333218792Snp	 */
2334220873Snp	rc = t4_setup_adapter_queues(sc);
2335218792Snp	if (rc != 0)
2336218792Snp		goto done;
2337218792Snp
2338228561Snp	for (i = 0; i < ARRAY_SIZE(sc->tq); i++) {
2339228561Snp		sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
2340228561Snp		    taskqueue_thread_enqueue, &sc->tq[i]);
2341228561Snp		if (sc->tq[i] == NULL) {
2342228561Snp			device_printf(sc->dev,
2343228561Snp			    "failed to allocate task queue %d\n", i);
2344228561Snp			rc = ENOMEM;
2345228561Snp			goto done;
2346228561Snp		}
2347228561Snp		taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
2348228561Snp		    device_get_nameunit(sc->dev), i);
2349228561Snp	}
2350228561Snp
2351218792Snp	/*
2352218792Snp	 * Setup interrupts.
2353218792Snp	 */
2354222510Snp	irq = &sc->irq[0];
2355222510Snp	rid = sc->intr_type == INTR_INTX ? 0 : 1;
2356218792Snp	if (sc->intr_count == 1) {
2357228561Snp		KASSERT(!(sc->flags & INTR_DIRECT),
2358228561Snp		    ("%s: single interrupt && INTR_DIRECT?", __func__));
2359222510Snp
2360222510Snp		T4_ALLOC_IRQ(sc, irq, rid, t4_intr_all, sc, "all");
2361218792Snp	} else {
2362228561Snp		/* Multiple interrupts. */
2363228561Snp		KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
2364228561Snp		    ("%s: too few intr.", __func__));
2365228561Snp
2366228561Snp		/* The first one is always error intr */
2367222510Snp		T4_ALLOC_IRQ(sc, irq, rid, t4_intr_err, sc, "err");
2368222510Snp		irq++;
2369222510Snp		rid++;
2370218792Snp
2371228561Snp		/* The second one is always the firmware event queue */
2372228561Snp		T4_ALLOC_IRQ(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
2373228561Snp		irq++;
2374228561Snp		rid++;
2375222510Snp
2376228561Snp		/*
2377228561Snp		 * Note that if INTR_DIRECT is not set then either the NIC rx
2378228561Snp		 * queues or (exclusive or) the TOE rx queueus will be taking
2379228561Snp		 * direct interrupts.
2380228561Snp		 *
2381228561Snp		 * There is no need to check for is_offload(sc) as nofldrxq
2382228561Snp		 * will be 0 if offload is disabled.
2383228561Snp		 */
2384228561Snp		for_each_port(sc, p) {
2385228561Snp			pi = sc->port[p];
2386222510Snp
2387237263Snp#ifdef TCP_OFFLOAD
2388228561Snp			/*
2389228561Snp			 * Skip over the NIC queues if they aren't taking direct
2390228561Snp			 * interrupts.
2391228561Snp			 */
2392228561Snp			if (!(sc->flags & INTR_DIRECT) &&
2393228561Snp			    pi->nofldrxq > pi->nrxq)
2394228561Snp				goto ofld_queues;
2395228561Snp#endif
2396228561Snp			rxq = &sc->sge.rxq[pi->first_rxq];
2397228561Snp			for (q = 0; q < pi->nrxq; q++, rxq++) {
2398228561Snp				snprintf(s, sizeof(s), "%d.%d", p, q);
2399228561Snp				T4_ALLOC_IRQ(sc, irq, rid, t4_intr, rxq, s);
2400222510Snp				irq++;
2401222510Snp				rid++;
2402218792Snp			}
2403218792Snp
2404237263Snp#ifdef TCP_OFFLOAD
2405228561Snp			/*
2406228561Snp			 * Skip over the offload queues if they aren't taking
2407228561Snp			 * direct interrupts.
2408228561Snp			 */
2409228561Snp			if (!(sc->flags & INTR_DIRECT))
2410228561Snp				continue;
2411228561Snpofld_queues:
2412228561Snp			ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
2413228561Snp			for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
2414228561Snp				snprintf(s, sizeof(s), "%d,%d", p, q);
2415228561Snp				T4_ALLOC_IRQ(sc, irq, rid, t4_intr, ofld_rxq, s);
2416228561Snp				irq++;
2417228561Snp				rid++;
2418218792Snp			}
2419228561Snp#endif
2420218792Snp		}
2421218792Snp	}
2422218792Snp
2423218792Snp	t4_intr_enable(sc);
2424218792Snp	sc->flags |= FULL_INIT_DONE;
2425218792Snpdone:
2426218792Snp	if (rc != 0)
2427228561Snp		adapter_full_uninit(sc);
2428218792Snp
2429218792Snp	return (rc);
2430218792Snp}
2431218792Snp#undef T4_ALLOC_IRQ
2432218792Snp
2433218792Snpstatic int
2434228561Snpadapter_full_uninit(struct adapter *sc)
2435218792Snp{
2436218792Snp	int i;
2437218792Snp
2438218792Snp	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2439218792Snp
2440220873Snp	t4_teardown_adapter_queues(sc);
2441218792Snp
2442218792Snp	for (i = 0; i < sc->intr_count; i++)
2443218792Snp		t4_free_irq(sc, &sc->irq[i]);
2444218792Snp
2445228561Snp	for (i = 0; i < ARRAY_SIZE(sc->tq) && sc->tq[i]; i++) {
2446228561Snp		taskqueue_free(sc->tq[i]);
2447228561Snp		sc->tq[i] = NULL;
2448228561Snp	}
2449228561Snp
2450218792Snp	sc->flags &= ~FULL_INIT_DONE;
2451218792Snp
2452218792Snp	return (0);
2453218792Snp}
2454218792Snp
2455218792Snpstatic int
2456228561Snpport_full_init(struct port_info *pi)
2457228561Snp{
2458228561Snp	struct adapter *sc = pi->adapter;
2459228561Snp	struct ifnet *ifp = pi->ifp;
2460228561Snp	uint16_t *rss;
2461228561Snp	struct sge_rxq *rxq;
2462228561Snp	int rc, i;
2463228561Snp
2464228561Snp	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
2465228561Snp	KASSERT((pi->flags & PORT_INIT_DONE) == 0,
2466228561Snp	    ("%s: PORT_INIT_DONE already", __func__));
2467228561Snp
2468228561Snp	sysctl_ctx_init(&pi->ctx);
2469228561Snp	pi->flags |= PORT_SYSCTL_CTX;
2470228561Snp
2471228561Snp	/*
2472228561Snp	 * Allocate tx/rx/fl queues for this port.
2473228561Snp	 */
2474228561Snp	rc = t4_setup_port_queues(pi);
2475228561Snp	if (rc != 0)
2476228561Snp		goto done;	/* error message displayed already */
2477228561Snp
2478228561Snp	/*
2479228561Snp	 * Setup RSS for this port.
2480228561Snp	 */
2481228561Snp	rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE,
2482228561Snp	    M_ZERO | M_WAITOK);
2483228561Snp	for_each_rxq(pi, i, rxq) {
2484228561Snp		rss[i] = rxq->iq.abs_id;
2485228561Snp	}
2486228561Snp	rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
2487228561Snp	    pi->rss_size, rss, pi->nrxq);
2488228561Snp	free(rss, M_CXGBE);
2489228561Snp	if (rc != 0) {
2490228561Snp		if_printf(ifp, "rss_config failed: %d\n", rc);
2491228561Snp		goto done;
2492228561Snp	}
2493228561Snp
2494228561Snp	pi->flags |= PORT_INIT_DONE;
2495228561Snpdone:
2496228561Snp	if (rc != 0)
2497228561Snp		port_full_uninit(pi);
2498228561Snp
2499228561Snp	return (rc);
2500228561Snp}
2501228561Snp
2502228561Snp/*
2503228561Snp * Idempotent.
2504228561Snp */
2505228561Snpstatic int
2506228561Snpport_full_uninit(struct port_info *pi)
2507228561Snp{
2508228561Snp	struct adapter *sc = pi->adapter;
2509228561Snp	int i;
2510228561Snp	struct sge_rxq *rxq;
2511228561Snp	struct sge_txq *txq;
2512237263Snp#ifdef TCP_OFFLOAD
2513228561Snp	struct sge_ofld_rxq *ofld_rxq;
2514228561Snp	struct sge_wrq *ofld_txq;
2515228561Snp#endif
2516228561Snp
2517228561Snp	if (pi->flags & PORT_INIT_DONE) {
2518228561Snp
2519228561Snp		/* Need to quiesce queues.  XXX: ctrl queues? */
2520228561Snp
2521228561Snp		for_each_txq(pi, i, txq) {
2522228561Snp			quiesce_eq(sc, &txq->eq);
2523228561Snp		}
2524228561Snp
2525237263Snp#ifdef TCP_OFFLOAD
2526228561Snp		for_each_ofld_txq(pi, i, ofld_txq) {
2527228561Snp			quiesce_eq(sc, &ofld_txq->eq);
2528228561Snp		}
2529228561Snp#endif
2530228561Snp
2531228561Snp		for_each_rxq(pi, i, rxq) {
2532228561Snp			quiesce_iq(sc, &rxq->iq);
2533228561Snp			quiesce_fl(sc, &rxq->fl);
2534228561Snp		}
2535228561Snp
2536237263Snp#ifdef TCP_OFFLOAD
2537228561Snp		for_each_ofld_rxq(pi, i, ofld_rxq) {
2538228561Snp			quiesce_iq(sc, &ofld_rxq->iq);
2539228561Snp			quiesce_fl(sc, &ofld_rxq->fl);
2540228561Snp		}
2541228561Snp#endif
2542228561Snp	}
2543228561Snp
2544228561Snp	t4_teardown_port_queues(pi);
2545228561Snp	pi->flags &= ~PORT_INIT_DONE;
2546228561Snp
2547228561Snp	return (0);
2548228561Snp}
2549228561Snp
2550228561Snpstatic void
2551228561Snpquiesce_eq(struct adapter *sc, struct sge_eq *eq)
2552228561Snp{
2553228561Snp	EQ_LOCK(eq);
2554228561Snp	eq->flags |= EQ_DOOMED;
2555228561Snp
2556228561Snp	/*
2557228561Snp	 * Wait for the response to a credit flush if one's
2558228561Snp	 * pending.
2559228561Snp	 */
2560228561Snp	while (eq->flags & EQ_CRFLUSHED)
2561228561Snp		mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
2562228561Snp	EQ_UNLOCK(eq);
2563228561Snp
2564228561Snp	callout_drain(&eq->tx_callout);	/* XXX: iffy */
2565228561Snp	pause("callout", 10);		/* Still iffy */
2566228561Snp
2567228561Snp	taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
2568228561Snp}
2569228561Snp
2570228561Snpstatic void
2571228561Snpquiesce_iq(struct adapter *sc, struct sge_iq *iq)
2572228561Snp{
2573228561Snp	(void) sc;	/* unused */
2574228561Snp
2575228561Snp	/* Synchronize with the interrupt handler */
2576228561Snp	while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
2577228561Snp		pause("iqfree", 1);
2578228561Snp}
2579228561Snp
2580228561Snpstatic void
2581228561Snpquiesce_fl(struct adapter *sc, struct sge_fl *fl)
2582228561Snp{
2583228561Snp	mtx_lock(&sc->sfl_lock);
2584228561Snp	FL_LOCK(fl);
2585228561Snp	fl->flags |= FL_DOOMED;
2586228561Snp	FL_UNLOCK(fl);
2587228561Snp	mtx_unlock(&sc->sfl_lock);
2588228561Snp
2589228561Snp	callout_drain(&sc->sfl_callout);
2590228561Snp	KASSERT((fl->flags & FL_STARVING) == 0,
2591228561Snp	    ("%s: still starving", __func__));
2592228561Snp}
2593228561Snp
2594228561Snpstatic int
2595218792Snpt4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
2596228561Snp    driver_intr_t *handler, void *arg, char *name)
2597218792Snp{
2598218792Snp	int rc;
2599218792Snp
2600218792Snp	irq->rid = rid;
2601218792Snp	irq->res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irq->rid,
2602218792Snp	    RF_SHAREABLE | RF_ACTIVE);
2603218792Snp	if (irq->res == NULL) {
2604218792Snp		device_printf(sc->dev,
2605218792Snp		    "failed to allocate IRQ for rid %d, name %s.\n", rid, name);
2606218792Snp		return (ENOMEM);
2607218792Snp	}
2608218792Snp
2609218792Snp	rc = bus_setup_intr(sc->dev, irq->res, INTR_MPSAFE | INTR_TYPE_NET,
2610218792Snp	    NULL, handler, arg, &irq->tag);
2611218792Snp	if (rc != 0) {
2612218792Snp		device_printf(sc->dev,
2613218792Snp		    "failed to setup interrupt for rid %d, name %s: %d\n",
2614218792Snp		    rid, name, rc);
2615218792Snp	} else if (name)
2616218792Snp		bus_describe_intr(sc->dev, irq->res, irq->tag, name);
2617218792Snp
2618218792Snp	return (rc);
2619218792Snp}
2620218792Snp
2621218792Snpstatic int
2622218792Snpt4_free_irq(struct adapter *sc, struct irq *irq)
2623218792Snp{
2624218792Snp	if (irq->tag)
2625218792Snp		bus_teardown_intr(sc->dev, irq->res, irq->tag);
2626218792Snp	if (irq->res)
2627218792Snp		bus_release_resource(sc->dev, SYS_RES_IRQ, irq->rid, irq->res);
2628218792Snp
2629218792Snp	bzero(irq, sizeof(*irq));
2630218792Snp
2631218792Snp	return (0);
2632218792Snp}
2633218792Snp
2634218792Snpstatic void
2635218792Snpreg_block_dump(struct adapter *sc, uint8_t *buf, unsigned int start,
2636218792Snp    unsigned int end)
2637218792Snp{
2638218792Snp	uint32_t *p = (uint32_t *)(buf + start);
2639218792Snp
2640218792Snp	for ( ; start <= end; start += sizeof(uint32_t))
2641218792Snp		*p++ = t4_read_reg(sc, start);
2642218792Snp}
2643218792Snp
2644218792Snpstatic void
2645218792Snpt4_get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
2646218792Snp{
2647218792Snp	int i;
2648218792Snp	static const unsigned int reg_ranges[] = {
2649218792Snp		0x1008, 0x1108,
2650218792Snp		0x1180, 0x11b4,
2651218792Snp		0x11fc, 0x123c,
2652218792Snp		0x1300, 0x173c,
2653218792Snp		0x1800, 0x18fc,
2654218792Snp		0x3000, 0x30d8,
2655218792Snp		0x30e0, 0x5924,
2656218792Snp		0x5960, 0x59d4,
2657218792Snp		0x5a00, 0x5af8,
2658218792Snp		0x6000, 0x6098,
2659218792Snp		0x6100, 0x6150,
2660218792Snp		0x6200, 0x6208,
2661218792Snp		0x6240, 0x6248,
2662218792Snp		0x6280, 0x6338,
2663218792Snp		0x6370, 0x638c,
2664218792Snp		0x6400, 0x643c,
2665218792Snp		0x6500, 0x6524,
2666218792Snp		0x6a00, 0x6a38,
2667218792Snp		0x6a60, 0x6a78,
2668218792Snp		0x6b00, 0x6b84,
2669218792Snp		0x6bf0, 0x6c84,
2670218792Snp		0x6cf0, 0x6d84,
2671218792Snp		0x6df0, 0x6e84,
2672218792Snp		0x6ef0, 0x6f84,
2673218792Snp		0x6ff0, 0x7084,
2674218792Snp		0x70f0, 0x7184,
2675218792Snp		0x71f0, 0x7284,
2676218792Snp		0x72f0, 0x7384,
2677218792Snp		0x73f0, 0x7450,
2678218792Snp		0x7500, 0x7530,
2679218792Snp		0x7600, 0x761c,
2680218792Snp		0x7680, 0x76cc,
2681218792Snp		0x7700, 0x7798,
2682218792Snp		0x77c0, 0x77fc,
2683218792Snp		0x7900, 0x79fc,
2684218792Snp		0x7b00, 0x7c38,
2685218792Snp		0x7d00, 0x7efc,
2686218792Snp		0x8dc0, 0x8e1c,
2687218792Snp		0x8e30, 0x8e78,
2688218792Snp		0x8ea0, 0x8f6c,
2689218792Snp		0x8fc0, 0x9074,
2690218792Snp		0x90fc, 0x90fc,
2691218792Snp		0x9400, 0x9458,
2692218792Snp		0x9600, 0x96bc,
2693218792Snp		0x9800, 0x9808,
2694218792Snp		0x9820, 0x983c,
2695218792Snp		0x9850, 0x9864,
2696218792Snp		0x9c00, 0x9c6c,
2697218792Snp		0x9c80, 0x9cec,
2698218792Snp		0x9d00, 0x9d6c,
2699218792Snp		0x9d80, 0x9dec,
2700218792Snp		0x9e00, 0x9e6c,
2701218792Snp		0x9e80, 0x9eec,
2702218792Snp		0x9f00, 0x9f6c,
2703218792Snp		0x9f80, 0x9fec,
2704218792Snp		0xd004, 0xd03c,
2705218792Snp		0xdfc0, 0xdfe0,
2706218792Snp		0xe000, 0xea7c,
2707218792Snp		0xf000, 0x11190,
2708237439Snp		0x19040, 0x1906c,
2709237439Snp		0x19078, 0x19080,
2710237439Snp		0x1908c, 0x19124,
2711218792Snp		0x19150, 0x191b0,
2712218792Snp		0x191d0, 0x191e8,
2713218792Snp		0x19238, 0x1924c,
2714218792Snp		0x193f8, 0x19474,
2715218792Snp		0x19490, 0x194f8,
2716218792Snp		0x19800, 0x19f30,
2717218792Snp		0x1a000, 0x1a06c,
2718218792Snp		0x1a0b0, 0x1a120,
2719218792Snp		0x1a128, 0x1a138,
2720218792Snp		0x1a190, 0x1a1c4,
2721218792Snp		0x1a1fc, 0x1a1fc,
2722218792Snp		0x1e040, 0x1e04c,
2723237439Snp		0x1e284, 0x1e28c,
2724218792Snp		0x1e2c0, 0x1e2c0,
2725218792Snp		0x1e2e0, 0x1e2e0,
2726218792Snp		0x1e300, 0x1e384,
2727218792Snp		0x1e3c0, 0x1e3c8,
2728218792Snp		0x1e440, 0x1e44c,
2729237439Snp		0x1e684, 0x1e68c,
2730218792Snp		0x1e6c0, 0x1e6c0,
2731218792Snp		0x1e6e0, 0x1e6e0,
2732218792Snp		0x1e700, 0x1e784,
2733218792Snp		0x1e7c0, 0x1e7c8,
2734218792Snp		0x1e840, 0x1e84c,
2735237439Snp		0x1ea84, 0x1ea8c,
2736218792Snp		0x1eac0, 0x1eac0,
2737218792Snp		0x1eae0, 0x1eae0,
2738218792Snp		0x1eb00, 0x1eb84,
2739218792Snp		0x1ebc0, 0x1ebc8,
2740218792Snp		0x1ec40, 0x1ec4c,
2741237439Snp		0x1ee84, 0x1ee8c,
2742218792Snp		0x1eec0, 0x1eec0,
2743218792Snp		0x1eee0, 0x1eee0,
2744218792Snp		0x1ef00, 0x1ef84,
2745218792Snp		0x1efc0, 0x1efc8,
2746218792Snp		0x1f040, 0x1f04c,
2747237439Snp		0x1f284, 0x1f28c,
2748218792Snp		0x1f2c0, 0x1f2c0,
2749218792Snp		0x1f2e0, 0x1f2e0,
2750218792Snp		0x1f300, 0x1f384,
2751218792Snp		0x1f3c0, 0x1f3c8,
2752218792Snp		0x1f440, 0x1f44c,
2753237439Snp		0x1f684, 0x1f68c,
2754218792Snp		0x1f6c0, 0x1f6c0,
2755218792Snp		0x1f6e0, 0x1f6e0,
2756218792Snp		0x1f700, 0x1f784,
2757218792Snp		0x1f7c0, 0x1f7c8,
2758218792Snp		0x1f840, 0x1f84c,
2759237439Snp		0x1fa84, 0x1fa8c,
2760218792Snp		0x1fac0, 0x1fac0,
2761218792Snp		0x1fae0, 0x1fae0,
2762218792Snp		0x1fb00, 0x1fb84,
2763218792Snp		0x1fbc0, 0x1fbc8,
2764218792Snp		0x1fc40, 0x1fc4c,
2765237439Snp		0x1fe84, 0x1fe8c,
2766218792Snp		0x1fec0, 0x1fec0,
2767218792Snp		0x1fee0, 0x1fee0,
2768218792Snp		0x1ff00, 0x1ff84,
2769218792Snp		0x1ffc0, 0x1ffc8,
2770218792Snp		0x20000, 0x2002c,
2771218792Snp		0x20100, 0x2013c,
2772218792Snp		0x20190, 0x201c8,
2773218792Snp		0x20200, 0x20318,
2774218792Snp		0x20400, 0x20528,
2775218792Snp		0x20540, 0x20614,
2776218792Snp		0x21000, 0x21040,
2777218792Snp		0x2104c, 0x21060,
2778218792Snp		0x210c0, 0x210ec,
2779218792Snp		0x21200, 0x21268,
2780218792Snp		0x21270, 0x21284,
2781218792Snp		0x212fc, 0x21388,
2782218792Snp		0x21400, 0x21404,
2783218792Snp		0x21500, 0x21518,
2784218792Snp		0x2152c, 0x2153c,
2785218792Snp		0x21550, 0x21554,
2786218792Snp		0x21600, 0x21600,
2787218792Snp		0x21608, 0x21628,
2788218792Snp		0x21630, 0x2163c,
2789218792Snp		0x21700, 0x2171c,
2790218792Snp		0x21780, 0x2178c,
2791218792Snp		0x21800, 0x21c38,
2792218792Snp		0x21c80, 0x21d7c,
2793218792Snp		0x21e00, 0x21e04,
2794218792Snp		0x22000, 0x2202c,
2795218792Snp		0x22100, 0x2213c,
2796218792Snp		0x22190, 0x221c8,
2797218792Snp		0x22200, 0x22318,
2798218792Snp		0x22400, 0x22528,
2799218792Snp		0x22540, 0x22614,
2800218792Snp		0x23000, 0x23040,
2801218792Snp		0x2304c, 0x23060,
2802218792Snp		0x230c0, 0x230ec,
2803218792Snp		0x23200, 0x23268,
2804218792Snp		0x23270, 0x23284,
2805218792Snp		0x232fc, 0x23388,
2806218792Snp		0x23400, 0x23404,
2807218792Snp		0x23500, 0x23518,
2808218792Snp		0x2352c, 0x2353c,
2809218792Snp		0x23550, 0x23554,
2810218792Snp		0x23600, 0x23600,
2811218792Snp		0x23608, 0x23628,
2812218792Snp		0x23630, 0x2363c,
2813218792Snp		0x23700, 0x2371c,
2814218792Snp		0x23780, 0x2378c,
2815218792Snp		0x23800, 0x23c38,
2816218792Snp		0x23c80, 0x23d7c,
2817218792Snp		0x23e00, 0x23e04,
2818218792Snp		0x24000, 0x2402c,
2819218792Snp		0x24100, 0x2413c,
2820218792Snp		0x24190, 0x241c8,
2821218792Snp		0x24200, 0x24318,
2822218792Snp		0x24400, 0x24528,
2823218792Snp		0x24540, 0x24614,
2824218792Snp		0x25000, 0x25040,
2825218792Snp		0x2504c, 0x25060,
2826218792Snp		0x250c0, 0x250ec,
2827218792Snp		0x25200, 0x25268,
2828218792Snp		0x25270, 0x25284,
2829218792Snp		0x252fc, 0x25388,
2830218792Snp		0x25400, 0x25404,
2831218792Snp		0x25500, 0x25518,
2832218792Snp		0x2552c, 0x2553c,
2833218792Snp		0x25550, 0x25554,
2834218792Snp		0x25600, 0x25600,
2835218792Snp		0x25608, 0x25628,
2836218792Snp		0x25630, 0x2563c,
2837218792Snp		0x25700, 0x2571c,
2838218792Snp		0x25780, 0x2578c,
2839218792Snp		0x25800, 0x25c38,
2840218792Snp		0x25c80, 0x25d7c,
2841218792Snp		0x25e00, 0x25e04,
2842218792Snp		0x26000, 0x2602c,
2843218792Snp		0x26100, 0x2613c,
2844218792Snp		0x26190, 0x261c8,
2845218792Snp		0x26200, 0x26318,
2846218792Snp		0x26400, 0x26528,
2847218792Snp		0x26540, 0x26614,
2848218792Snp		0x27000, 0x27040,
2849218792Snp		0x2704c, 0x27060,
2850218792Snp		0x270c0, 0x270ec,
2851218792Snp		0x27200, 0x27268,
2852218792Snp		0x27270, 0x27284,
2853218792Snp		0x272fc, 0x27388,
2854218792Snp		0x27400, 0x27404,
2855218792Snp		0x27500, 0x27518,
2856218792Snp		0x2752c, 0x2753c,
2857218792Snp		0x27550, 0x27554,
2858218792Snp		0x27600, 0x27600,
2859218792Snp		0x27608, 0x27628,
2860218792Snp		0x27630, 0x2763c,
2861218792Snp		0x27700, 0x2771c,
2862218792Snp		0x27780, 0x2778c,
2863218792Snp		0x27800, 0x27c38,
2864218792Snp		0x27c80, 0x27d7c,
2865218792Snp		0x27e00, 0x27e04
2866218792Snp	};
2867218792Snp
2868218792Snp	regs->version = 4 | (sc->params.rev << 10);
2869218792Snp	for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
2870218792Snp		reg_block_dump(sc, buf, reg_ranges[i], reg_ranges[i + 1]);
2871218792Snp}
2872218792Snp
2873218792Snpstatic void
2874218792Snpcxgbe_tick(void *arg)
2875218792Snp{
2876218792Snp	struct port_info *pi = arg;
2877218792Snp	struct ifnet *ifp = pi->ifp;
2878218792Snp	struct sge_txq *txq;
2879218792Snp	int i, drops;
2880218792Snp	struct port_stats *s = &pi->stats;
2881218792Snp
2882218792Snp	PORT_LOCK(pi);
2883218792Snp	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
2884218792Snp		PORT_UNLOCK(pi);
2885218792Snp		return;	/* without scheduling another callout */
2886218792Snp	}
2887218792Snp
2888218792Snp	t4_get_port_stats(pi->adapter, pi->tx_chan, s);
2889218792Snp
2890228561Snp	ifp->if_opackets = s->tx_frames - s->tx_pause;
2891228561Snp	ifp->if_ipackets = s->rx_frames - s->rx_pause;
2892228561Snp	ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
2893228561Snp	ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
2894228561Snp	ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
2895228561Snp	ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
2896218792Snp	ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
2897218792Snp	    s->rx_ovflow3;
2898218792Snp
2899218792Snp	drops = s->tx_drop;
2900218792Snp	for_each_txq(pi, i, txq)
2901220873Snp		drops += txq->br->br_drops;
2902218792Snp	ifp->if_snd.ifq_drops = drops;
2903218792Snp
2904218792Snp	ifp->if_oerrors = s->tx_error_frames;
2905218792Snp	ifp->if_ierrors = s->rx_jabber + s->rx_runt + s->rx_too_long +
2906218792Snp	    s->rx_fcs_err + s->rx_len_err;
2907218792Snp
2908218792Snp	callout_schedule(&pi->tick, hz);
2909218792Snp	PORT_UNLOCK(pi);
2910218792Snp}
2911218792Snp
2912237263Snpstatic void
2913237263Snpcxgbe_vlan_config(void *arg, struct ifnet *ifp, uint16_t vid)
2914237263Snp{
2915237263Snp	struct ifnet *vlan;
2916237263Snp
2917237263Snp	if (arg != ifp)
2918237263Snp		return;
2919237263Snp
2920237263Snp	vlan = VLAN_DEVAT(ifp, vid);
2921237263Snp	VLAN_SETCOOKIE(vlan, ifp);
2922237263Snp}
2923237263Snp
2924218792Snpstatic int
2925228561Snpcpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
2926228561Snp{
2927237263Snp
2928228561Snp#ifdef INVARIANTS
2929237263Snp	panic("%s: opcode 0x%02x on iq %p with payload %p",
2930228561Snp	    __func__, rss->opcode, iq, m);
2931228561Snp#else
2932237263Snp	log(LOG_ERR, "%s: opcode 0x%02x on iq %p with payload %p",
2933228561Snp	    __func__, rss->opcode, iq, m);
2934228561Snp	m_freem(m);
2935228561Snp#endif
2936228561Snp	return (EDOOFUS);
2937228561Snp}
2938228561Snp
2939228561Snpint
2940228561Snpt4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
2941228561Snp{
2942228561Snp	uintptr_t *loc, new;
2943228561Snp
2944228561Snp	if (opcode >= ARRAY_SIZE(sc->cpl_handler))
2945228561Snp		return (EINVAL);
2946228561Snp
2947228561Snp	new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
2948228561Snp	loc = (uintptr_t *) &sc->cpl_handler[opcode];
2949228561Snp	atomic_store_rel_ptr(loc, new);
2950228561Snp
2951228561Snp	return (0);
2952228561Snp}
2953228561Snp
2954228561Snpstatic int
2955237263Snpan_not_handled(struct sge_iq *iq, const struct rsp_ctrl *ctrl)
2956237263Snp{
2957237263Snp
2958237263Snp#ifdef INVARIANTS
2959237263Snp	panic("%s: async notification on iq %p (ctrl %p)", __func__, iq, ctrl);
2960237263Snp#else
2961237263Snp	log(LOG_ERR, "%s: async notification on iq %p (ctrl %p)",
2962237263Snp	    __func__, iq, ctrl);
2963237263Snp#endif
2964237263Snp	return (EDOOFUS);
2965237263Snp}
2966237263Snp
2967237263Snpint
2968237263Snpt4_register_an_handler(struct adapter *sc, an_handler_t h)
2969237263Snp{
2970237263Snp	uintptr_t *loc, new;
2971237263Snp
2972237263Snp	new = h ? (uintptr_t)h : (uintptr_t)an_not_handled;
2973237263Snp	loc = (uintptr_t *) &sc->an_handler;
2974237263Snp	atomic_store_rel_ptr(loc, new);
2975237263Snp
2976237263Snp	return (0);
2977237263Snp}
2978237263Snp
2979237263Snpstatic int
2980218792Snpt4_sysctls(struct adapter *sc)
2981218792Snp{
2982218792Snp	struct sysctl_ctx_list *ctx;
2983218792Snp	struct sysctl_oid *oid;
2984228561Snp	struct sysctl_oid_list *children, *c0;
2985228561Snp	static char *caps[] = {
2986228561Snp		"\20\1PPP\2QFC\3DCBX",			/* caps[0] linkcaps */
2987228561Snp		"\20\1NIC\2VM\3IDS\4UM\5UM_ISGL",	/* caps[1] niccaps */
2988228561Snp		"\20\1TOE",				/* caps[2] toecaps */
2989228561Snp		"\20\1RDDP\2RDMAC",			/* caps[3] rdmacaps */
2990228561Snp		"\20\1INITIATOR_PDU\2TARGET_PDU"	/* caps[4] iscsicaps */
2991228561Snp		    "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
2992228561Snp		    "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
2993228561Snp		"\20\1INITIATOR\2TARGET\3CTRL_OFLD"	/* caps[5] fcoecaps */
2994228561Snp	};
2995218792Snp
2996218792Snp	ctx = device_get_sysctl_ctx(sc->dev);
2997228561Snp
2998228561Snp	/*
2999228561Snp	 * dev.t4nex.X.
3000228561Snp	 */
3001218792Snp	oid = device_get_sysctl_tree(sc->dev);
3002228561Snp	c0 = children = SYSCTL_CHILDREN(oid);
3003218792Snp
3004218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD,
3005218792Snp	    &sc->params.nports, 0, "# of ports");
3006218792Snp
3007218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "hw_revision", CTLFLAG_RD,
3008218792Snp	    &sc->params.rev, 0, "chip hardware revision");
3009218792Snp
3010218792Snp	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
3011218792Snp	    CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
3012218792Snp
3013228561Snp	SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
3014228561Snp	    CTLFLAG_RD, &t4_cfg_file, 0, "configuration file");
3015218792Snp
3016228561Snp	SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD,
3017228561Snp	    &sc->cfcsum, 0, "config file checksum");
3018228561Snp
3019228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
3020228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
3021228561Snp	    sysctl_bitfield, "A", "available link capabilities");
3022228561Snp
3023228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
3024228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
3025228561Snp	    sysctl_bitfield, "A", "available NIC capabilities");
3026228561Snp
3027228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
3028228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
3029228561Snp	    sysctl_bitfield, "A", "available TCP offload capabilities");
3030228561Snp
3031228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
3032228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
3033228561Snp	    sysctl_bitfield, "A", "available RDMA capabilities");
3034228561Snp
3035228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
3036228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
3037228561Snp	    sysctl_bitfield, "A", "available iSCSI capabilities");
3038228561Snp
3039228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
3040228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
3041228561Snp	    sysctl_bitfield, "A", "available FCoE capabilities");
3042228561Snp
3043218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD,
3044218792Snp	    &sc->params.vpd.cclk, 0, "core clock frequency (in KHz)");
3045218792Snp
3046219436Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
3047228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
3048228561Snp	    sizeof(sc->sge.timer_val), sysctl_int_array, "A",
3049228561Snp	    "interrupt holdoff timer values (us)");
3050218792Snp
3051219436Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
3052228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
3053228561Snp	    sizeof(sc->sge.counter_val), sysctl_int_array, "A",
3054228561Snp	    "interrupt holdoff packet counter values");
3055218792Snp
3056231115Snp#ifdef SBUF_DRAIN
3057228561Snp	/*
3058228561Snp	 * dev.t4nex.X.misc.  Marked CTLFLAG_SKIP to avoid information overload.
3059228561Snp	 */
3060228561Snp	oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
3061228561Snp	    CTLFLAG_RD | CTLFLAG_SKIP, NULL,
3062228561Snp	    "logs and miscellaneous information");
3063228561Snp	children = SYSCTL_CHILDREN(oid);
3064228561Snp
3065228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
3066228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3067228561Snp	    sysctl_cctrl, "A", "congestion control");
3068228561Snp
3069228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
3070228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3071228561Snp	    sysctl_cpl_stats, "A", "CPL statistics");
3072228561Snp
3073228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
3074228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3075228561Snp	    sysctl_ddp_stats, "A", "DDP statistics");
3076228561Snp
3077222551Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
3078222551Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3079228561Snp	    sysctl_devlog, "A", "firmware's device log");
3080222551Snp
3081228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
3082228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3083228561Snp	    sysctl_fcoe_stats, "A", "FCoE statistics");
3084228561Snp
3085228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
3086228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3087228561Snp	    sysctl_hw_sched, "A", "hardware scheduler ");
3088228561Snp
3089228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
3090228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3091228561Snp	    sysctl_l2t, "A", "hardware L2 table");
3092228561Snp
3093228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
3094228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3095228561Snp	    sysctl_lb_stats, "A", "loopback statistics");
3096228561Snp
3097228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
3098228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3099228561Snp	    sysctl_meminfo, "A", "memory regions");
3100228561Snp
3101228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
3102228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3103228561Snp	    sysctl_path_mtus, "A", "path MTUs");
3104228561Snp
3105228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
3106228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3107228561Snp	    sysctl_pm_stats, "A", "PM statistics");
3108228561Snp
3109228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
3110228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3111228561Snp	    sysctl_rdma_stats, "A", "RDMA statistics");
3112228561Snp
3113228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
3114228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3115228561Snp	    sysctl_tcp_stats, "A", "TCP statistics");
3116228561Snp
3117228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
3118228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3119228561Snp	    sysctl_tids, "A", "TID information");
3120228561Snp
3121228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
3122228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3123228561Snp	    sysctl_tp_err_stats, "A", "TP error statistics");
3124228561Snp
3125228561Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
3126228561Snp	    CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
3127228561Snp	    sysctl_tx_rate, "A", "Tx rate");
3128231115Snp#endif
3129228561Snp
3130237263Snp#ifdef TCP_OFFLOAD
3131228561Snp	if (is_offload(sc)) {
3132228561Snp		/*
3133228561Snp		 * dev.t4nex.X.toe.
3134228561Snp		 */
3135228561Snp		oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
3136228561Snp		    NULL, "TOE parameters");
3137228561Snp		children = SYSCTL_CHILDREN(oid);
3138228561Snp
3139228561Snp		sc->tt.sndbuf = 256 * 1024;
3140228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
3141228561Snp		    &sc->tt.sndbuf, 0, "max hardware send buffer size");
3142228561Snp
3143228561Snp		sc->tt.ddp = 0;
3144228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
3145228561Snp		    &sc->tt.ddp, 0, "DDP allowed");
3146228561Snp		sc->tt.indsz = M_INDICATESIZE;
3147228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
3148228561Snp		    &sc->tt.indsz, 0, "DDP max indicate size allowed");
3149228561Snp		sc->tt.ddp_thres = 3*4096;
3150228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
3151228561Snp		    &sc->tt.ddp_thres, 0, "DDP threshold");
3152228561Snp	}
3153228561Snp#endif
3154228561Snp
3155228561Snp
3156218792Snp	return (0);
3157218792Snp}
3158218792Snp
3159218792Snpstatic int
3160218792Snpcxgbe_sysctls(struct port_info *pi)
3161218792Snp{
3162218792Snp	struct sysctl_ctx_list *ctx;
3163218792Snp	struct sysctl_oid *oid;
3164218792Snp	struct sysctl_oid_list *children;
3165218792Snp
3166218792Snp	ctx = device_get_sysctl_ctx(pi->dev);
3167218792Snp
3168218792Snp	/*
3169218792Snp	 * dev.cxgbe.X.
3170218792Snp	 */
3171218792Snp	oid = device_get_sysctl_tree(pi->dev);
3172218792Snp	children = SYSCTL_CHILDREN(oid);
3173218792Snp
3174218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nrxq", CTLFLAG_RD,
3175218792Snp	    &pi->nrxq, 0, "# of rx queues");
3176218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ntxq", CTLFLAG_RD,
3177218792Snp	    &pi->ntxq, 0, "# of tx queues");
3178218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_rxq", CTLFLAG_RD,
3179218792Snp	    &pi->first_rxq, 0, "index of first rx queue");
3180218792Snp	SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
3181218792Snp	    &pi->first_txq, 0, "index of first tx queue");
3182218792Snp
3183237263Snp#ifdef TCP_OFFLOAD
3184228561Snp	if (is_offload(pi->adapter)) {
3185228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
3186228561Snp		    &pi->nofldrxq, 0,
3187228561Snp		    "# of rx queues for offloaded TCP connections");
3188228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
3189228561Snp		    &pi->nofldtxq, 0,
3190228561Snp		    "# of tx queues for offloaded TCP connections");
3191228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
3192228561Snp		    CTLFLAG_RD, &pi->first_ofld_rxq, 0,
3193228561Snp		    "index of first TOE rx queue");
3194228561Snp		SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
3195228561Snp		    CTLFLAG_RD, &pi->first_ofld_txq, 0,
3196228561Snp		    "index of first TOE tx queue");
3197228561Snp	}
3198228561Snp#endif
3199228561Snp
3200218792Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
3201218792Snp	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
3202218792Snp	    "holdoff timer index");
3203218792Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pktc_idx",
3204218792Snp	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_pktc_idx, "I",
3205218792Snp	    "holdoff packet counter index");
3206218792Snp
3207218792Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_rxq",
3208218792Snp	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_rxq, "I",
3209218792Snp	    "rx queue size");
3210218792Snp	SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "qsize_txq",
3211218792Snp	    CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_qsize_txq, "I",
3212218792Snp	    "tx queue size");
3213218792Snp
3214218792Snp	/*
3215218792Snp	 * dev.cxgbe.X.stats.
3216218792Snp	 */
3217218792Snp	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats", CTLFLAG_RD,
3218218792Snp	    NULL, "port statistics");
3219218792Snp	children = SYSCTL_CHILDREN(oid);
3220218792Snp
3221218792Snp#define SYSCTL_ADD_T4_REG64(pi, name, desc, reg) \
3222218792Snp	SYSCTL_ADD_OID(ctx, children, OID_AUTO, name, \
3223218792Snp	    CTLTYPE_U64 | CTLFLAG_RD, pi->adapter, reg, \
3224218792Snp	    sysctl_handle_t4_reg64, "QU", desc)
3225218792Snp
3226218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_octets", "# of octets in good frames",
3227218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BYTES_L));
3228218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames", "total # of good frames",
3229218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_FRAMES_L));
3230218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_bcast_frames", "# of broadcast frames",
3231218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_BCAST_L));
3232218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_mcast_frames", "# of multicast frames",
3233218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_MCAST_L));
3234218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ucast_frames", "# of unicast frames",
3235218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_UCAST_L));
3236218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_error_frames", "# of error frames",
3237218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_ERROR_L));
3238218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_64",
3239218792Snp	    "# of tx frames in this range",
3240218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_64B_L));
3241218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_65_127",
3242218792Snp	    "# of tx frames in this range",
3243218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_65B_127B_L));
3244218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_128_255",
3245218792Snp	    "# of tx frames in this range",
3246218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_128B_255B_L));
3247218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_256_511",
3248218792Snp	    "# of tx frames in this range",
3249218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_256B_511B_L));
3250218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_512_1023",
3251218792Snp	    "# of tx frames in this range",
3252218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_512B_1023B_L));
3253218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1024_1518",
3254218792Snp	    "# of tx frames in this range",
3255218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1024B_1518B_L));
3256218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_frames_1519_max",
3257218792Snp	    "# of tx frames in this range",
3258218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_1519B_MAX_L));
3259218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_drop", "# of dropped tx frames",
3260218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_DROP_L));
3261218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_pause", "# of pause frames transmitted",
3262218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PAUSE_L));
3263218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp0", "# of PPP prio 0 frames transmitted",
3264218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP0_L));
3265218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp1", "# of PPP prio 1 frames transmitted",
3266218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP1_L));
3267218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp2", "# of PPP prio 2 frames transmitted",
3268218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP2_L));
3269218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp3", "# of PPP prio 3 frames transmitted",
3270218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP3_L));
3271218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp4", "# of PPP prio 4 frames transmitted",
3272218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP4_L));
3273218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp5", "# of PPP prio 5 frames transmitted",
3274218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP5_L));
3275218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp6", "# of PPP prio 6 frames transmitted",
3276218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP6_L));
3277218792Snp	SYSCTL_ADD_T4_REG64(pi, "tx_ppp7", "# of PPP prio 7 frames transmitted",
3278218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_TX_PORT_PPP7_L));
3279218792Snp
3280218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_octets", "# of octets in good frames",
3281218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BYTES_L));
3282218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames", "total # of good frames",
3283218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_FRAMES_L));
3284218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_bcast_frames", "# of broadcast frames",
3285218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_BCAST_L));
3286218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_mcast_frames", "# of multicast frames",
3287218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MCAST_L));
3288218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ucast_frames", "# of unicast frames",
3289218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_UCAST_L));
3290218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_too_long", "# of frames exceeding MTU",
3291218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_ERROR_L));
3292218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_jabber", "# of jabber frames",
3293218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_MTU_CRC_ERROR_L));
3294218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_fcs_err",
3295218792Snp	    "# of frames received with bad FCS",
3296218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_CRC_ERROR_L));
3297218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_len_err",
3298218792Snp	    "# of frames received with length error",
3299218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LEN_ERROR_L));
3300218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_symbol_err", "symbol errors",
3301218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_SYM_ERROR_L));
3302218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_runt", "# of short frames received",
3303218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_LESS_64B_L));
3304218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_64",
3305218792Snp	    "# of rx frames in this range",
3306218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_64B_L));
3307218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_65_127",
3308218792Snp	    "# of rx frames in this range",
3309218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_65B_127B_L));
3310218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_128_255",
3311218792Snp	    "# of rx frames in this range",
3312218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_128B_255B_L));
3313218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_256_511",
3314218792Snp	    "# of rx frames in this range",
3315218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_256B_511B_L));
3316218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_512_1023",
3317218792Snp	    "# of rx frames in this range",
3318218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_512B_1023B_L));
3319218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1024_1518",
3320218792Snp	    "# of rx frames in this range",
3321218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1024B_1518B_L));
3322218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_frames_1519_max",
3323218792Snp	    "# of rx frames in this range",
3324218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_1519B_MAX_L));
3325218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_pause", "# of pause frames received",
3326218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PAUSE_L));
3327218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp0", "# of PPP prio 0 frames received",
3328218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP0_L));
3329218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp1", "# of PPP prio 1 frames received",
3330218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP1_L));
3331218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp2", "# of PPP prio 2 frames received",
3332218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP2_L));
3333218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp3", "# of PPP prio 3 frames received",
3334218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP3_L));
3335218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp4", "# of PPP prio 4 frames received",
3336218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP4_L));
3337218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp5", "# of PPP prio 5 frames received",
3338218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP5_L));
3339218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp6", "# of PPP prio 6 frames received",
3340218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP6_L));
3341218792Snp	SYSCTL_ADD_T4_REG64(pi, "rx_ppp7", "# of PPP prio 7 frames received",
3342218792Snp	    PORT_REG(pi->tx_chan, A_MPS_PORT_STAT_RX_PORT_PPP7_L));
3343218792Snp
3344218792Snp#undef SYSCTL_ADD_T4_REG64
3345218792Snp
3346218792Snp#define SYSCTL_ADD_T4_PORTSTAT(name, desc) \
3347218792Snp	SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, #name, CTLFLAG_RD, \
3348218792Snp	    &pi->stats.name, desc)
3349218792Snp
3350218792Snp	/* We get these from port_stats and they may be stale by upto 1s */
3351218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow0,
3352218792Snp	    "# drops due to buffer-group 0 overflows");
3353218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow1,
3354218792Snp	    "# drops due to buffer-group 1 overflows");
3355218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow2,
3356218792Snp	    "# drops due to buffer-group 2 overflows");
3357218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_ovflow3,
3358218792Snp	    "# drops due to buffer-group 3 overflows");
3359218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_trunc0,
3360218792Snp	    "# of buffer-group 0 truncated packets");
3361218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_trunc1,
3362218792Snp	    "# of buffer-group 1 truncated packets");
3363218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_trunc2,
3364218792Snp	    "# of buffer-group 2 truncated packets");
3365218792Snp	SYSCTL_ADD_T4_PORTSTAT(rx_trunc3,
3366218792Snp	    "# of buffer-group 3 truncated packets");
3367218792Snp
3368218792Snp#undef SYSCTL_ADD_T4_PORTSTAT
3369218792Snp
3370218792Snp	return (0);
3371218792Snp}
3372218792Snp
3373218792Snpstatic int
3374219436Snpsysctl_int_array(SYSCTL_HANDLER_ARGS)
3375219436Snp{
3376219436Snp	int rc, *i;
3377219436Snp	struct sbuf sb;
3378219436Snp
3379219436Snp	sbuf_new(&sb, NULL, 32, SBUF_AUTOEXTEND);
3380219436Snp	for (i = arg1; arg2; arg2 -= sizeof(int), i++)
3381219436Snp		sbuf_printf(&sb, "%d ", *i);
3382219436Snp	sbuf_trim(&sb);
3383219436Snp	sbuf_finish(&sb);
3384219436Snp	rc = sysctl_handle_string(oidp, sbuf_data(&sb), sbuf_len(&sb), req);
3385219436Snp	sbuf_delete(&sb);
3386219436Snp	return (rc);
3387219436Snp}
3388219436Snp
3389219436Snpstatic int
3390228561Snpsysctl_bitfield(SYSCTL_HANDLER_ARGS)
3391228561Snp{
3392228561Snp	int rc;
3393228561Snp	struct sbuf *sb;
3394228561Snp
3395228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3396228561Snp	if (rc != 0)
3397228561Snp		return(rc);
3398228561Snp
3399228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
3400228561Snp	if (sb == NULL)
3401228561Snp		return (ENOMEM);
3402228561Snp
3403228561Snp	sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
3404228561Snp	rc = sbuf_finish(sb);
3405228561Snp	sbuf_delete(sb);
3406228561Snp
3407228561Snp	return (rc);
3408228561Snp}
3409228561Snp
3410228561Snpstatic int
3411218792Snpsysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
3412218792Snp{
3413218792Snp	struct port_info *pi = arg1;
3414218792Snp	struct adapter *sc = pi->adapter;
3415218792Snp	int idx, rc, i;
3416218792Snp
3417218792Snp	idx = pi->tmr_idx;
3418218792Snp
3419218792Snp	rc = sysctl_handle_int(oidp, &idx, 0, req);
3420218792Snp	if (rc != 0 || req->newptr == NULL)
3421218792Snp		return (rc);
3422218792Snp
3423218792Snp	if (idx < 0 || idx >= SGE_NTIMERS)
3424218792Snp		return (EINVAL);
3425218792Snp
3426218792Snp	ADAPTER_LOCK(sc);
3427218792Snp	rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3428218792Snp	if (rc == 0) {
3429228561Snp		struct sge_rxq *rxq;
3430228561Snp		uint8_t v;
3431228561Snp
3432228561Snp		v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
3433218792Snp		for_each_rxq(pi, i, rxq) {
3434228561Snp#ifdef atomic_store_rel_8
3435228561Snp			atomic_store_rel_8(&rxq->iq.intr_params, v);
3436228561Snp#else
3437228561Snp			rxq->iq.intr_params = v;
3438228561Snp#endif
3439218792Snp		}
3440218792Snp		pi->tmr_idx = idx;
3441218792Snp	}
3442218792Snp
3443218792Snp	ADAPTER_UNLOCK(sc);
3444218792Snp	return (rc);
3445218792Snp}
3446218792Snp
3447218792Snpstatic int
3448218792Snpsysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
3449218792Snp{
3450218792Snp	struct port_info *pi = arg1;
3451218792Snp	struct adapter *sc = pi->adapter;
3452218792Snp	int idx, rc;
3453218792Snp
3454218792Snp	idx = pi->pktc_idx;
3455218792Snp
3456218792Snp	rc = sysctl_handle_int(oidp, &idx, 0, req);
3457218792Snp	if (rc != 0 || req->newptr == NULL)
3458218792Snp		return (rc);
3459218792Snp
3460218792Snp	if (idx < -1 || idx >= SGE_NCOUNTERS)
3461218792Snp		return (EINVAL);
3462218792Snp
3463218792Snp	ADAPTER_LOCK(sc);
3464218792Snp	rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3465228561Snp	if (rc == 0 && pi->flags & PORT_INIT_DONE)
3466228561Snp		rc = EBUSY; /* cannot be changed once the queues are created */
3467218792Snp
3468218792Snp	if (rc == 0)
3469218792Snp		pi->pktc_idx = idx;
3470218792Snp
3471218792Snp	ADAPTER_UNLOCK(sc);
3472218792Snp	return (rc);
3473218792Snp}
3474218792Snp
3475218792Snpstatic int
3476218792Snpsysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
3477218792Snp{
3478218792Snp	struct port_info *pi = arg1;
3479218792Snp	struct adapter *sc = pi->adapter;
3480218792Snp	int qsize, rc;
3481218792Snp
3482218792Snp	qsize = pi->qsize_rxq;
3483218792Snp
3484218792Snp	rc = sysctl_handle_int(oidp, &qsize, 0, req);
3485218792Snp	if (rc != 0 || req->newptr == NULL)
3486218792Snp		return (rc);
3487218792Snp
3488218792Snp	if (qsize < 128 || (qsize & 7))
3489218792Snp		return (EINVAL);
3490218792Snp
3491218792Snp	ADAPTER_LOCK(sc);
3492218792Snp	rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3493228561Snp	if (rc == 0 && pi->flags & PORT_INIT_DONE)
3494228561Snp		rc = EBUSY; /* cannot be changed once the queues are created */
3495218792Snp
3496218792Snp	if (rc == 0)
3497218792Snp		pi->qsize_rxq = qsize;
3498218792Snp
3499218792Snp	ADAPTER_UNLOCK(sc);
3500218792Snp	return (rc);
3501218792Snp}
3502218792Snp
3503218792Snpstatic int
3504218792Snpsysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
3505218792Snp{
3506218792Snp	struct port_info *pi = arg1;
3507218792Snp	struct adapter *sc = pi->adapter;
3508218792Snp	int qsize, rc;
3509218792Snp
3510218792Snp	qsize = pi->qsize_txq;
3511218792Snp
3512218792Snp	rc = sysctl_handle_int(oidp, &qsize, 0, req);
3513218792Snp	if (rc != 0 || req->newptr == NULL)
3514218792Snp		return (rc);
3515218792Snp
3516218792Snp	if (qsize < 128)
3517218792Snp		return (EINVAL);
3518218792Snp
3519218792Snp	ADAPTER_LOCK(sc);
3520218792Snp	rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
3521228561Snp	if (rc == 0 && pi->flags & PORT_INIT_DONE)
3522228561Snp		rc = EBUSY; /* cannot be changed once the queues are created */
3523218792Snp
3524218792Snp	if (rc == 0)
3525218792Snp		pi->qsize_txq = qsize;
3526218792Snp
3527218792Snp	ADAPTER_UNLOCK(sc);
3528218792Snp	return (rc);
3529218792Snp}
3530218792Snp
3531218792Snpstatic int
3532218792Snpsysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
3533218792Snp{
3534218792Snp	struct adapter *sc = arg1;
3535218792Snp	int reg = arg2;
3536218792Snp	uint64_t val;
3537218792Snp
3538218792Snp	val = t4_read_reg64(sc, reg);
3539218792Snp
3540218792Snp	return (sysctl_handle_64(oidp, &val, 0, req));
3541218792Snp}
3542218792Snp
3543231115Snp#ifdef SBUF_DRAIN
3544228561Snpstatic int
3545228561Snpsysctl_cctrl(SYSCTL_HANDLER_ARGS)
3546228561Snp{
3547228561Snp	struct adapter *sc = arg1;
3548228561Snp	struct sbuf *sb;
3549228561Snp	int rc, i;
3550228561Snp	uint16_t incr[NMTUS][NCCTRL_WIN];
3551228561Snp	static const char *dec_fac[] = {
3552228561Snp		"0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
3553228561Snp		"0.9375"
3554228561Snp	};
3555228561Snp
3556228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3557228561Snp	if (rc != 0)
3558228561Snp		return (rc);
3559228561Snp
3560228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3561228561Snp	if (sb == NULL)
3562228561Snp		return (ENOMEM);
3563228561Snp
3564228561Snp	t4_read_cong_tbl(sc, incr);
3565228561Snp
3566228561Snp	for (i = 0; i < NCCTRL_WIN; ++i) {
3567228561Snp		sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
3568228561Snp		    incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
3569228561Snp		    incr[5][i], incr[6][i], incr[7][i]);
3570228561Snp		sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
3571228561Snp		    incr[8][i], incr[9][i], incr[10][i], incr[11][i],
3572228561Snp		    incr[12][i], incr[13][i], incr[14][i], incr[15][i],
3573228561Snp		    sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
3574228561Snp	}
3575228561Snp
3576228561Snp	rc = sbuf_finish(sb);
3577228561Snp	sbuf_delete(sb);
3578228561Snp
3579228561Snp	return (rc);
3580228561Snp}
3581228561Snp
3582228561Snpstatic int
3583228561Snpsysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
3584228561Snp{
3585228561Snp	struct adapter *sc = arg1;
3586228561Snp	struct sbuf *sb;
3587228561Snp	int rc;
3588228561Snp	struct tp_cpl_stats stats;
3589228561Snp
3590228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3591228561Snp	if (rc != 0)
3592228561Snp		return (rc);
3593228561Snp
3594228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3595228561Snp	if (sb == NULL)
3596228561Snp		return (ENOMEM);
3597228561Snp
3598228561Snp	t4_tp_get_cpl_stats(sc, &stats);
3599228561Snp
3600228561Snp	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
3601228561Snp	    "channel 3\n");
3602228561Snp	sbuf_printf(sb, "CPL requests:   %10u %10u %10u %10u\n",
3603228561Snp		   stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
3604228561Snp	sbuf_printf(sb, "CPL responses:  %10u %10u %10u %10u",
3605228561Snp		   stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
3606228561Snp
3607228561Snp	rc = sbuf_finish(sb);
3608228561Snp	sbuf_delete(sb);
3609228561Snp
3610228561Snp	return (rc);
3611228561Snp}
3612228561Snp
3613228561Snpstatic int
3614228561Snpsysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
3615228561Snp{
3616228561Snp	struct adapter *sc = arg1;
3617228561Snp	struct sbuf *sb;
3618228561Snp	int rc;
3619228561Snp	struct tp_usm_stats stats;
3620228561Snp
3621228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3622228561Snp	if (rc != 0)
3623228561Snp		return(rc);
3624228561Snp
3625228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3626228561Snp	if (sb == NULL)
3627228561Snp		return (ENOMEM);
3628228561Snp
3629228561Snp	t4_get_usm_stats(sc, &stats);
3630228561Snp
3631228561Snp	sbuf_printf(sb, "Frames: %u\n", stats.frames);
3632228561Snp	sbuf_printf(sb, "Octets: %ju\n", stats.octets);
3633228561Snp	sbuf_printf(sb, "Drops:  %u", stats.drops);
3634228561Snp
3635228561Snp	rc = sbuf_finish(sb);
3636228561Snp	sbuf_delete(sb);
3637228561Snp
3638228561Snp	return (rc);
3639228561Snp}
3640228561Snp
3641222551Snpconst char *devlog_level_strings[] = {
3642222551Snp	[FW_DEVLOG_LEVEL_EMERG]		= "EMERG",
3643222551Snp	[FW_DEVLOG_LEVEL_CRIT]		= "CRIT",
3644222551Snp	[FW_DEVLOG_LEVEL_ERR]		= "ERR",
3645222551Snp	[FW_DEVLOG_LEVEL_NOTICE]	= "NOTICE",
3646222551Snp	[FW_DEVLOG_LEVEL_INFO]		= "INFO",
3647222551Snp	[FW_DEVLOG_LEVEL_DEBUG]		= "DEBUG"
3648222551Snp};
3649222551Snp
3650222551Snpconst char *devlog_facility_strings[] = {
3651222551Snp	[FW_DEVLOG_FACILITY_CORE]	= "CORE",
3652222551Snp	[FW_DEVLOG_FACILITY_SCHED]	= "SCHED",
3653222551Snp	[FW_DEVLOG_FACILITY_TIMER]	= "TIMER",
3654222551Snp	[FW_DEVLOG_FACILITY_RES]	= "RES",
3655222551Snp	[FW_DEVLOG_FACILITY_HW]		= "HW",
3656222551Snp	[FW_DEVLOG_FACILITY_FLR]	= "FLR",
3657222551Snp	[FW_DEVLOG_FACILITY_DMAQ]	= "DMAQ",
3658222551Snp	[FW_DEVLOG_FACILITY_PHY]	= "PHY",
3659222551Snp	[FW_DEVLOG_FACILITY_MAC]	= "MAC",
3660222551Snp	[FW_DEVLOG_FACILITY_PORT]	= "PORT",
3661222551Snp	[FW_DEVLOG_FACILITY_VI]		= "VI",
3662222551Snp	[FW_DEVLOG_FACILITY_FILTER]	= "FILTER",
3663222551Snp	[FW_DEVLOG_FACILITY_ACL]	= "ACL",
3664222551Snp	[FW_DEVLOG_FACILITY_TM]		= "TM",
3665222551Snp	[FW_DEVLOG_FACILITY_QFC]	= "QFC",
3666222551Snp	[FW_DEVLOG_FACILITY_DCB]	= "DCB",
3667222551Snp	[FW_DEVLOG_FACILITY_ETH]	= "ETH",
3668222551Snp	[FW_DEVLOG_FACILITY_OFLD]	= "OFLD",
3669222551Snp	[FW_DEVLOG_FACILITY_RI]		= "RI",
3670222551Snp	[FW_DEVLOG_FACILITY_ISCSI]	= "ISCSI",
3671222551Snp	[FW_DEVLOG_FACILITY_FCOE]	= "FCOE",
3672222551Snp	[FW_DEVLOG_FACILITY_FOISCSI]	= "FOISCSI",
3673222551Snp	[FW_DEVLOG_FACILITY_FOFCOE]	= "FOFCOE"
3674222551Snp};
3675222551Snp
3676222551Snpstatic int
3677222551Snpsysctl_devlog(SYSCTL_HANDLER_ARGS)
3678222551Snp{
3679222551Snp	struct adapter *sc = arg1;
3680222551Snp	struct devlog_params *dparams = &sc->params.devlog;
3681222551Snp	struct fw_devlog_e *buf, *e;
3682222551Snp	int i, j, rc, nentries, first = 0;
3683222551Snp	struct sbuf *sb;
3684222551Snp	uint64_t ftstamp = UINT64_MAX;
3685222551Snp
3686222551Snp	if (dparams->start == 0)
3687222551Snp		return (ENXIO);
3688222551Snp
3689222551Snp	nentries = dparams->size / sizeof(struct fw_devlog_e);
3690222551Snp
3691222551Snp	buf = malloc(dparams->size, M_CXGBE, M_NOWAIT);
3692222551Snp	if (buf == NULL)
3693222551Snp		return (ENOMEM);
3694222551Snp
3695222551Snp	rc = -t4_mem_read(sc, dparams->memtype, dparams->start, dparams->size,
3696222551Snp	    (void *)buf);
3697222551Snp	if (rc != 0)
3698222551Snp		goto done;
3699222551Snp
3700222551Snp	for (i = 0; i < nentries; i++) {
3701222551Snp		e = &buf[i];
3702222551Snp
3703222551Snp		if (e->timestamp == 0)
3704222551Snp			break;	/* end */
3705222551Snp
3706222551Snp		e->timestamp = be64toh(e->timestamp);
3707222551Snp		e->seqno = be32toh(e->seqno);
3708222551Snp		for (j = 0; j < 8; j++)
3709222551Snp			e->params[j] = be32toh(e->params[j]);
3710222551Snp
3711222551Snp		if (e->timestamp < ftstamp) {
3712222551Snp			ftstamp = e->timestamp;
3713222551Snp			first = i;
3714222551Snp		}
3715222551Snp	}
3716222551Snp
3717222551Snp	if (buf[first].timestamp == 0)
3718222551Snp		goto done;	/* nothing in the log */
3719222551Snp
3720222551Snp	rc = sysctl_wire_old_buffer(req, 0);
3721222551Snp	if (rc != 0)
3722222551Snp		goto done;
3723222551Snp
3724222551Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3725228561Snp	if (sb == NULL) {
3726228561Snp		rc = ENOMEM;
3727228561Snp		goto done;
3728228561Snp	}
3729228561Snp	sbuf_printf(sb, "%10s  %15s  %8s  %8s  %s\n",
3730222551Snp	    "Seq#", "Tstamp", "Level", "Facility", "Message");
3731222551Snp
3732222551Snp	i = first;
3733222551Snp	do {
3734222551Snp		e = &buf[i];
3735222551Snp		if (e->timestamp == 0)
3736222551Snp			break;	/* end */
3737222551Snp
3738222551Snp		sbuf_printf(sb, "%10d  %15ju  %8s  %8s  ",
3739222551Snp		    e->seqno, e->timestamp,
3740222551Snp		    (e->level < ARRAY_SIZE(devlog_level_strings) ?
3741222551Snp			devlog_level_strings[e->level] : "UNKNOWN"),
3742222551Snp		    (e->facility < ARRAY_SIZE(devlog_facility_strings) ?
3743222551Snp			devlog_facility_strings[e->facility] : "UNKNOWN"));
3744222551Snp		sbuf_printf(sb, e->fmt, e->params[0], e->params[1],
3745222551Snp		    e->params[2], e->params[3], e->params[4],
3746222551Snp		    e->params[5], e->params[6], e->params[7]);
3747222551Snp
3748222551Snp		if (++i == nentries)
3749222551Snp			i = 0;
3750222551Snp	} while (i != first);
3751222551Snp
3752222551Snp	rc = sbuf_finish(sb);
3753222551Snp	sbuf_delete(sb);
3754222551Snpdone:
3755222551Snp	free(buf, M_CXGBE);
3756222551Snp	return (rc);
3757222551Snp}
3758222551Snp
3759228561Snpstatic int
3760228561Snpsysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
3761228561Snp{
3762228561Snp	struct adapter *sc = arg1;
3763228561Snp	struct sbuf *sb;
3764228561Snp	int rc;
3765228561Snp	struct tp_fcoe_stats stats[4];
3766228561Snp
3767228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3768228561Snp	if (rc != 0)
3769228561Snp		return (rc);
3770228561Snp
3771228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3772228561Snp	if (sb == NULL)
3773228561Snp		return (ENOMEM);
3774228561Snp
3775228561Snp	t4_get_fcoe_stats(sc, 0, &stats[0]);
3776228561Snp	t4_get_fcoe_stats(sc, 1, &stats[1]);
3777228561Snp	t4_get_fcoe_stats(sc, 2, &stats[2]);
3778228561Snp	t4_get_fcoe_stats(sc, 3, &stats[3]);
3779228561Snp
3780228561Snp	sbuf_printf(sb, "                   channel 0        channel 1        "
3781228561Snp	    "channel 2        channel 3\n");
3782228561Snp	sbuf_printf(sb, "octetsDDP:  %16ju %16ju %16ju %16ju\n",
3783228561Snp	    stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
3784228561Snp	    stats[3].octetsDDP);
3785228561Snp	sbuf_printf(sb, "framesDDP:  %16u %16u %16u %16u\n", stats[0].framesDDP,
3786228561Snp	    stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
3787228561Snp	sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
3788228561Snp	    stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
3789228561Snp	    stats[3].framesDrop);
3790228561Snp
3791228561Snp	rc = sbuf_finish(sb);
3792228561Snp	sbuf_delete(sb);
3793228561Snp
3794228561Snp	return (rc);
3795228561Snp}
3796228561Snp
3797228561Snpstatic int
3798228561Snpsysctl_hw_sched(SYSCTL_HANDLER_ARGS)
3799228561Snp{
3800228561Snp	struct adapter *sc = arg1;
3801228561Snp	struct sbuf *sb;
3802228561Snp	int rc, i;
3803228561Snp	unsigned int map, kbps, ipg, mode;
3804228561Snp	unsigned int pace_tab[NTX_SCHED];
3805228561Snp
3806228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3807228561Snp	if (rc != 0)
3808228561Snp		return (rc);
3809228561Snp
3810228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
3811228561Snp	if (sb == NULL)
3812228561Snp		return (ENOMEM);
3813228561Snp
3814228561Snp	map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
3815228561Snp	mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
3816228561Snp	t4_read_pace_tbl(sc, pace_tab);
3817228561Snp
3818228561Snp	sbuf_printf(sb, "Scheduler  Mode   Channel  Rate (Kbps)   "
3819228561Snp	    "Class IPG (0.1 ns)   Flow IPG (us)");
3820228561Snp
3821228561Snp	for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
3822228561Snp		t4_get_tx_sched(sc, i, &kbps, &ipg);
3823228561Snp		sbuf_printf(sb, "\n    %u      %-5s     %u     ", i,
3824228561Snp		    (mode & (1 << i)) ? "flow" : "class", map & 3);
3825228561Snp		if (kbps)
3826228561Snp			sbuf_printf(sb, "%9u     ", kbps);
3827228561Snp		else
3828228561Snp			sbuf_printf(sb, " disabled     ");
3829228561Snp
3830228561Snp		if (ipg)
3831228561Snp			sbuf_printf(sb, "%13u        ", ipg);
3832228561Snp		else
3833228561Snp			sbuf_printf(sb, "     disabled        ");
3834228561Snp
3835228561Snp		if (pace_tab[i])
3836228561Snp			sbuf_printf(sb, "%10u", pace_tab[i]);
3837228561Snp		else
3838228561Snp			sbuf_printf(sb, "  disabled");
3839228561Snp	}
3840228561Snp
3841228561Snp	rc = sbuf_finish(sb);
3842228561Snp	sbuf_delete(sb);
3843228561Snp
3844228561Snp	return (rc);
3845228561Snp}
3846228561Snp
3847228561Snpstatic int
3848228561Snpsysctl_lb_stats(SYSCTL_HANDLER_ARGS)
3849228561Snp{
3850228561Snp	struct adapter *sc = arg1;
3851228561Snp	struct sbuf *sb;
3852228561Snp	int rc, i, j;
3853228561Snp	uint64_t *p0, *p1;
3854228561Snp	struct lb_port_stats s[2];
3855228561Snp	static const char *stat_name[] = {
3856228561Snp		"OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
3857228561Snp		"UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
3858228561Snp		"Frames128To255:", "Frames256To511:", "Frames512To1023:",
3859228561Snp		"Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
3860228561Snp		"BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
3861228561Snp		"BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
3862228561Snp		"BG2FramesTrunc:", "BG3FramesTrunc:"
3863228561Snp	};
3864228561Snp
3865228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3866228561Snp	if (rc != 0)
3867228561Snp		return (rc);
3868228561Snp
3869228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3870228561Snp	if (sb == NULL)
3871228561Snp		return (ENOMEM);
3872228561Snp
3873228561Snp	memset(s, 0, sizeof(s));
3874228561Snp
3875228561Snp	for (i = 0; i < 4; i += 2) {
3876228561Snp		t4_get_lb_stats(sc, i, &s[0]);
3877228561Snp		t4_get_lb_stats(sc, i + 1, &s[1]);
3878228561Snp
3879228561Snp		p0 = &s[0].octets;
3880228561Snp		p1 = &s[1].octets;
3881228561Snp		sbuf_printf(sb, "%s                       Loopback %u"
3882228561Snp		    "           Loopback %u", i == 0 ? "" : "\n", i, i + 1);
3883228561Snp
3884228561Snp		for (j = 0; j < ARRAY_SIZE(stat_name); j++)
3885228561Snp			sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
3886228561Snp				   *p0++, *p1++);
3887228561Snp	}
3888228561Snp
3889228561Snp	rc = sbuf_finish(sb);
3890228561Snp	sbuf_delete(sb);
3891228561Snp
3892228561Snp	return (rc);
3893228561Snp}
3894228561Snp
3895228561Snpstruct mem_desc {
3896228561Snp	unsigned int base;
3897228561Snp	unsigned int limit;
3898228561Snp	unsigned int idx;
3899228561Snp};
3900228561Snp
3901228561Snpstatic int
3902228561Snpmem_desc_cmp(const void *a, const void *b)
3903228561Snp{
3904228561Snp	return ((const struct mem_desc *)a)->base -
3905228561Snp	       ((const struct mem_desc *)b)->base;
3906228561Snp}
3907228561Snp
3908228561Snpstatic void
3909228561Snpmem_region_show(struct sbuf *sb, const char *name, unsigned int from,
3910228561Snp    unsigned int to)
3911228561Snp{
3912228561Snp	unsigned int size;
3913228561Snp
3914228561Snp	size = to - from + 1;
3915228561Snp	if (size == 0)
3916228561Snp		return;
3917228561Snp
3918228561Snp	/* XXX: need humanize_number(3) in libkern for a more readable 'size' */
3919228561Snp	sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
3920228561Snp}
3921228561Snp
3922228561Snpstatic int
3923228561Snpsysctl_meminfo(SYSCTL_HANDLER_ARGS)
3924228561Snp{
3925228561Snp	struct adapter *sc = arg1;
3926228561Snp	struct sbuf *sb;
3927228561Snp	int rc, i, n;
3928228561Snp	uint32_t lo, hi;
3929228561Snp	static const char *memory[] = { "EDC0:", "EDC1:", "MC:" };
3930228561Snp	static const char *region[] = {
3931228561Snp		"DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
3932228561Snp		"Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
3933228561Snp		"Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
3934228561Snp		"TDDP region:", "TPT region:", "STAG region:", "RQ region:",
3935228561Snp		"RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:",
3936228561Snp		"ULPTX state:", "On-chip queues:"
3937228561Snp	};
3938228561Snp	struct mem_desc avail[3];
3939228561Snp	struct mem_desc mem[ARRAY_SIZE(region) + 3];	/* up to 3 holes */
3940228561Snp	struct mem_desc *md = mem;
3941228561Snp
3942228561Snp	rc = sysctl_wire_old_buffer(req, 0);
3943228561Snp	if (rc != 0)
3944228561Snp		return (rc);
3945228561Snp
3946228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
3947228561Snp	if (sb == NULL)
3948228561Snp		return (ENOMEM);
3949228561Snp
3950228561Snp	for (i = 0; i < ARRAY_SIZE(mem); i++) {
3951228561Snp		mem[i].limit = 0;
3952228561Snp		mem[i].idx = i;
3953228561Snp	}
3954228561Snp
3955228561Snp	/* Find and sort the populated memory ranges */
3956228561Snp	i = 0;
3957228561Snp	lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
3958228561Snp	if (lo & F_EDRAM0_ENABLE) {
3959228561Snp		hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
3960228561Snp		avail[i].base = G_EDRAM0_BASE(hi) << 20;
3961228561Snp		avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
3962228561Snp		avail[i].idx = 0;
3963228561Snp		i++;
3964228561Snp	}
3965228561Snp	if (lo & F_EDRAM1_ENABLE) {
3966228561Snp		hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
3967228561Snp		avail[i].base = G_EDRAM1_BASE(hi) << 20;
3968228561Snp		avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
3969228561Snp		avail[i].idx = 1;
3970228561Snp		i++;
3971228561Snp	}
3972228561Snp	if (lo & F_EXT_MEM_ENABLE) {
3973228561Snp		hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
3974228561Snp		avail[i].base = G_EXT_MEM_BASE(hi) << 20;
3975228561Snp		avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20);
3976228561Snp		avail[i].idx = 2;
3977228561Snp		i++;
3978228561Snp	}
3979228561Snp	if (!i)                                    /* no memory available */
3980228561Snp		return 0;
3981228561Snp	qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
3982228561Snp
3983228561Snp	(md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
3984228561Snp	(md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
3985228561Snp	(md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
3986228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
3987228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
3988228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
3989228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
3990228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
3991228561Snp	(md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
3992228561Snp
3993228561Snp	/* the next few have explicit upper bounds */
3994228561Snp	md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
3995228561Snp	md->limit = md->base - 1 +
3996228561Snp		    t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
3997228561Snp		    G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
3998228561Snp	md++;
3999228561Snp
4000228561Snp	md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
4001228561Snp	md->limit = md->base - 1 +
4002228561Snp		    t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
4003228561Snp		    G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
4004228561Snp	md++;
4005228561Snp
4006228561Snp	if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4007228561Snp		hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
4008228561Snp		md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
4009228561Snp		md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
4010228561Snp	} else {
4011228561Snp		md->base = 0;
4012228561Snp		md->idx = ARRAY_SIZE(region);  /* hide it */
4013228561Snp	}
4014228561Snp	md++;
4015228561Snp
4016228561Snp#define ulp_region(reg) \
4017228561Snp	md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
4018228561Snp	(md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
4019228561Snp
4020228561Snp	ulp_region(RX_ISCSI);
4021228561Snp	ulp_region(RX_TDDP);
4022228561Snp	ulp_region(TX_TPT);
4023228561Snp	ulp_region(RX_STAG);
4024228561Snp	ulp_region(RX_RQ);
4025228561Snp	ulp_region(RX_RQUDP);
4026228561Snp	ulp_region(RX_PBL);
4027228561Snp	ulp_region(TX_PBL);
4028228561Snp#undef ulp_region
4029228561Snp
4030228561Snp	md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
4031228561Snp	md->limit = md->base + sc->tids.ntids - 1;
4032228561Snp	md++;
4033228561Snp	md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
4034228561Snp	md->limit = md->base + sc->tids.ntids - 1;
4035228561Snp	md++;
4036228561Snp
4037228561Snp	md->base = sc->vres.ocq.start;
4038228561Snp	if (sc->vres.ocq.size)
4039228561Snp		md->limit = md->base + sc->vres.ocq.size - 1;
4040228561Snp	else
4041228561Snp		md->idx = ARRAY_SIZE(region);  /* hide it */
4042228561Snp	md++;
4043228561Snp
4044228561Snp	/* add any address-space holes, there can be up to 3 */
4045228561Snp	for (n = 0; n < i - 1; n++)
4046228561Snp		if (avail[n].limit < avail[n + 1].base)
4047228561Snp			(md++)->base = avail[n].limit;
4048228561Snp	if (avail[n].limit)
4049228561Snp		(md++)->base = avail[n].limit;
4050228561Snp
4051228561Snp	n = md - mem;
4052228561Snp	qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
4053228561Snp
4054228561Snp	for (lo = 0; lo < i; lo++)
4055228561Snp		mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
4056228561Snp				avail[lo].limit - 1);
4057228561Snp
4058228561Snp	sbuf_printf(sb, "\n");
4059228561Snp	for (i = 0; i < n; i++) {
4060228561Snp		if (mem[i].idx >= ARRAY_SIZE(region))
4061228561Snp			continue;                        /* skip holes */
4062228561Snp		if (!mem[i].limit)
4063228561Snp			mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
4064228561Snp		mem_region_show(sb, region[mem[i].idx], mem[i].base,
4065228561Snp				mem[i].limit);
4066228561Snp	}
4067228561Snp
4068228561Snp	sbuf_printf(sb, "\n");
4069228561Snp	lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
4070228561Snp	hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
4071228561Snp	mem_region_show(sb, "uP RAM:", lo, hi);
4072228561Snp
4073228561Snp	lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
4074228561Snp	hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
4075228561Snp	mem_region_show(sb, "uP Extmem2:", lo, hi);
4076228561Snp
4077228561Snp	lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
4078228561Snp	sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
4079228561Snp		   G_PMRXMAXPAGE(lo),
4080228561Snp		   t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
4081228561Snp		   (lo & F_PMRXNUMCHN) ? 2 : 1);
4082228561Snp
4083228561Snp	lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
4084228561Snp	hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
4085228561Snp	sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
4086228561Snp		   G_PMTXMAXPAGE(lo),
4087228561Snp		   hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
4088228561Snp		   hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
4089228561Snp	sbuf_printf(sb, "%u p-structs\n",
4090228561Snp		   t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
4091228561Snp
4092228561Snp	for (i = 0; i < 4; i++) {
4093228561Snp		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
4094228561Snp		sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
4095228561Snp			   i, G_USED(lo), G_ALLOC(lo));
4096228561Snp	}
4097228561Snp	for (i = 0; i < 4; i++) {
4098228561Snp		lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
4099228561Snp		sbuf_printf(sb,
4100228561Snp			   "\nLoopback %d using %u pages out of %u allocated",
4101228561Snp			   i, G_USED(lo), G_ALLOC(lo));
4102228561Snp	}
4103228561Snp
4104228561Snp	rc = sbuf_finish(sb);
4105228561Snp	sbuf_delete(sb);
4106228561Snp
4107228561Snp	return (rc);
4108228561Snp}
4109228561Snp
4110228561Snpstatic int
4111228561Snpsysctl_path_mtus(SYSCTL_HANDLER_ARGS)
4112228561Snp{
4113228561Snp	struct adapter *sc = arg1;
4114228561Snp	struct sbuf *sb;
4115228561Snp	int rc;
4116228561Snp	uint16_t mtus[NMTUS];
4117228561Snp
4118228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4119228561Snp	if (rc != 0)
4120228561Snp		return (rc);
4121228561Snp
4122228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4123228561Snp	if (sb == NULL)
4124228561Snp		return (ENOMEM);
4125228561Snp
4126228561Snp	t4_read_mtu_tbl(sc, mtus, NULL);
4127228561Snp
4128228561Snp	sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
4129228561Snp	    mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
4130228561Snp	    mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
4131228561Snp	    mtus[14], mtus[15]);
4132228561Snp
4133228561Snp	rc = sbuf_finish(sb);
4134228561Snp	sbuf_delete(sb);
4135228561Snp
4136228561Snp	return (rc);
4137228561Snp}
4138228561Snp
4139228561Snpstatic int
4140228561Snpsysctl_pm_stats(SYSCTL_HANDLER_ARGS)
4141228561Snp{
4142228561Snp	struct adapter *sc = arg1;
4143228561Snp	struct sbuf *sb;
4144228561Snp	int rc, i;
4145228561Snp	uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS];
4146228561Snp	uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS];
4147228561Snp	static const char *pm_stats[] = {
4148228561Snp		"Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:"
4149228561Snp	};
4150228561Snp
4151228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4152228561Snp	if (rc != 0)
4153228561Snp		return (rc);
4154228561Snp
4155228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4156228561Snp	if (sb == NULL)
4157228561Snp		return (ENOMEM);
4158228561Snp
4159228561Snp	t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
4160228561Snp	t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
4161228561Snp
4162228561Snp	sbuf_printf(sb, "                Tx count            Tx cycles    "
4163228561Snp	    "Rx count            Rx cycles");
4164228561Snp	for (i = 0; i < PM_NSTATS; i++)
4165228561Snp		sbuf_printf(sb, "\n%-13s %10u %20ju  %10u %20ju",
4166228561Snp		    pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]);
4167228561Snp
4168228561Snp	rc = sbuf_finish(sb);
4169228561Snp	sbuf_delete(sb);
4170228561Snp
4171228561Snp	return (rc);
4172228561Snp}
4173228561Snp
4174228561Snpstatic int
4175228561Snpsysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
4176228561Snp{
4177228561Snp	struct adapter *sc = arg1;
4178228561Snp	struct sbuf *sb;
4179228561Snp	int rc;
4180228561Snp	struct tp_rdma_stats stats;
4181228561Snp
4182228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4183228561Snp	if (rc != 0)
4184228561Snp		return (rc);
4185228561Snp
4186228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4187228561Snp	if (sb == NULL)
4188228561Snp		return (ENOMEM);
4189228561Snp
4190228561Snp	t4_tp_get_rdma_stats(sc, &stats);
4191228561Snp	sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
4192228561Snp	sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
4193228561Snp
4194228561Snp	rc = sbuf_finish(sb);
4195228561Snp	sbuf_delete(sb);
4196228561Snp
4197228561Snp	return (rc);
4198228561Snp}
4199228561Snp
4200228561Snpstatic int
4201228561Snpsysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
4202228561Snp{
4203228561Snp	struct adapter *sc = arg1;
4204228561Snp	struct sbuf *sb;
4205228561Snp	int rc;
4206228561Snp	struct tp_tcp_stats v4, v6;
4207228561Snp
4208228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4209228561Snp	if (rc != 0)
4210228561Snp		return (rc);
4211228561Snp
4212228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4213228561Snp	if (sb == NULL)
4214228561Snp		return (ENOMEM);
4215228561Snp
4216228561Snp	t4_tp_get_tcp_stats(sc, &v4, &v6);
4217228561Snp	sbuf_printf(sb,
4218228561Snp	    "                                IP                 IPv6\n");
4219228561Snp	sbuf_printf(sb, "OutRsts:      %20u %20u\n",
4220228561Snp	    v4.tcpOutRsts, v6.tcpOutRsts);
4221228561Snp	sbuf_printf(sb, "InSegs:       %20ju %20ju\n",
4222228561Snp	    v4.tcpInSegs, v6.tcpInSegs);
4223228561Snp	sbuf_printf(sb, "OutSegs:      %20ju %20ju\n",
4224228561Snp	    v4.tcpOutSegs, v6.tcpOutSegs);
4225228561Snp	sbuf_printf(sb, "RetransSegs:  %20ju %20ju",
4226228561Snp	    v4.tcpRetransSegs, v6.tcpRetransSegs);
4227228561Snp
4228228561Snp	rc = sbuf_finish(sb);
4229228561Snp	sbuf_delete(sb);
4230228561Snp
4231228561Snp	return (rc);
4232228561Snp}
4233228561Snp
4234228561Snpstatic int
4235228561Snpsysctl_tids(SYSCTL_HANDLER_ARGS)
4236228561Snp{
4237228561Snp	struct adapter *sc = arg1;
4238228561Snp	struct sbuf *sb;
4239228561Snp	int rc;
4240228561Snp	struct tid_info *t = &sc->tids;
4241228561Snp
4242228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4243228561Snp	if (rc != 0)
4244228561Snp		return (rc);
4245228561Snp
4246228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4247228561Snp	if (sb == NULL)
4248228561Snp		return (ENOMEM);
4249228561Snp
4250228561Snp	if (t->natids) {
4251228561Snp		sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
4252228561Snp		    t->atids_in_use);
4253228561Snp	}
4254228561Snp
4255228561Snp	if (t->ntids) {
4256228561Snp		if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
4257228561Snp			uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
4258228561Snp
4259228561Snp			if (b) {
4260228561Snp				sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
4261228561Snp				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4262228561Snp				    t->ntids - 1);
4263228561Snp			} else {
4264228561Snp				sbuf_printf(sb, "TID range: %u-%u",
4265228561Snp				    t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
4266228561Snp				    t->ntids - 1);
4267228561Snp			}
4268228561Snp		} else
4269228561Snp			sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
4270228561Snp		sbuf_printf(sb, ", in use: %u\n",
4271228561Snp		    atomic_load_acq_int(&t->tids_in_use));
4272228561Snp	}
4273228561Snp
4274228561Snp	if (t->nstids) {
4275228561Snp		sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
4276228561Snp		    t->stid_base + t->nstids - 1, t->stids_in_use);
4277228561Snp	}
4278228561Snp
4279228561Snp	if (t->nftids) {
4280228561Snp		sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
4281228561Snp		    t->ftid_base + t->nftids - 1);
4282228561Snp	}
4283228561Snp
4284228561Snp	sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
4285228561Snp	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
4286228561Snp	    t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
4287228561Snp
4288228561Snp	rc = sbuf_finish(sb);
4289228561Snp	sbuf_delete(sb);
4290228561Snp
4291228561Snp	return (rc);
4292228561Snp}
4293228561Snp
4294228561Snpstatic int
4295228561Snpsysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
4296228561Snp{
4297228561Snp	struct adapter *sc = arg1;
4298228561Snp	struct sbuf *sb;
4299228561Snp	int rc;
4300228561Snp	struct tp_err_stats stats;
4301228561Snp
4302228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4303228561Snp	if (rc != 0)
4304228561Snp		return (rc);
4305228561Snp
4306228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4307228561Snp	if (sb == NULL)
4308228561Snp		return (ENOMEM);
4309228561Snp
4310228561Snp	t4_tp_get_err_stats(sc, &stats);
4311228561Snp
4312228561Snp	sbuf_printf(sb, "                 channel 0  channel 1  channel 2  "
4313228561Snp		      "channel 3\n");
4314228561Snp	sbuf_printf(sb, "macInErrs:      %10u %10u %10u %10u\n",
4315228561Snp	    stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
4316228561Snp	    stats.macInErrs[3]);
4317228561Snp	sbuf_printf(sb, "hdrInErrs:      %10u %10u %10u %10u\n",
4318228561Snp	    stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
4319228561Snp	    stats.hdrInErrs[3]);
4320228561Snp	sbuf_printf(sb, "tcpInErrs:      %10u %10u %10u %10u\n",
4321228561Snp	    stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
4322228561Snp	    stats.tcpInErrs[3]);
4323228561Snp	sbuf_printf(sb, "tcp6InErrs:     %10u %10u %10u %10u\n",
4324228561Snp	    stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
4325228561Snp	    stats.tcp6InErrs[3]);
4326228561Snp	sbuf_printf(sb, "tnlCongDrops:   %10u %10u %10u %10u\n",
4327228561Snp	    stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
4328228561Snp	    stats.tnlCongDrops[3]);
4329228561Snp	sbuf_printf(sb, "tnlTxDrops:     %10u %10u %10u %10u\n",
4330228561Snp	    stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
4331228561Snp	    stats.tnlTxDrops[3]);
4332228561Snp	sbuf_printf(sb, "ofldVlanDrops:  %10u %10u %10u %10u\n",
4333228561Snp	    stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
4334228561Snp	    stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
4335228561Snp	sbuf_printf(sb, "ofldChanDrops:  %10u %10u %10u %10u\n\n",
4336228561Snp	    stats.ofldChanDrops[0], stats.ofldChanDrops[1],
4337228561Snp	    stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
4338228561Snp	sbuf_printf(sb, "ofldNoNeigh:    %u\nofldCongDefer:  %u",
4339228561Snp	    stats.ofldNoNeigh, stats.ofldCongDefer);
4340228561Snp
4341228561Snp	rc = sbuf_finish(sb);
4342228561Snp	sbuf_delete(sb);
4343228561Snp
4344228561Snp	return (rc);
4345228561Snp}
4346228561Snp
4347228561Snpstatic int
4348228561Snpsysctl_tx_rate(SYSCTL_HANDLER_ARGS)
4349228561Snp{
4350228561Snp	struct adapter *sc = arg1;
4351228561Snp	struct sbuf *sb;
4352228561Snp	int rc;
4353228561Snp	u64 nrate[NCHAN], orate[NCHAN];
4354228561Snp
4355228561Snp	rc = sysctl_wire_old_buffer(req, 0);
4356228561Snp	if (rc != 0)
4357228561Snp		return (rc);
4358228561Snp
4359228561Snp	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
4360228561Snp	if (sb == NULL)
4361228561Snp		return (ENOMEM);
4362228561Snp
4363228561Snp	t4_get_chan_txrate(sc, nrate, orate);
4364228561Snp	sbuf_printf(sb, "              channel 0   channel 1   channel 2   "
4365228561Snp		 "channel 3\n");
4366228561Snp	sbuf_printf(sb, "NIC B/s:     %10ju  %10ju  %10ju  %10ju\n",
4367228561Snp	    nrate[0], nrate[1], nrate[2], nrate[3]);
4368228561Snp	sbuf_printf(sb, "Offload B/s: %10ju  %10ju  %10ju  %10ju",
4369228561Snp	    orate[0], orate[1], orate[2], orate[3]);
4370228561Snp
4371228561Snp	rc = sbuf_finish(sb);
4372228561Snp	sbuf_delete(sb);
4373228561Snp
4374228561Snp	return (rc);
4375228561Snp}
4376231115Snp#endif
4377228561Snp
4378219286Snpstatic inline void
4379219286Snptxq_start(struct ifnet *ifp, struct sge_txq *txq)
4380219286Snp{
4381219286Snp	struct buf_ring *br;
4382219286Snp	struct mbuf *m;
4383219286Snp
4384219286Snp	TXQ_LOCK_ASSERT_OWNED(txq);
4385219286Snp
4386220873Snp	br = txq->br;
4387219286Snp	m = txq->m ? txq->m : drbr_dequeue(ifp, br);
4388219286Snp	if (m)
4389219286Snp		t4_eth_tx(ifp, txq, m);
4390219286Snp}
4391219286Snp
4392219286Snpvoid
4393228561Snpt4_tx_callout(void *arg)
4394219286Snp{
4395228561Snp	struct sge_eq *eq = arg;
4396228561Snp	struct adapter *sc;
4397219286Snp
4398228561Snp	if (EQ_TRYLOCK(eq) == 0)
4399228561Snp		goto reschedule;
4400228561Snp
4401228561Snp	if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
4402228561Snp		EQ_UNLOCK(eq);
4403228561Snpreschedule:
4404228561Snp		if (__predict_true(!(eq->flags && EQ_DOOMED)))
4405228561Snp			callout_schedule(&eq->tx_callout, 1);
4406228561Snp		return;
4407228561Snp	}
4408228561Snp
4409228561Snp	EQ_LOCK_ASSERT_OWNED(eq);
4410228561Snp
4411228561Snp	if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
4412228561Snp
4413228561Snp		if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4414228561Snp			struct sge_txq *txq = arg;
4415228561Snp			struct port_info *pi = txq->ifp->if_softc;
4416228561Snp
4417228561Snp			sc = pi->adapter;
4418228561Snp		} else {
4419228561Snp			struct sge_wrq *wrq = arg;
4420228561Snp
4421228561Snp			sc = wrq->adapter;
4422228561Snp		}
4423228561Snp
4424228561Snp		taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
4425228561Snp	}
4426228561Snp
4427228561Snp	EQ_UNLOCK(eq);
4428228561Snp}
4429228561Snp
4430228561Snpvoid
4431228561Snpt4_tx_task(void *arg, int count)
4432228561Snp{
4433228561Snp	struct sge_eq *eq = arg;
4434228561Snp
4435228561Snp	EQ_LOCK(eq);
4436228561Snp	if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
4437228561Snp		struct sge_txq *txq = arg;
4438220649Snp		txq_start(txq->ifp, txq);
4439228561Snp	} else {
4440228561Snp		struct sge_wrq *wrq = arg;
4441228561Snp		t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
4442228561Snp	}
4443228561Snp	EQ_UNLOCK(eq);
4444219286Snp}
4445219286Snp
4446221474Snpstatic uint32_t
4447221474Snpfconf_to_mode(uint32_t fconf)
4448221474Snp{
4449221474Snp	uint32_t mode;
4450221474Snp
4451221474Snp	mode = T4_FILTER_IPv4 | T4_FILTER_IPv6 | T4_FILTER_IP_SADDR |
4452221474Snp	    T4_FILTER_IP_DADDR | T4_FILTER_IP_SPORT | T4_FILTER_IP_DPORT;
4453221474Snp
4454221474Snp	if (fconf & F_FRAGMENTATION)
4455221474Snp		mode |= T4_FILTER_IP_FRAGMENT;
4456221474Snp
4457221474Snp	if (fconf & F_MPSHITTYPE)
4458221474Snp		mode |= T4_FILTER_MPS_HIT_TYPE;
4459221474Snp
4460221474Snp	if (fconf & F_MACMATCH)
4461221474Snp		mode |= T4_FILTER_MAC_IDX;
4462221474Snp
4463221474Snp	if (fconf & F_ETHERTYPE)
4464221474Snp		mode |= T4_FILTER_ETH_TYPE;
4465221474Snp
4466221474Snp	if (fconf & F_PROTOCOL)
4467221474Snp		mode |= T4_FILTER_IP_PROTO;
4468221474Snp
4469221474Snp	if (fconf & F_TOS)
4470221474Snp		mode |= T4_FILTER_IP_TOS;
4471221474Snp
4472221474Snp	if (fconf & F_VLAN)
4473228561Snp		mode |= T4_FILTER_VLAN;
4474221474Snp
4475221474Snp	if (fconf & F_VNIC_ID)
4476228561Snp		mode |= T4_FILTER_VNIC;
4477221474Snp
4478221474Snp	if (fconf & F_PORT)
4479221474Snp		mode |= T4_FILTER_PORT;
4480221474Snp
4481221474Snp	if (fconf & F_FCOE)
4482221474Snp		mode |= T4_FILTER_FCoE;
4483221474Snp
4484221474Snp	return (mode);
4485221474Snp}
4486221474Snp
4487221474Snpstatic uint32_t
4488221474Snpmode_to_fconf(uint32_t mode)
4489221474Snp{
4490221474Snp	uint32_t fconf = 0;
4491221474Snp
4492221474Snp	if (mode & T4_FILTER_IP_FRAGMENT)
4493221474Snp		fconf |= F_FRAGMENTATION;
4494221474Snp
4495221474Snp	if (mode & T4_FILTER_MPS_HIT_TYPE)
4496221474Snp		fconf |= F_MPSHITTYPE;
4497221474Snp
4498221474Snp	if (mode & T4_FILTER_MAC_IDX)
4499221474Snp		fconf |= F_MACMATCH;
4500221474Snp
4501221474Snp	if (mode & T4_FILTER_ETH_TYPE)
4502221474Snp		fconf |= F_ETHERTYPE;
4503221474Snp
4504221474Snp	if (mode & T4_FILTER_IP_PROTO)
4505221474Snp		fconf |= F_PROTOCOL;
4506221474Snp
4507221474Snp	if (mode & T4_FILTER_IP_TOS)
4508221474Snp		fconf |= F_TOS;
4509221474Snp
4510228561Snp	if (mode & T4_FILTER_VLAN)
4511221474Snp		fconf |= F_VLAN;
4512221474Snp
4513228561Snp	if (mode & T4_FILTER_VNIC)
4514221474Snp		fconf |= F_VNIC_ID;
4515221474Snp
4516221474Snp	if (mode & T4_FILTER_PORT)
4517221474Snp		fconf |= F_PORT;
4518221474Snp
4519221474Snp	if (mode & T4_FILTER_FCoE)
4520221474Snp		fconf |= F_FCOE;
4521221474Snp
4522221474Snp	return (fconf);
4523221474Snp}
4524221474Snp
4525221474Snpstatic uint32_t
4526221474Snpfspec_to_fconf(struct t4_filter_specification *fs)
4527221474Snp{
4528221474Snp	uint32_t fconf = 0;
4529221474Snp
4530221474Snp	if (fs->val.frag || fs->mask.frag)
4531221474Snp		fconf |= F_FRAGMENTATION;
4532221474Snp
4533221474Snp	if (fs->val.matchtype || fs->mask.matchtype)
4534221474Snp		fconf |= F_MPSHITTYPE;
4535221474Snp
4536221474Snp	if (fs->val.macidx || fs->mask.macidx)
4537221474Snp		fconf |= F_MACMATCH;
4538221474Snp
4539221474Snp	if (fs->val.ethtype || fs->mask.ethtype)
4540221474Snp		fconf |= F_ETHERTYPE;
4541221474Snp
4542221474Snp	if (fs->val.proto || fs->mask.proto)
4543221474Snp		fconf |= F_PROTOCOL;
4544221474Snp
4545221474Snp	if (fs->val.tos || fs->mask.tos)
4546221474Snp		fconf |= F_TOS;
4547221474Snp
4548228561Snp	if (fs->val.vlan_vld || fs->mask.vlan_vld)
4549221474Snp		fconf |= F_VLAN;
4550221474Snp
4551228561Snp	if (fs->val.vnic_vld || fs->mask.vnic_vld)
4552221474Snp		fconf |= F_VNIC_ID;
4553221474Snp
4554221474Snp	if (fs->val.iport || fs->mask.iport)
4555221474Snp		fconf |= F_PORT;
4556221474Snp
4557221474Snp	if (fs->val.fcoe || fs->mask.fcoe)
4558221474Snp		fconf |= F_FCOE;
4559221474Snp
4560221474Snp	return (fconf);
4561221474Snp}
4562221474Snp
4563221474Snpstatic int
4564221474Snpget_filter_mode(struct adapter *sc, uint32_t *mode)
4565221474Snp{
4566221474Snp	uint32_t fconf;
4567221474Snp
4568221474Snp	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
4569221474Snp	    A_TP_VLAN_PRI_MAP);
4570221474Snp
4571228561Snp	if (sc->filter_mode != fconf) {
4572228561Snp		log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
4573228561Snp		    device_get_nameunit(sc->dev), sc->filter_mode, fconf);
4574228561Snp		sc->filter_mode = fconf;
4575228561Snp	}
4576221474Snp
4577228561Snp	*mode = fconf_to_mode(sc->filter_mode);
4578228561Snp
4579221474Snp	return (0);
4580221474Snp}
4581221474Snp
4582221474Snpstatic int
4583221474Snpset_filter_mode(struct adapter *sc, uint32_t mode)
4584221474Snp{
4585221474Snp	uint32_t fconf;
4586221474Snp	int rc;
4587221474Snp
4588221474Snp	fconf = mode_to_fconf(mode);
4589221474Snp
4590221474Snp	ADAPTER_LOCK(sc);
4591221474Snp	if (IS_BUSY(sc)) {
4592221474Snp		rc = EAGAIN;
4593221474Snp		goto done;
4594221474Snp	}
4595221474Snp
4596221474Snp	if (sc->tids.ftids_in_use > 0) {
4597221474Snp		rc = EBUSY;
4598221474Snp		goto done;
4599221474Snp	}
4600221474Snp
4601237263Snp#ifdef TCP_OFFLOAD
4602228561Snp	if (sc->offload_map) {
4603228561Snp		rc = EBUSY;
4604228561Snp		goto done;
4605228561Snp	}
4606228561Snp#endif
4607228561Snp
4608228561Snp#ifdef notyet
4609221474Snp	rc = -t4_set_filter_mode(sc, fconf);
4610228561Snp	if (rc == 0)
4611228561Snp		sc->filter_mode = fconf;
4612228561Snp#else
4613228561Snp	rc = ENOTSUP;
4614228561Snp#endif
4615228561Snp
4616221474Snpdone:
4617221474Snp	ADAPTER_UNLOCK(sc);
4618221474Snp	return (rc);
4619221474Snp}
4620221474Snp
4621222552Snpstatic inline uint64_t
4622222552Snpget_filter_hits(struct adapter *sc, uint32_t fid)
4623222552Snp{
4624222552Snp	uint32_t tcb_base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
4625222552Snp	uint64_t hits;
4626222552Snp
4627222552Snp	t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0),
4628222552Snp	    tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
4629222552Snp	t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 0));
4630222552Snp	hits = t4_read_reg64(sc, MEMWIN0_BASE + 16);
4631222552Snp
4632222552Snp	return (be64toh(hits));
4633222552Snp}
4634222552Snp
4635221474Snpstatic int
4636221474Snpget_filter(struct adapter *sc, struct t4_filter *t)
4637221474Snp{
4638221474Snp	int i, nfilters = sc->tids.nftids;
4639221474Snp	struct filter_entry *f;
4640221474Snp
4641221474Snp	ADAPTER_LOCK_ASSERT_OWNED(sc);
4642221474Snp
4643221474Snp	if (IS_BUSY(sc))
4644221474Snp		return (EAGAIN);
4645221474Snp
4646221474Snp	if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
4647221474Snp	    t->idx >= nfilters) {
4648221474Snp		t->idx = 0xffffffff;
4649221474Snp		return (0);
4650221474Snp	}
4651221474Snp
4652221474Snp	f = &sc->tids.ftid_tab[t->idx];
4653221474Snp	for (i = t->idx; i < nfilters; i++, f++) {
4654221474Snp		if (f->valid) {
4655221474Snp			t->idx = i;
4656222509Snp			t->l2tidx = f->l2t ? f->l2t->idx : 0;
4657222509Snp			t->smtidx = f->smtidx;
4658222552Snp			if (f->fs.hitcnts)
4659222552Snp				t->hits = get_filter_hits(sc, t->idx);
4660222552Snp			else
4661222552Snp				t->hits = UINT64_MAX;
4662221474Snp			t->fs = f->fs;
4663221474Snp
4664221474Snp			return (0);
4665221474Snp		}
4666221474Snp	}
4667221474Snp
4668221474Snp	t->idx = 0xffffffff;
4669221474Snp	return (0);
4670221474Snp}
4671221474Snp
4672221474Snpstatic int
4673221474Snpset_filter(struct adapter *sc, struct t4_filter *t)
4674221474Snp{
4675221474Snp	unsigned int nfilters, nports;
4676221474Snp	struct filter_entry *f;
4677221474Snp	int i;
4678221474Snp
4679221474Snp	ADAPTER_LOCK_ASSERT_OWNED(sc);
4680221474Snp
4681221474Snp	nfilters = sc->tids.nftids;
4682221474Snp	nports = sc->params.nports;
4683221474Snp
4684221474Snp	if (nfilters == 0)
4685221474Snp		return (ENOTSUP);
4686221474Snp
4687221474Snp	if (!(sc->flags & FULL_INIT_DONE))
4688221474Snp		return (EAGAIN);
4689221474Snp
4690221474Snp	if (t->idx >= nfilters)
4691221474Snp		return (EINVAL);
4692221474Snp
4693221474Snp	/* Validate against the global filter mode */
4694228561Snp	if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode)
4695221474Snp		return (E2BIG);
4696221474Snp
4697221474Snp	if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports)
4698221474Snp		return (EINVAL);
4699221474Snp
4700221474Snp	if (t->fs.val.iport >= nports)
4701221474Snp		return (EINVAL);
4702221474Snp
4703221474Snp	/* Can't specify an iq if not steering to it */
4704221474Snp	if (!t->fs.dirsteer && t->fs.iq)
4705221474Snp		return (EINVAL);
4706221474Snp
4707221474Snp	/* IPv6 filter idx must be 4 aligned */
4708221474Snp	if (t->fs.type == 1 &&
4709221474Snp	    ((t->idx & 0x3) || t->idx + 4 >= nfilters))
4710221474Snp		return (EINVAL);
4711221474Snp
4712221474Snp	if (sc->tids.ftid_tab == NULL) {
4713221474Snp		KASSERT(sc->tids.ftids_in_use == 0,
4714221474Snp		    ("%s: no memory allocated but filters_in_use > 0",
4715221474Snp		    __func__));
4716221474Snp
4717221474Snp		sc->tids.ftid_tab = malloc(sizeof (struct filter_entry) *
4718221474Snp		    nfilters, M_CXGBE, M_NOWAIT | M_ZERO);
4719221474Snp		if (sc->tids.ftid_tab == NULL)
4720221474Snp			return (ENOMEM);
4721221474Snp	}
4722221474Snp
4723221474Snp	for (i = 0; i < 4; i++) {
4724221474Snp		f = &sc->tids.ftid_tab[t->idx + i];
4725221474Snp
4726221474Snp		if (f->pending || f->valid)
4727221474Snp			return (EBUSY);
4728221474Snp		if (f->locked)
4729221474Snp			return (EPERM);
4730221474Snp
4731221474Snp		if (t->fs.type == 0)
4732221474Snp			break;
4733221474Snp	}
4734221474Snp
4735221474Snp	f = &sc->tids.ftid_tab[t->idx];
4736221474Snp	f->fs = t->fs;
4737221474Snp
4738221474Snp	return set_filter_wr(sc, t->idx);
4739221474Snp}
4740221474Snp
4741221474Snpstatic int
4742221474Snpdel_filter(struct adapter *sc, struct t4_filter *t)
4743221474Snp{
4744221474Snp	unsigned int nfilters;
4745221474Snp	struct filter_entry *f;
4746221474Snp
4747221474Snp	ADAPTER_LOCK_ASSERT_OWNED(sc);
4748221474Snp
4749221474Snp	if (IS_BUSY(sc))
4750221474Snp		return (EAGAIN);
4751221474Snp
4752221474Snp	nfilters = sc->tids.nftids;
4753221474Snp
4754221474Snp	if (nfilters == 0)
4755221474Snp		return (ENOTSUP);
4756221474Snp
4757221474Snp	if (sc->tids.ftid_tab == NULL || sc->tids.ftids_in_use == 0 ||
4758221474Snp	    t->idx >= nfilters)
4759221474Snp		return (EINVAL);
4760221474Snp
4761221474Snp	if (!(sc->flags & FULL_INIT_DONE))
4762221474Snp		return (EAGAIN);
4763221474Snp
4764221474Snp	f = &sc->tids.ftid_tab[t->idx];
4765221474Snp
4766221474Snp	if (f->pending)
4767221474Snp		return (EBUSY);
4768221474Snp	if (f->locked)
4769221474Snp		return (EPERM);
4770221474Snp
4771221474Snp	if (f->valid) {
4772221474Snp		t->fs = f->fs;	/* extra info for the caller */
4773221474Snp		return del_filter_wr(sc, t->idx);
4774221474Snp	}
4775221474Snp
4776221474Snp	return (0);
4777221474Snp}
4778221474Snp
4779221474Snpstatic void
4780222509Snpclear_filter(struct filter_entry *f)
4781221474Snp{
4782222509Snp	if (f->l2t)
4783222509Snp		t4_l2t_release(f->l2t);
4784222509Snp
4785221474Snp	bzero(f, sizeof (*f));
4786221474Snp}
4787221474Snp
4788221474Snpstatic int
4789221474Snpset_filter_wr(struct adapter *sc, int fidx)
4790221474Snp{
4791221474Snp	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
4792237263Snp	struct wrqe *wr;
4793221474Snp	struct fw_filter_wr *fwr;
4794221474Snp	unsigned int ftid;
4795221474Snp
4796221474Snp	ADAPTER_LOCK_ASSERT_OWNED(sc);
4797221474Snp
4798222509Snp	if (f->fs.newdmac || f->fs.newvlan) {
4799222509Snp		/* This filter needs an L2T entry; allocate one. */
4800222509Snp		f->l2t = t4_l2t_alloc_switching(sc->l2t);
4801222509Snp		if (f->l2t == NULL)
4802222509Snp			return (EAGAIN);
4803222509Snp		if (t4_l2t_set_switching(sc, f->l2t, f->fs.vlan, f->fs.eport,
4804222509Snp		    f->fs.dmac)) {
4805222509Snp			t4_l2t_release(f->l2t);
4806222509Snp			f->l2t = NULL;
4807222509Snp			return (ENOMEM);
4808222509Snp		}
4809222509Snp	}
4810221474Snp
4811221474Snp	ftid = sc->tids.ftid_base + fidx;
4812221474Snp
4813237263Snp	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
4814237263Snp	if (wr == NULL)
4815221474Snp		return (ENOMEM);
4816221474Snp
4817237263Snp	fwr = wrtod(wr);
4818221474Snp	bzero(fwr, sizeof (*fwr));
4819221474Snp
4820221474Snp	fwr->op_pkd = htobe32(V_FW_WR_OP(FW_FILTER_WR));
4821221474Snp	fwr->len16_pkd = htobe32(FW_LEN16(*fwr));
4822221474Snp	fwr->tid_to_iq =
4823221474Snp	    htobe32(V_FW_FILTER_WR_TID(ftid) |
4824221474Snp		V_FW_FILTER_WR_RQTYPE(f->fs.type) |
4825221474Snp		V_FW_FILTER_WR_NOREPLY(0) |
4826221474Snp		V_FW_FILTER_WR_IQ(f->fs.iq));
4827221474Snp	fwr->del_filter_to_l2tix =
4828221474Snp	    htobe32(V_FW_FILTER_WR_RPTTID(f->fs.rpttid) |
4829221474Snp		V_FW_FILTER_WR_DROP(f->fs.action == FILTER_DROP) |
4830221474Snp		V_FW_FILTER_WR_DIRSTEER(f->fs.dirsteer) |
4831221474Snp		V_FW_FILTER_WR_MASKHASH(f->fs.maskhash) |
4832221474Snp		V_FW_FILTER_WR_DIRSTEERHASH(f->fs.dirsteerhash) |
4833221474Snp		V_FW_FILTER_WR_LPBK(f->fs.action == FILTER_SWITCH) |
4834221474Snp		V_FW_FILTER_WR_DMAC(f->fs.newdmac) |
4835221474Snp		V_FW_FILTER_WR_SMAC(f->fs.newsmac) |
4836221474Snp		V_FW_FILTER_WR_INSVLAN(f->fs.newvlan == VLAN_INSERT ||
4837221474Snp		    f->fs.newvlan == VLAN_REWRITE) |
4838221474Snp		V_FW_FILTER_WR_RMVLAN(f->fs.newvlan == VLAN_REMOVE ||
4839221474Snp		    f->fs.newvlan == VLAN_REWRITE) |
4840221474Snp		V_FW_FILTER_WR_HITCNTS(f->fs.hitcnts) |
4841221474Snp		V_FW_FILTER_WR_TXCHAN(f->fs.eport) |
4842221474Snp		V_FW_FILTER_WR_PRIO(f->fs.prio) |
4843222509Snp		V_FW_FILTER_WR_L2TIX(f->l2t ? f->l2t->idx : 0));
4844221474Snp	fwr->ethtype = htobe16(f->fs.val.ethtype);
4845221474Snp	fwr->ethtypem = htobe16(f->fs.mask.ethtype);
4846221474Snp	fwr->frag_to_ovlan_vldm =
4847221474Snp	    (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
4848221474Snp		V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
4849228561Snp		V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
4850228561Snp		V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
4851228561Snp		V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
4852228561Snp		V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
4853221474Snp	fwr->smac_sel = 0;
4854221474Snp	fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
4855228561Snp	    V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
4856221474Snp	fwr->maci_to_matchtypem =
4857221474Snp	    htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
4858221474Snp		V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
4859221474Snp		V_FW_FILTER_WR_FCOE(f->fs.val.fcoe) |
4860221474Snp		V_FW_FILTER_WR_FCOEM(f->fs.mask.fcoe) |
4861221474Snp		V_FW_FILTER_WR_PORT(f->fs.val.iport) |
4862221474Snp		V_FW_FILTER_WR_PORTM(f->fs.mask.iport) |
4863221474Snp		V_FW_FILTER_WR_MATCHTYPE(f->fs.val.matchtype) |
4864221474Snp		V_FW_FILTER_WR_MATCHTYPEM(f->fs.mask.matchtype));
4865221474Snp	fwr->ptcl = f->fs.val.proto;
4866221474Snp	fwr->ptclm = f->fs.mask.proto;
4867221474Snp	fwr->ttyp = f->fs.val.tos;
4868221474Snp	fwr->ttypm = f->fs.mask.tos;
4869228561Snp	fwr->ivlan = htobe16(f->fs.val.vlan);
4870228561Snp	fwr->ivlanm = htobe16(f->fs.mask.vlan);
4871228561Snp	fwr->ovlan = htobe16(f->fs.val.vnic);
4872228561Snp	fwr->ovlanm = htobe16(f->fs.mask.vnic);
4873221474Snp	bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
4874221474Snp	bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
4875221474Snp	bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
4876221474Snp	bcopy(f->fs.mask.sip, fwr->fipm, sizeof (fwr->fipm));
4877221474Snp	fwr->lp = htobe16(f->fs.val.dport);
4878221474Snp	fwr->lpm = htobe16(f->fs.mask.dport);
4879221474Snp	fwr->fp = htobe16(f->fs.val.sport);
4880221474Snp	fwr->fpm = htobe16(f->fs.mask.sport);
4881221474Snp	if (f->fs.newsmac)
4882221474Snp		bcopy(f->fs.smac, fwr->sma, sizeof (fwr->sma));
4883221474Snp
4884221474Snp	f->pending = 1;
4885221474Snp	sc->tids.ftids_in_use++;
4886228561Snp
4887237263Snp	t4_wrq_tx(sc, wr);
4888228561Snp	return (0);
4889221474Snp}
4890221474Snp
4891221474Snpstatic int
4892221474Snpdel_filter_wr(struct adapter *sc, int fidx)
4893221474Snp{
4894221474Snp	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
4895237263Snp	struct wrqe *wr;
4896221474Snp	struct fw_filter_wr *fwr;
4897228561Snp	unsigned int ftid;
4898221474Snp
4899221474Snp	ADAPTER_LOCK_ASSERT_OWNED(sc);
4900221474Snp
4901221474Snp	ftid = sc->tids.ftid_base + fidx;
4902221474Snp
4903237263Snp	wr = alloc_wrqe(sizeof(*fwr), &sc->sge.mgmtq);
4904237263Snp	if (wr == NULL)
4905221474Snp		return (ENOMEM);
4906237263Snp	fwr = wrtod(wr);
4907221474Snp	bzero(fwr, sizeof (*fwr));
4908221474Snp
4909228561Snp	t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
4910221474Snp
4911221474Snp	f->pending = 1;
4912237263Snp	t4_wrq_tx(sc, wr);
4913228561Snp	return (0);
4914221474Snp}
4915221474Snp
4916228561Snpstatic int
4917228561Snpfilter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
4918221474Snp{
4919228561Snp	struct adapter *sc = iq->adapter;
4920228561Snp	const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
4921221474Snp	unsigned int idx = GET_TID(rpl);
4922221474Snp
4923228561Snp	KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
4924228561Snp	    rss->opcode));
4925228561Snp
4926221474Snp	if (idx >= sc->tids.ftid_base &&
4927221474Snp	    (idx -= sc->tids.ftid_base) < sc->tids.nftids) {
4928221474Snp		unsigned int rc = G_COOKIE(rpl->cookie);
4929221474Snp		struct filter_entry *f = &sc->tids.ftid_tab[idx];
4930221474Snp
4931231120Snp		ADAPTER_LOCK(sc);
4932228561Snp		if (rc == FW_FILTER_WR_FLT_ADDED) {
4933221474Snp			f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
4934221474Snp			f->pending = 0;  /* asynchronous setup completed */
4935221474Snp			f->valid = 1;
4936231120Snp		} else {
4937231120Snp			if (rc != FW_FILTER_WR_FLT_DELETED) {
4938231120Snp				/* Add or delete failed, display an error */
4939231120Snp				log(LOG_ERR,
4940231120Snp				    "filter %u setup failed with error %u\n",
4941231120Snp				    idx, rc);
4942231120Snp			}
4943228561Snp
4944231120Snp			clear_filter(f);
4945231120Snp			sc->tids.ftids_in_use--;
4946221474Snp		}
4947228561Snp		ADAPTER_UNLOCK(sc);
4948221474Snp	}
4949228561Snp
4950228561Snp	return (0);
4951221474Snp}
4952221474Snp
4953222973Snpstatic int
4954222973Snpget_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
4955222973Snp{
4956222973Snp	int rc = EINVAL;
4957222973Snp
4958222973Snp	if (cntxt->cid > M_CTXTQID)
4959222973Snp		return (rc);
4960222973Snp
4961222973Snp	if (cntxt->mem_id != CTXT_EGRESS && cntxt->mem_id != CTXT_INGRESS &&
4962222973Snp	    cntxt->mem_id != CTXT_FLM && cntxt->mem_id != CTXT_CNM)
4963222973Snp		return (rc);
4964222973Snp
4965222973Snp	if (sc->flags & FW_OK) {
4966222973Snp		ADAPTER_LOCK(sc);	/* Avoid parallel t4_wr_mbox */
4967222973Snp		rc = -t4_sge_ctxt_rd(sc, sc->mbox, cntxt->cid, cntxt->mem_id,
4968222973Snp		    &cntxt->data[0]);
4969222973Snp		ADAPTER_UNLOCK(sc);
4970222973Snp	}
4971222973Snp
4972222973Snp	if (rc != 0) {
4973222973Snp		/* Read via firmware failed or wasn't even attempted */
4974222973Snp
4975222973Snp		rc = -t4_sge_ctxt_rd_bd(sc, cntxt->cid, cntxt->mem_id,
4976222973Snp		    &cntxt->data[0]);
4977222973Snp	}
4978222973Snp
4979222973Snp	return (rc);
4980222973Snp}
4981222973Snp
4982228561Snpstatic int
4983228561Snpread_card_mem(struct adapter *sc, struct t4_mem_range *mr)
4984228561Snp{
4985228561Snp	uint32_t base, size, lo, hi, win, off, remaining, i, n;
4986228561Snp	uint32_t *buf, *b;
4987228561Snp	int rc;
4988228561Snp
4989228561Snp	/* reads are in multiples of 32 bits */
4990228561Snp	if (mr->addr & 3 || mr->len & 3 || mr->len == 0)
4991228561Snp		return (EINVAL);
4992228561Snp
4993228561Snp	/*
4994228561Snp	 * We don't want to deal with potential holes so we mandate that the
4995228561Snp	 * requested region must lie entirely within one of the 3 memories.
4996228561Snp	 */
4997228561Snp	lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
4998228561Snp	if (lo & F_EDRAM0_ENABLE) {
4999228561Snp		hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
5000228561Snp		base = G_EDRAM0_BASE(hi) << 20;
5001228561Snp		size = G_EDRAM0_SIZE(hi) << 20;
5002228561Snp		if (size > 0 &&
5003228561Snp		    mr->addr >= base && mr->addr < base + size &&
5004228561Snp		    mr->addr + mr->len <= base + size)
5005228561Snp			goto proceed;
5006228561Snp	}
5007228561Snp	if (lo & F_EDRAM1_ENABLE) {
5008228561Snp		hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
5009228561Snp		base = G_EDRAM1_BASE(hi) << 20;
5010228561Snp		size = G_EDRAM1_SIZE(hi) << 20;
5011228561Snp		if (size > 0 &&
5012228561Snp		    mr->addr >= base && mr->addr < base + size &&
5013228561Snp		    mr->addr + mr->len <= base + size)
5014228561Snp			goto proceed;
5015228561Snp	}
5016228561Snp	if (lo & F_EXT_MEM_ENABLE) {
5017228561Snp		hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
5018228561Snp		base = G_EXT_MEM_BASE(hi) << 20;
5019228561Snp		size = G_EXT_MEM_SIZE(hi) << 20;
5020228561Snp		if (size > 0 &&
5021228561Snp		    mr->addr >= base && mr->addr < base + size &&
5022228561Snp		    mr->addr + mr->len <= base + size)
5023228561Snp			goto proceed;
5024228561Snp	}
5025228561Snp	return (ENXIO);
5026228561Snp
5027228561Snpproceed:
5028228561Snp	buf = b = malloc(mr->len, M_CXGBE, M_WAITOK);
5029228561Snp
5030228561Snp	/*
5031228561Snp	 * Position the PCIe window (we use memwin2) to the 16B aligned area
5032228561Snp	 * just at/before the requested region.
5033228561Snp	 */
5034228561Snp	win = mr->addr & ~0xf;
5035228561Snp	off = mr->addr - win;  /* offset of the requested region in the win */
5036228561Snp	remaining = mr->len;
5037228561Snp
5038228561Snp	while (remaining) {
5039228561Snp		t4_write_reg(sc,
5040228561Snp		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
5041228561Snp		t4_read_reg(sc,
5042228561Snp		    PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
5043228561Snp
5044228561Snp		/* number of bytes that we'll copy in the inner loop */
5045228561Snp		n = min(remaining, MEMWIN2_APERTURE - off);
5046228561Snp
5047228561Snp		for (i = 0; i < n; i += 4, remaining -= 4)
5048228561Snp			*b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i);
5049228561Snp
5050228561Snp		win += MEMWIN2_APERTURE;
5051228561Snp		off = 0;
5052228561Snp	}
5053228561Snp
5054228561Snp	rc = copyout(buf, mr->data, mr->len);
5055228561Snp	free(buf, M_CXGBE);
5056228561Snp
5057228561Snp	return (rc);
5058228561Snp}
5059228561Snp
5060218792Snpint
5061218792Snpt4_os_find_pci_capability(struct adapter *sc, int cap)
5062218792Snp{
5063222102Snp	int i;
5064218792Snp
5065222102Snp	return (pci_find_cap(sc->dev, cap, &i) == 0 ? i : 0);
5066218792Snp}
5067218792Snp
5068218792Snpint
5069218792Snpt4_os_pci_save_state(struct adapter *sc)
5070218792Snp{
5071218792Snp	device_t dev;
5072218792Snp	struct pci_devinfo *dinfo;
5073218792Snp
5074218792Snp	dev = sc->dev;
5075218792Snp	dinfo = device_get_ivars(dev);
5076218792Snp
5077218792Snp	pci_cfg_save(dev, dinfo, 0);
5078218792Snp	return (0);
5079218792Snp}
5080218792Snp
5081218792Snpint
5082218792Snpt4_os_pci_restore_state(struct adapter *sc)
5083218792Snp{
5084218792Snp	device_t dev;
5085218792Snp	struct pci_devinfo *dinfo;
5086218792Snp
5087218792Snp	dev = sc->dev;
5088218792Snp	dinfo = device_get_ivars(dev);
5089218792Snp
5090218792Snp	pci_cfg_restore(dev, dinfo);
5091218792Snp	return (0);
5092218792Snp}
5093219299Snp
5094218792Snpvoid
5095218792Snpt4_os_portmod_changed(const struct adapter *sc, int idx)
5096218792Snp{
5097218792Snp	struct port_info *pi = sc->port[idx];
5098218792Snp	static const char *mod_str[] = {
5099220232Snp		NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM"
5100218792Snp	};
5101218792Snp
5102218792Snp	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
5103218792Snp		if_printf(pi->ifp, "transceiver unplugged.\n");
5104220232Snp	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
5105220232Snp		if_printf(pi->ifp, "unknown transceiver inserted.\n");
5106220232Snp	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
5107220232Snp		if_printf(pi->ifp, "unsupported transceiver inserted.\n");
5108219299Snp	else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) {
5109218792Snp		if_printf(pi->ifp, "%s transceiver inserted.\n",
5110218792Snp		    mod_str[pi->mod_type]);
5111219299Snp	} else {
5112219299Snp		if_printf(pi->ifp, "transceiver (type %d) inserted.\n",
5113219299Snp		    pi->mod_type);
5114219299Snp	}
5115218792Snp}
5116218792Snp
5117218792Snpvoid
5118218792Snpt4_os_link_changed(struct adapter *sc, int idx, int link_stat)
5119218792Snp{
5120218792Snp	struct port_info *pi = sc->port[idx];
5121218792Snp	struct ifnet *ifp = pi->ifp;
5122218792Snp
5123218792Snp	if (link_stat) {
5124218792Snp		ifp->if_baudrate = IF_Mbps(pi->link_cfg.speed);
5125218792Snp		if_link_state_change(ifp, LINK_STATE_UP);
5126218792Snp	} else
5127218792Snp		if_link_state_change(ifp, LINK_STATE_DOWN);
5128218792Snp}
5129218792Snp
5130228561Snpvoid
5131228561Snpt4_iterate(void (*func)(struct adapter *, void *), void *arg)
5132228561Snp{
5133228561Snp	struct adapter *sc;
5134228561Snp
5135228561Snp	mtx_lock(&t4_list_lock);
5136228561Snp	SLIST_FOREACH(sc, &t4_list, link) {
5137228561Snp		/*
5138228561Snp		 * func should not make any assumptions about what state sc is
5139228561Snp		 * in - the only guarantee is that sc->sc_lock is a valid lock.
5140228561Snp		 */
5141228561Snp		func(sc, arg);
5142228561Snp	}
5143228561Snp	mtx_unlock(&t4_list_lock);
5144228561Snp}
5145228561Snp
5146218792Snpstatic int
5147218792Snpt4_open(struct cdev *dev, int flags, int type, struct thread *td)
5148218792Snp{
5149218792Snp       return (0);
5150218792Snp}
5151218792Snp
5152218792Snpstatic int
5153218792Snpt4_close(struct cdev *dev, int flags, int type, struct thread *td)
5154218792Snp{
5155218792Snp       return (0);
5156218792Snp}
5157218792Snp
5158218792Snpstatic int
5159218792Snpt4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
5160218792Snp    struct thread *td)
5161218792Snp{
5162218792Snp	int rc;
5163218792Snp	struct adapter *sc = dev->si_drv1;
5164218792Snp
5165218792Snp	rc = priv_check(td, PRIV_DRIVER);
5166218792Snp	if (rc != 0)
5167218792Snp		return (rc);
5168218792Snp
5169218792Snp	switch (cmd) {
5170220410Snp	case CHELSIO_T4_GETREG: {
5171220410Snp		struct t4_reg *edata = (struct t4_reg *)data;
5172220410Snp
5173218792Snp		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5174218792Snp			return (EFAULT);
5175220410Snp
5176220410Snp		if (edata->size == 4)
5177220410Snp			edata->val = t4_read_reg(sc, edata->addr);
5178220410Snp		else if (edata->size == 8)
5179220410Snp			edata->val = t4_read_reg64(sc, edata->addr);
5180220410Snp		else
5181220410Snp			return (EINVAL);
5182220410Snp
5183218792Snp		break;
5184218792Snp	}
5185220410Snp	case CHELSIO_T4_SETREG: {
5186220410Snp		struct t4_reg *edata = (struct t4_reg *)data;
5187220410Snp
5188218792Snp		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
5189218792Snp			return (EFAULT);
5190220410Snp
5191220410Snp		if (edata->size == 4) {
5192220410Snp			if (edata->val & 0xffffffff00000000)
5193220410Snp				return (EINVAL);
5194220410Snp			t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
5195220410Snp		} else if (edata->size == 8)
5196220410Snp			t4_write_reg64(sc, edata->addr, edata->val);
5197220410Snp		else
5198220410Snp			return (EINVAL);
5199218792Snp		break;
5200218792Snp	}
5201218792Snp	case CHELSIO_T4_REGDUMP: {
5202218792Snp		struct t4_regdump *regs = (struct t4_regdump *)data;
5203218792Snp		int reglen = T4_REGDUMP_SIZE;
5204218792Snp		uint8_t *buf;
5205218792Snp
5206218792Snp		if (regs->len < reglen) {
5207218792Snp			regs->len = reglen; /* hint to the caller */
5208218792Snp			return (ENOBUFS);
5209218792Snp		}
5210218792Snp
5211218792Snp		regs->len = reglen;
5212218792Snp		buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
5213218792Snp		t4_get_regs(sc, regs, buf);
5214218792Snp		rc = copyout(buf, regs->data, reglen);
5215218792Snp		free(buf, M_CXGBE);
5216218792Snp		break;
5217218792Snp	}
5218221474Snp	case CHELSIO_T4_GET_FILTER_MODE:
5219221474Snp		rc = get_filter_mode(sc, (uint32_t *)data);
5220221474Snp		break;
5221221474Snp	case CHELSIO_T4_SET_FILTER_MODE:
5222221474Snp		rc = set_filter_mode(sc, *(uint32_t *)data);
5223221474Snp		break;
5224221474Snp	case CHELSIO_T4_GET_FILTER:
5225221474Snp		ADAPTER_LOCK(sc);
5226221474Snp		rc = get_filter(sc, (struct t4_filter *)data);
5227221474Snp		ADAPTER_UNLOCK(sc);
5228221474Snp		break;
5229221474Snp	case CHELSIO_T4_SET_FILTER:
5230221474Snp		ADAPTER_LOCK(sc);
5231221474Snp		rc = set_filter(sc, (struct t4_filter *)data);
5232221474Snp		ADAPTER_UNLOCK(sc);
5233221474Snp		break;
5234221474Snp	case CHELSIO_T4_DEL_FILTER:
5235221474Snp		ADAPTER_LOCK(sc);
5236221474Snp		rc = del_filter(sc, (struct t4_filter *)data);
5237221474Snp		ADAPTER_UNLOCK(sc);
5238221474Snp		break;
5239222973Snp	case CHELSIO_T4_GET_SGE_CONTEXT:
5240222973Snp		rc = get_sge_context(sc, (struct t4_sge_context *)data);
5241222973Snp		break;
5242228561Snp	case CHELSIO_T4_LOAD_FW: {
5243228561Snp		struct t4_data *fw = (struct t4_data *)data;
5244228561Snp		uint8_t *fw_data;
5245228561Snp
5246228561Snp		if (sc->flags & FULL_INIT_DONE)
5247228561Snp			return (EBUSY);
5248228561Snp
5249228561Snp		fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT);
5250228561Snp		if (fw_data == NULL)
5251228561Snp			return (ENOMEM);
5252228561Snp
5253228561Snp		rc = copyin(fw->data, fw_data, fw->len);
5254228561Snp		if (rc == 0)
5255228561Snp			rc = -t4_load_fw(sc, fw_data, fw->len);
5256228561Snp
5257228561Snp		free(fw_data, M_CXGBE);
5258228561Snp		break;
5259228561Snp	}
5260228561Snp	case CHELSIO_T4_GET_MEM:
5261228561Snp		rc = read_card_mem(sc, (struct t4_mem_range *)data);
5262228561Snp		break;
5263218792Snp	default:
5264218792Snp		rc = EINVAL;
5265218792Snp	}
5266218792Snp
5267218792Snp	return (rc);
5268218792Snp}
5269218792Snp
5270237263Snp#ifdef TCP_OFFLOAD
5271219392Snpstatic int
5272228561Snptoe_capability(struct port_info *pi, int enable)
5273228561Snp{
5274228561Snp	int rc;
5275228561Snp	struct adapter *sc = pi->adapter;
5276228561Snp
5277228561Snp	ADAPTER_LOCK_ASSERT_OWNED(sc);
5278228561Snp
5279228561Snp	if (!is_offload(sc))
5280228561Snp		return (ENODEV);
5281228561Snp
5282228561Snp	if (enable) {
5283237263Snp		if (!(sc->flags & FULL_INIT_DONE)) {
5284237263Snp			log(LOG_WARNING,
5285237263Snp			    "You must enable a cxgbe interface first\n");
5286237263Snp			return (EAGAIN);
5287237263Snp		}
5288237263Snp
5289228561Snp		if (isset(&sc->offload_map, pi->port_id))
5290228561Snp			return (0);
5291228561Snp
5292237263Snp		if (!(sc->flags & TOM_INIT_DONE)) {
5293237263Snp			rc = t4_activate_uld(sc, ULD_TOM);
5294237263Snp			if (rc == EAGAIN) {
5295237263Snp				log(LOG_WARNING,
5296237263Snp				    "You must kldload t4_tom.ko before trying "
5297237263Snp				    "to enable TOE on a cxgbe interface.\n");
5298237263Snp			}
5299228561Snp			if (rc != 0)
5300228561Snp				return (rc);
5301237263Snp			KASSERT(sc->tom_softc != NULL,
5302237263Snp			    ("%s: TOM activated but softc NULL", __func__));
5303237263Snp			KASSERT(sc->flags & TOM_INIT_DONE,
5304237263Snp			    ("%s: TOM activated but flag not set", __func__));
5305228561Snp		}
5306228561Snp
5307228561Snp		setbit(&sc->offload_map, pi->port_id);
5308228561Snp	} else {
5309228561Snp		if (!isset(&sc->offload_map, pi->port_id))
5310228561Snp			return (0);
5311228561Snp
5312237263Snp		KASSERT(sc->flags & TOM_INIT_DONE,
5313237263Snp		    ("%s: TOM never initialized?", __func__));
5314228561Snp		clrbit(&sc->offload_map, pi->port_id);
5315228561Snp	}
5316228561Snp
5317228561Snp	return (0);
5318228561Snp}
5319228561Snp
5320228561Snp/*
5321228561Snp * Add an upper layer driver to the global list.
5322228561Snp */
5323228561Snpint
5324228561Snpt4_register_uld(struct uld_info *ui)
5325228561Snp{
5326228561Snp	int rc = 0;
5327228561Snp	struct uld_info *u;
5328228561Snp
5329228561Snp	mtx_lock(&t4_uld_list_lock);
5330228561Snp	SLIST_FOREACH(u, &t4_uld_list, link) {
5331228561Snp	    if (u->uld_id == ui->uld_id) {
5332228561Snp		    rc = EEXIST;
5333228561Snp		    goto done;
5334228561Snp	    }
5335228561Snp	}
5336228561Snp
5337228561Snp	SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
5338228561Snp	ui->refcount = 0;
5339228561Snpdone:
5340228561Snp	mtx_unlock(&t4_uld_list_lock);
5341228561Snp	return (rc);
5342228561Snp}
5343228561Snp
5344228561Snpint
5345228561Snpt4_unregister_uld(struct uld_info *ui)
5346228561Snp{
5347228561Snp	int rc = EINVAL;
5348228561Snp	struct uld_info *u;
5349228561Snp
5350228561Snp	mtx_lock(&t4_uld_list_lock);
5351228561Snp
5352228561Snp	SLIST_FOREACH(u, &t4_uld_list, link) {
5353228561Snp	    if (u == ui) {
5354228561Snp		    if (ui->refcount > 0) {
5355228561Snp			    rc = EBUSY;
5356228561Snp			    goto done;
5357228561Snp		    }
5358228561Snp
5359228561Snp		    SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
5360228561Snp		    rc = 0;
5361228561Snp		    goto done;
5362228561Snp	    }
5363228561Snp	}
5364228561Snpdone:
5365228561Snp	mtx_unlock(&t4_uld_list_lock);
5366228561Snp	return (rc);
5367228561Snp}
5368228561Snp
5369237263Snpint
5370237263Snpt4_activate_uld(struct adapter *sc, int id)
5371228561Snp{
5372228561Snp	int rc = EAGAIN;
5373228561Snp	struct uld_info *ui;
5374228561Snp
5375228561Snp	mtx_lock(&t4_uld_list_lock);
5376228561Snp
5377228561Snp	SLIST_FOREACH(ui, &t4_uld_list, link) {
5378228561Snp		if (ui->uld_id == id) {
5379237263Snp			rc = ui->activate(sc);
5380237263Snp			if (rc == 0)
5381228561Snp				ui->refcount++;
5382228561Snp			goto done;
5383228561Snp		}
5384228561Snp	}
5385228561Snpdone:
5386228561Snp	mtx_unlock(&t4_uld_list_lock);
5387228561Snp
5388228561Snp	return (rc);
5389228561Snp}
5390228561Snp
5391237263Snpint
5392237263Snpt4_deactivate_uld(struct adapter *sc, int id)
5393228561Snp{
5394237263Snp	int rc = EINVAL;
5395237263Snp	struct uld_info *ui;
5396228561Snp
5397228561Snp	mtx_lock(&t4_uld_list_lock);
5398228561Snp
5399237263Snp	SLIST_FOREACH(ui, &t4_uld_list, link) {
5400237263Snp		if (ui->uld_id == id) {
5401237263Snp			rc = ui->deactivate(sc);
5402237263Snp			if (rc == 0)
5403237263Snp				ui->refcount--;
5404237263Snp			goto done;
5405237263Snp		}
5406228561Snp	}
5407228561Snpdone:
5408228561Snp	mtx_unlock(&t4_uld_list_lock);
5409228561Snp
5410228561Snp	return (rc);
5411228561Snp}
5412228561Snp#endif
5413228561Snp
5414228561Snp/*
5415228561Snp * Come up with reasonable defaults for some of the tunables, provided they're
5416228561Snp * not set by the user (in which case we'll use the values as is).
5417228561Snp */
5418228561Snpstatic void
5419228561Snptweak_tunables(void)
5420228561Snp{
5421228561Snp	int nc = mp_ncpus;	/* our snapshot of the number of CPUs */
5422228561Snp
5423228561Snp	if (t4_ntxq10g < 1)
5424228561Snp		t4_ntxq10g = min(nc, NTXQ_10G);
5425228561Snp
5426228561Snp	if (t4_ntxq1g < 1)
5427228561Snp		t4_ntxq1g = min(nc, NTXQ_1G);
5428228561Snp
5429228561Snp	if (t4_nrxq10g < 1)
5430228561Snp		t4_nrxq10g = min(nc, NRXQ_10G);
5431228561Snp
5432228561Snp	if (t4_nrxq1g < 1)
5433228561Snp		t4_nrxq1g = min(nc, NRXQ_1G);
5434228561Snp
5435237263Snp#ifdef TCP_OFFLOAD
5436228561Snp	if (t4_nofldtxq10g < 1)
5437228561Snp		t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
5438228561Snp
5439228561Snp	if (t4_nofldtxq1g < 1)
5440228561Snp		t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
5441228561Snp
5442228561Snp	if (t4_nofldrxq10g < 1)
5443228561Snp		t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
5444228561Snp
5445228561Snp	if (t4_nofldrxq1g < 1)
5446228561Snp		t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
5447228561Snp#endif
5448228561Snp
5449228561Snp	if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
5450228561Snp		t4_tmr_idx_10g = TMR_IDX_10G;
5451228561Snp
5452228561Snp	if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
5453228561Snp		t4_pktc_idx_10g = PKTC_IDX_10G;
5454228561Snp
5455228561Snp	if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
5456228561Snp		t4_tmr_idx_1g = TMR_IDX_1G;
5457228561Snp
5458228561Snp	if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
5459228561Snp		t4_pktc_idx_1g = PKTC_IDX_1G;
5460228561Snp
5461228561Snp	if (t4_qsize_txq < 128)
5462228561Snp		t4_qsize_txq = 128;
5463228561Snp
5464228561Snp	if (t4_qsize_rxq < 128)
5465228561Snp		t4_qsize_rxq = 128;
5466228561Snp	while (t4_qsize_rxq & 7)
5467228561Snp		t4_qsize_rxq++;
5468228561Snp
5469228561Snp	t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
5470228561Snp}
5471228561Snp
5472228561Snpstatic int
5473219392Snpt4_mod_event(module_t mod, int cmd, void *arg)
5474219392Snp{
5475228561Snp	int rc = 0;
5476219392Snp
5477228561Snp	switch (cmd) {
5478228561Snp	case MOD_LOAD:
5479219392Snp		t4_sge_modload();
5480228561Snp		mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF);
5481228561Snp		SLIST_INIT(&t4_list);
5482237263Snp#ifdef TCP_OFFLOAD
5483228561Snp		mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF);
5484228561Snp		SLIST_INIT(&t4_uld_list);
5485228561Snp#endif
5486228561Snp		tweak_tunables();
5487228561Snp		break;
5488219392Snp
5489228561Snp	case MOD_UNLOAD:
5490237263Snp#ifdef TCP_OFFLOAD
5491228561Snp		mtx_lock(&t4_uld_list_lock);
5492228561Snp		if (!SLIST_EMPTY(&t4_uld_list)) {
5493228561Snp			rc = EBUSY;
5494228561Snp			mtx_unlock(&t4_uld_list_lock);
5495228561Snp			break;
5496228561Snp		}
5497228561Snp		mtx_unlock(&t4_uld_list_lock);
5498228561Snp		mtx_destroy(&t4_uld_list_lock);
5499228561Snp#endif
5500228561Snp		mtx_lock(&t4_list_lock);
5501228561Snp		if (!SLIST_EMPTY(&t4_list)) {
5502228561Snp			rc = EBUSY;
5503228561Snp			mtx_unlock(&t4_list_lock);
5504228561Snp			break;
5505228561Snp		}
5506228561Snp		mtx_unlock(&t4_list_lock);
5507228561Snp		mtx_destroy(&t4_list_lock);
5508228561Snp		break;
5509228561Snp	}
5510228561Snp
5511228561Snp	return (rc);
5512219392Snp}
5513219392Snp
5514218792Snpstatic devclass_t t4_devclass;
5515218792Snpstatic devclass_t cxgbe_devclass;
5516218792Snp
5517219392SnpDRIVER_MODULE(t4nex, pci, t4_driver, t4_devclass, t4_mod_event, 0);
5518218792SnpMODULE_VERSION(t4nex, 1);
5519218792Snp
5520218792SnpDRIVER_MODULE(cxgbe, t4nex, cxgbe_driver, cxgbe_devclass, 0, 0);
5521218792SnpMODULE_VERSION(cxgbe, 1);
5522