if_ixl.c revision 295826 
1/******************************************************************************
2
3  Copyright (c) 2013-2015, Intel Corporation
4  All rights reserved.
5
6  Redistribution and use in source and binary forms, with or without
7  modification, are permitted provided that the following conditions are met:
8
9   1. Redistributions of source code must retain the above copyright notice,
10      this list of conditions and the following disclaimer.
11
12   2. Redistributions in binary form must reproduce the above copyright
13      notice, this list of conditions and the following disclaimer in the
14      documentation and/or other materials provided with the distribution.
15
16   3. Neither the name of the Intel Corporation nor the names of its
17      contributors may be used to endorse or promote products derived from
18      this software without specific prior written permission.
19
20  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  POSSIBILITY OF SUCH DAMAGE.
31
32******************************************************************************/
33/*$FreeBSD: head/sys/dev/ixl/if_ixl.c 295826 2016-02-19 21:58:14Z erj $*/
34
35#ifndef IXL_STANDALONE_BUILD
36#include "opt_inet.h"
37#include "opt_inet6.h"
38#include "opt_rss.h"
39#endif
40
41#include "ixl.h"
42#include "ixl_pf.h"
43
44#ifdef RSS
45#include <net/rss_config.h>
46#endif
47
48/*********************************************************************
49 *  Driver version
50 *********************************************************************/
51char ixl_driver_version[] = "1.4.3";
52
53/*********************************************************************
54 *  PCI Device ID Table
55 *
56 *  Used by probe to select devices to load on
57 *  Last field stores an index into ixl_strings
58 *  Last entry must be all 0s
59 *
60 *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
61 *********************************************************************/
62
63static ixl_vendor_info_t ixl_vendor_info_array[] =
64{
65	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_XL710, 0, 0, 0},
66	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_A, 0, 0, 0},
67	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_B, 0, 0, 0},
68	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_KX_C, 0, 0, 0},
69	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_A, 0, 0, 0},
70	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_B, 0, 0, 0},
71	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_QSFP_C, 0, 0, 0},
72	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T, 0, 0, 0},
73	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T4, 0, 0, 0},
74	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2, 0, 0, 0},
75	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_20G_KR2_A, 0, 0, 0},
76#ifdef X722_SUPPORT
77	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_SFP_X722, 0, 0, 0},
78	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_1G_BASE_T_X722, 0, 0, 0},
79	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_10G_BASE_T_X722, 0, 0, 0},
80#endif
81	/* required last entry */
82	{0, 0, 0, 0, 0}
83};
84
85/*********************************************************************
86 *  Table of branding strings
87 *********************************************************************/
88
89static char    *ixl_strings[] = {
90	"Intel(R) Ethernet Connection XL710 Driver"
91};
92
93
94/*********************************************************************
95 *  Function prototypes
96 *********************************************************************/
97static int      ixl_probe(device_t);
98static int      ixl_attach(device_t);
99static int      ixl_detach(device_t);
100static int      ixl_shutdown(device_t);
101static int	ixl_get_hw_capabilities(struct ixl_pf *);
102static void	ixl_cap_txcsum_tso(struct ixl_vsi *, struct ifnet *, int);
103static int      ixl_ioctl(struct ifnet *, u_long, caddr_t);
104static void	ixl_init(void *);
105static void	ixl_init_locked(struct ixl_pf *);
106static void     ixl_stop(struct ixl_pf *);
107static void     ixl_media_status(struct ifnet *, struct ifmediareq *);
108static int      ixl_media_change(struct ifnet *);
109static void     ixl_update_link_status(struct ixl_pf *);
110static int      ixl_allocate_pci_resources(struct ixl_pf *);
111static u16	ixl_get_bus_info(struct i40e_hw *, device_t);
112static int	ixl_setup_stations(struct ixl_pf *);
113static int	ixl_switch_config(struct ixl_pf *);
114static int	ixl_initialize_vsi(struct ixl_vsi *);
115static int	ixl_assign_vsi_msix(struct ixl_pf *);
116static int	ixl_assign_vsi_legacy(struct ixl_pf *);
117static int	ixl_init_msix(struct ixl_pf *);
118static void	ixl_configure_msix(struct ixl_pf *);
119static void	ixl_configure_itr(struct ixl_pf *);
120static void	ixl_configure_legacy(struct ixl_pf *);
121static void	ixl_free_pci_resources(struct ixl_pf *);
122static void	ixl_local_timer(void *);
123static int	ixl_setup_interface(device_t, struct ixl_vsi *);
124static void	ixl_link_event(struct ixl_pf *, struct i40e_arq_event_info *);
125static void	ixl_config_rss(struct ixl_vsi *);
126static void	ixl_set_queue_rx_itr(struct ixl_queue *);
127static void	ixl_set_queue_tx_itr(struct ixl_queue *);
128static int	ixl_set_advertised_speeds(struct ixl_pf *, int);
129
130static int	ixl_enable_rings(struct ixl_vsi *);
131static int	ixl_disable_rings(struct ixl_vsi *);
132static void	ixl_enable_intr(struct ixl_vsi *);
133static void	ixl_disable_intr(struct ixl_vsi *);
134static void	ixl_disable_rings_intr(struct ixl_vsi *);
135
136static void     ixl_enable_adminq(struct i40e_hw *);
137static void     ixl_disable_adminq(struct i40e_hw *);
138static void     ixl_enable_queue(struct i40e_hw *, int);
139static void     ixl_disable_queue(struct i40e_hw *, int);
140static void     ixl_enable_legacy(struct i40e_hw *);
141static void     ixl_disable_legacy(struct i40e_hw *);
142
143static void     ixl_set_promisc(struct ixl_vsi *);
144static void     ixl_add_multi(struct ixl_vsi *);
145static void     ixl_del_multi(struct ixl_vsi *);
146static void	ixl_register_vlan(void *, struct ifnet *, u16);
147static void	ixl_unregister_vlan(void *, struct ifnet *, u16);
148static void	ixl_setup_vlan_filters(struct ixl_vsi *);
149
150static void	ixl_init_filters(struct ixl_vsi *);
151static void	ixl_reconfigure_filters(struct ixl_vsi *vsi);
152static void	ixl_add_filter(struct ixl_vsi *, u8 *, s16 vlan);
153static void	ixl_del_filter(struct ixl_vsi *, u8 *, s16 vlan);
154static void	ixl_add_hw_filters(struct ixl_vsi *, int, int);
155static void	ixl_del_hw_filters(struct ixl_vsi *, int);
156static struct ixl_mac_filter *
157		ixl_find_filter(struct ixl_vsi *, u8 *, s16);
158static void	ixl_add_mc_filter(struct ixl_vsi *, u8 *);
159static void	ixl_free_mac_filters(struct ixl_vsi *vsi);
160
161
162/* Sysctl debug interface */
163#ifdef IXL_DEBUG_SYSCTL
164static int	ixl_debug_info(SYSCTL_HANDLER_ARGS);
165static void	ixl_print_debug_info(struct ixl_pf *);
166#endif
167
168/* The MSI/X Interrupt handlers */
169static void	ixl_intr(void *);
170static void	ixl_msix_que(void *);
171static void	ixl_msix_adminq(void *);
172static void	ixl_handle_mdd_event(struct ixl_pf *);
173
174/* Deferred interrupt tasklets */
175static void	ixl_do_adminq(void *, int);
176
177/* Sysctl handlers */
178static int	ixl_set_flowcntl(SYSCTL_HANDLER_ARGS);
179static int	ixl_set_advertise(SYSCTL_HANDLER_ARGS);
180static int	ixl_current_speed(SYSCTL_HANDLER_ARGS);
181static int	ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
182
183/* Statistics */
184static void     ixl_add_hw_stats(struct ixl_pf *);
185static void	ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *,
186		    struct sysctl_oid_list *, struct i40e_hw_port_stats *);
187static void	ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *,
188		    struct sysctl_oid_list *,
189		    struct i40e_eth_stats *);
190static void	ixl_update_stats_counters(struct ixl_pf *);
191static void	ixl_update_eth_stats(struct ixl_vsi *);
192static void	ixl_update_vsi_stats(struct ixl_vsi *);
193static void	ixl_pf_reset_stats(struct ixl_pf *);
194static void	ixl_vsi_reset_stats(struct ixl_vsi *);
195static void	ixl_stat_update48(struct i40e_hw *, u32, u32, bool,
196		    u64 *, u64 *);
197static void	ixl_stat_update32(struct i40e_hw *, u32, bool,
198		    u64 *, u64 *);
199
200#ifdef IXL_DEBUG_SYSCTL
201static int 	ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS);
202static int	ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS);
203static int	ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS);
204static int	ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS);
205static int	ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS);
206#endif
207
208#ifdef PCI_IOV
209static int	ixl_adminq_err_to_errno(enum i40e_admin_queue_err err);
210
211static int	ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t*);
212static void	ixl_iov_uninit(device_t dev);
213static int	ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t*);
214
215static void	ixl_handle_vf_msg(struct ixl_pf *,
216		    struct i40e_arq_event_info *);
217static void	ixl_handle_vflr(void *arg, int pending);
218
219static void	ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf);
220static void	ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf);
221#endif
222
223/*********************************************************************
224 *  FreeBSD Device Interface Entry Points
225 *********************************************************************/
226
227static device_method_t ixl_methods[] = {
228	/* Device interface */
229	DEVMETHOD(device_probe, ixl_probe),
230	DEVMETHOD(device_attach, ixl_attach),
231	DEVMETHOD(device_detach, ixl_detach),
232	DEVMETHOD(device_shutdown, ixl_shutdown),
233#ifdef PCI_IOV
234	DEVMETHOD(pci_iov_init, ixl_iov_init),
235	DEVMETHOD(pci_iov_uninit, ixl_iov_uninit),
236	DEVMETHOD(pci_iov_add_vf, ixl_add_vf),
237#endif
238	{0, 0}
239};
240
241static driver_t ixl_driver = {
242	"ixl", ixl_methods, sizeof(struct ixl_pf),
243};
244
245devclass_t ixl_devclass;
246DRIVER_MODULE(ixl, pci, ixl_driver, ixl_devclass, 0, 0);
247
248MODULE_DEPEND(ixl, pci, 1, 1, 1);
249MODULE_DEPEND(ixl, ether, 1, 1, 1);
250#ifdef DEV_NETMAP
251MODULE_DEPEND(ixl, netmap, 1, 1, 1);
252#endif /* DEV_NETMAP */
253
254/*
255** Global reset mutex
256*/
257static struct mtx ixl_reset_mtx;
258
259/*
260** TUNEABLE PARAMETERS:
261*/
262
263static SYSCTL_NODE(_hw, OID_AUTO, ixl, CTLFLAG_RD, 0,
264                   "IXL driver parameters");
265
266/*
267 * MSIX should be the default for best performance,
268 * but this allows it to be forced off for testing.
269 */
270static int ixl_enable_msix = 1;
271TUNABLE_INT("hw.ixl.enable_msix", &ixl_enable_msix);
272SYSCTL_INT(_hw_ixl, OID_AUTO, enable_msix, CTLFLAG_RDTUN, &ixl_enable_msix, 0,
273    "Enable MSI-X interrupts");
274
275/*
276** Number of descriptors per ring:
277**   - TX and RX are the same size
278*/
279static int ixl_ringsz = DEFAULT_RING;
280TUNABLE_INT("hw.ixl.ringsz", &ixl_ringsz);
281SYSCTL_INT(_hw_ixl, OID_AUTO, ring_size, CTLFLAG_RDTUN,
282    &ixl_ringsz, 0, "Descriptor Ring Size");
283
284/*
285** This can be set manually, if left as 0 the
286** number of queues will be calculated based
287** on cpus and msix vectors available.
288*/
289int ixl_max_queues = 0;
290TUNABLE_INT("hw.ixl.max_queues", &ixl_max_queues);
291SYSCTL_INT(_hw_ixl, OID_AUTO, max_queues, CTLFLAG_RDTUN,
292    &ixl_max_queues, 0, "Number of Queues");
293
294/*
295** Controls for Interrupt Throttling
296**	- true/false for dynamic adjustment
297** 	- default values for static ITR
298*/
299int ixl_dynamic_rx_itr = 0;
300TUNABLE_INT("hw.ixl.dynamic_rx_itr", &ixl_dynamic_rx_itr);
301SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
302    &ixl_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
303
304int ixl_dynamic_tx_itr = 0;
305TUNABLE_INT("hw.ixl.dynamic_tx_itr", &ixl_dynamic_tx_itr);
306SYSCTL_INT(_hw_ixl, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
307    &ixl_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
308
309int ixl_rx_itr = IXL_ITR_8K;
310TUNABLE_INT("hw.ixl.rx_itr", &ixl_rx_itr);
311SYSCTL_INT(_hw_ixl, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
312    &ixl_rx_itr, 0, "RX Interrupt Rate");
313
314int ixl_tx_itr = IXL_ITR_4K;
315TUNABLE_INT("hw.ixl.tx_itr", &ixl_tx_itr);
316SYSCTL_INT(_hw_ixl, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
317    &ixl_tx_itr, 0, "TX Interrupt Rate");
318
319#ifdef IXL_FDIR
320static int ixl_enable_fdir = 1;
321TUNABLE_INT("hw.ixl.enable_fdir", &ixl_enable_fdir);
322/* Rate at which we sample */
323int ixl_atr_rate = 20;
324TUNABLE_INT("hw.ixl.atr_rate", &ixl_atr_rate);
325#endif
326
327#ifdef DEV_NETMAP
328#define NETMAP_IXL_MAIN /* only bring in one part of the netmap code */
329#include <dev/netmap/if_ixl_netmap.h>
330#endif /* DEV_NETMAP */
331
332static char *ixl_fc_string[6] = {
333	"None",
334	"Rx",
335	"Tx",
336	"Full",
337	"Priority",
338	"Default"
339};
340
341static MALLOC_DEFINE(M_IXL, "ixl", "ixl driver allocations");
342
343static uint8_t ixl_bcast_addr[ETHER_ADDR_LEN] =
344    {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
345
346/*********************************************************************
347 *  Device identification routine
348 *
349 *  ixl_probe determines if the driver should be loaded on
350 *  the hardware based on PCI vendor/device id of the device.
351 *
352 *  return BUS_PROBE_DEFAULT on success, positive on failure
353 *********************************************************************/
354
355static int
356ixl_probe(device_t dev)
357{
358	ixl_vendor_info_t *ent;
359
360	u16	pci_vendor_id, pci_device_id;
361	u16	pci_subvendor_id, pci_subdevice_id;
362	char	device_name[256];
363	static bool lock_init = FALSE;
364
365	INIT_DEBUGOUT("ixl_probe: begin");
366
367	pci_vendor_id = pci_get_vendor(dev);
368	if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
369		return (ENXIO);
370
371	pci_device_id = pci_get_device(dev);
372	pci_subvendor_id = pci_get_subvendor(dev);
373	pci_subdevice_id = pci_get_subdevice(dev);
374
375	ent = ixl_vendor_info_array;
376	while (ent->vendor_id != 0) {
377		if ((pci_vendor_id == ent->vendor_id) &&
378		    (pci_device_id == ent->device_id) &&
379
380		    ((pci_subvendor_id == ent->subvendor_id) ||
381		     (ent->subvendor_id == 0)) &&
382
383		    ((pci_subdevice_id == ent->subdevice_id) ||
384		     (ent->subdevice_id == 0))) {
385			sprintf(device_name, "%s, Version - %s",
386				ixl_strings[ent->index],
387				ixl_driver_version);
388			device_set_desc_copy(dev, device_name);
389			/* One shot mutex init */
390			if (lock_init == FALSE) {
391				lock_init = TRUE;
392				mtx_init(&ixl_reset_mtx,
393				    "ixl_reset",
394				    "IXL RESET Lock", MTX_DEF);
395			}
396			return (BUS_PROBE_DEFAULT);
397		}
398		ent++;
399	}
400	return (ENXIO);
401}
402
403/*********************************************************************
404 *  Device initialization routine
405 *
406 *  The attach entry point is called when the driver is being loaded.
407 *  This routine identifies the type of hardware, allocates all resources
408 *  and initializes the hardware.
409 *
410 *  return 0 on success, positive on failure
411 *********************************************************************/
412
413static int
414ixl_attach(device_t dev)
415{
416	struct ixl_pf	*pf;
417	struct i40e_hw	*hw;
418	struct ixl_vsi *vsi;
419	u16		bus;
420	int             error = 0;
421#ifdef PCI_IOV
422	nvlist_t	*pf_schema, *vf_schema;
423	int		iov_error;
424#endif
425
426	INIT_DEBUGOUT("ixl_attach: begin");
427
428	/* Allocate, clear, and link in our primary soft structure */
429	pf = device_get_softc(dev);
430	pf->dev = pf->osdep.dev = dev;
431	hw = &pf->hw;
432
433	/*
434	** Note this assumes we have a single embedded VSI,
435	** this could be enhanced later to allocate multiple
436	*/
437	vsi = &pf->vsi;
438	vsi->dev = pf->dev;
439
440	/* Core Lock Init*/
441	IXL_PF_LOCK_INIT(pf, device_get_nameunit(dev));
442
443	/* Set up the timer callout */
444	callout_init_mtx(&pf->timer, &pf->pf_mtx, 0);
445
446	/* Set up sysctls */
447	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
448	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
449	    OID_AUTO, "fc", CTLTYPE_INT | CTLFLAG_RW,
450	    pf, 0, ixl_set_flowcntl, "I", "Flow Control");
451
452	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
453	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
454	    OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
455	    pf, 0, ixl_set_advertise, "I", "Advertised Speed");
456
457	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
458	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
459	    OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
460	    pf, 0, ixl_current_speed, "A", "Current Port Speed");
461
462	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
463	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
464	    OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
465	    pf, 0, ixl_sysctl_show_fw, "A", "Firmware version");
466
467	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
468	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
469	    OID_AUTO, "rx_itr", CTLFLAG_RW,
470	    &ixl_rx_itr, IXL_ITR_8K, "RX ITR");
471
472	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
473	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
474	    OID_AUTO, "dynamic_rx_itr", CTLFLAG_RW,
475	    &ixl_dynamic_rx_itr, 0, "Dynamic RX ITR");
476
477	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
478	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
479	    OID_AUTO, "tx_itr", CTLFLAG_RW,
480	    &ixl_tx_itr, IXL_ITR_4K, "TX ITR");
481
482	SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
483	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
484	    OID_AUTO, "dynamic_tx_itr", CTLFLAG_RW,
485	    &ixl_dynamic_tx_itr, 0, "Dynamic TX ITR");
486
487#ifdef IXL_DEBUG_SYSCTL
488	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
489	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
490	    OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, pf, 0,
491	    ixl_debug_info, "I", "Debug Information");
492
493	/* Debug shared-code message level */
494	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
495	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
496	    OID_AUTO, "debug_mask", CTLFLAG_RW,
497	    &pf->hw.debug_mask, 0, "Debug Message Level");
498
499	SYSCTL_ADD_UINT(device_get_sysctl_ctx(dev),
500	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
501	    OID_AUTO, "vc_debug_level", CTLFLAG_RW, &pf->vc_debug_lvl,
502	    0, "PF/VF Virtual Channel debug level");
503
504	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
505	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
506	    OID_AUTO, "link_status", CTLTYPE_STRING | CTLFLAG_RD,
507	    pf, 0, ixl_sysctl_link_status, "A", "Current Link Status");
508
509	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
510	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
511	    OID_AUTO, "phy_abilities", CTLTYPE_STRING | CTLFLAG_RD,
512	    pf, 0, ixl_sysctl_phy_abilities, "A", "PHY Abilities");
513
514	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
515	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
516	    OID_AUTO, "filter_list", CTLTYPE_STRING | CTLFLAG_RD,
517	    pf, 0, ixl_sysctl_sw_filter_list, "A", "SW Filter List");
518
519	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
520	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
521	    OID_AUTO, "hw_res_alloc", CTLTYPE_STRING | CTLFLAG_RD,
522	    pf, 0, ixl_sysctl_hw_res_alloc, "A", "HW Resource Allocation");
523
524	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
525	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
526	    OID_AUTO, "switch_config", CTLTYPE_STRING | CTLFLAG_RD,
527	    pf, 0, ixl_sysctl_switch_config, "A", "HW Switch Configuration");
528#endif
529
530	/* Save off the PCI information */
531	hw->vendor_id = pci_get_vendor(dev);
532	hw->device_id = pci_get_device(dev);
533	hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
534	hw->subsystem_vendor_id =
535	    pci_read_config(dev, PCIR_SUBVEND_0, 2);
536	hw->subsystem_device_id =
537	    pci_read_config(dev, PCIR_SUBDEV_0, 2);
538
539	hw->bus.device = pci_get_slot(dev);
540	hw->bus.func = pci_get_function(dev);
541
542	pf->vc_debug_lvl = 1;
543
544	/* Do PCI setup - map BAR0, etc */
545	if (ixl_allocate_pci_resources(pf)) {
546		device_printf(dev, "Allocation of PCI resources failed\n");
547		error = ENXIO;
548		goto err_out;
549	}
550
551	/* Establish a clean starting point */
552	i40e_clear_hw(hw);
553	error = i40e_pf_reset(hw);
554	if (error) {
555		device_printf(dev,"PF reset failure %x\n", error);
556		error = EIO;
557		goto err_out;
558	}
559
560	/* Set admin queue parameters */
561	hw->aq.num_arq_entries = IXL_AQ_LEN;
562	hw->aq.num_asq_entries = IXL_AQ_LEN;
563	hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
564	hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
565
566	/* Initialize the shared code */
567	error = i40e_init_shared_code(hw);
568	if (error) {
569		device_printf(dev,"Unable to initialize the shared code\n");
570		error = EIO;
571		goto err_out;
572	}
573
574	/* Set up the admin queue */
575	error = i40e_init_adminq(hw);
576	if (error) {
577		device_printf(dev, "The driver for the device stopped "
578		    "because the NVM image is newer than expected.\n"
579		    "You must install the most recent version of "
580		    " the network driver.\n");
581		goto err_out;
582	}
583	device_printf(dev, "%s\n", ixl_fw_version_str(hw));
584
585        if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
586	    hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
587		device_printf(dev, "The driver for the device detected "
588		    "a newer version of the NVM image than expected.\n"
589		    "Please install the most recent version of the network driver.\n");
590	else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
591	    hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
592		device_printf(dev, "The driver for the device detected "
593		    "an older version of the NVM image than expected.\n"
594		    "Please update the NVM image.\n");
595
596	/* Clear PXE mode */
597	i40e_clear_pxe_mode(hw);
598
599	/* Get capabilities from the device */
600	error = ixl_get_hw_capabilities(pf);
601	if (error) {
602		device_printf(dev, "HW capabilities failure!\n");
603		goto err_get_cap;
604	}
605
606	/* Set up host memory cache */
607	error = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
608	    hw->func_caps.num_rx_qp, 0, 0);
609	if (error) {
610		device_printf(dev, "init_lan_hmc failed: %d\n", error);
611		goto err_get_cap;
612	}
613
614	error = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
615	if (error) {
616		device_printf(dev, "configure_lan_hmc failed: %d\n", error);
617		goto err_mac_hmc;
618	}
619
620	/* Disable LLDP from the firmware */
621	i40e_aq_stop_lldp(hw, TRUE, NULL);
622
623	i40e_get_mac_addr(hw, hw->mac.addr);
624	error = i40e_validate_mac_addr(hw->mac.addr);
625	if (error) {
626		device_printf(dev, "validate_mac_addr failed: %d\n", error);
627		goto err_mac_hmc;
628	}
629	bcopy(hw->mac.addr, hw->mac.perm_addr, ETHER_ADDR_LEN);
630	i40e_get_port_mac_addr(hw, hw->mac.port_addr);
631
632	/* Set up VSI and queues */
633	if (ixl_setup_stations(pf) != 0) {
634		device_printf(dev, "setup stations failed!\n");
635		error = ENOMEM;
636		goto err_mac_hmc;
637	}
638
639	/* Initialize mac filter list for VSI */
640	SLIST_INIT(&vsi->ftl);
641
642	/* Set up interrupt routing here */
643	if (pf->msix > 1)
644		error = ixl_assign_vsi_msix(pf);
645	else
646		error = ixl_assign_vsi_legacy(pf);
647	if (error)
648		goto err_late;
649
650	if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) ||
651	    (hw->aq.fw_maj_ver < 4)) {
652		i40e_msec_delay(75);
653		error = i40e_aq_set_link_restart_an(hw, TRUE, NULL);
654		if (error)
655			device_printf(dev, "link restart failed, aq_err=%d\n",
656			    pf->hw.aq.asq_last_status);
657	}
658
659	/* Determine link state */
660	i40e_aq_get_link_info(hw, TRUE, NULL, NULL);
661	i40e_get_link_status(hw, &pf->link_up);
662
663	/* Setup OS specific network interface */
664	if (ixl_setup_interface(dev, vsi) != 0) {
665		device_printf(dev, "interface setup failed!\n");
666		error = EIO;
667		goto err_late;
668	}
669
670	error = ixl_switch_config(pf);
671	if (error) {
672		device_printf(dev, "Initial switch config failed: %d\n", error);
673		goto err_mac_hmc;
674	}
675
676	/* Limit phy interrupts to link and modules failure */
677	error = i40e_aq_set_phy_int_mask(hw, ~(I40E_AQ_EVENT_LINK_UPDOWN |
678		I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
679	if (error)
680		device_printf(dev, "set phy mask failed: %d\n", error);
681
682	/* Get the bus configuration and set the shared code */
683	bus = ixl_get_bus_info(hw, dev);
684	i40e_set_pci_config_data(hw, bus);
685
686	/* Initialize statistics */
687	ixl_pf_reset_stats(pf);
688	ixl_update_stats_counters(pf);
689	ixl_add_hw_stats(pf);
690
691	/* Register for VLAN events */
692	vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
693	    ixl_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
694	vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
695	    ixl_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
696
697#ifdef PCI_IOV
698	/* SR-IOV is only supported when MSI-X is in use. */
699	if (pf->msix > 1) {
700		pf_schema = pci_iov_schema_alloc_node();
701		vf_schema = pci_iov_schema_alloc_node();
702		pci_iov_schema_add_unicast_mac(vf_schema, "mac-addr", 0, NULL);
703		pci_iov_schema_add_bool(vf_schema, "mac-anti-spoof",
704		    IOV_SCHEMA_HASDEFAULT, TRUE);
705		pci_iov_schema_add_bool(vf_schema, "allow-set-mac",
706		    IOV_SCHEMA_HASDEFAULT, FALSE);
707		pci_iov_schema_add_bool(vf_schema, "allow-promisc",
708		    IOV_SCHEMA_HASDEFAULT, FALSE);
709
710		iov_error = pci_iov_attach(dev, pf_schema, vf_schema);
711		if (iov_error != 0)
712			device_printf(dev,
713			    "Failed to initialize SR-IOV (error=%d)\n",
714			    iov_error);
715	}
716#endif
717
718#ifdef DEV_NETMAP
719	ixl_netmap_attach(vsi);
720#endif /* DEV_NETMAP */
721	INIT_DEBUGOUT("ixl_attach: end");
722	return (0);
723
724err_late:
725	if (vsi->ifp != NULL)
726		if_free(vsi->ifp);
727err_mac_hmc:
728	i40e_shutdown_lan_hmc(hw);
729err_get_cap:
730	i40e_shutdown_adminq(hw);
731err_out:
732	ixl_free_pci_resources(pf);
733	ixl_free_vsi(vsi);
734	IXL_PF_LOCK_DESTROY(pf);
735	return (error);
736}
737
738/*********************************************************************
739 *  Device removal routine
740 *
741 *  The detach entry point is called when the driver is being removed.
742 *  This routine stops the adapter and deallocates all the resources
743 *  that were allocated for driver operation.
744 *
745 *  return 0 on success, positive on failure
746 *********************************************************************/
747
748static int
749ixl_detach(device_t dev)
750{
751	struct ixl_pf		*pf = device_get_softc(dev);
752	struct i40e_hw		*hw = &pf->hw;
753	struct ixl_vsi		*vsi = &pf->vsi;
754	struct ixl_queue	*que = vsi->queues;
755	i40e_status		status;
756#ifdef PCI_IOV
757	int			error;
758#endif
759
760	INIT_DEBUGOUT("ixl_detach: begin");
761
762	/* Make sure VLANS are not using driver */
763	if (vsi->ifp->if_vlantrunk != NULL) {
764		device_printf(dev,"Vlan in use, detach first\n");
765		return (EBUSY);
766	}
767
768#ifdef PCI_IOV
769	error = pci_iov_detach(dev);
770	if (error != 0) {
771		device_printf(dev, "SR-IOV in use; detach first.\n");
772		return (error);
773	}
774#endif
775
776	ether_ifdetach(vsi->ifp);
777	if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) {
778		IXL_PF_LOCK(pf);
779		ixl_stop(pf);
780		IXL_PF_UNLOCK(pf);
781	}
782
783	for (int i = 0; i < vsi->num_queues; i++, que++) {
784		if (que->tq) {
785			taskqueue_drain(que->tq, &que->task);
786			taskqueue_drain(que->tq, &que->tx_task);
787			taskqueue_free(que->tq);
788		}
789	}
790
791	/* Shutdown LAN HMC */
792	status = i40e_shutdown_lan_hmc(hw);
793	if (status)
794		device_printf(dev,
795		    "Shutdown LAN HMC failed with code %d\n", status);
796
797	/* Shutdown admin queue */
798	status = i40e_shutdown_adminq(hw);
799	if (status)
800		device_printf(dev,
801		    "Shutdown Admin queue failed with code %d\n", status);
802
803	/* Unregister VLAN events */
804	if (vsi->vlan_attach != NULL)
805		EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
806	if (vsi->vlan_detach != NULL)
807		EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
808
809	callout_drain(&pf->timer);
810#ifdef DEV_NETMAP
811	netmap_detach(vsi->ifp);
812#endif /* DEV_NETMAP */
813	ixl_free_pci_resources(pf);
814	bus_generic_detach(dev);
815	if_free(vsi->ifp);
816	ixl_free_vsi(vsi);
817	IXL_PF_LOCK_DESTROY(pf);
818	return (0);
819}
820
821/*********************************************************************
822 *
823 *  Shutdown entry point
824 *
825 **********************************************************************/
826
827static int
828ixl_shutdown(device_t dev)
829{
830	struct ixl_pf *pf = device_get_softc(dev);
831	IXL_PF_LOCK(pf);
832	ixl_stop(pf);
833	IXL_PF_UNLOCK(pf);
834	return (0);
835}
836
837
838/*********************************************************************
839 *
840 *  Get the hardware capabilities
841 *
842 **********************************************************************/
843
844static int
845ixl_get_hw_capabilities(struct ixl_pf *pf)
846{
847	struct i40e_aqc_list_capabilities_element_resp *buf;
848	struct i40e_hw	*hw = &pf->hw;
849	device_t 	dev = pf->dev;
850	int             error, len;
851	u16		needed;
852	bool		again = TRUE;
853
854	len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
855retry:
856	if (!(buf = (struct i40e_aqc_list_capabilities_element_resp *)
857	    malloc(len, M_DEVBUF, M_NOWAIT | M_ZERO))) {
858		device_printf(dev, "Unable to allocate cap memory\n");
859                return (ENOMEM);
860	}
861
862	/* This populates the hw struct */
863        error = i40e_aq_discover_capabilities(hw, buf, len,
864	    &needed, i40e_aqc_opc_list_func_capabilities, NULL);
865	free(buf, M_DEVBUF);
866	if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) &&
867	    (again == TRUE)) {
868		/* retry once with a larger buffer */
869		again = FALSE;
870		len = needed;
871		goto retry;
872	} else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
873		device_printf(dev, "capability discovery failed: %d\n",
874		    pf->hw.aq.asq_last_status);
875		return (ENODEV);
876	}
877
878	/* Capture this PF's starting queue pair */
879	pf->qbase = hw->func_caps.base_queue;
880
881#ifdef IXL_DEBUG
882	device_printf(dev,"pf_id=%d, num_vfs=%d, msix_pf=%d, "
883	    "msix_vf=%d, fd_g=%d, fd_b=%d, tx_qp=%d rx_qp=%d qbase=%d\n",
884	    hw->pf_id, hw->func_caps.num_vfs,
885	    hw->func_caps.num_msix_vectors,
886	    hw->func_caps.num_msix_vectors_vf,
887	    hw->func_caps.fd_filters_guaranteed,
888	    hw->func_caps.fd_filters_best_effort,
889	    hw->func_caps.num_tx_qp,
890	    hw->func_caps.num_rx_qp,
891	    hw->func_caps.base_queue);
892#endif
893	return (error);
894}
895
896static void
897ixl_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
898{
899	device_t 	dev = vsi->dev;
900
901	/* Enable/disable TXCSUM/TSO4 */
902	if (!(ifp->if_capenable & IFCAP_TXCSUM)
903	    && !(ifp->if_capenable & IFCAP_TSO4)) {
904		if (mask & IFCAP_TXCSUM) {
905			ifp->if_capenable |= IFCAP_TXCSUM;
906			/* enable TXCSUM, restore TSO if previously enabled */
907			if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
908				vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
909				ifp->if_capenable |= IFCAP_TSO4;
910			}
911		}
912		else if (mask & IFCAP_TSO4) {
913			ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
914			vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
915			device_printf(dev,
916			    "TSO4 requires txcsum, enabling both...\n");
917		}
918	} else if((ifp->if_capenable & IFCAP_TXCSUM)
919	    && !(ifp->if_capenable & IFCAP_TSO4)) {
920		if (mask & IFCAP_TXCSUM)
921			ifp->if_capenable &= ~IFCAP_TXCSUM;
922		else if (mask & IFCAP_TSO4)
923			ifp->if_capenable |= IFCAP_TSO4;
924	} else if((ifp->if_capenable & IFCAP_TXCSUM)
925	    && (ifp->if_capenable & IFCAP_TSO4)) {
926		if (mask & IFCAP_TXCSUM) {
927			vsi->flags |= IXL_FLAGS_KEEP_TSO4;
928			ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
929			device_printf(dev,
930			    "TSO4 requires txcsum, disabling both...\n");
931		} else if (mask & IFCAP_TSO4)
932			ifp->if_capenable &= ~IFCAP_TSO4;
933	}
934
935	/* Enable/disable TXCSUM_IPV6/TSO6 */
936	if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
937	    && !(ifp->if_capenable & IFCAP_TSO6)) {
938		if (mask & IFCAP_TXCSUM_IPV6) {
939			ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
940			if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
941				vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
942				ifp->if_capenable |= IFCAP_TSO6;
943			}
944		} else if (mask & IFCAP_TSO6) {
945			ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
946			vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
947			device_printf(dev,
948			    "TSO6 requires txcsum6, enabling both...\n");
949		}
950	} else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
951	    && !(ifp->if_capenable & IFCAP_TSO6)) {
952		if (mask & IFCAP_TXCSUM_IPV6)
953			ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
954		else if (mask & IFCAP_TSO6)
955			ifp->if_capenable |= IFCAP_TSO6;
956	} else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
957	    && (ifp->if_capenable & IFCAP_TSO6)) {
958		if (mask & IFCAP_TXCSUM_IPV6) {
959			vsi->flags |= IXL_FLAGS_KEEP_TSO6;
960			ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
961			device_printf(dev,
962			    "TSO6 requires txcsum6, disabling both...\n");
963		} else if (mask & IFCAP_TSO6)
964			ifp->if_capenable &= ~IFCAP_TSO6;
965	}
966}
967
968/*********************************************************************
969 *  Ioctl entry point
970 *
971 *  ixl_ioctl is called when the user wants to configure the
972 *  interface.
973 *
974 *  return 0 on success, positive on failure
975 **********************************************************************/
976
977static int
978ixl_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
979{
980	struct ixl_vsi	*vsi = ifp->if_softc;
981	struct ixl_pf	*pf = vsi->back;
982	struct ifreq	*ifr = (struct ifreq *) data;
983#if defined(INET) || defined(INET6)
984	struct ifaddr *ifa = (struct ifaddr *)data;
985	bool		avoid_reset = FALSE;
986#endif
987	int             error = 0;
988
989	switch (command) {
990
991        case SIOCSIFADDR:
992#ifdef INET
993		if (ifa->ifa_addr->sa_family == AF_INET)
994			avoid_reset = TRUE;
995#endif
996#ifdef INET6
997		if (ifa->ifa_addr->sa_family == AF_INET6)
998			avoid_reset = TRUE;
999#endif
1000#if defined(INET) || defined(INET6)
1001		/*
1002		** Calling init results in link renegotiation,
1003		** so we avoid doing it when possible.
1004		*/
1005		if (avoid_reset) {
1006			ifp->if_flags |= IFF_UP;
1007			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1008				ixl_init(pf);
1009#ifdef INET
1010			if (!(ifp->if_flags & IFF_NOARP))
1011				arp_ifinit(ifp, ifa);
1012#endif
1013		} else
1014			error = ether_ioctl(ifp, command, data);
1015		break;
1016#endif
1017	case SIOCSIFMTU:
1018		IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)");
1019		if (ifr->ifr_mtu > IXL_MAX_FRAME -
1020		   ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
1021			error = EINVAL;
1022		} else {
1023			IXL_PF_LOCK(pf);
1024			ifp->if_mtu = ifr->ifr_mtu;
1025			vsi->max_frame_size =
1026				ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
1027			    + ETHER_VLAN_ENCAP_LEN;
1028			ixl_init_locked(pf);
1029			IXL_PF_UNLOCK(pf);
1030		}
1031		break;
1032	case SIOCSIFFLAGS:
1033		IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
1034		IXL_PF_LOCK(pf);
1035		if (ifp->if_flags & IFF_UP) {
1036			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1037				if ((ifp->if_flags ^ pf->if_flags) &
1038				    (IFF_PROMISC | IFF_ALLMULTI)) {
1039					ixl_set_promisc(vsi);
1040				}
1041			} else
1042				ixl_init_locked(pf);
1043		} else
1044			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1045				ixl_stop(pf);
1046		pf->if_flags = ifp->if_flags;
1047		IXL_PF_UNLOCK(pf);
1048		break;
1049	case SIOCADDMULTI:
1050		IOCTL_DEBUGOUT("ioctl: SIOCADDMULTI");
1051		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1052			IXL_PF_LOCK(pf);
1053			ixl_disable_intr(vsi);
1054			ixl_add_multi(vsi);
1055			ixl_enable_intr(vsi);
1056			IXL_PF_UNLOCK(pf);
1057		}
1058		break;
1059	case SIOCDELMULTI:
1060		IOCTL_DEBUGOUT("ioctl: SIOCDELMULTI");
1061		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1062			IXL_PF_LOCK(pf);
1063			ixl_disable_intr(vsi);
1064			ixl_del_multi(vsi);
1065			ixl_enable_intr(vsi);
1066			IXL_PF_UNLOCK(pf);
1067		}
1068		break;
1069	case SIOCSIFMEDIA:
1070	case SIOCGIFMEDIA:
1071#ifdef IFM_ETH_XTYPE
1072	case SIOCGIFXMEDIA:
1073#endif
1074		IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
1075		error = ifmedia_ioctl(ifp, ifr, &vsi->media, command);
1076		break;
1077	case SIOCSIFCAP:
1078	{
1079		int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1080		IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)");
1081
1082		ixl_cap_txcsum_tso(vsi, ifp, mask);
1083
1084		if (mask & IFCAP_RXCSUM)
1085			ifp->if_capenable ^= IFCAP_RXCSUM;
1086		if (mask & IFCAP_RXCSUM_IPV6)
1087			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1088		if (mask & IFCAP_LRO)
1089			ifp->if_capenable ^= IFCAP_LRO;
1090		if (mask & IFCAP_VLAN_HWTAGGING)
1091			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1092		if (mask & IFCAP_VLAN_HWFILTER)
1093			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
1094		if (mask & IFCAP_VLAN_HWTSO)
1095			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
1096		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1097			IXL_PF_LOCK(pf);
1098			ixl_init_locked(pf);
1099			IXL_PF_UNLOCK(pf);
1100		}
1101		VLAN_CAPABILITIES(ifp);
1102
1103		break;
1104	}
1105
1106	default:
1107		IOCTL_DEBUGOUT("ioctl: UNKNOWN (0x%X)\n", (int)command);
1108		error = ether_ioctl(ifp, command, data);
1109		break;
1110	}
1111
1112	return (error);
1113}
1114
1115
1116/*********************************************************************
1117 *  Init entry point
1118 *
1119 *  This routine is used in two ways. It is used by the stack as
1120 *  init entry point in network interface structure. It is also used
1121 *  by the driver as a hw/sw initialization routine to get to a
1122 *  consistent state.
1123 *
1124 *  return 0 on success, positive on failure
1125 **********************************************************************/
1126
1127static void
1128ixl_init_locked(struct ixl_pf *pf)
1129{
1130	struct i40e_hw	*hw = &pf->hw;
1131	struct ixl_vsi	*vsi = &pf->vsi;
1132	struct ifnet	*ifp = vsi->ifp;
1133	device_t 	dev = pf->dev;
1134	struct i40e_filter_control_settings	filter;
1135	u8		tmpaddr[ETHER_ADDR_LEN];
1136	int		ret;
1137
1138	mtx_assert(&pf->pf_mtx, MA_OWNED);
1139	INIT_DEBUGOUT("ixl_init: begin");
1140	ixl_stop(pf);
1141
1142	/* Get the latest mac address... User might use a LAA */
1143	bcopy(IF_LLADDR(vsi->ifp), tmpaddr,
1144	      I40E_ETH_LENGTH_OF_ADDRESS);
1145	if (!cmp_etheraddr(hw->mac.addr, tmpaddr) &&
1146	    (i40e_validate_mac_addr(tmpaddr) == I40E_SUCCESS)) {
1147		ixl_del_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1148		bcopy(tmpaddr, hw->mac.addr,
1149		    I40E_ETH_LENGTH_OF_ADDRESS);
1150		ret = i40e_aq_mac_address_write(hw,
1151		    I40E_AQC_WRITE_TYPE_LAA_ONLY,
1152		    hw->mac.addr, NULL);
1153		if (ret) {
1154			device_printf(dev, "LLA address"
1155			 "change failed!!\n");
1156			return;
1157		} else {
1158			ixl_add_filter(vsi, hw->mac.addr, IXL_VLAN_ANY);
1159		}
1160	}
1161
1162	/* Set the various hardware offload abilities */
1163	ifp->if_hwassist = 0;
1164	if (ifp->if_capenable & IFCAP_TSO)
1165		ifp->if_hwassist |= CSUM_TSO;
1166	if (ifp->if_capenable & IFCAP_TXCSUM)
1167		ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1168	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
1169		ifp->if_hwassist |= (CSUM_TCP_IPV6 | CSUM_UDP_IPV6);
1170
1171	/* Set up the device filtering */
1172	bzero(&filter, sizeof(filter));
1173	filter.enable_ethtype = TRUE;
1174	filter.enable_macvlan = TRUE;
1175#ifdef IXL_FDIR
1176	filter.enable_fdir = TRUE;
1177#endif
1178	filter.hash_lut_size = I40E_HASH_LUT_SIZE_512;
1179	if (i40e_set_filter_control(hw, &filter))
1180		device_printf(dev, "set_filter_control() failed\n");
1181
1182	/* Set up RSS */
1183	ixl_config_rss(vsi);
1184
1185	/*
1186	** Prepare the VSI: rings, hmc contexts, etc...
1187	*/
1188	if (ixl_initialize_vsi(vsi)) {
1189		device_printf(dev, "initialize vsi failed!!\n");
1190		return;
1191	}
1192
1193	/* Add protocol filters to list */
1194	ixl_init_filters(vsi);
1195
1196	/* Setup vlan's if needed */
1197	ixl_setup_vlan_filters(vsi);
1198
1199	/* Start the local timer */
1200	callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1201
1202	/* Set up MSI/X routing and the ITR settings */
1203	if (ixl_enable_msix) {
1204		ixl_configure_msix(pf);
1205		ixl_configure_itr(pf);
1206	} else
1207		ixl_configure_legacy(pf);
1208
1209	ixl_enable_rings(vsi);
1210
1211	i40e_aq_set_default_vsi(hw, vsi->seid, NULL);
1212
1213	ixl_reconfigure_filters(vsi);
1214
1215	/* Set MTU in hardware*/
1216	int aq_error = i40e_aq_set_mac_config(hw, vsi->max_frame_size,
1217	    TRUE, 0, NULL);
1218	if (aq_error)
1219		device_printf(vsi->dev,
1220			"aq_set_mac_config in init error, code %d\n",
1221		    aq_error);
1222
1223	/* And now turn on interrupts */
1224	ixl_enable_intr(vsi);
1225
1226	/* Now inform the stack we're ready */
1227	ifp->if_drv_flags |= IFF_DRV_RUNNING;
1228	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1229
1230	return;
1231}
1232
1233static void
1234ixl_init(void *arg)
1235{
1236	struct ixl_pf *pf = arg;
1237
1238	IXL_PF_LOCK(pf);
1239	ixl_init_locked(pf);
1240	IXL_PF_UNLOCK(pf);
1241	return;
1242}
1243
1244/*
1245**
1246** MSIX Interrupt Handlers and Tasklets
1247**
1248*/
1249static void
1250ixl_handle_que(void *context, int pending)
1251{
1252	struct ixl_queue *que = context;
1253	struct ixl_vsi *vsi = que->vsi;
1254	struct i40e_hw  *hw = vsi->hw;
1255	struct tx_ring  *txr = &que->txr;
1256	struct ifnet    *ifp = vsi->ifp;
1257	bool		more;
1258
1259	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1260		more = ixl_rxeof(que, IXL_RX_LIMIT);
1261		IXL_TX_LOCK(txr);
1262		ixl_txeof(que);
1263		if (!drbr_empty(ifp, txr->br))
1264			ixl_mq_start_locked(ifp, txr);
1265		IXL_TX_UNLOCK(txr);
1266		if (more) {
1267			taskqueue_enqueue(que->tq, &que->task);
1268			return;
1269		}
1270	}
1271
1272	/* Reenable this interrupt - hmmm */
1273	ixl_enable_queue(hw, que->me);
1274	return;
1275}
1276
1277
1278/*********************************************************************
1279 *
1280 *  Legacy Interrupt Service routine
1281 *
1282 **********************************************************************/
1283void
1284ixl_intr(void *arg)
1285{
1286	struct ixl_pf		*pf = arg;
1287	struct i40e_hw		*hw =  &pf->hw;
1288	struct ixl_vsi		*vsi = &pf->vsi;
1289	struct ixl_queue	*que = vsi->queues;
1290	struct ifnet		*ifp = vsi->ifp;
1291	struct tx_ring		*txr = &que->txr;
1292        u32			reg, icr0, mask;
1293	bool			more_tx, more_rx;
1294
1295	++que->irqs;
1296
1297	/* Protect against spurious interrupts */
1298	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
1299		return;
1300
1301	icr0 = rd32(hw, I40E_PFINT_ICR0);
1302
1303	reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1304	reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1305	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1306
1307        mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1308
1309#ifdef PCI_IOV
1310	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK)
1311		taskqueue_enqueue(pf->tq, &pf->vflr_task);
1312#endif
1313
1314	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
1315		taskqueue_enqueue(pf->tq, &pf->adminq);
1316		return;
1317	}
1318
1319	more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1320
1321	IXL_TX_LOCK(txr);
1322	more_tx = ixl_txeof(que);
1323	if (!drbr_empty(vsi->ifp, txr->br))
1324		more_tx = 1;
1325	IXL_TX_UNLOCK(txr);
1326
1327	/* re-enable other interrupt causes */
1328	wr32(hw, I40E_PFINT_ICR0_ENA, mask);
1329
1330	/* And now the queues */
1331	reg = rd32(hw, I40E_QINT_RQCTL(0));
1332	reg |= I40E_QINT_RQCTL_CAUSE_ENA_MASK;
1333	wr32(hw, I40E_QINT_RQCTL(0), reg);
1334
1335	reg = rd32(hw, I40E_QINT_TQCTL(0));
1336	reg |= I40E_QINT_TQCTL_CAUSE_ENA_MASK;
1337	reg &= ~I40E_PFINT_ICR0_INTEVENT_MASK;
1338	wr32(hw, I40E_QINT_TQCTL(0), reg);
1339
1340	ixl_enable_legacy(hw);
1341
1342	return;
1343}
1344
1345
1346/*********************************************************************
1347 *
1348 *  MSIX VSI Interrupt Service routine
1349 *
1350 **********************************************************************/
1351void
1352ixl_msix_que(void *arg)
1353{
1354	struct ixl_queue	*que = arg;
1355	struct ixl_vsi	*vsi = que->vsi;
1356	struct i40e_hw	*hw = vsi->hw;
1357	struct tx_ring	*txr = &que->txr;
1358	bool		more_tx, more_rx;
1359
1360	/* Protect against spurious interrupts */
1361	if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
1362		return;
1363
1364	++que->irqs;
1365
1366	more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
1367
1368	IXL_TX_LOCK(txr);
1369	more_tx = ixl_txeof(que);
1370	/*
1371	** Make certain that if the stack
1372	** has anything queued the task gets
1373	** scheduled to handle it.
1374	*/
1375	if (!drbr_empty(vsi->ifp, txr->br))
1376		more_tx = 1;
1377	IXL_TX_UNLOCK(txr);
1378
1379	ixl_set_queue_rx_itr(que);
1380	ixl_set_queue_tx_itr(que);
1381
1382	if (more_tx || more_rx)
1383		taskqueue_enqueue(que->tq, &que->task);
1384	else
1385		ixl_enable_queue(hw, que->me);
1386
1387	return;
1388}
1389
1390
1391/*********************************************************************
1392 *
1393 *  MSIX Admin Queue Interrupt Service routine
1394 *
1395 **********************************************************************/
1396static void
1397ixl_msix_adminq(void *arg)
1398{
1399	struct ixl_pf	*pf = arg;
1400	struct i40e_hw	*hw = &pf->hw;
1401	u32		reg, mask;
1402
1403	++pf->admin_irq;
1404
1405	reg = rd32(hw, I40E_PFINT_ICR0);
1406	mask = rd32(hw, I40E_PFINT_ICR0_ENA);
1407
1408	/* Check on the cause */
1409	if (reg & I40E_PFINT_ICR0_ADMINQ_MASK)
1410		mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
1411
1412	if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
1413		ixl_handle_mdd_event(pf);
1414		mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
1415	}
1416
1417#ifdef PCI_IOV
1418	if (reg & I40E_PFINT_ICR0_VFLR_MASK) {
1419		mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
1420		taskqueue_enqueue(pf->tq, &pf->vflr_task);
1421	}
1422#endif
1423
1424	reg = rd32(hw, I40E_PFINT_DYN_CTL0);
1425	reg = reg | I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
1426	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
1427
1428	taskqueue_enqueue(pf->tq, &pf->adminq);
1429	return;
1430}
1431
1432/*********************************************************************
1433 *
1434 *  Media Ioctl callback
1435 *
1436 *  This routine is called whenever the user queries the status of
1437 *  the interface using ifconfig.
1438 *
1439 **********************************************************************/
1440static void
1441ixl_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
1442{
1443	struct ixl_vsi	*vsi = ifp->if_softc;
1444	struct ixl_pf	*pf = vsi->back;
1445	struct i40e_hw  *hw = &pf->hw;
1446
1447	INIT_DEBUGOUT("ixl_media_status: begin");
1448	IXL_PF_LOCK(pf);
1449
1450	hw->phy.get_link_info = TRUE;
1451	i40e_get_link_status(hw, &pf->link_up);
1452	ixl_update_link_status(pf);
1453
1454	ifmr->ifm_status = IFM_AVALID;
1455	ifmr->ifm_active = IFM_ETHER;
1456
1457	if (!pf->link_up) {
1458		IXL_PF_UNLOCK(pf);
1459		return;
1460	}
1461
1462	ifmr->ifm_status |= IFM_ACTIVE;
1463	/* Hardware is always full-duplex */
1464	ifmr->ifm_active |= IFM_FDX;
1465
1466	switch (hw->phy.link_info.phy_type) {
1467		/* 100 M */
1468		case I40E_PHY_TYPE_100BASE_TX:
1469			ifmr->ifm_active |= IFM_100_TX;
1470			break;
1471		/* 1 G */
1472		case I40E_PHY_TYPE_1000BASE_T:
1473			ifmr->ifm_active |= IFM_1000_T;
1474			break;
1475		case I40E_PHY_TYPE_1000BASE_SX:
1476			ifmr->ifm_active |= IFM_1000_SX;
1477			break;
1478		case I40E_PHY_TYPE_1000BASE_LX:
1479			ifmr->ifm_active |= IFM_1000_LX;
1480			break;
1481		/* 10 G */
1482		case I40E_PHY_TYPE_10GBASE_SFPP_CU:
1483			ifmr->ifm_active |= IFM_10G_TWINAX;
1484			break;
1485		case I40E_PHY_TYPE_10GBASE_SR:
1486			ifmr->ifm_active |= IFM_10G_SR;
1487			break;
1488		case I40E_PHY_TYPE_10GBASE_LR:
1489			ifmr->ifm_active |= IFM_10G_LR;
1490			break;
1491		case I40E_PHY_TYPE_10GBASE_T:
1492			ifmr->ifm_active |= IFM_10G_T;
1493			break;
1494		/* 40 G */
1495		case I40E_PHY_TYPE_40GBASE_CR4:
1496		case I40E_PHY_TYPE_40GBASE_CR4_CU:
1497			ifmr->ifm_active |= IFM_40G_CR4;
1498			break;
1499		case I40E_PHY_TYPE_40GBASE_SR4:
1500			ifmr->ifm_active |= IFM_40G_SR4;
1501			break;
1502		case I40E_PHY_TYPE_40GBASE_LR4:
1503			ifmr->ifm_active |= IFM_40G_LR4;
1504			break;
1505#ifndef IFM_ETH_XTYPE
1506		case I40E_PHY_TYPE_1000BASE_KX:
1507			ifmr->ifm_active |= IFM_1000_CX;
1508			break;
1509		case I40E_PHY_TYPE_10GBASE_CR1_CU:
1510		case I40E_PHY_TYPE_10GBASE_CR1:
1511			ifmr->ifm_active |= IFM_10G_TWINAX;
1512			break;
1513		case I40E_PHY_TYPE_10GBASE_KX4:
1514			ifmr->ifm_active |= IFM_10G_CX4;
1515			break;
1516		case I40E_PHY_TYPE_10GBASE_KR:
1517			ifmr->ifm_active |= IFM_10G_SR;
1518			break;
1519		case I40E_PHY_TYPE_40GBASE_KR4:
1520		case I40E_PHY_TYPE_XLPPI:
1521			ifmr->ifm_active |= IFM_40G_SR4;
1522			break;
1523#else
1524		case I40E_PHY_TYPE_1000BASE_KX:
1525			ifmr->ifm_active |= IFM_1000_KX;
1526			break;
1527		/* ERJ: What's the difference between these? */
1528		case I40E_PHY_TYPE_10GBASE_CR1_CU:
1529		case I40E_PHY_TYPE_10GBASE_CR1:
1530			ifmr->ifm_active |= IFM_10G_CR1;
1531			break;
1532		case I40E_PHY_TYPE_10GBASE_KX4:
1533			ifmr->ifm_active |= IFM_10G_KX4;
1534			break;
1535		case I40E_PHY_TYPE_10GBASE_KR:
1536			ifmr->ifm_active |= IFM_10G_KR;
1537			break;
1538		case I40E_PHY_TYPE_20GBASE_KR2:
1539			ifmr->ifm_active |= IFM_20G_KR2;
1540			break;
1541		case I40E_PHY_TYPE_40GBASE_KR4:
1542			ifmr->ifm_active |= IFM_40G_KR4;
1543			break;
1544		case I40E_PHY_TYPE_XLPPI:
1545			ifmr->ifm_active |= IFM_40G_XLPPI;
1546			break;
1547#endif
1548		default:
1549			ifmr->ifm_active |= IFM_UNKNOWN;
1550			break;
1551	}
1552	/* Report flow control status as well */
1553	if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX)
1554		ifmr->ifm_active |= IFM_ETH_TXPAUSE;
1555	if (hw->phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)
1556		ifmr->ifm_active |= IFM_ETH_RXPAUSE;
1557
1558	IXL_PF_UNLOCK(pf);
1559
1560	return;
1561}
1562
1563/*********************************************************************
1564 *
1565 *  Media Ioctl callback
1566 *
1567 *  This routine is called when the user changes speed/duplex using
1568 *  media/mediopt option with ifconfig.
1569 *
1570 **********************************************************************/
1571static int
1572ixl_media_change(struct ifnet * ifp)
1573{
1574	struct ixl_vsi *vsi = ifp->if_softc;
1575	struct ifmedia *ifm = &vsi->media;
1576
1577	INIT_DEBUGOUT("ixl_media_change: begin");
1578
1579	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1580		return (EINVAL);
1581
1582	if_printf(ifp, "Media change is currently not supported.\n");
1583
1584	return (ENODEV);
1585}
1586
1587
1588#ifdef IXL_FDIR
1589/*
1590** ATR: Application Targetted Receive - creates a filter
1591**	based on TX flow info that will keep the receive
1592**	portion of the flow on the same queue. Based on the
1593**	implementation this is only available for TCP connections
1594*/
1595void
1596ixl_atr(struct ixl_queue *que, struct tcphdr *th, int etype)
1597{
1598	struct ixl_vsi			*vsi = que->vsi;
1599	struct tx_ring			*txr = &que->txr;
1600	struct i40e_filter_program_desc	*FDIR;
1601	u32				ptype, dtype;
1602	int				idx;
1603
1604	/* check if ATR is enabled and sample rate */
1605	if ((!ixl_enable_fdir) || (!txr->atr_rate))
1606		return;
1607	/*
1608	** We sample all TCP SYN/FIN packets,
1609	** or at the selected sample rate
1610	*/
1611	txr->atr_count++;
1612	if (((th->th_flags & (TH_FIN | TH_SYN)) == 0) &&
1613	    (txr->atr_count < txr->atr_rate))
1614                return;
1615	txr->atr_count = 0;
1616
1617	/* Get a descriptor to use */
1618	idx = txr->next_avail;
1619	FDIR = (struct i40e_filter_program_desc *) &txr->base[idx];
1620	if (++idx == que->num_desc)
1621		idx = 0;
1622	txr->avail--;
1623	txr->next_avail = idx;
1624
1625	ptype = (que->me << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) &
1626	    I40E_TXD_FLTR_QW0_QINDEX_MASK;
1627
1628	ptype |= (etype == ETHERTYPE_IP) ?
1629	    (I40E_FILTER_PCTYPE_NONF_IPV4_TCP <<
1630	    I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) :
1631	    (I40E_FILTER_PCTYPE_NONF_IPV6_TCP <<
1632	    I40E_TXD_FLTR_QW0_PCTYPE_SHIFT);
1633
1634	ptype |= vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT;
1635
1636	dtype = I40E_TX_DESC_DTYPE_FILTER_PROG;
1637
1638	/*
1639	** We use the TCP TH_FIN as a trigger to remove
1640	** the filter, otherwise its an update.
1641	*/
1642	dtype |= (th->th_flags & TH_FIN) ?
1643	    (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE <<
1644	    I40E_TXD_FLTR_QW1_PCMD_SHIFT) :
1645	    (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE <<
1646	    I40E_TXD_FLTR_QW1_PCMD_SHIFT);
1647
1648	dtype |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX <<
1649	    I40E_TXD_FLTR_QW1_DEST_SHIFT;
1650
1651	dtype |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID <<
1652	    I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT;
1653
1654	FDIR->qindex_flex_ptype_vsi = htole32(ptype);
1655	FDIR->dtype_cmd_cntindex = htole32(dtype);
1656	return;
1657}
1658#endif
1659
1660
1661static void
1662ixl_set_promisc(struct ixl_vsi *vsi)
1663{
1664	struct ifnet	*ifp = vsi->ifp;
1665	struct i40e_hw	*hw = vsi->hw;
1666	int		err, mcnt = 0;
1667	bool		uni = FALSE, multi = FALSE;
1668
1669	if (ifp->if_flags & IFF_ALLMULTI)
1670                multi = TRUE;
1671	else { /* Need to count the multicast addresses */
1672		struct  ifmultiaddr *ifma;
1673		if_maddr_rlock(ifp);
1674		TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1675                        if (ifma->ifma_addr->sa_family != AF_LINK)
1676                                continue;
1677                        if (mcnt == MAX_MULTICAST_ADDR)
1678                                break;
1679                        mcnt++;
1680		}
1681		if_maddr_runlock(ifp);
1682	}
1683
1684	if (mcnt >= MAX_MULTICAST_ADDR)
1685                multi = TRUE;
1686        if (ifp->if_flags & IFF_PROMISC)
1687		uni = TRUE;
1688
1689	err = i40e_aq_set_vsi_unicast_promiscuous(hw,
1690	    vsi->seid, uni, NULL);
1691	err = i40e_aq_set_vsi_multicast_promiscuous(hw,
1692	    vsi->seid, multi, NULL);
1693	return;
1694}
1695
1696/*********************************************************************
1697 * 	Filter Routines
1698 *
1699 *	Routines for multicast and vlan filter management.
1700 *
1701 *********************************************************************/
1702static void
1703ixl_add_multi(struct ixl_vsi *vsi)
1704{
1705	struct	ifmultiaddr	*ifma;
1706	struct ifnet		*ifp = vsi->ifp;
1707	struct i40e_hw		*hw = vsi->hw;
1708	int			mcnt = 0, flags;
1709
1710	IOCTL_DEBUGOUT("ixl_add_multi: begin");
1711
1712	if_maddr_rlock(ifp);
1713	/*
1714	** First just get a count, to decide if we
1715	** we simply use multicast promiscuous.
1716	*/
1717	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1718		if (ifma->ifma_addr->sa_family != AF_LINK)
1719			continue;
1720		mcnt++;
1721	}
1722	if_maddr_runlock(ifp);
1723
1724	if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
1725		/* delete existing MC filters */
1726		ixl_del_hw_filters(vsi, mcnt);
1727		i40e_aq_set_vsi_multicast_promiscuous(hw,
1728		    vsi->seid, TRUE, NULL);
1729		return;
1730	}
1731
1732	mcnt = 0;
1733	if_maddr_rlock(ifp);
1734	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1735		if (ifma->ifma_addr->sa_family != AF_LINK)
1736			continue;
1737		ixl_add_mc_filter(vsi,
1738		    (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr));
1739		mcnt++;
1740	}
1741	if_maddr_runlock(ifp);
1742	if (mcnt > 0) {
1743		flags = (IXL_FILTER_ADD | IXL_FILTER_USED | IXL_FILTER_MC);
1744		ixl_add_hw_filters(vsi, flags, mcnt);
1745	}
1746
1747	IOCTL_DEBUGOUT("ixl_add_multi: end");
1748	return;
1749}
1750
1751static void
1752ixl_del_multi(struct ixl_vsi *vsi)
1753{
1754	struct ifnet		*ifp = vsi->ifp;
1755	struct ifmultiaddr	*ifma;
1756	struct ixl_mac_filter	*f;
1757	int			mcnt = 0;
1758	bool		match = FALSE;
1759
1760	IOCTL_DEBUGOUT("ixl_del_multi: begin");
1761
1762	/* Search for removed multicast addresses */
1763	if_maddr_rlock(ifp);
1764	SLIST_FOREACH(f, &vsi->ftl, next) {
1765		if ((f->flags & IXL_FILTER_USED) && (f->flags & IXL_FILTER_MC)) {
1766			match = FALSE;
1767			TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1768				if (ifma->ifma_addr->sa_family != AF_LINK)
1769					continue;
1770				u8 *mc_addr = (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
1771				if (cmp_etheraddr(f->macaddr, mc_addr)) {
1772					match = TRUE;
1773					break;
1774				}
1775			}
1776			if (match == FALSE) {
1777				f->flags |= IXL_FILTER_DEL;
1778				mcnt++;
1779			}
1780		}
1781	}
1782	if_maddr_runlock(ifp);
1783
1784	if (mcnt > 0)
1785		ixl_del_hw_filters(vsi, mcnt);
1786}
1787
1788
1789/*********************************************************************
1790 *  Timer routine
1791 *
1792 *  This routine checks for link status,updates statistics,
1793 *  and runs the watchdog check.
1794 *
1795 **********************************************************************/
1796
1797static void
1798ixl_local_timer(void *arg)
1799{
1800	struct ixl_pf		*pf = arg;
1801	struct i40e_hw		*hw = &pf->hw;
1802	struct ixl_vsi		*vsi = &pf->vsi;
1803	struct ixl_queue	*que = vsi->queues;
1804	device_t		dev = pf->dev;
1805	int			hung = 0;
1806	u32			mask;
1807
1808	mtx_assert(&pf->pf_mtx, MA_OWNED);
1809
1810	/* Fire off the adminq task */
1811	taskqueue_enqueue(pf->tq, &pf->adminq);
1812
1813	/* Update stats */
1814	ixl_update_stats_counters(pf);
1815
1816	/*
1817	** Check status of the queues
1818	*/
1819	mask = (I40E_PFINT_DYN_CTLN_INTENA_MASK |
1820		I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK);
1821
1822	for (int i = 0; i < vsi->num_queues; i++,que++) {
1823		/* Any queues with outstanding work get a sw irq */
1824		if (que->busy)
1825			wr32(hw, I40E_PFINT_DYN_CTLN(que->me), mask);
1826		/*
1827		** Each time txeof runs without cleaning, but there
1828		** are uncleaned descriptors it increments busy. If
1829		** we get to 5 we declare it hung.
1830		*/
1831		if (que->busy == IXL_QUEUE_HUNG) {
1832			++hung;
1833			/* Mark the queue as inactive */
1834			vsi->active_queues &= ~((u64)1 << que->me);
1835			continue;
1836		} else {
1837			/* Check if we've come back from hung */
1838			if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
1839				vsi->active_queues |= ((u64)1 << que->me);
1840		}
1841		if (que->busy >= IXL_MAX_TX_BUSY) {
1842#ifdef IXL_DEBUG
1843			device_printf(dev,"Warning queue %d "
1844			    "appears to be hung!\n", i);
1845#endif
1846			que->busy = IXL_QUEUE_HUNG;
1847			++hung;
1848		}
1849	}
1850	/* Only reinit if all queues show hung */
1851	if (hung == vsi->num_queues)
1852		goto hung;
1853
1854	callout_reset(&pf->timer, hz, ixl_local_timer, pf);
1855	return;
1856
1857hung:
1858	device_printf(dev, "Local Timer: HANG DETECT - Resetting!!\n");
1859	ixl_init_locked(pf);
1860}
1861
1862/*
1863** Note: this routine updates the OS on the link state
1864**	the real check of the hardware only happens with
1865**	a link interrupt.
1866*/
1867static void
1868ixl_update_link_status(struct ixl_pf *pf)
1869{
1870	struct ixl_vsi		*vsi = &pf->vsi;
1871	struct i40e_hw		*hw = &pf->hw;
1872	struct ifnet		*ifp = vsi->ifp;
1873	device_t		dev = pf->dev;
1874
1875	if (pf->link_up){
1876		if (vsi->link_active == FALSE) {
1877			pf->fc = hw->fc.current_mode;
1878			if (bootverbose) {
1879				device_printf(dev,"Link is up %d Gbps %s,"
1880				    " Flow Control: %s\n",
1881				    ((pf->link_speed ==
1882				    I40E_LINK_SPEED_40GB)? 40:10),
1883				    "Full Duplex", ixl_fc_string[pf->fc]);
1884			}
1885			vsi->link_active = TRUE;
1886			/*
1887			** Warn user if link speed on NPAR enabled
1888			** partition is not at least 10GB
1889			*/
1890			if (hw->func_caps.npar_enable &&
1891			   (hw->phy.link_info.link_speed ==
1892			   I40E_LINK_SPEED_1GB ||
1893			   hw->phy.link_info.link_speed ==
1894			   I40E_LINK_SPEED_100MB))
1895				device_printf(dev, "The partition detected"
1896				    "link speed that is less than 10Gbps\n");
1897			if_link_state_change(ifp, LINK_STATE_UP);
1898		}
1899	} else { /* Link down */
1900		if (vsi->link_active == TRUE) {
1901			if (bootverbose)
1902				device_printf(dev,"Link is Down\n");
1903			if_link_state_change(ifp, LINK_STATE_DOWN);
1904			vsi->link_active = FALSE;
1905		}
1906	}
1907
1908	return;
1909}
1910
1911/*********************************************************************
1912 *
1913 *  This routine disables all traffic on the adapter by issuing a
1914 *  global reset on the MAC and deallocates TX/RX buffers.
1915 *
1916 **********************************************************************/
1917
1918static void
1919ixl_stop(struct ixl_pf *pf)
1920{
1921	struct ixl_vsi	*vsi = &pf->vsi;
1922	struct ifnet	*ifp = vsi->ifp;
1923
1924	mtx_assert(&pf->pf_mtx, MA_OWNED);
1925
1926	INIT_DEBUGOUT("ixl_stop: begin\n");
1927	if (pf->num_vfs == 0)
1928		ixl_disable_intr(vsi);
1929	else
1930		ixl_disable_rings_intr(vsi);
1931	ixl_disable_rings(vsi);
1932
1933	/* Tell the stack that the interface is no longer active */
1934	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1935
1936	/* Stop the local timer */
1937	callout_stop(&pf->timer);
1938
1939	return;
1940}
1941
1942
1943/*********************************************************************
1944 *
1945 *  Setup MSIX Interrupt resources and handlers for the VSI
1946 *
1947 **********************************************************************/
1948static int
1949ixl_assign_vsi_legacy(struct ixl_pf *pf)
1950{
1951	device_t        dev = pf->dev;
1952	struct 		ixl_vsi *vsi = &pf->vsi;
1953	struct		ixl_queue *que = vsi->queues;
1954	int 		error, rid = 0;
1955
1956	if (pf->msix == 1)
1957		rid = 1;
1958	pf->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
1959	    &rid, RF_SHAREABLE | RF_ACTIVE);
1960	if (pf->res == NULL) {
1961		device_printf(dev,"Unable to allocate"
1962		    " bus resource: vsi legacy/msi interrupt\n");
1963		return (ENXIO);
1964	}
1965
1966	/* Set the handler function */
1967	error = bus_setup_intr(dev, pf->res,
1968	    INTR_TYPE_NET | INTR_MPSAFE, NULL,
1969	    ixl_intr, pf, &pf->tag);
1970	if (error) {
1971		pf->res = NULL;
1972		device_printf(dev, "Failed to register legacy/msi handler");
1973		return (error);
1974	}
1975	bus_describe_intr(dev, pf->res, pf->tag, "irq0");
1976	TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
1977	TASK_INIT(&que->task, 0, ixl_handle_que, que);
1978	que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
1979	    taskqueue_thread_enqueue, &que->tq);
1980	taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
1981	    device_get_nameunit(dev));
1982	TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
1983
1984#ifdef PCI_IOV
1985	TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
1986#endif
1987
1988	pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
1989	    taskqueue_thread_enqueue, &pf->tq);
1990	taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
1991	    device_get_nameunit(dev));
1992
1993	return (0);
1994}
1995
1996
1997/*********************************************************************
1998 *
1999 *  Setup MSIX Interrupt resources and handlers for the VSI
2000 *
2001 **********************************************************************/
2002static int
2003ixl_assign_vsi_msix(struct ixl_pf *pf)
2004{
2005	device_t	dev = pf->dev;
2006	struct 		ixl_vsi *vsi = &pf->vsi;
2007	struct 		ixl_queue *que = vsi->queues;
2008	struct		tx_ring	 *txr;
2009	int 		error, rid, vector = 0;
2010#ifdef	RSS
2011	cpuset_t cpu_mask;
2012#endif
2013
2014	/* Admin Que is vector 0*/
2015	rid = vector + 1;
2016	pf->res = bus_alloc_resource_any(dev,
2017    	    SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2018	if (!pf->res) {
2019		device_printf(dev,"Unable to allocate"
2020    	    " bus resource: Adminq interrupt [%d]\n", rid);
2021		return (ENXIO);
2022	}
2023	/* Set the adminq vector and handler */
2024	error = bus_setup_intr(dev, pf->res,
2025	    INTR_TYPE_NET | INTR_MPSAFE, NULL,
2026	    ixl_msix_adminq, pf, &pf->tag);
2027	if (error) {
2028		pf->res = NULL;
2029		device_printf(dev, "Failed to register Admin que handler");
2030		return (error);
2031	}
2032	bus_describe_intr(dev, pf->res, pf->tag, "aq");
2033	pf->admvec = vector;
2034	/* Tasklet for Admin Queue */
2035	TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf);
2036
2037#ifdef PCI_IOV
2038	TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf);
2039#endif
2040
2041	pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
2042	    taskqueue_thread_enqueue, &pf->tq);
2043	taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq",
2044	    device_get_nameunit(pf->dev));
2045	++vector;
2046
2047	/* Now set up the stations */
2048	for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
2049		int cpu_id = i;
2050		rid = vector + 1;
2051		txr = &que->txr;
2052		que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2053		    RF_SHAREABLE | RF_ACTIVE);
2054		if (que->res == NULL) {
2055			device_printf(dev,"Unable to allocate"
2056		    	    " bus resource: que interrupt [%d]\n", vector);
2057			return (ENXIO);
2058		}
2059		/* Set the handler function */
2060		error = bus_setup_intr(dev, que->res,
2061		    INTR_TYPE_NET | INTR_MPSAFE, NULL,
2062		    ixl_msix_que, que, &que->tag);
2063		if (error) {
2064			que->res = NULL;
2065			device_printf(dev, "Failed to register que handler");
2066			return (error);
2067		}
2068		bus_describe_intr(dev, que->res, que->tag, "q%d", i);
2069		/* Bind the vector to a CPU */
2070#ifdef RSS
2071		cpu_id = rss_getcpu(i % rss_getnumbuckets());
2072#endif
2073		bus_bind_intr(dev, que->res, cpu_id);
2074		que->msix = vector;
2075		TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
2076		TASK_INIT(&que->task, 0, ixl_handle_que, que);
2077		que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT,
2078		    taskqueue_thread_enqueue, &que->tq);
2079#ifdef RSS
2080		CPU_SETOF(cpu_id, &cpu_mask);
2081		taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET,
2082		    &cpu_mask, "%s (bucket %d)",
2083		    device_get_nameunit(dev), cpu_id);
2084#else
2085		taskqueue_start_threads(&que->tq, 1, PI_NET,
2086		    "%s que", device_get_nameunit(dev));
2087#endif
2088	}
2089
2090	return (0);
2091}
2092
2093
2094/*
2095 * Allocate MSI/X vectors
2096 */
2097static int
2098ixl_init_msix(struct ixl_pf *pf)
2099{
2100	device_t dev = pf->dev;
2101	int rid, want, vectors, queues, available;
2102
2103	/* Override by tuneable */
2104	if (ixl_enable_msix == 0)
2105		goto msi;
2106
2107	/*
2108	** When used in a virtualized environment
2109	** PCI BUSMASTER capability may not be set
2110	** so explicity set it here and rewrite
2111	** the ENABLE in the MSIX control register
2112	** at this point to cause the host to
2113	** successfully initialize us.
2114	*/
2115	{
2116		u16 pci_cmd_word;
2117		int msix_ctrl;
2118		pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2119		pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
2120		pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
2121		pci_find_cap(dev, PCIY_MSIX, &rid);
2122		rid += PCIR_MSIX_CTRL;
2123		msix_ctrl = pci_read_config(dev, rid, 2);
2124		msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
2125		pci_write_config(dev, rid, msix_ctrl, 2);
2126	}
2127
2128	/* First try MSI/X */
2129	rid = PCIR_BAR(IXL_BAR);
2130	pf->msix_mem = bus_alloc_resource_any(dev,
2131	    SYS_RES_MEMORY, &rid, RF_ACTIVE);
2132       	if (!pf->msix_mem) {
2133		/* May not be enabled */
2134		device_printf(pf->dev,
2135		    "Unable to map MSIX table \n");
2136		goto msi;
2137	}
2138
2139	available = pci_msix_count(dev);
2140	if (available == 0) { /* system has msix disabled */
2141		bus_release_resource(dev, SYS_RES_MEMORY,
2142		    rid, pf->msix_mem);
2143		pf->msix_mem = NULL;
2144		goto msi;
2145	}
2146
2147	/* Figure out a reasonable auto config value */
2148	queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
2149
2150	/* Override with hardcoded value if sane */
2151	if ((ixl_max_queues != 0) && (ixl_max_queues <= queues))
2152		queues = ixl_max_queues;
2153
2154#ifdef  RSS
2155	/* If we're doing RSS, clamp at the number of RSS buckets */
2156	if (queues > rss_getnumbuckets())
2157		queues = rss_getnumbuckets();
2158#endif
2159
2160	/*
2161	** Want one vector (RX/TX pair) per queue
2162	** plus an additional for the admin queue.
2163	*/
2164	want = queues + 1;
2165	if (want <= available)	/* Have enough */
2166		vectors = want;
2167	else {
2168               	device_printf(pf->dev,
2169		    "MSIX Configuration Problem, "
2170		    "%d vectors available but %d wanted!\n",
2171		    available, want);
2172		return (0); /* Will go to Legacy setup */
2173	}
2174
2175	if (pci_alloc_msix(dev, &vectors) == 0) {
2176               	device_printf(pf->dev,
2177		    "Using MSIX interrupts with %d vectors\n", vectors);
2178		pf->msix = vectors;
2179		pf->vsi.num_queues = queues;
2180#ifdef RSS
2181		/*
2182		 * If we're doing RSS, the number of queues needs to
2183		 * match the number of RSS buckets that are configured.
2184		 *
2185		 * + If there's more queues than RSS buckets, we'll end
2186		 *   up with queues that get no traffic.
2187		 *
2188		 * + If there's more RSS buckets than queues, we'll end
2189		 *   up having multiple RSS buckets map to the same queue,
2190		 *   so there'll be some contention.
2191		 */
2192		if (queues != rss_getnumbuckets()) {
2193			device_printf(dev,
2194			    "%s: queues (%d) != RSS buckets (%d)"
2195			    "; performance will be impacted.\n",
2196			    __func__, queues, rss_getnumbuckets());
2197		}
2198#endif
2199		return (vectors);
2200	}
2201msi:
2202       	vectors = pci_msi_count(dev);
2203	pf->vsi.num_queues = 1;
2204	pf->msix = 1;
2205	ixl_max_queues = 1;
2206	ixl_enable_msix = 0;
2207       	if (vectors == 1 && pci_alloc_msi(dev, &vectors) == 0)
2208               	device_printf(pf->dev,"Using an MSI interrupt\n");
2209	else {
2210		pf->msix = 0;
2211               	device_printf(pf->dev,"Using a Legacy interrupt\n");
2212	}
2213	return (vectors);
2214}
2215
2216
2217/*
2218 * Plumb MSI/X vectors
2219 */
2220static void
2221ixl_configure_msix(struct ixl_pf *pf)
2222{
2223	struct i40e_hw	*hw = &pf->hw;
2224	struct ixl_vsi *vsi = &pf->vsi;
2225	u32		reg;
2226	u16		vector = 1;
2227
2228	/* First set up the adminq - vector 0 */
2229	wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
2230	rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
2231
2232	reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
2233	    I40E_PFINT_ICR0_ENA_GRST_MASK |
2234	    I40E_PFINT_ICR0_HMC_ERR_MASK |
2235	    I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
2236	    I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
2237	    I40E_PFINT_ICR0_ENA_VFLR_MASK |
2238	    I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK;
2239	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2240
2241	wr32(hw, I40E_PFINT_LNKLST0, 0x7FF);
2242	wr32(hw, I40E_PFINT_ITR0(IXL_RX_ITR), 0x003E);
2243
2244	wr32(hw, I40E_PFINT_DYN_CTL0,
2245	    I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
2246	    I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
2247
2248	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2249
2250	/* Next configure the queues */
2251	for (int i = 0; i < vsi->num_queues; i++, vector++) {
2252		wr32(hw, I40E_PFINT_DYN_CTLN(i), i);
2253		wr32(hw, I40E_PFINT_LNKLSTN(i), i);
2254
2255		reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
2256		(IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
2257		(vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
2258		(i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
2259		(I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
2260		wr32(hw, I40E_QINT_RQCTL(i), reg);
2261
2262		reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
2263		(IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
2264		(vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
2265		((i+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
2266		(I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2267		if (i == (vsi->num_queues - 1))
2268			reg |= (IXL_QUEUE_EOL
2269			    << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2270		wr32(hw, I40E_QINT_TQCTL(i), reg);
2271	}
2272}
2273
2274/*
2275 * Configure for MSI single vector operation
2276 */
2277static void
2278ixl_configure_legacy(struct ixl_pf *pf)
2279{
2280	struct i40e_hw	*hw = &pf->hw;
2281	u32		reg;
2282
2283
2284	wr32(hw, I40E_PFINT_ITR0(0), 0);
2285	wr32(hw, I40E_PFINT_ITR0(1), 0);
2286
2287
2288	/* Setup "other" causes */
2289	reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK
2290	    | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK
2291	    | I40E_PFINT_ICR0_ENA_GRST_MASK
2292	    | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK
2293	    | I40E_PFINT_ICR0_ENA_GPIO_MASK
2294	    | I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK
2295	    | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK
2296	    | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK
2297	    | I40E_PFINT_ICR0_ENA_VFLR_MASK
2298	    | I40E_PFINT_ICR0_ENA_ADMINQ_MASK
2299	    ;
2300	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
2301
2302	/* SW_ITR_IDX = 0, but don't change INTENA */
2303	wr32(hw, I40E_PFINT_DYN_CTL0,
2304	    I40E_PFINT_DYN_CTLN_SW_ITR_INDX_MASK |
2305	    I40E_PFINT_DYN_CTLN_INTENA_MSK_MASK);
2306	/* SW_ITR_IDX = 0, OTHER_ITR_IDX = 0 */
2307	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
2308
2309	/* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
2310	wr32(hw, I40E_PFINT_LNKLST0, 0);
2311
2312	/* Associate the queue pair to the vector and enable the q int */
2313	reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK
2314	    | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)
2315	    | (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
2316	wr32(hw, I40E_QINT_RQCTL(0), reg);
2317
2318	reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK
2319	    | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)
2320	    | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
2321	wr32(hw, I40E_QINT_TQCTL(0), reg);
2322
2323	/* Next enable the queue pair */
2324	reg = rd32(hw, I40E_QTX_ENA(0));
2325	reg |= I40E_QTX_ENA_QENA_REQ_MASK;
2326	wr32(hw, I40E_QTX_ENA(0), reg);
2327
2328	reg = rd32(hw, I40E_QRX_ENA(0));
2329	reg |= I40E_QRX_ENA_QENA_REQ_MASK;
2330	wr32(hw, I40E_QRX_ENA(0), reg);
2331}
2332
2333
2334/*
2335 * Set the Initial ITR state
2336 */
2337static void
2338ixl_configure_itr(struct ixl_pf *pf)
2339{
2340	struct i40e_hw		*hw = &pf->hw;
2341	struct ixl_vsi		*vsi = &pf->vsi;
2342	struct ixl_queue	*que = vsi->queues;
2343
2344	vsi->rx_itr_setting = ixl_rx_itr;
2345	if (ixl_dynamic_rx_itr)
2346		vsi->rx_itr_setting |= IXL_ITR_DYNAMIC;
2347	vsi->tx_itr_setting = ixl_tx_itr;
2348	if (ixl_dynamic_tx_itr)
2349		vsi->tx_itr_setting |= IXL_ITR_DYNAMIC;
2350
2351	for (int i = 0; i < vsi->num_queues; i++, que++) {
2352		struct tx_ring	*txr = &que->txr;
2353		struct rx_ring 	*rxr = &que->rxr;
2354
2355		wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i),
2356		    vsi->rx_itr_setting);
2357		rxr->itr = vsi->rx_itr_setting;
2358		rxr->latency = IXL_AVE_LATENCY;
2359		wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i),
2360		    vsi->tx_itr_setting);
2361		txr->itr = vsi->tx_itr_setting;
2362		txr->latency = IXL_AVE_LATENCY;
2363	}
2364}
2365
2366
2367static int
2368ixl_allocate_pci_resources(struct ixl_pf *pf)
2369{
2370	int             rid;
2371	device_t        dev = pf->dev;
2372
2373	rid = PCIR_BAR(0);
2374	pf->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2375	    &rid, RF_ACTIVE);
2376
2377	if (!(pf->pci_mem)) {
2378		device_printf(dev,"Unable to allocate bus resource: memory\n");
2379		return (ENXIO);
2380	}
2381
2382	pf->osdep.mem_bus_space_tag =
2383		rman_get_bustag(pf->pci_mem);
2384	pf->osdep.mem_bus_space_handle =
2385		rman_get_bushandle(pf->pci_mem);
2386	pf->osdep.mem_bus_space_size = rman_get_size(pf->pci_mem);
2387	pf->osdep.flush_reg = I40E_GLGEN_STAT;
2388	pf->hw.hw_addr = (u8 *) &pf->osdep.mem_bus_space_handle;
2389
2390	pf->hw.back = &pf->osdep;
2391
2392	/*
2393	** Now setup MSI or MSI/X, should
2394	** return us the number of supported
2395	** vectors. (Will be 1 for MSI)
2396	*/
2397	pf->msix = ixl_init_msix(pf);
2398	return (0);
2399}
2400
2401static void
2402ixl_free_pci_resources(struct ixl_pf * pf)
2403{
2404	struct ixl_vsi		*vsi = &pf->vsi;
2405	struct ixl_queue	*que = vsi->queues;
2406	device_t		dev = pf->dev;
2407	int			rid, memrid;
2408
2409	memrid = PCIR_BAR(IXL_BAR);
2410
2411	/* We may get here before stations are setup */
2412	if ((!ixl_enable_msix) || (que == NULL))
2413		goto early;
2414
2415	/*
2416	**  Release all msix VSI resources:
2417	*/
2418	for (int i = 0; i < vsi->num_queues; i++, que++) {
2419		rid = que->msix + 1;
2420		if (que->tag != NULL) {
2421			bus_teardown_intr(dev, que->res, que->tag);
2422			que->tag = NULL;
2423		}
2424		if (que->res != NULL)
2425			bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
2426	}
2427
2428early:
2429	/* Clean the AdminQ interrupt last */
2430	if (pf->admvec) /* we are doing MSIX */
2431		rid = pf->admvec + 1;
2432	else
2433		(pf->msix != 0) ? (rid = 1):(rid = 0);
2434
2435	if (pf->tag != NULL) {
2436		bus_teardown_intr(dev, pf->res, pf->tag);
2437		pf->tag = NULL;
2438	}
2439	if (pf->res != NULL)
2440		bus_release_resource(dev, SYS_RES_IRQ, rid, pf->res);
2441
2442	if (pf->msix)
2443		pci_release_msi(dev);
2444
2445	if (pf->msix_mem != NULL)
2446		bus_release_resource(dev, SYS_RES_MEMORY,
2447		    memrid, pf->msix_mem);
2448
2449	if (pf->pci_mem != NULL)
2450		bus_release_resource(dev, SYS_RES_MEMORY,
2451		    PCIR_BAR(0), pf->pci_mem);
2452
2453	return;
2454}
2455
2456static void
2457ixl_add_ifmedia(struct ixl_vsi *vsi, u32 phy_type)
2458{
2459	/* Display supported media types */
2460	if (phy_type & (1 << I40E_PHY_TYPE_100BASE_TX))
2461		ifmedia_add(&vsi->media, IFM_ETHER | IFM_100_TX, 0, NULL);
2462
2463	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_T))
2464		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2465	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_SX))
2466		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2467	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_LX))
2468		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_LX, 0, NULL);
2469
2470	if (phy_type & (1 << I40E_PHY_TYPE_XAUI) ||
2471	    phy_type & (1 << I40E_PHY_TYPE_XFI) ||
2472	    phy_type & (1 << I40E_PHY_TYPE_10GBASE_SFPP_CU))
2473		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2474
2475	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_SR))
2476		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2477	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_LR))
2478		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
2479	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_T))
2480		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_T, 0, NULL);
2481
2482	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4) ||
2483	    phy_type & (1 << I40E_PHY_TYPE_40GBASE_CR4_CU) ||
2484	    phy_type & (1 << I40E_PHY_TYPE_40GBASE_AOC) ||
2485	    phy_type & (1 << I40E_PHY_TYPE_XLAUI) ||
2486	    phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2487		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2488	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_SR4))
2489		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2490	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_LR4))
2491		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_LR4, 0, NULL);
2492
2493#ifndef IFM_ETH_XTYPE
2494	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2495		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_CX, 0, NULL);
2496
2497	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU) ||
2498	    phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1) ||
2499	    phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC) ||
2500	    phy_type & (1 << I40E_PHY_TYPE_SFI))
2501		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX, 0, NULL);
2502	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2503		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2504	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2505		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2506
2507	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2508		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_SR4, 0, NULL);
2509	if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2510		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_CR4, 0, NULL);
2511#else
2512	if (phy_type & (1 << I40E_PHY_TYPE_1000BASE_KX))
2513		ifmedia_add(&vsi->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
2514
2515	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1_CU)
2516	    || phy_type & (1 << I40E_PHY_TYPE_10GBASE_CR1))
2517		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_CR1, 0, NULL);
2518	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_AOC))
2519		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_TWINAX_LONG, 0, NULL);
2520	if (phy_type & (1 << I40E_PHY_TYPE_SFI))
2521		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_SFI, 0, NULL);
2522	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KX4))
2523		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
2524	if (phy_type & (1 << I40E_PHY_TYPE_10GBASE_KR))
2525		ifmedia_add(&vsi->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
2526
2527	if (phy_type & (1 << I40E_PHY_TYPE_20GBASE_KR2))
2528		ifmedia_add(&vsi->media, IFM_ETHER | IFM_20G_KR2, 0, NULL);
2529
2530	if (phy_type & (1 << I40E_PHY_TYPE_40GBASE_KR4))
2531		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_KR4, 0, NULL);
2532	if (phy_type & (1 << I40E_PHY_TYPE_XLPPI))
2533		ifmedia_add(&vsi->media, IFM_ETHER | IFM_40G_XLPPI, 0, NULL);
2534#endif
2535}
2536
2537/*********************************************************************
2538 *
2539 *  Setup networking device structure and register an interface.
2540 *
2541 **********************************************************************/
2542static int
2543ixl_setup_interface(device_t dev, struct ixl_vsi *vsi)
2544{
2545	struct ifnet		*ifp;
2546	struct i40e_hw		*hw = vsi->hw;
2547	struct ixl_queue	*que = vsi->queues;
2548	struct i40e_aq_get_phy_abilities_resp abilities;
2549	enum i40e_status_code aq_error = 0;
2550
2551	INIT_DEBUGOUT("ixl_setup_interface: begin");
2552
2553	ifp = vsi->ifp = if_alloc(IFT_ETHER);
2554	if (ifp == NULL) {
2555		device_printf(dev, "can not allocate ifnet structure\n");
2556		return (-1);
2557	}
2558	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2559	ifp->if_mtu = ETHERMTU;
2560	ifp->if_baudrate = IF_Gbps(40);
2561	ifp->if_init = ixl_init;
2562	ifp->if_softc = vsi;
2563	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2564	ifp->if_ioctl = ixl_ioctl;
2565
2566#if __FreeBSD_version >= 1100036
2567	if_setgetcounterfn(ifp, ixl_get_counter);
2568#endif
2569
2570	ifp->if_transmit = ixl_mq_start;
2571
2572	ifp->if_qflush = ixl_qflush;
2573
2574	ifp->if_snd.ifq_maxlen = que->num_desc - 2;
2575
2576	vsi->max_frame_size =
2577	    ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
2578	    + ETHER_VLAN_ENCAP_LEN;
2579
2580	/*
2581	 * Tell the upper layer(s) we support long frames.
2582	 */
2583	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
2584
2585	ifp->if_capabilities |= IFCAP_HWCSUM;
2586	ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
2587	ifp->if_capabilities |= IFCAP_TSO;
2588	ifp->if_capabilities |= IFCAP_JUMBO_MTU;
2589	ifp->if_capabilities |= IFCAP_LRO;
2590
2591	/* VLAN capabilties */
2592	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
2593			     |  IFCAP_VLAN_HWTSO
2594			     |  IFCAP_VLAN_MTU
2595			     |  IFCAP_VLAN_HWCSUM;
2596	ifp->if_capenable = ifp->if_capabilities;
2597
2598	/*
2599	** Don't turn this on by default, if vlans are
2600	** created on another pseudo device (eg. lagg)
2601	** then vlan events are not passed thru, breaking
2602	** operation, but with HW FILTER off it works. If
2603	** using vlans directly on the ixl driver you can
2604	** enable this and get full hardware tag filtering.
2605	*/
2606	ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
2607
2608	/*
2609	 * Specify the media types supported by this adapter and register
2610	 * callbacks to update media and link information
2611	 */
2612	ifmedia_init(&vsi->media, IFM_IMASK, ixl_media_change,
2613		     ixl_media_status);
2614
2615	aq_error = i40e_aq_get_phy_capabilities(hw,
2616	    FALSE, TRUE, &abilities, NULL);
2617	/* May need delay to detect fiber correctly */
2618	if (aq_error == I40E_ERR_UNKNOWN_PHY) {
2619		i40e_msec_delay(200);
2620		aq_error = i40e_aq_get_phy_capabilities(hw, FALSE,
2621		    TRUE, &abilities, NULL);
2622	}
2623	if (aq_error) {
2624		if (aq_error == I40E_ERR_UNKNOWN_PHY)
2625			device_printf(dev, "Unknown PHY type detected!\n");
2626		else
2627			device_printf(dev,
2628			    "Error getting supported media types, err %d,"
2629			    " AQ error %d\n", aq_error, hw->aq.asq_last_status);
2630		return (0);
2631	}
2632
2633	ixl_add_ifmedia(vsi, abilities.phy_type);
2634
2635	/* Use autoselect media by default */
2636	ifmedia_add(&vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2637	ifmedia_set(&vsi->media, IFM_ETHER | IFM_AUTO);
2638
2639	ether_ifattach(ifp, hw->mac.addr);
2640
2641	return (0);
2642}
2643
2644/*
2645** Run when the Admin Queue gets a
2646** link transition interrupt.
2647*/
2648static void
2649ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e)
2650{
2651	struct i40e_hw	*hw = &pf->hw;
2652	struct i40e_aqc_get_link_status *status =
2653	    (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
2654	bool check;
2655
2656	hw->phy.get_link_info = TRUE;
2657	i40e_get_link_status(hw, &check);
2658	pf->link_up = check;
2659#ifdef IXL_DEBUG
2660	printf("Link is %s\n", check ? "up":"down");
2661#endif
2662	/* Report if Unqualified modules are found */
2663	if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
2664	    (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
2665	    (!(status->link_info & I40E_AQ_LINK_UP)))
2666		device_printf(pf->dev, "Link failed because "
2667		    "an unqualified module was detected\n");
2668
2669	return;
2670}
2671
2672/*********************************************************************
2673 *
2674 *  Get Firmware Switch configuration
2675 *	- this will need to be more robust when more complex
2676 *	  switch configurations are enabled.
2677 *
2678 **********************************************************************/
2679static int
2680ixl_switch_config(struct ixl_pf *pf)
2681{
2682	struct i40e_hw	*hw = &pf->hw;
2683	struct ixl_vsi	*vsi = &pf->vsi;
2684	device_t 	dev = vsi->dev;
2685	struct i40e_aqc_get_switch_config_resp *sw_config;
2686	u8	aq_buf[I40E_AQ_LARGE_BUF];
2687	int	ret;
2688	u16	next = 0;
2689
2690	memset(&aq_buf, 0, sizeof(aq_buf));
2691	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
2692	ret = i40e_aq_get_switch_config(hw, sw_config,
2693	    sizeof(aq_buf), &next, NULL);
2694	if (ret) {
2695		device_printf(dev,"aq_get_switch_config failed (ret=%d)!!\n",
2696		    ret);
2697		return (ret);
2698	}
2699#ifdef IXL_DEBUG
2700	device_printf(dev,
2701	    "Switch config: header reported: %d in structure, %d total\n",
2702    	    sw_config->header.num_reported, sw_config->header.num_total);
2703	for (int i = 0; i < sw_config->header.num_reported; i++) {
2704		device_printf(dev,
2705		    "%d: type=%d seid=%d uplink=%d downlink=%d\n", i,
2706		    sw_config->element[i].element_type,
2707		    sw_config->element[i].seid,
2708		    sw_config->element[i].uplink_seid,
2709		    sw_config->element[i].downlink_seid);
2710	}
2711#endif
2712	/* Simplified due to a single VSI at the moment */
2713	vsi->uplink_seid = sw_config->element[0].uplink_seid;
2714	vsi->downlink_seid = sw_config->element[0].downlink_seid;
2715	vsi->seid = sw_config->element[0].seid;
2716	return (ret);
2717}
2718
2719/*********************************************************************
2720 *
2721 *  Initialize the VSI:  this handles contexts, which means things
2722 *  			 like the number of descriptors, buffer size,
2723 *			 plus we init the rings thru this function.
2724 *
2725 **********************************************************************/
2726static int
2727ixl_initialize_vsi(struct ixl_vsi *vsi)
2728{
2729	struct ixl_pf		*pf = vsi->back;
2730	struct ixl_queue	*que = vsi->queues;
2731	device_t		dev = vsi->dev;
2732	struct i40e_hw		*hw = vsi->hw;
2733	struct i40e_vsi_context	ctxt;
2734	int			err = 0;
2735
2736	memset(&ctxt, 0, sizeof(ctxt));
2737	ctxt.seid = vsi->seid;
2738	if (pf->veb_seid != 0)
2739		ctxt.uplink_seid = pf->veb_seid;
2740	ctxt.pf_num = hw->pf_id;
2741	err = i40e_aq_get_vsi_params(hw, &ctxt, NULL);
2742	if (err) {
2743		device_printf(dev,"get vsi params failed %x!!\n", err);
2744		return (err);
2745	}
2746#ifdef IXL_DEBUG
2747	printf("get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, "
2748	    "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, "
2749	    "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid,
2750	    ctxt.uplink_seid, ctxt.vsi_number,
2751	    ctxt.vsis_allocated, ctxt.vsis_unallocated,
2752	    ctxt.flags, ctxt.pf_num, ctxt.vf_num,
2753	    ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits);
2754#endif
2755	/*
2756	** Set the queue and traffic class bits
2757	**  - when multiple traffic classes are supported
2758	**    this will need to be more robust.
2759	*/
2760	ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2761	ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG;
2762	/* In contig mode, que_mapping[0] is first queue index used by this VSI */
2763	ctxt.info.queue_mapping[0] = 0;
2764	/*
2765	 * This VSI will only use traffic class 0; start traffic class 0's
2766	 * queue allocation at queue 0, and assign it 64 (2^6) queues (though
2767	 * the driver may not use all of them).
2768	 */
2769	ctxt.info.tc_mapping[0] = ((0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT)
2770	    & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) |
2771	    ((6 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT)
2772	    & I40E_AQ_VSI_TC_QUE_NUMBER_MASK);
2773
2774	/* Set VLAN receive stripping mode */
2775	ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID;
2776	ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL;
2777	if (vsi->ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
2778	    ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2779	else
2780	    ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2781
2782	/* Keep copy of VSI info in VSI for statistic counters */
2783	memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info));
2784
2785	/* Reset VSI statistics */
2786	ixl_vsi_reset_stats(vsi);
2787	vsi->hw_filters_add = 0;
2788	vsi->hw_filters_del = 0;
2789
2790	ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF);
2791
2792	err = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
2793	if (err) {
2794		device_printf(dev,"update vsi params failed %x!!\n",
2795		   hw->aq.asq_last_status);
2796		return (err);
2797	}
2798
2799	for (int i = 0; i < vsi->num_queues; i++, que++) {
2800		struct tx_ring		*txr = &que->txr;
2801		struct rx_ring 		*rxr = &que->rxr;
2802		struct i40e_hmc_obj_txq tctx;
2803		struct i40e_hmc_obj_rxq rctx;
2804		u32			txctl;
2805		u16			size;
2806
2807
2808		/* Setup the HMC TX Context  */
2809		size = que->num_desc * sizeof(struct i40e_tx_desc);
2810		memset(&tctx, 0, sizeof(struct i40e_hmc_obj_txq));
2811		tctx.new_context = 1;
2812		tctx.base = (txr->dma.pa/IXL_TX_CTX_BASE_UNITS);
2813		tctx.qlen = que->num_desc;
2814		tctx.fc_ena = 0;
2815		tctx.rdylist = vsi->info.qs_handle[0]; /* index is TC */
2816		/* Enable HEAD writeback */
2817		tctx.head_wb_ena = 1;
2818		tctx.head_wb_addr = txr->dma.pa +
2819		    (que->num_desc * sizeof(struct i40e_tx_desc));
2820		tctx.rdylist_act = 0;
2821		err = i40e_clear_lan_tx_queue_context(hw, i);
2822		if (err) {
2823			device_printf(dev, "Unable to clear TX context\n");
2824			break;
2825		}
2826		err = i40e_set_lan_tx_queue_context(hw, i, &tctx);
2827		if (err) {
2828			device_printf(dev, "Unable to set TX context\n");
2829			break;
2830		}
2831		/* Associate the ring with this PF */
2832		txctl = I40E_QTX_CTL_PF_QUEUE;
2833		txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2834		    I40E_QTX_CTL_PF_INDX_MASK);
2835		wr32(hw, I40E_QTX_CTL(i), txctl);
2836		ixl_flush(hw);
2837
2838		/* Do ring (re)init */
2839		ixl_init_tx_ring(que);
2840
2841		/* Next setup the HMC RX Context  */
2842		if (vsi->max_frame_size <= MCLBYTES)
2843			rxr->mbuf_sz = MCLBYTES;
2844		else
2845			rxr->mbuf_sz = MJUMPAGESIZE;
2846
2847		u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len;
2848
2849		/* Set up an RX context for the HMC */
2850		memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq));
2851		rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT;
2852		/* ignore header split for now */
2853		rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT;
2854		rctx.rxmax = (vsi->max_frame_size < max_rxmax) ?
2855		    vsi->max_frame_size : max_rxmax;
2856		rctx.dtype = 0;
2857		rctx.dsize = 1;	/* do 32byte descriptors */
2858		rctx.hsplit_0 = 0;  /* no HDR split initially */
2859		rctx.base = (rxr->dma.pa/IXL_RX_CTX_BASE_UNITS);
2860		rctx.qlen = que->num_desc;
2861		rctx.tphrdesc_ena = 1;
2862		rctx.tphwdesc_ena = 1;
2863		rctx.tphdata_ena = 0;
2864		rctx.tphhead_ena = 0;
2865		rctx.lrxqthresh = 2;
2866		rctx.crcstrip = 1;
2867		rctx.l2tsel = 1;
2868		rctx.showiv = 1;
2869		rctx.fc_ena = 0;
2870		rctx.prefena = 1;
2871
2872		err = i40e_clear_lan_rx_queue_context(hw, i);
2873		if (err) {
2874			device_printf(dev,
2875			    "Unable to clear RX context %d\n", i);
2876			break;
2877		}
2878		err = i40e_set_lan_rx_queue_context(hw, i, &rctx);
2879		if (err) {
2880			device_printf(dev, "Unable to set RX context %d\n", i);
2881			break;
2882		}
2883		err = ixl_init_rx_ring(que);
2884		if (err) {
2885			device_printf(dev, "Fail in init_rx_ring %d\n", i);
2886			break;
2887		}
2888		wr32(vsi->hw, I40E_QRX_TAIL(que->me), 0);
2889#ifdef DEV_NETMAP
2890		/* preserve queue */
2891		if (vsi->ifp->if_capenable & IFCAP_NETMAP) {
2892			struct netmap_adapter *na = NA(vsi->ifp);
2893			struct netmap_kring *kring = &na->rx_rings[i];
2894			int t = na->num_rx_desc - 1 - nm_kr_rxspace(kring);
2895			wr32(vsi->hw, I40E_QRX_TAIL(que->me), t);
2896		} else
2897#endif /* DEV_NETMAP */
2898		wr32(vsi->hw, I40E_QRX_TAIL(que->me), que->num_desc - 1);
2899	}
2900	return (err);
2901}
2902
2903
2904/*********************************************************************
2905 *
2906 *  Free all VSI structs.
2907 *
2908 **********************************************************************/
2909void
2910ixl_free_vsi(struct ixl_vsi *vsi)
2911{
2912	struct ixl_pf		*pf = (struct ixl_pf *)vsi->back;
2913	struct ixl_queue	*que = vsi->queues;
2914
2915	/* Free station queues */
2916	for (int i = 0; i < vsi->num_queues; i++, que++) {
2917		struct tx_ring *txr = &que->txr;
2918		struct rx_ring *rxr = &que->rxr;
2919
2920		if (!mtx_initialized(&txr->mtx)) /* uninitialized */
2921			continue;
2922		IXL_TX_LOCK(txr);
2923		ixl_free_que_tx(que);
2924		if (txr->base)
2925			i40e_free_dma_mem(&pf->hw, &txr->dma);
2926		IXL_TX_UNLOCK(txr);
2927		IXL_TX_LOCK_DESTROY(txr);
2928
2929		if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
2930			continue;
2931		IXL_RX_LOCK(rxr);
2932		ixl_free_que_rx(que);
2933		if (rxr->base)
2934			i40e_free_dma_mem(&pf->hw, &rxr->dma);
2935		IXL_RX_UNLOCK(rxr);
2936		IXL_RX_LOCK_DESTROY(rxr);
2937
2938	}
2939	free(vsi->queues, M_DEVBUF);
2940
2941	/* Free VSI filter list */
2942	ixl_free_mac_filters(vsi);
2943}
2944
2945static void
2946ixl_free_mac_filters(struct ixl_vsi *vsi)
2947{
2948	struct ixl_mac_filter *f;
2949
2950	while (!SLIST_EMPTY(&vsi->ftl)) {
2951		f = SLIST_FIRST(&vsi->ftl);
2952		SLIST_REMOVE_HEAD(&vsi->ftl, next);
2953		free(f, M_DEVBUF);
2954	}
2955}
2956
2957
2958/*********************************************************************
2959 *
2960 *  Allocate memory for the VSI (virtual station interface) and their
2961 *  associated queues, rings and the descriptors associated with each,
2962 *  called only once at attach.
2963 *
2964 **********************************************************************/
2965static int
2966ixl_setup_stations(struct ixl_pf *pf)
2967{
2968	device_t		dev = pf->dev;
2969	struct ixl_vsi		*vsi;
2970	struct ixl_queue	*que;
2971	struct tx_ring		*txr;
2972	struct rx_ring		*rxr;
2973	int 			rsize, tsize;
2974	int			error = I40E_SUCCESS;
2975
2976	vsi = &pf->vsi;
2977	vsi->back = (void *)pf;
2978	vsi->hw = &pf->hw;
2979	vsi->id = 0;
2980	vsi->num_vlans = 0;
2981	vsi->back = pf;
2982
2983	/* Get memory for the station queues */
2984        if (!(vsi->queues =
2985            (struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
2986            vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
2987                device_printf(dev, "Unable to allocate queue memory\n");
2988                error = ENOMEM;
2989                goto early;
2990        }
2991
2992	for (int i = 0; i < vsi->num_queues; i++) {
2993		que = &vsi->queues[i];
2994		que->num_desc = ixl_ringsz;
2995		que->me = i;
2996		que->vsi = vsi;
2997		/* mark the queue as active */
2998		vsi->active_queues |= (u64)1 << que->me;
2999		txr = &que->txr;
3000		txr->que = que;
3001		txr->tail = I40E_QTX_TAIL(que->me);
3002
3003		/* Initialize the TX lock */
3004		snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
3005		    device_get_nameunit(dev), que->me);
3006		mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
3007		/* Create the TX descriptor ring */
3008		tsize = roundup2((que->num_desc *
3009		    sizeof(struct i40e_tx_desc)) +
3010		    sizeof(u32), DBA_ALIGN);
3011		if (i40e_allocate_dma_mem(&pf->hw,
3012		    &txr->dma, i40e_mem_reserved, tsize, DBA_ALIGN)) {
3013			device_printf(dev,
3014			    "Unable to allocate TX Descriptor memory\n");
3015			error = ENOMEM;
3016			goto fail;
3017		}
3018		txr->base = (struct i40e_tx_desc *)txr->dma.va;
3019		bzero((void *)txr->base, tsize);
3020       		/* Now allocate transmit soft structs for the ring */
3021       		if (ixl_allocate_tx_data(que)) {
3022			device_printf(dev,
3023			    "Critical Failure setting up TX structures\n");
3024			error = ENOMEM;
3025			goto fail;
3026       		}
3027		/* Allocate a buf ring */
3028		txr->br = buf_ring_alloc(4096, M_DEVBUF,
3029		    M_WAITOK, &txr->mtx);
3030		if (txr->br == NULL) {
3031			device_printf(dev,
3032			    "Critical Failure setting up TX buf ring\n");
3033			error = ENOMEM;
3034			goto fail;
3035       		}
3036
3037		/*
3038		 * Next the RX queues...
3039		 */
3040		rsize = roundup2(que->num_desc *
3041		    sizeof(union i40e_rx_desc), DBA_ALIGN);
3042		rxr = &que->rxr;
3043		rxr->que = que;
3044		rxr->tail = I40E_QRX_TAIL(que->me);
3045
3046		/* Initialize the RX side lock */
3047		snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
3048		    device_get_nameunit(dev), que->me);
3049		mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
3050
3051		if (i40e_allocate_dma_mem(&pf->hw,
3052		    &rxr->dma, i40e_mem_reserved, rsize, 4096)) {
3053			device_printf(dev,
3054			    "Unable to allocate RX Descriptor memory\n");
3055			error = ENOMEM;
3056			goto fail;
3057		}
3058		rxr->base = (union i40e_rx_desc *)rxr->dma.va;
3059		bzero((void *)rxr->base, rsize);
3060
3061        	/* Allocate receive soft structs for the ring*/
3062		if (ixl_allocate_rx_data(que)) {
3063			device_printf(dev,
3064			    "Critical Failure setting up receive structs\n");
3065			error = ENOMEM;
3066			goto fail;
3067		}
3068	}
3069
3070	return (0);
3071
3072fail:
3073	for (int i = 0; i < vsi->num_queues; i++) {
3074		que = &vsi->queues[i];
3075		rxr = &que->rxr;
3076		txr = &que->txr;
3077		if (rxr->base)
3078			i40e_free_dma_mem(&pf->hw, &rxr->dma);
3079		if (txr->base)
3080			i40e_free_dma_mem(&pf->hw, &txr->dma);
3081	}
3082
3083early:
3084	return (error);
3085}
3086
3087/*
3088** Provide a update to the queue RX
3089** interrupt moderation value.
3090*/
3091static void
3092ixl_set_queue_rx_itr(struct ixl_queue *que)
3093{
3094	struct ixl_vsi	*vsi = que->vsi;
3095	struct i40e_hw	*hw = vsi->hw;
3096	struct rx_ring	*rxr = &que->rxr;
3097	u16		rx_itr;
3098	u16		rx_latency = 0;
3099	int		rx_bytes;
3100
3101
3102	/* Idle, do nothing */
3103	if (rxr->bytes == 0)
3104		return;
3105
3106	if (ixl_dynamic_rx_itr) {
3107		rx_bytes = rxr->bytes/rxr->itr;
3108		rx_itr = rxr->itr;
3109
3110		/* Adjust latency range */
3111		switch (rxr->latency) {
3112		case IXL_LOW_LATENCY:
3113			if (rx_bytes > 10) {
3114				rx_latency = IXL_AVE_LATENCY;
3115				rx_itr = IXL_ITR_20K;
3116			}
3117			break;
3118		case IXL_AVE_LATENCY:
3119			if (rx_bytes > 20) {
3120				rx_latency = IXL_BULK_LATENCY;
3121				rx_itr = IXL_ITR_8K;
3122			} else if (rx_bytes <= 10) {
3123				rx_latency = IXL_LOW_LATENCY;
3124				rx_itr = IXL_ITR_100K;
3125			}
3126			break;
3127		case IXL_BULK_LATENCY:
3128			if (rx_bytes <= 20) {
3129				rx_latency = IXL_AVE_LATENCY;
3130				rx_itr = IXL_ITR_20K;
3131			}
3132			break;
3133       		 }
3134
3135		rxr->latency = rx_latency;
3136
3137		if (rx_itr != rxr->itr) {
3138			/* do an exponential smoothing */
3139			rx_itr = (10 * rx_itr * rxr->itr) /
3140			    ((9 * rx_itr) + rxr->itr);
3141			rxr->itr = rx_itr & IXL_MAX_ITR;
3142			wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3143			    que->me), rxr->itr);
3144		}
3145	} else { /* We may have have toggled to non-dynamic */
3146		if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
3147			vsi->rx_itr_setting = ixl_rx_itr;
3148		/* Update the hardware if needed */
3149		if (rxr->itr != vsi->rx_itr_setting) {
3150			rxr->itr = vsi->rx_itr_setting;
3151			wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR,
3152			    que->me), rxr->itr);
3153		}
3154	}
3155	rxr->bytes = 0;
3156	rxr->packets = 0;
3157	return;
3158}
3159
3160
3161/*
3162** Provide a update to the queue TX
3163** interrupt moderation value.
3164*/
3165static void
3166ixl_set_queue_tx_itr(struct ixl_queue *que)
3167{
3168	struct ixl_vsi	*vsi = que->vsi;
3169	struct i40e_hw	*hw = vsi->hw;
3170	struct tx_ring	*txr = &que->txr;
3171	u16		tx_itr;
3172	u16		tx_latency = 0;
3173	int		tx_bytes;
3174
3175
3176	/* Idle, do nothing */
3177	if (txr->bytes == 0)
3178		return;
3179
3180	if (ixl_dynamic_tx_itr) {
3181		tx_bytes = txr->bytes/txr->itr;
3182		tx_itr = txr->itr;
3183
3184		switch (txr->latency) {
3185		case IXL_LOW_LATENCY:
3186			if (tx_bytes > 10) {
3187				tx_latency = IXL_AVE_LATENCY;
3188				tx_itr = IXL_ITR_20K;
3189			}
3190			break;
3191		case IXL_AVE_LATENCY:
3192			if (tx_bytes > 20) {
3193				tx_latency = IXL_BULK_LATENCY;
3194				tx_itr = IXL_ITR_8K;
3195			} else if (tx_bytes <= 10) {
3196				tx_latency = IXL_LOW_LATENCY;
3197				tx_itr = IXL_ITR_100K;
3198			}
3199			break;
3200		case IXL_BULK_LATENCY:
3201			if (tx_bytes <= 20) {
3202				tx_latency = IXL_AVE_LATENCY;
3203				tx_itr = IXL_ITR_20K;
3204			}
3205			break;
3206		}
3207
3208		txr->latency = tx_latency;
3209
3210		if (tx_itr != txr->itr) {
3211       	         /* do an exponential smoothing */
3212			tx_itr = (10 * tx_itr * txr->itr) /
3213			    ((9 * tx_itr) + txr->itr);
3214			txr->itr = tx_itr & IXL_MAX_ITR;
3215			wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3216			    que->me), txr->itr);
3217		}
3218
3219	} else { /* We may have have toggled to non-dynamic */
3220		if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
3221			vsi->tx_itr_setting = ixl_tx_itr;
3222		/* Update the hardware if needed */
3223		if (txr->itr != vsi->tx_itr_setting) {
3224			txr->itr = vsi->tx_itr_setting;
3225			wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR,
3226			    que->me), txr->itr);
3227		}
3228	}
3229	txr->bytes = 0;
3230	txr->packets = 0;
3231	return;
3232}
3233
3234#define QUEUE_NAME_LEN 32
3235
3236static void
3237ixl_add_vsi_sysctls(struct ixl_pf *pf, struct ixl_vsi *vsi,
3238    struct sysctl_ctx_list *ctx, const char *sysctl_name)
3239{
3240	struct sysctl_oid *tree;
3241	struct sysctl_oid_list *child;
3242	struct sysctl_oid_list *vsi_list;
3243
3244	tree = device_get_sysctl_tree(pf->dev);
3245	child = SYSCTL_CHILDREN(tree);
3246	vsi->vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, sysctl_name,
3247				   CTLFLAG_RD, NULL, "VSI Number");
3248	vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);
3249
3250	ixl_add_sysctls_eth_stats(ctx, vsi_list, &vsi->eth_stats);
3251}
3252
3253static void
3254ixl_add_hw_stats(struct ixl_pf *pf)
3255{
3256	device_t dev = pf->dev;
3257	struct ixl_vsi *vsi = &pf->vsi;
3258	struct ixl_queue *queues = vsi->queues;
3259	struct i40e_hw_port_stats *pf_stats = &pf->stats;
3260
3261	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
3262	struct sysctl_oid *tree = device_get_sysctl_tree(dev);
3263	struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
3264	struct sysctl_oid_list *vsi_list;
3265
3266	struct sysctl_oid *queue_node;
3267	struct sysctl_oid_list *queue_list;
3268
3269	struct tx_ring *txr;
3270	struct rx_ring *rxr;
3271	char queue_namebuf[QUEUE_NAME_LEN];
3272
3273	/* Driver statistics */
3274	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
3275			CTLFLAG_RD, &pf->watchdog_events,
3276			"Watchdog timeouts");
3277	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
3278			CTLFLAG_RD, &pf->admin_irq,
3279			"Admin Queue IRQ Handled");
3280
3281	ixl_add_vsi_sysctls(pf, &pf->vsi, ctx, "pf");
3282	vsi_list = SYSCTL_CHILDREN(pf->vsi.vsi_node);
3283
3284	/* Queue statistics */
3285	for (int q = 0; q < vsi->num_queues; q++) {
3286		snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
3287		queue_node = SYSCTL_ADD_NODE(ctx, vsi_list,
3288		    OID_AUTO, queue_namebuf, CTLFLAG_RD, NULL, "Queue #");
3289		queue_list = SYSCTL_CHILDREN(queue_node);
3290
3291		txr = &(queues[q].txr);
3292		rxr = &(queues[q].rxr);
3293
3294		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
3295				CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
3296				"m_defrag() failed");
3297		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped",
3298				CTLFLAG_RD, &(queues[q].dropped_pkts),
3299				"Driver dropped packets");
3300		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
3301				CTLFLAG_RD, &(queues[q].irqs),
3302				"irqs on this queue");
3303		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
3304				CTLFLAG_RD, &(queues[q].tso),
3305				"TSO");
3306		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
3307				CTLFLAG_RD, &(queues[q].tx_dma_setup),
3308				"Driver tx dma failure in xmit");
3309		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
3310				CTLFLAG_RD, &(txr->no_desc),
3311				"Queue No Descriptor Available");
3312		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
3313				CTLFLAG_RD, &(txr->total_packets),
3314				"Queue Packets Transmitted");
3315		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
3316				CTLFLAG_RD, &(txr->tx_bytes),
3317				"Queue Bytes Transmitted");
3318		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
3319				CTLFLAG_RD, &(rxr->rx_packets),
3320				"Queue Packets Received");
3321		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
3322				CTLFLAG_RD, &(rxr->rx_bytes),
3323				"Queue Bytes Received");
3324	}
3325
3326	/* MAC stats */
3327	ixl_add_sysctls_mac_stats(ctx, child, pf_stats);
3328}
3329
3330static void
3331ixl_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
3332	struct sysctl_oid_list *child,
3333	struct i40e_eth_stats *eth_stats)
3334{
3335	struct ixl_sysctl_info ctls[] =
3336	{
3337		{&eth_stats->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
3338		{&eth_stats->rx_unicast, "ucast_pkts_rcvd",
3339			"Unicast Packets Received"},
3340		{&eth_stats->rx_multicast, "mcast_pkts_rcvd",
3341			"Multicast Packets Received"},
3342		{&eth_stats->rx_broadcast, "bcast_pkts_rcvd",
3343			"Broadcast Packets Received"},
3344		{&eth_stats->rx_discards, "rx_discards", "Discarded RX packets"},
3345		{&eth_stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
3346		{&eth_stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
3347		{&eth_stats->tx_multicast, "mcast_pkts_txd",
3348			"Multicast Packets Transmitted"},
3349		{&eth_stats->tx_broadcast, "bcast_pkts_txd",
3350			"Broadcast Packets Transmitted"},
3351		// end
3352		{0,0,0}
3353	};
3354
3355	struct ixl_sysctl_info *entry = ctls;
3356	while (entry->stat != 0)
3357	{
3358		SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
3359				CTLFLAG_RD, entry->stat,
3360				entry->description);
3361		entry++;
3362	}
3363}
3364
3365static void
3366ixl_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
3367	struct sysctl_oid_list *child,
3368	struct i40e_hw_port_stats *stats)
3369{
3370	struct sysctl_oid *stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac",
3371				    CTLFLAG_RD, NULL, "Mac Statistics");
3372	struct sysctl_oid_list *stat_list = SYSCTL_CHILDREN(stat_node);
3373
3374	struct i40e_eth_stats *eth_stats = &stats->eth;
3375	ixl_add_sysctls_eth_stats(ctx, stat_list, eth_stats);
3376
3377	struct ixl_sysctl_info ctls[] =
3378	{
3379		{&stats->crc_errors, "crc_errors", "CRC Errors"},
3380		{&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
3381		{&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
3382		{&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
3383		{&stats->rx_length_errors, "rx_length_errors", "Receive Length Errors"},
3384		/* Packet Reception Stats */
3385		{&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
3386		{&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
3387		{&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
3388		{&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
3389		{&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
3390		{&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
3391		{&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
3392		{&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
3393		{&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
3394		{&stats->rx_oversize, "rx_oversized", "Oversized packets received"},
3395		{&stats->rx_jabber, "rx_jabber", "Received Jabber"},
3396		{&stats->checksum_error, "checksum_errors", "Checksum Errors"},
3397		/* Packet Transmission Stats */
3398		{&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
3399		{&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
3400		{&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
3401		{&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
3402		{&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
3403		{&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
3404		{&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
3405		/* Flow control */
3406		{&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
3407		{&stats->link_xon_rx, "xon_recvd", "Link XON received"},
3408		{&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
3409		{&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
3410		/* End */
3411		{0,0,0}
3412	};
3413
3414	struct ixl_sysctl_info *entry = ctls;
3415	while (entry->stat != 0)
3416	{
3417		SYSCTL_ADD_UQUAD(ctx, stat_list, OID_AUTO, entry->name,
3418				CTLFLAG_RD, entry->stat,
3419				entry->description);
3420		entry++;
3421	}
3422}
3423
3424
3425/*
3426** ixl_config_rss - setup RSS
3427**  - note this is done for the single vsi
3428*/
3429static void ixl_config_rss(struct ixl_vsi *vsi)
3430{
3431	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
3432	struct i40e_hw	*hw = vsi->hw;
3433	u32		lut = 0;
3434	u64		set_hena = 0, hena;
3435	int		i, j, que_id;
3436#ifdef RSS
3437	u32		rss_hash_config;
3438	u32		rss_seed[IXL_KEYSZ];
3439#else
3440	u32             rss_seed[IXL_KEYSZ] = {0x41b01687,
3441			    0x183cfd8c, 0xce880440, 0x580cbc3c,
3442			    0x35897377, 0x328b25e1, 0x4fa98922,
3443			    0xb7d90c14, 0xd5bad70d, 0xcd15a2c1};
3444#endif
3445
3446#ifdef RSS
3447        /* Fetch the configured RSS key */
3448        rss_getkey((uint8_t *) &rss_seed);
3449#endif
3450
3451	/* Fill out hash function seed */
3452	for (i = 0; i < IXL_KEYSZ; i++)
3453                wr32(hw, I40E_PFQF_HKEY(i), rss_seed[i]);
3454
3455	/* Enable PCTYPES for RSS: */
3456#ifdef RSS
3457	rss_hash_config = rss_gethashconfig();
3458	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4)
3459                set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER);
3460	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4)
3461                set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP);
3462	if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4)
3463                set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP);
3464	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6)
3465                set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER);
3466	if (rss_hash_config & RSS_HASHTYPE_RSS_IPV6_EX)
3467		set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6);
3468	if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6)
3469                set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP);
3470        if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6)
3471                set_hena |= ((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP);
3472#else
3473	set_hena =
3474		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
3475		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
3476		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
3477		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
3478		((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
3479		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
3480		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
3481		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
3482		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
3483		((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
3484		((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
3485#endif
3486	hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) |
3487	    ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32);
3488	hena |= set_hena;
3489	wr32(hw, I40E_PFQF_HENA(0), (u32)hena);
3490	wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
3491
3492	/* Populate the LUT with max no. of queues in round robin fashion */
3493	for (i = j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) {
3494		if (j == vsi->num_queues)
3495			j = 0;
3496#ifdef RSS
3497		/*
3498		 * Fetch the RSS bucket id for the given indirection entry.
3499		 * Cap it at the number of configured buckets (which is
3500		 * num_queues.)
3501		 */
3502		que_id = rss_get_indirection_to_bucket(i);
3503		que_id = que_id % vsi->num_queues;
3504#else
3505		que_id = j;
3506#endif
3507		/* lut = 4-byte sliding window of 4 lut entries */
3508		lut = (lut << 8) | (que_id &
3509		    ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1));
3510		/* On i = 3, we have 4 entries in lut; write to the register */
3511		if ((i & 3) == 3)
3512			wr32(hw, I40E_PFQF_HLUT(i >> 2), lut);
3513	}
3514	ixl_flush(hw);
3515}
3516
3517
3518/*
3519** This routine is run via an vlan config EVENT,
3520** it enables us to use the HW Filter table since
3521** we can get the vlan id. This just creates the
3522** entry in the soft version of the VFTA, init will
3523** repopulate the real table.
3524*/
3525static void
3526ixl_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3527{
3528	struct ixl_vsi	*vsi = ifp->if_softc;
3529	struct i40e_hw	*hw = vsi->hw;
3530	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
3531
3532	if (ifp->if_softc !=  arg)   /* Not our event */
3533		return;
3534
3535	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
3536		return;
3537
3538	IXL_PF_LOCK(pf);
3539	++vsi->num_vlans;
3540	ixl_add_filter(vsi, hw->mac.addr, vtag);
3541	IXL_PF_UNLOCK(pf);
3542}
3543
3544/*
3545** This routine is run via an vlan
3546** unconfig EVENT, remove our entry
3547** in the soft vfta.
3548*/
3549static void
3550ixl_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
3551{
3552	struct ixl_vsi	*vsi = ifp->if_softc;
3553	struct i40e_hw	*hw = vsi->hw;
3554	struct ixl_pf	*pf = (struct ixl_pf *)vsi->back;
3555
3556	if (ifp->if_softc !=  arg)
3557		return;
3558
3559	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
3560		return;
3561
3562	IXL_PF_LOCK(pf);
3563	--vsi->num_vlans;
3564	ixl_del_filter(vsi, hw->mac.addr, vtag);
3565	IXL_PF_UNLOCK(pf);
3566}
3567
3568/*
3569** This routine updates vlan filters, called by init
3570** it scans the filter table and then updates the hw
3571** after a soft reset.
3572*/
3573static void
3574ixl_setup_vlan_filters(struct ixl_vsi *vsi)
3575{
3576	struct ixl_mac_filter	*f;
3577	int			cnt = 0, flags;
3578
3579	if (vsi->num_vlans == 0)
3580		return;
3581	/*
3582	** Scan the filter list for vlan entries,
3583	** mark them for addition and then call
3584	** for the AQ update.
3585	*/
3586	SLIST_FOREACH(f, &vsi->ftl, next) {
3587		if (f->flags & IXL_FILTER_VLAN) {
3588			f->flags |=
3589			    (IXL_FILTER_ADD |
3590			    IXL_FILTER_USED);
3591			cnt++;
3592		}
3593	}
3594	if (cnt == 0) {
3595		printf("setup vlan: no filters found!\n");
3596		return;
3597	}
3598	flags = IXL_FILTER_VLAN;
3599	flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3600	ixl_add_hw_filters(vsi, flags, cnt);
3601	return;
3602}
3603
3604/*
3605** Initialize filter list and add filters that the hardware
3606** needs to know about.
3607*/
3608static void
3609ixl_init_filters(struct ixl_vsi *vsi)
3610{
3611	/* Add broadcast address */
3612	ixl_add_filter(vsi, ixl_bcast_addr, IXL_VLAN_ANY);
3613}
3614
3615/*
3616** This routine adds mulicast filters
3617*/
3618static void
3619ixl_add_mc_filter(struct ixl_vsi *vsi, u8 *macaddr)
3620{
3621	struct ixl_mac_filter *f;
3622
3623	/* Does one already exist */
3624	f = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3625	if (f != NULL)
3626		return;
3627
3628	f = ixl_get_filter(vsi);
3629	if (f == NULL) {
3630		printf("WARNING: no filter available!!\n");
3631		return;
3632	}
3633	bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3634	f->vlan = IXL_VLAN_ANY;
3635	f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED
3636	    | IXL_FILTER_MC);
3637
3638	return;
3639}
3640
3641static void
3642ixl_reconfigure_filters(struct ixl_vsi *vsi)
3643{
3644
3645	ixl_add_hw_filters(vsi, IXL_FILTER_USED, vsi->num_macs);
3646}
3647
3648/*
3649** This routine adds macvlan filters
3650*/
3651static void
3652ixl_add_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3653{
3654	struct ixl_mac_filter	*f, *tmp;
3655	struct ixl_pf		*pf;
3656	device_t		dev;
3657
3658	DEBUGOUT("ixl_add_filter: begin");
3659
3660	pf = vsi->back;
3661	dev = pf->dev;
3662
3663	/* Does one already exist */
3664	f = ixl_find_filter(vsi, macaddr, vlan);
3665	if (f != NULL)
3666		return;
3667	/*
3668	** Is this the first vlan being registered, if so we
3669	** need to remove the ANY filter that indicates we are
3670	** not in a vlan, and replace that with a 0 filter.
3671	*/
3672	if ((vlan != IXL_VLAN_ANY) && (vsi->num_vlans == 1)) {
3673		tmp = ixl_find_filter(vsi, macaddr, IXL_VLAN_ANY);
3674		if (tmp != NULL) {
3675			ixl_del_filter(vsi, macaddr, IXL_VLAN_ANY);
3676			ixl_add_filter(vsi, macaddr, 0);
3677		}
3678	}
3679
3680	f = ixl_get_filter(vsi);
3681	if (f == NULL) {
3682		device_printf(dev, "WARNING: no filter available!!\n");
3683		return;
3684	}
3685	bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
3686	f->vlan = vlan;
3687	f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
3688	if (f->vlan != IXL_VLAN_ANY)
3689		f->flags |= IXL_FILTER_VLAN;
3690	else
3691		vsi->num_macs++;
3692
3693	ixl_add_hw_filters(vsi, f->flags, 1);
3694	return;
3695}
3696
3697static void
3698ixl_del_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3699{
3700	struct ixl_mac_filter *f;
3701
3702	f = ixl_find_filter(vsi, macaddr, vlan);
3703	if (f == NULL)
3704		return;
3705
3706	f->flags |= IXL_FILTER_DEL;
3707	ixl_del_hw_filters(vsi, 1);
3708	vsi->num_macs--;
3709
3710	/* Check if this is the last vlan removal */
3711	if (vlan != IXL_VLAN_ANY && vsi->num_vlans == 0) {
3712		/* Switch back to a non-vlan filter */
3713		ixl_del_filter(vsi, macaddr, 0);
3714		ixl_add_filter(vsi, macaddr, IXL_VLAN_ANY);
3715	}
3716	return;
3717}
3718
3719/*
3720** Find the filter with both matching mac addr and vlan id
3721*/
3722static struct ixl_mac_filter *
3723ixl_find_filter(struct ixl_vsi *vsi, u8 *macaddr, s16 vlan)
3724{
3725	struct ixl_mac_filter	*f;
3726	bool			match = FALSE;
3727
3728	SLIST_FOREACH(f, &vsi->ftl, next) {
3729		if (!cmp_etheraddr(f->macaddr, macaddr))
3730			continue;
3731		if (f->vlan == vlan) {
3732			match = TRUE;
3733			break;
3734		}
3735	}
3736
3737	if (!match)
3738		f = NULL;
3739	return (f);
3740}
3741
3742/*
3743** This routine takes additions to the vsi filter
3744** table and creates an Admin Queue call to create
3745** the filters in the hardware.
3746*/
3747static void
3748ixl_add_hw_filters(struct ixl_vsi *vsi, int flags, int cnt)
3749{
3750	struct i40e_aqc_add_macvlan_element_data *a, *b;
3751	struct ixl_mac_filter	*f;
3752	struct ixl_pf		*pf;
3753	struct i40e_hw		*hw;
3754	device_t		dev;
3755	int			err, j = 0;
3756
3757	pf = vsi->back;
3758	dev = pf->dev;
3759	hw = &pf->hw;
3760	IXL_PF_LOCK_ASSERT(pf);
3761
3762	a = malloc(sizeof(struct i40e_aqc_add_macvlan_element_data) * cnt,
3763	    M_DEVBUF, M_NOWAIT | M_ZERO);
3764	if (a == NULL) {
3765		device_printf(dev, "add_hw_filters failed to get memory\n");
3766		return;
3767	}
3768
3769	/*
3770	** Scan the filter list, each time we find one
3771	** we add it to the admin queue array and turn off
3772	** the add bit.
3773	*/
3774	SLIST_FOREACH(f, &vsi->ftl, next) {
3775		if (f->flags == flags) {
3776			b = &a[j]; // a pox on fvl long names :)
3777			bcopy(f->macaddr, b->mac_addr, ETHER_ADDR_LEN);
3778			if (f->vlan == IXL_VLAN_ANY) {
3779				b->vlan_tag = 0;
3780				b->flags = I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
3781			} else {
3782				b->vlan_tag = f->vlan;
3783				b->flags = 0;
3784			}
3785			b->flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
3786			f->flags &= ~IXL_FILTER_ADD;
3787			j++;
3788		}
3789		if (j == cnt)
3790			break;
3791	}
3792	if (j > 0) {
3793		err = i40e_aq_add_macvlan(hw, vsi->seid, a, j, NULL);
3794		if (err)
3795			device_printf(dev, "aq_add_macvlan err %d, "
3796			    "aq_error %d\n", err, hw->aq.asq_last_status);
3797		else
3798			vsi->hw_filters_add += j;
3799	}
3800	free(a, M_DEVBUF);
3801	return;
3802}
3803
3804/*
3805** This routine takes removals in the vsi filter
3806** table and creates an Admin Queue call to delete
3807** the filters in the hardware.
3808*/
3809static void
3810ixl_del_hw_filters(struct ixl_vsi *vsi, int cnt)
3811{
3812	struct i40e_aqc_remove_macvlan_element_data *d, *e;
3813	struct ixl_pf		*pf;
3814	struct i40e_hw		*hw;
3815	device_t		dev;
3816	struct ixl_mac_filter	*f, *f_temp;
3817	int			err, j = 0;
3818
3819	DEBUGOUT("ixl_del_hw_filters: begin\n");
3820
3821	pf = vsi->back;
3822	hw = &pf->hw;
3823	dev = pf->dev;
3824
3825	d = malloc(sizeof(struct i40e_aqc_remove_macvlan_element_data) * cnt,
3826	    M_DEVBUF, M_NOWAIT | M_ZERO);
3827	if (d == NULL) {
3828		printf("del hw filter failed to get memory\n");
3829		return;
3830	}
3831
3832	SLIST_FOREACH_SAFE(f, &vsi->ftl, next, f_temp) {
3833		if (f->flags & IXL_FILTER_DEL) {
3834			e = &d[j]; // a pox on fvl long names :)
3835			bcopy(f->macaddr, e->mac_addr, ETHER_ADDR_LEN);
3836			e->vlan_tag = (f->vlan == IXL_VLAN_ANY ? 0 : f->vlan);
3837			e->flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
3838			/* delete entry from vsi list */
3839			SLIST_REMOVE(&vsi->ftl, f, ixl_mac_filter, next);
3840			free(f, M_DEVBUF);
3841			j++;
3842		}
3843		if (j == cnt)
3844			break;
3845	}
3846	if (j > 0) {
3847		err = i40e_aq_remove_macvlan(hw, vsi->seid, d, j, NULL);
3848		/* NOTE: returns ENOENT every time but seems to work fine,
3849		   so we'll ignore that specific error. */
3850		// TODO: Does this still occur on current firmwares?
3851		if (err && hw->aq.asq_last_status != I40E_AQ_RC_ENOENT) {
3852			int sc = 0;
3853			for (int i = 0; i < j; i++)
3854				sc += (!d[i].error_code);
3855			vsi->hw_filters_del += sc;
3856			device_printf(dev,
3857			    "Failed to remove %d/%d filters, aq error %d\n",
3858			    j - sc, j, hw->aq.asq_last_status);
3859		} else
3860			vsi->hw_filters_del += j;
3861	}
3862	free(d, M_DEVBUF);
3863
3864	DEBUGOUT("ixl_del_hw_filters: end\n");
3865	return;
3866}
3867
3868static int
3869ixl_enable_rings(struct ixl_vsi *vsi)
3870{
3871	struct ixl_pf	*pf = vsi->back;
3872	struct i40e_hw	*hw = &pf->hw;
3873	int		index, error;
3874	u32		reg;
3875
3876	error = 0;
3877	for (int i = 0; i < vsi->num_queues; i++) {
3878		index = vsi->first_queue + i;
3879		i40e_pre_tx_queue_cfg(hw, index, TRUE);
3880
3881		reg = rd32(hw, I40E_QTX_ENA(index));
3882		reg |= I40E_QTX_ENA_QENA_REQ_MASK |
3883		    I40E_QTX_ENA_QENA_STAT_MASK;
3884		wr32(hw, I40E_QTX_ENA(index), reg);
3885		/* Verify the enable took */
3886		for (int j = 0; j < 10; j++) {
3887			reg = rd32(hw, I40E_QTX_ENA(index));
3888			if (reg & I40E_QTX_ENA_QENA_STAT_MASK)
3889				break;
3890			i40e_msec_delay(10);
3891		}
3892		if ((reg & I40E_QTX_ENA_QENA_STAT_MASK) == 0) {
3893			device_printf(pf->dev, "TX queue %d disabled!\n",
3894			    index);
3895			error = ETIMEDOUT;
3896		}
3897
3898		reg = rd32(hw, I40E_QRX_ENA(index));
3899		reg |= I40E_QRX_ENA_QENA_REQ_MASK |
3900		    I40E_QRX_ENA_QENA_STAT_MASK;
3901		wr32(hw, I40E_QRX_ENA(index), reg);
3902		/* Verify the enable took */
3903		for (int j = 0; j < 10; j++) {
3904			reg = rd32(hw, I40E_QRX_ENA(index));
3905			if (reg & I40E_QRX_ENA_QENA_STAT_MASK)
3906				break;
3907			i40e_msec_delay(10);
3908		}
3909		if ((reg & I40E_QRX_ENA_QENA_STAT_MASK) == 0) {
3910			device_printf(pf->dev, "RX queue %d disabled!\n",
3911			    index);
3912			error = ETIMEDOUT;
3913		}
3914	}
3915
3916	return (error);
3917}
3918
3919static int
3920ixl_disable_rings(struct ixl_vsi *vsi)
3921{
3922	struct ixl_pf	*pf = vsi->back;
3923	struct i40e_hw	*hw = &pf->hw;
3924	int		index, error;
3925	u32		reg;
3926
3927	error = 0;
3928	for (int i = 0; i < vsi->num_queues; i++) {
3929		index = vsi->first_queue + i;
3930
3931		i40e_pre_tx_queue_cfg(hw, index, FALSE);
3932		i40e_usec_delay(500);
3933
3934		reg = rd32(hw, I40E_QTX_ENA(index));
3935		reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
3936		wr32(hw, I40E_QTX_ENA(index), reg);
3937		/* Verify the disable took */
3938		for (int j = 0; j < 10; j++) {
3939			reg = rd32(hw, I40E_QTX_ENA(index));
3940			if (!(reg & I40E_QTX_ENA_QENA_STAT_MASK))
3941				break;
3942			i40e_msec_delay(10);
3943		}
3944		if (reg & I40E_QTX_ENA_QENA_STAT_MASK) {
3945			device_printf(pf->dev, "TX queue %d still enabled!\n",
3946			    index);
3947			error = ETIMEDOUT;
3948		}
3949
3950		reg = rd32(hw, I40E_QRX_ENA(index));
3951		reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
3952		wr32(hw, I40E_QRX_ENA(index), reg);
3953		/* Verify the disable took */
3954		for (int j = 0; j < 10; j++) {
3955			reg = rd32(hw, I40E_QRX_ENA(index));
3956			if (!(reg & I40E_QRX_ENA_QENA_STAT_MASK))
3957				break;
3958			i40e_msec_delay(10);
3959		}
3960		if (reg & I40E_QRX_ENA_QENA_STAT_MASK) {
3961			device_printf(pf->dev, "RX queue %d still enabled!\n",
3962			    index);
3963			error = ETIMEDOUT;
3964		}
3965	}
3966
3967	return (error);
3968}
3969
3970/**
3971 * ixl_handle_mdd_event
3972 *
3973 * Called from interrupt handler to identify possibly malicious vfs
3974 * (But also detects events from the PF, as well)
3975 **/
3976static void ixl_handle_mdd_event(struct ixl_pf *pf)
3977{
3978	struct i40e_hw *hw = &pf->hw;
3979	device_t dev = pf->dev;
3980	bool mdd_detected = false;
3981	bool pf_mdd_detected = false;
3982	u32 reg;
3983
3984	/* find what triggered the MDD event */
3985	reg = rd32(hw, I40E_GL_MDET_TX);
3986	if (reg & I40E_GL_MDET_TX_VALID_MASK) {
3987		u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
3988				I40E_GL_MDET_TX_PF_NUM_SHIFT;
3989		u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
3990				I40E_GL_MDET_TX_EVENT_SHIFT;
3991		u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
3992				I40E_GL_MDET_TX_QUEUE_SHIFT;
3993		device_printf(dev,
3994			 "Malicious Driver Detection event 0x%02x"
3995			 " on TX queue %d pf number 0x%02x\n",
3996			 event, queue, pf_num);
3997		wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
3998		mdd_detected = true;
3999	}
4000	reg = rd32(hw, I40E_GL_MDET_RX);
4001	if (reg & I40E_GL_MDET_RX_VALID_MASK) {
4002		u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
4003				I40E_GL_MDET_RX_FUNCTION_SHIFT;
4004		u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
4005				I40E_GL_MDET_RX_EVENT_SHIFT;
4006		u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
4007				I40E_GL_MDET_RX_QUEUE_SHIFT;
4008		device_printf(dev,
4009			 "Malicious Driver Detection event 0x%02x"
4010			 " on RX queue %d of function 0x%02x\n",
4011			 event, queue, func);
4012		wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
4013		mdd_detected = true;
4014	}
4015
4016	if (mdd_detected) {
4017		reg = rd32(hw, I40E_PF_MDET_TX);
4018		if (reg & I40E_PF_MDET_TX_VALID_MASK) {
4019			wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
4020			device_printf(dev,
4021				 "MDD TX event is for this function 0x%08x",
4022				 reg);
4023			pf_mdd_detected = true;
4024		}
4025		reg = rd32(hw, I40E_PF_MDET_RX);
4026		if (reg & I40E_PF_MDET_RX_VALID_MASK) {
4027			wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
4028			device_printf(dev,
4029				 "MDD RX event is for this function 0x%08x",
4030				 reg);
4031			pf_mdd_detected = true;
4032		}
4033	}
4034
4035	/* re-enable mdd interrupt cause */
4036	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4037	reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4038	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4039	ixl_flush(hw);
4040}
4041
4042static void
4043ixl_enable_intr(struct ixl_vsi *vsi)
4044{
4045	struct i40e_hw		*hw = vsi->hw;
4046	struct ixl_queue	*que = vsi->queues;
4047
4048	if (ixl_enable_msix) {
4049		ixl_enable_adminq(hw);
4050		for (int i = 0; i < vsi->num_queues; i++, que++)
4051			ixl_enable_queue(hw, que->me);
4052	} else
4053		ixl_enable_legacy(hw);
4054}
4055
4056static void
4057ixl_disable_rings_intr(struct ixl_vsi *vsi)
4058{
4059	struct i40e_hw		*hw = vsi->hw;
4060	struct ixl_queue	*que = vsi->queues;
4061
4062	for (int i = 0; i < vsi->num_queues; i++, que++)
4063		ixl_disable_queue(hw, que->me);
4064}
4065
4066static void
4067ixl_disable_intr(struct ixl_vsi *vsi)
4068{
4069	struct i40e_hw		*hw = vsi->hw;
4070
4071	if (ixl_enable_msix)
4072		ixl_disable_adminq(hw);
4073	else
4074		ixl_disable_legacy(hw);
4075}
4076
4077static void
4078ixl_enable_adminq(struct i40e_hw *hw)
4079{
4080	u32		reg;
4081
4082	reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4083	    I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4084	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4085	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4086	ixl_flush(hw);
4087	return;
4088}
4089
4090static void
4091ixl_disable_adminq(struct i40e_hw *hw)
4092{
4093	u32		reg;
4094
4095	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4096	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4097
4098	return;
4099}
4100
4101static void
4102ixl_enable_queue(struct i40e_hw *hw, int id)
4103{
4104	u32		reg;
4105
4106	reg = I40E_PFINT_DYN_CTLN_INTENA_MASK |
4107	    I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
4108	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4109	wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4110}
4111
4112static void
4113ixl_disable_queue(struct i40e_hw *hw, int id)
4114{
4115	u32		reg;
4116
4117	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
4118	wr32(hw, I40E_PFINT_DYN_CTLN(id), reg);
4119
4120	return;
4121}
4122
4123static void
4124ixl_enable_legacy(struct i40e_hw *hw)
4125{
4126	u32		reg;
4127	reg = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4128	    I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4129	    (IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4130	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4131}
4132
4133static void
4134ixl_disable_legacy(struct i40e_hw *hw)
4135{
4136	u32		reg;
4137
4138	reg = IXL_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT;
4139	wr32(hw, I40E_PFINT_DYN_CTL0, reg);
4140
4141	return;
4142}
4143
4144static void
4145ixl_update_stats_counters(struct ixl_pf *pf)
4146{
4147	struct i40e_hw	*hw = &pf->hw;
4148	struct ixl_vsi	*vsi = &pf->vsi;
4149	struct ixl_vf	*vf;
4150
4151	struct i40e_hw_port_stats *nsd = &pf->stats;
4152	struct i40e_hw_port_stats *osd = &pf->stats_offsets;
4153
4154	/* Update hw stats */
4155	ixl_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
4156			   pf->stat_offsets_loaded,
4157			   &osd->crc_errors, &nsd->crc_errors);
4158	ixl_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
4159			   pf->stat_offsets_loaded,
4160			   &osd->illegal_bytes, &nsd->illegal_bytes);
4161	ixl_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
4162			   I40E_GLPRT_GORCL(hw->port),
4163			   pf->stat_offsets_loaded,
4164			   &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
4165	ixl_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
4166			   I40E_GLPRT_GOTCL(hw->port),
4167			   pf->stat_offsets_loaded,
4168			   &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
4169	ixl_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
4170			   pf->stat_offsets_loaded,
4171			   &osd->eth.rx_discards,
4172			   &nsd->eth.rx_discards);
4173	ixl_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
4174			   I40E_GLPRT_UPRCL(hw->port),
4175			   pf->stat_offsets_loaded,
4176			   &osd->eth.rx_unicast,
4177			   &nsd->eth.rx_unicast);
4178	ixl_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
4179			   I40E_GLPRT_UPTCL(hw->port),
4180			   pf->stat_offsets_loaded,
4181			   &osd->eth.tx_unicast,
4182			   &nsd->eth.tx_unicast);
4183	ixl_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
4184			   I40E_GLPRT_MPRCL(hw->port),
4185			   pf->stat_offsets_loaded,
4186			   &osd->eth.rx_multicast,
4187			   &nsd->eth.rx_multicast);
4188	ixl_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
4189			   I40E_GLPRT_MPTCL(hw->port),
4190			   pf->stat_offsets_loaded,
4191			   &osd->eth.tx_multicast,
4192			   &nsd->eth.tx_multicast);
4193	ixl_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
4194			   I40E_GLPRT_BPRCL(hw->port),
4195			   pf->stat_offsets_loaded,
4196			   &osd->eth.rx_broadcast,
4197			   &nsd->eth.rx_broadcast);
4198	ixl_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
4199			   I40E_GLPRT_BPTCL(hw->port),
4200			   pf->stat_offsets_loaded,
4201			   &osd->eth.tx_broadcast,
4202			   &nsd->eth.tx_broadcast);
4203
4204	ixl_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
4205			   pf->stat_offsets_loaded,
4206			   &osd->tx_dropped_link_down,
4207			   &nsd->tx_dropped_link_down);
4208	ixl_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
4209			   pf->stat_offsets_loaded,
4210			   &osd->mac_local_faults,
4211			   &nsd->mac_local_faults);
4212	ixl_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
4213			   pf->stat_offsets_loaded,
4214			   &osd->mac_remote_faults,
4215			   &nsd->mac_remote_faults);
4216	ixl_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
4217			   pf->stat_offsets_loaded,
4218			   &osd->rx_length_errors,
4219			   &nsd->rx_length_errors);
4220
4221	/* Flow control (LFC) stats */
4222	ixl_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
4223			   pf->stat_offsets_loaded,
4224			   &osd->link_xon_rx, &nsd->link_xon_rx);
4225	ixl_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
4226			   pf->stat_offsets_loaded,
4227			   &osd->link_xon_tx, &nsd->link_xon_tx);
4228	ixl_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
4229			   pf->stat_offsets_loaded,
4230			   &osd->link_xoff_rx, &nsd->link_xoff_rx);
4231	ixl_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
4232			   pf->stat_offsets_loaded,
4233			   &osd->link_xoff_tx, &nsd->link_xoff_tx);
4234
4235	/* Packet size stats rx */
4236	ixl_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
4237			   I40E_GLPRT_PRC64L(hw->port),
4238			   pf->stat_offsets_loaded,
4239			   &osd->rx_size_64, &nsd->rx_size_64);
4240	ixl_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
4241			   I40E_GLPRT_PRC127L(hw->port),
4242			   pf->stat_offsets_loaded,
4243			   &osd->rx_size_127, &nsd->rx_size_127);
4244	ixl_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
4245			   I40E_GLPRT_PRC255L(hw->port),
4246			   pf->stat_offsets_loaded,
4247			   &osd->rx_size_255, &nsd->rx_size_255);
4248	ixl_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
4249			   I40E_GLPRT_PRC511L(hw->port),
4250			   pf->stat_offsets_loaded,
4251			   &osd->rx_size_511, &nsd->rx_size_511);
4252	ixl_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
4253			   I40E_GLPRT_PRC1023L(hw->port),
4254			   pf->stat_offsets_loaded,
4255			   &osd->rx_size_1023, &nsd->rx_size_1023);
4256	ixl_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
4257			   I40E_GLPRT_PRC1522L(hw->port),
4258			   pf->stat_offsets_loaded,
4259			   &osd->rx_size_1522, &nsd->rx_size_1522);
4260	ixl_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
4261			   I40E_GLPRT_PRC9522L(hw->port),
4262			   pf->stat_offsets_loaded,
4263			   &osd->rx_size_big, &nsd->rx_size_big);
4264
4265	/* Packet size stats tx */
4266	ixl_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
4267			   I40E_GLPRT_PTC64L(hw->port),
4268			   pf->stat_offsets_loaded,
4269			   &osd->tx_size_64, &nsd->tx_size_64);
4270	ixl_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
4271			   I40E_GLPRT_PTC127L(hw->port),
4272			   pf->stat_offsets_loaded,
4273			   &osd->tx_size_127, &nsd->tx_size_127);
4274	ixl_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
4275			   I40E_GLPRT_PTC255L(hw->port),
4276			   pf->stat_offsets_loaded,
4277			   &osd->tx_size_255, &nsd->tx_size_255);
4278	ixl_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
4279			   I40E_GLPRT_PTC511L(hw->port),
4280			   pf->stat_offsets_loaded,
4281			   &osd->tx_size_511, &nsd->tx_size_511);
4282	ixl_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
4283			   I40E_GLPRT_PTC1023L(hw->port),
4284			   pf->stat_offsets_loaded,
4285			   &osd->tx_size_1023, &nsd->tx_size_1023);
4286	ixl_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
4287			   I40E_GLPRT_PTC1522L(hw->port),
4288			   pf->stat_offsets_loaded,
4289			   &osd->tx_size_1522, &nsd->tx_size_1522);
4290	ixl_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
4291			   I40E_GLPRT_PTC9522L(hw->port),
4292			   pf->stat_offsets_loaded,
4293			   &osd->tx_size_big, &nsd->tx_size_big);
4294
4295	ixl_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
4296			   pf->stat_offsets_loaded,
4297			   &osd->rx_undersize, &nsd->rx_undersize);
4298	ixl_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
4299			   pf->stat_offsets_loaded,
4300			   &osd->rx_fragments, &nsd->rx_fragments);
4301	ixl_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
4302			   pf->stat_offsets_loaded,
4303			   &osd->rx_oversize, &nsd->rx_oversize);
4304	ixl_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
4305			   pf->stat_offsets_loaded,
4306			   &osd->rx_jabber, &nsd->rx_jabber);
4307	pf->stat_offsets_loaded = true;
4308	/* End hw stats */
4309
4310	/* Update vsi stats */
4311	ixl_update_vsi_stats(vsi);
4312
4313	for (int i = 0; i < pf->num_vfs; i++) {
4314		vf = &pf->vfs[i];
4315		if (vf->vf_flags & VF_FLAG_ENABLED)
4316			ixl_update_eth_stats(&pf->vfs[i].vsi);
4317	}
4318}
4319
4320/*
4321** Tasklet handler for MSIX Adminq interrupts
4322**  - do outside interrupt since it might sleep
4323*/
4324static void
4325ixl_do_adminq(void *context, int pending)
4326{
4327	struct ixl_pf			*pf = context;
4328	struct i40e_hw			*hw = &pf->hw;
4329	struct ixl_vsi			*vsi = &pf->vsi;
4330	struct i40e_arq_event_info	event;
4331	i40e_status			ret;
4332	u32				reg, loop = 0;
4333	u16				opcode, result;
4334
4335	event.buf_len = IXL_AQ_BUF_SZ;
4336	event.msg_buf = malloc(event.buf_len,
4337	    M_DEVBUF, M_NOWAIT | M_ZERO);
4338	if (!event.msg_buf) {
4339		printf("Unable to allocate adminq memory\n");
4340		return;
4341	}
4342
4343	IXL_PF_LOCK(pf);
4344	/* clean and process any events */
4345	do {
4346		ret = i40e_clean_arq_element(hw, &event, &result);
4347		if (ret)
4348			break;
4349		opcode = LE16_TO_CPU(event.desc.opcode);
4350		switch (opcode) {
4351		case i40e_aqc_opc_get_link_status:
4352			ixl_link_event(pf, &event);
4353			ixl_update_link_status(pf);
4354			break;
4355		case i40e_aqc_opc_send_msg_to_pf:
4356#ifdef PCI_IOV
4357			ixl_handle_vf_msg(pf, &event);
4358#endif
4359			break;
4360		case i40e_aqc_opc_event_lan_overflow:
4361			break;
4362		default:
4363#ifdef IXL_DEBUG
4364			printf("AdminQ unknown event %x\n", opcode);
4365#endif
4366			break;
4367		}
4368
4369	} while (result && (loop++ < IXL_ADM_LIMIT));
4370
4371	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4372	reg |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4373	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4374	free(event.msg_buf, M_DEVBUF);
4375
4376	/*
4377	 * If there are still messages to process, reschedule ourselves.
4378	 * Otherwise, re-enable our interrupt and go to sleep.
4379	 */
4380	if (result > 0)
4381		taskqueue_enqueue(pf->tq, &pf->adminq);
4382	else
4383		ixl_enable_intr(vsi);
4384
4385	IXL_PF_UNLOCK(pf);
4386}
4387
4388#ifdef IXL_DEBUG_SYSCTL
4389static int
4390ixl_debug_info(SYSCTL_HANDLER_ARGS)
4391{
4392	struct ixl_pf	*pf;
4393	int		error, input = 0;
4394
4395	error = sysctl_handle_int(oidp, &input, 0, req);
4396
4397	if (error || !req->newptr)
4398		return (error);
4399
4400	if (input == 1) {
4401		pf = (struct ixl_pf *)arg1;
4402		ixl_print_debug_info(pf);
4403	}
4404
4405	return (error);
4406}
4407
4408static void
4409ixl_print_debug_info(struct ixl_pf *pf)
4410{
4411	struct i40e_hw		*hw = &pf->hw;
4412	struct ixl_vsi		*vsi = &pf->vsi;
4413	struct ixl_queue	*que = vsi->queues;
4414	struct rx_ring		*rxr = &que->rxr;
4415	struct tx_ring		*txr = &que->txr;
4416	u32			reg;
4417
4418
4419	printf("Queue irqs = %jx\n", (uintmax_t)que->irqs);
4420	printf("AdminQ irqs = %jx\n", (uintmax_t)pf->admin_irq);
4421	printf("RX next check = %x\n", rxr->next_check);
4422	printf("RX not ready = %jx\n", (uintmax_t)rxr->not_done);
4423	printf("RX packets = %jx\n", (uintmax_t)rxr->rx_packets);
4424	printf("TX desc avail = %x\n", txr->avail);
4425
4426	reg = rd32(hw, I40E_GLV_GORCL(0xc));
4427	 printf("RX Bytes = %x\n", reg);
4428	reg = rd32(hw, I40E_GLPRT_GORCL(hw->port));
4429	 printf("Port RX Bytes = %x\n", reg);
4430	reg = rd32(hw, I40E_GLV_RDPC(0xc));
4431	 printf("RX discard = %x\n", reg);
4432	reg = rd32(hw, I40E_GLPRT_RDPC(hw->port));
4433	 printf("Port RX discard = %x\n", reg);
4434
4435	reg = rd32(hw, I40E_GLV_TEPC(0xc));
4436	 printf("TX errors = %x\n", reg);
4437	reg = rd32(hw, I40E_GLV_GOTCL(0xc));
4438	 printf("TX Bytes = %x\n", reg);
4439
4440	reg = rd32(hw, I40E_GLPRT_RUC(hw->port));
4441	 printf("RX undersize = %x\n", reg);
4442	reg = rd32(hw, I40E_GLPRT_RFC(hw->port));
4443	 printf("RX fragments = %x\n", reg);
4444	reg = rd32(hw, I40E_GLPRT_ROC(hw->port));
4445	 printf("RX oversize = %x\n", reg);
4446	reg = rd32(hw, I40E_GLPRT_RLEC(hw->port));
4447	 printf("RX length error = %x\n", reg);
4448	reg = rd32(hw, I40E_GLPRT_MRFC(hw->port));
4449	 printf("mac remote fault = %x\n", reg);
4450	reg = rd32(hw, I40E_GLPRT_MLFC(hw->port));
4451	 printf("mac local fault = %x\n", reg);
4452}
4453#endif
4454
4455/**
4456 * Update VSI-specific ethernet statistics counters.
4457 **/
4458void ixl_update_eth_stats(struct ixl_vsi *vsi)
4459{
4460	struct ixl_pf *pf = (struct ixl_pf *)vsi->back;
4461	struct i40e_hw *hw = &pf->hw;
4462	struct i40e_eth_stats *es;
4463	struct i40e_eth_stats *oes;
4464	struct i40e_hw_port_stats *nsd;
4465	u16 stat_idx = vsi->info.stat_counter_idx;
4466
4467	es = &vsi->eth_stats;
4468	oes = &vsi->eth_stats_offsets;
4469	nsd = &pf->stats;
4470
4471	/* Gather up the stats that the hw collects */
4472	ixl_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
4473			   vsi->stat_offsets_loaded,
4474			   &oes->tx_errors, &es->tx_errors);
4475	ixl_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
4476			   vsi->stat_offsets_loaded,
4477			   &oes->rx_discards, &es->rx_discards);
4478
4479	ixl_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
4480			   I40E_GLV_GORCL(stat_idx),
4481			   vsi->stat_offsets_loaded,
4482			   &oes->rx_bytes, &es->rx_bytes);
4483	ixl_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
4484			   I40E_GLV_UPRCL(stat_idx),
4485			   vsi->stat_offsets_loaded,
4486			   &oes->rx_unicast, &es->rx_unicast);
4487	ixl_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
4488			   I40E_GLV_MPRCL(stat_idx),
4489			   vsi->stat_offsets_loaded,
4490			   &oes->rx_multicast, &es->rx_multicast);
4491	ixl_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
4492			   I40E_GLV_BPRCL(stat_idx),
4493			   vsi->stat_offsets_loaded,
4494			   &oes->rx_broadcast, &es->rx_broadcast);
4495
4496	ixl_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
4497			   I40E_GLV_GOTCL(stat_idx),
4498			   vsi->stat_offsets_loaded,
4499			   &oes->tx_bytes, &es->tx_bytes);
4500	ixl_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
4501			   I40E_GLV_UPTCL(stat_idx),
4502			   vsi->stat_offsets_loaded,
4503			   &oes->tx_unicast, &es->tx_unicast);
4504	ixl_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
4505			   I40E_GLV_MPTCL(stat_idx),
4506			   vsi->stat_offsets_loaded,
4507			   &oes->tx_multicast, &es->tx_multicast);
4508	ixl_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
4509			   I40E_GLV_BPTCL(stat_idx),
4510			   vsi->stat_offsets_loaded,
4511			   &oes->tx_broadcast, &es->tx_broadcast);
4512	vsi->stat_offsets_loaded = true;
4513}
4514
4515static void
4516ixl_update_vsi_stats(struct ixl_vsi *vsi)
4517{
4518	struct ixl_pf		*pf;
4519	struct ifnet		*ifp;
4520	struct i40e_eth_stats	*es;
4521	u64			tx_discards;
4522
4523	struct i40e_hw_port_stats *nsd;
4524
4525	pf = vsi->back;
4526	ifp = vsi->ifp;
4527	es = &vsi->eth_stats;
4528	nsd = &pf->stats;
4529
4530	ixl_update_eth_stats(vsi);
4531
4532	tx_discards = es->tx_discards + nsd->tx_dropped_link_down;
4533	for (int i = 0; i < vsi->num_queues; i++)
4534		tx_discards += vsi->queues[i].txr.br->br_drops;
4535
4536	/* Update ifnet stats */
4537	IXL_SET_IPACKETS(vsi, es->rx_unicast +
4538	                   es->rx_multicast +
4539			   es->rx_broadcast);
4540	IXL_SET_OPACKETS(vsi, es->tx_unicast +
4541	                   es->tx_multicast +
4542			   es->tx_broadcast);
4543	IXL_SET_IBYTES(vsi, es->rx_bytes);
4544	IXL_SET_OBYTES(vsi, es->tx_bytes);
4545	IXL_SET_IMCASTS(vsi, es->rx_multicast);
4546	IXL_SET_OMCASTS(vsi, es->tx_multicast);
4547
4548	IXL_SET_IERRORS(vsi, nsd->crc_errors + nsd->illegal_bytes +
4549	    nsd->rx_undersize + nsd->rx_oversize + nsd->rx_fragments +
4550	    nsd->rx_jabber);
4551	IXL_SET_OERRORS(vsi, es->tx_errors);
4552	IXL_SET_IQDROPS(vsi, es->rx_discards + nsd->eth.rx_discards);
4553	IXL_SET_OQDROPS(vsi, tx_discards);
4554	IXL_SET_NOPROTO(vsi, es->rx_unknown_protocol);
4555	IXL_SET_COLLISIONS(vsi, 0);
4556}
4557
4558/**
4559 * Reset all of the stats for the given pf
4560 **/
4561void ixl_pf_reset_stats(struct ixl_pf *pf)
4562{
4563	bzero(&pf->stats, sizeof(struct i40e_hw_port_stats));
4564	bzero(&pf->stats_offsets, sizeof(struct i40e_hw_port_stats));
4565	pf->stat_offsets_loaded = false;
4566}
4567
4568/**
4569 * Resets all stats of the given vsi
4570 **/
4571void ixl_vsi_reset_stats(struct ixl_vsi *vsi)
4572{
4573	bzero(&vsi->eth_stats, sizeof(struct i40e_eth_stats));
4574	bzero(&vsi->eth_stats_offsets, sizeof(struct i40e_eth_stats));
4575	vsi->stat_offsets_loaded = false;
4576}
4577
4578/**
4579 * Read and update a 48 bit stat from the hw
4580 *
4581 * Since the device stats are not reset at PFReset, they likely will not
4582 * be zeroed when the driver starts.  We'll save the first values read
4583 * and use them as offsets to be subtracted from the raw values in order
4584 * to report stats that count from zero.
4585 **/
4586static void
4587ixl_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
4588	bool offset_loaded, u64 *offset, u64 *stat)
4589{
4590	u64 new_data;
4591
4592#if defined(__FreeBSD__) && (__FreeBSD_version >= 1000000) && defined(__amd64__)
4593	new_data = rd64(hw, loreg);
4594#else
4595	/*
4596	 * Use two rd32's instead of one rd64; FreeBSD versions before
4597	 * 10 don't support 8 byte bus reads/writes.
4598	 */
4599	new_data = rd32(hw, loreg);
4600	new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
4601#endif
4602
4603	if (!offset_loaded)
4604		*offset = new_data;
4605	if (new_data >= *offset)
4606		*stat = new_data - *offset;
4607	else
4608		*stat = (new_data + ((u64)1 << 48)) - *offset;
4609	*stat &= 0xFFFFFFFFFFFFULL;
4610}
4611
4612/**
4613 * Read and update a 32 bit stat from the hw
4614 **/
4615static void
4616ixl_stat_update32(struct i40e_hw *hw, u32 reg,
4617	bool offset_loaded, u64 *offset, u64 *stat)
4618{
4619	u32 new_data;
4620
4621	new_data = rd32(hw, reg);
4622	if (!offset_loaded)
4623		*offset = new_data;
4624	if (new_data >= *offset)
4625		*stat = (u32)(new_data - *offset);
4626	else
4627		*stat = (u32)((new_data + ((u64)1 << 32)) - *offset);
4628}
4629
4630/*
4631** Set flow control using sysctl:
4632** 	0 - off
4633**	1 - rx pause
4634**	2 - tx pause
4635**	3 - full
4636*/
4637static int
4638ixl_set_flowcntl(SYSCTL_HANDLER_ARGS)
4639{
4640	/*
4641	 * TODO: ensure flow control is disabled if
4642	 * priority flow control is enabled
4643	 *
4644	 * TODO: ensure tx CRC by hardware should be enabled
4645	 * if tx flow control is enabled.
4646	 */
4647	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4648	struct i40e_hw *hw = &pf->hw;
4649	device_t dev = pf->dev;
4650	int error = 0;
4651	enum i40e_status_code aq_error = 0;
4652	u8 fc_aq_err = 0;
4653
4654	/* Get request */
4655	error = sysctl_handle_int(oidp, &pf->fc, 0, req);
4656	if ((error) || (req->newptr == NULL))
4657		return (error);
4658	if (pf->fc < 0 || pf->fc > 3) {
4659		device_printf(dev,
4660		    "Invalid fc mode; valid modes are 0 through 3\n");
4661		return (EINVAL);
4662	}
4663
4664	/*
4665	** Changing flow control mode currently does not work on
4666	** 40GBASE-CR4 PHYs
4667	*/
4668	if (hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4
4669	    || hw->phy.link_info.phy_type == I40E_PHY_TYPE_40GBASE_CR4_CU) {
4670		device_printf(dev, "Changing flow control mode unsupported"
4671		    " on 40GBase-CR4 media.\n");
4672		return (ENODEV);
4673	}
4674
4675	/* Set fc ability for port */
4676	hw->fc.requested_mode = pf->fc;
4677	aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE);
4678	if (aq_error) {
4679		device_printf(dev,
4680		    "%s: Error setting new fc mode %d; fc_err %#x\n",
4681		    __func__, aq_error, fc_aq_err);
4682		return (EAGAIN);
4683	}
4684
4685	return (0);
4686}
4687
4688static int
4689ixl_current_speed(SYSCTL_HANDLER_ARGS)
4690{
4691	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4692	struct i40e_hw *hw = &pf->hw;
4693	int error = 0, index = 0;
4694
4695	char *speeds[] = {
4696		"Unknown",
4697		"100M",
4698		"1G",
4699		"10G",
4700		"40G",
4701		"20G"
4702	};
4703
4704	ixl_update_link_status(pf);
4705
4706	switch (hw->phy.link_info.link_speed) {
4707	case I40E_LINK_SPEED_100MB:
4708		index = 1;
4709		break;
4710	case I40E_LINK_SPEED_1GB:
4711		index = 2;
4712		break;
4713	case I40E_LINK_SPEED_10GB:
4714		index = 3;
4715		break;
4716	case I40E_LINK_SPEED_40GB:
4717		index = 4;
4718		break;
4719	case I40E_LINK_SPEED_20GB:
4720		index = 5;
4721		break;
4722	case I40E_LINK_SPEED_UNKNOWN:
4723	default:
4724		index = 0;
4725		break;
4726	}
4727
4728	error = sysctl_handle_string(oidp, speeds[index],
4729	    strlen(speeds[index]), req);
4730	return (error);
4731}
4732
4733static int
4734ixl_set_advertised_speeds(struct ixl_pf *pf, int speeds)
4735{
4736	struct i40e_hw *hw = &pf->hw;
4737	device_t dev = pf->dev;
4738	struct i40e_aq_get_phy_abilities_resp abilities;
4739	struct i40e_aq_set_phy_config config;
4740	enum i40e_status_code aq_error = 0;
4741
4742	/* Get current capability information */
4743	aq_error = i40e_aq_get_phy_capabilities(hw,
4744	    FALSE, FALSE, &abilities, NULL);
4745	if (aq_error) {
4746		device_printf(dev,
4747		    "%s: Error getting phy capabilities %d,"
4748		    " aq error: %d\n", __func__, aq_error,
4749		    hw->aq.asq_last_status);
4750		return (EAGAIN);
4751	}
4752
4753	/* Prepare new config */
4754	bzero(&config, sizeof(config));
4755	config.phy_type = abilities.phy_type;
4756	config.abilities = abilities.abilities
4757	    | I40E_AQ_PHY_ENABLE_ATOMIC_LINK;
4758	config.eee_capability = abilities.eee_capability;
4759	config.eeer = abilities.eeer_val;
4760	config.low_power_ctrl = abilities.d3_lpan;
4761	/* Translate into aq cmd link_speed */
4762	if (speeds & 0x8)
4763		config.link_speed |= I40E_LINK_SPEED_20GB;
4764	if (speeds & 0x4)
4765		config.link_speed |= I40E_LINK_SPEED_10GB;
4766	if (speeds & 0x2)
4767		config.link_speed |= I40E_LINK_SPEED_1GB;
4768	if (speeds & 0x1)
4769		config.link_speed |= I40E_LINK_SPEED_100MB;
4770
4771	/* Do aq command & restart link */
4772	aq_error = i40e_aq_set_phy_config(hw, &config, NULL);
4773	if (aq_error) {
4774		device_printf(dev,
4775		    "%s: Error setting new phy config %d,"
4776		    " aq error: %d\n", __func__, aq_error,
4777		    hw->aq.asq_last_status);
4778		return (EAGAIN);
4779	}
4780
4781	/*
4782	** This seems a bit heavy handed, but we
4783	** need to get a reinit on some devices
4784	*/
4785	IXL_PF_LOCK(pf);
4786	ixl_stop(pf);
4787	ixl_init_locked(pf);
4788	IXL_PF_UNLOCK(pf);
4789
4790	return (0);
4791}
4792
4793/*
4794** Control link advertise speed:
4795**	Flags:
4796**	0x1 - advertise 100 Mb
4797**	0x2 - advertise 1G
4798**	0x4 - advertise 10G
4799**	0x8 - advertise 20G
4800**
4801** Does not work on 40G devices.
4802*/
4803static int
4804ixl_set_advertise(SYSCTL_HANDLER_ARGS)
4805{
4806	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4807	struct i40e_hw *hw = &pf->hw;
4808	device_t dev = pf->dev;
4809	int requested_ls = 0;
4810	int error = 0;
4811
4812	/*
4813	** FW doesn't support changing advertised speed
4814	** for 40G devices; speed is always 40G.
4815	*/
4816	if (i40e_is_40G_device(hw->device_id))
4817		return (ENODEV);
4818
4819	/* Read in new mode */
4820	requested_ls = pf->advertised_speed;
4821	error = sysctl_handle_int(oidp, &requested_ls, 0, req);
4822	if ((error) || (req->newptr == NULL))
4823		return (error);
4824	/* Check for sane value */
4825	if (requested_ls < 0x1 || requested_ls > 0xE) {
4826		device_printf(dev, "Invalid advertised speed; "
4827		    "valid modes are 0x1 through 0xE\n");
4828		return (EINVAL);
4829	}
4830	/* Then check for validity based on adapter type */
4831	switch (hw->device_id) {
4832	case I40E_DEV_ID_10G_BASE_T:
4833		if (requested_ls & 0x8) {
4834			device_printf(dev,
4835			    "20Gbs speed not supported on this device.\n");
4836			return (EINVAL);
4837		}
4838		break;
4839	case I40E_DEV_ID_20G_KR2:
4840		if (requested_ls & 0x1) {
4841			device_printf(dev,
4842			    "100Mbs speed not supported on this device.\n");
4843			return (EINVAL);
4844		}
4845		break;
4846	default:
4847		if (requested_ls & ~0x6) {
4848			device_printf(dev,
4849			    "Only 1/10Gbs speeds are supported on this device.\n");
4850			return (EINVAL);
4851		}
4852		break;
4853	}
4854
4855	/* Exit if no change */
4856	if (pf->advertised_speed == requested_ls)
4857		return (0);
4858
4859	error = ixl_set_advertised_speeds(pf, requested_ls);
4860	if (error)
4861		return (error);
4862
4863	pf->advertised_speed = requested_ls;
4864	ixl_update_link_status(pf);
4865	return (0);
4866}
4867
4868/*
4869** Get the width and transaction speed of
4870** the bus this adapter is plugged into.
4871*/
4872static u16
4873ixl_get_bus_info(struct i40e_hw *hw, device_t dev)
4874{
4875        u16                     link;
4876        u32                     offset;
4877
4878
4879        /* Get the PCI Express Capabilities offset */
4880        pci_find_cap(dev, PCIY_EXPRESS, &offset);
4881
4882        /* ...and read the Link Status Register */
4883        link = pci_read_config(dev, offset + PCIER_LINK_STA, 2);
4884
4885        switch (link & I40E_PCI_LINK_WIDTH) {
4886        case I40E_PCI_LINK_WIDTH_1:
4887                hw->bus.width = i40e_bus_width_pcie_x1;
4888                break;
4889        case I40E_PCI_LINK_WIDTH_2:
4890                hw->bus.width = i40e_bus_width_pcie_x2;
4891                break;
4892        case I40E_PCI_LINK_WIDTH_4:
4893                hw->bus.width = i40e_bus_width_pcie_x4;
4894                break;
4895        case I40E_PCI_LINK_WIDTH_8:
4896                hw->bus.width = i40e_bus_width_pcie_x8;
4897                break;
4898        default:
4899                hw->bus.width = i40e_bus_width_unknown;
4900                break;
4901        }
4902
4903        switch (link & I40E_PCI_LINK_SPEED) {
4904        case I40E_PCI_LINK_SPEED_2500:
4905                hw->bus.speed = i40e_bus_speed_2500;
4906                break;
4907        case I40E_PCI_LINK_SPEED_5000:
4908                hw->bus.speed = i40e_bus_speed_5000;
4909                break;
4910        case I40E_PCI_LINK_SPEED_8000:
4911                hw->bus.speed = i40e_bus_speed_8000;
4912                break;
4913        default:
4914                hw->bus.speed = i40e_bus_speed_unknown;
4915                break;
4916        }
4917
4918
4919        device_printf(dev,"PCI Express Bus: Speed %s %s\n",
4920            ((hw->bus.speed == i40e_bus_speed_8000) ? "8.0GT/s":
4921            (hw->bus.speed == i40e_bus_speed_5000) ? "5.0GT/s":
4922            (hw->bus.speed == i40e_bus_speed_2500) ? "2.5GT/s":"Unknown"),
4923            (hw->bus.width == i40e_bus_width_pcie_x8) ? "Width x8" :
4924            (hw->bus.width == i40e_bus_width_pcie_x4) ? "Width x4" :
4925            (hw->bus.width == i40e_bus_width_pcie_x1) ? "Width x1" :
4926            ("Unknown"));
4927
4928        if ((hw->bus.width <= i40e_bus_width_pcie_x8) &&
4929            (hw->bus.speed < i40e_bus_speed_8000)) {
4930                device_printf(dev, "PCI-Express bandwidth available"
4931                    " for this device\n     may be insufficient for"
4932                    " optimal performance.\n");
4933                device_printf(dev, "For expected performance a x8 "
4934                    "PCIE Gen3 slot is required.\n");
4935        }
4936
4937        return (link);
4938}
4939
4940static int
4941ixl_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
4942{
4943	struct ixl_pf	*pf = (struct ixl_pf *)arg1;
4944	struct i40e_hw	*hw = &pf->hw;
4945	char		buf[32];
4946
4947	snprintf(buf, sizeof(buf),
4948	    "f%d.%d a%d.%d n%02x.%02x e%08x",
4949	    hw->aq.fw_maj_ver, hw->aq.fw_min_ver,
4950	    hw->aq.api_maj_ver, hw->aq.api_min_ver,
4951	    (hw->nvm.version & IXL_NVM_VERSION_HI_MASK) >>
4952	    IXL_NVM_VERSION_HI_SHIFT,
4953	    (hw->nvm.version & IXL_NVM_VERSION_LO_MASK) >>
4954	    IXL_NVM_VERSION_LO_SHIFT,
4955	    hw->nvm.eetrack);
4956	return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4957}
4958
4959
4960#ifdef IXL_DEBUG_SYSCTL
4961static int
4962ixl_sysctl_link_status(SYSCTL_HANDLER_ARGS)
4963{
4964	struct ixl_pf *pf = (struct ixl_pf *)arg1;
4965	struct i40e_hw *hw = &pf->hw;
4966	struct i40e_link_status link_status;
4967	char buf[512];
4968
4969	enum i40e_status_code aq_error = 0;
4970
4971	aq_error = i40e_aq_get_link_info(hw, TRUE, &link_status, NULL);
4972	if (aq_error) {
4973		printf("i40e_aq_get_link_info() error %d\n", aq_error);
4974		return (EPERM);
4975	}
4976
4977	sprintf(buf, "\n"
4978	    "PHY Type : %#04x\n"
4979	    "Speed    : %#04x\n"
4980	    "Link info: %#04x\n"
4981	    "AN info  : %#04x\n"
4982	    "Ext info : %#04x",
4983	    link_status.phy_type, link_status.link_speed,
4984	    link_status.link_info, link_status.an_info,
4985	    link_status.ext_info);
4986
4987	return (sysctl_handle_string(oidp, buf, strlen(buf), req));
4988}
4989
4990static int
4991ixl_sysctl_phy_abilities(SYSCTL_HANDLER_ARGS)
4992{
4993	struct ixl_pf		*pf = (struct ixl_pf *)arg1;
4994	struct i40e_hw		*hw = &pf->hw;
4995	char			buf[512];
4996	enum i40e_status_code	aq_error = 0;
4997
4998	struct i40e_aq_get_phy_abilities_resp abilities;
4999
5000	aq_error = i40e_aq_get_phy_capabilities(hw,
5001	    TRUE, FALSE, &abilities, NULL);
5002	if (aq_error) {
5003		printf("i40e_aq_get_phy_capabilities() error %d\n", aq_error);
5004		return (EPERM);
5005	}
5006
5007	sprintf(buf, "\n"
5008	    "PHY Type : %#010x\n"
5009	    "Speed    : %#04x\n"
5010	    "Abilities: %#04x\n"
5011	    "EEE cap  : %#06x\n"
5012	    "EEER reg : %#010x\n"
5013	    "D3 Lpan  : %#04x",
5014	    abilities.phy_type, abilities.link_speed,
5015	    abilities.abilities, abilities.eee_capability,
5016	    abilities.eeer_val, abilities.d3_lpan);
5017
5018	return (sysctl_handle_string(oidp, buf, strlen(buf), req));
5019}
5020
5021static int
5022ixl_sysctl_sw_filter_list(SYSCTL_HANDLER_ARGS)
5023{
5024	struct ixl_pf *pf = (struct ixl_pf *)arg1;
5025	struct ixl_vsi *vsi = &pf->vsi;
5026	struct ixl_mac_filter *f;
5027	char *buf, *buf_i;
5028
5029	int error = 0;
5030	int ftl_len = 0;
5031	int ftl_counter = 0;
5032	int buf_len = 0;
5033	int entry_len = 42;
5034
5035	SLIST_FOREACH(f, &vsi->ftl, next) {
5036		ftl_len++;
5037	}
5038
5039	if (ftl_len < 1) {
5040		sysctl_handle_string(oidp, "(none)", 6, req);
5041		return (0);
5042	}
5043
5044	buf_len = sizeof(char) * (entry_len + 1) * ftl_len + 2;
5045	buf = buf_i = malloc(buf_len, M_DEVBUF, M_NOWAIT);
5046
5047	sprintf(buf_i++, "\n");
5048	SLIST_FOREACH(f, &vsi->ftl, next) {
5049		sprintf(buf_i,
5050		    MAC_FORMAT ", vlan %4d, flags %#06x",
5051		    MAC_FORMAT_ARGS(f->macaddr), f->vlan, f->flags);
5052		buf_i += entry_len;
5053		/* don't print '\n' for last entry */
5054		if (++ftl_counter != ftl_len) {
5055			sprintf(buf_i, "\n");
5056			buf_i++;
5057		}
5058	}
5059
5060	error = sysctl_handle_string(oidp, buf, strlen(buf), req);
5061	if (error)
5062		printf("sysctl error: %d\n", error);
5063	free(buf, M_DEVBUF);
5064	return error;
5065}
5066
5067#define IXL_SW_RES_SIZE 0x14
5068static int
5069ixl_res_alloc_cmp(const void *a, const void *b)
5070{
5071	const struct i40e_aqc_switch_resource_alloc_element_resp *one, *two;
5072	one = (const struct i40e_aqc_switch_resource_alloc_element_resp *)a;
5073	two = (const struct i40e_aqc_switch_resource_alloc_element_resp *)b;
5074
5075	return ((int)one->resource_type - (int)two->resource_type);
5076}
5077
5078static int
5079ixl_sysctl_hw_res_alloc(SYSCTL_HANDLER_ARGS)
5080{
5081	struct ixl_pf *pf = (struct ixl_pf *)arg1;
5082	struct i40e_hw *hw = &pf->hw;
5083	device_t dev = pf->dev;
5084	struct sbuf *buf;
5085	int error = 0;
5086
5087	u8 num_entries;
5088	struct i40e_aqc_switch_resource_alloc_element_resp resp[IXL_SW_RES_SIZE];
5089
5090	buf = sbuf_new_for_sysctl(NULL, NULL, 0, req);
5091	if (!buf) {
5092		device_printf(dev, "Could not allocate sbuf for output.\n");
5093		return (ENOMEM);
5094	}
5095
5096	bzero(resp, sizeof(resp));
5097	error = i40e_aq_get_switch_resource_alloc(hw, &num_entries,
5098				resp,
5099				IXL_SW_RES_SIZE,
5100				NULL);
5101	if (error) {
5102		device_printf(dev,
5103		    "%s: get_switch_resource_alloc() error %d, aq error %d\n",
5104		    __func__, error, hw->aq.asq_last_status);
5105		sbuf_delete(buf);
5106		return error;
5107	}
5108
5109	/* Sort entries by type for display */
5110	qsort(resp, num_entries,
5111	    sizeof(struct i40e_aqc_switch_resource_alloc_element_resp),
5112	    &ixl_res_alloc_cmp);
5113
5114	sbuf_cat(buf, "\n");
5115	sbuf_printf(buf, "# of entries: %d\n", num_entries);
5116	sbuf_printf(buf,
5117	    "Type | Guaranteed | Total | Used   | Un-allocated\n"
5118	    "     | (this)     | (all) | (this) | (all)       \n");
5119	for (int i = 0; i < num_entries; i++) {
5120		sbuf_printf(buf,
5121		    "%#4x | %10d   %5d   %6d   %12d",
5122		    resp[i].resource_type,
5123		    resp[i].guaranteed,
5124		    resp[i].total,
5125		    resp[i].used,
5126		    resp[i].total_unalloced);
5127		if (i < num_entries - 1)
5128			sbuf_cat(buf, "\n");
5129	}
5130
5131	error = sbuf_finish(buf);
5132	sbuf_delete(buf);
5133
5134	return (error);
5135}
5136
5137/*
5138** Caller must init and delete sbuf; this function will clear and
5139** finish it for caller.
5140*/
5141static char *
5142ixl_switch_element_string(struct sbuf *s, u16 seid, bool uplink)
5143{
5144	sbuf_clear(s);
5145
5146	if (seid == 0 && uplink)
5147		sbuf_cat(s, "Network");
5148	else if (seid == 0)
5149		sbuf_cat(s, "Host");
5150	else if (seid == 1)
5151		sbuf_cat(s, "EMP");
5152	else if (seid <= 5)
5153		sbuf_printf(s, "MAC %d", seid - 2);
5154	else if (seid <= 15)
5155		sbuf_cat(s, "Reserved");
5156	else if (seid <= 31)
5157		sbuf_printf(s, "PF %d", seid - 16);
5158	else if (seid <= 159)
5159		sbuf_printf(s, "VF %d", seid - 32);
5160	else if (seid <= 287)
5161		sbuf_cat(s, "Reserved");
5162	else if (seid <= 511)
5163		sbuf_cat(s, "Other"); // for other structures
5164	else if (seid <= 895)
5165		sbuf_printf(s, "VSI %d", seid - 512);
5166	else if (seid <= 1023)
5167		sbuf_printf(s, "Reserved");
5168	else
5169		sbuf_cat(s, "Invalid");
5170
5171	sbuf_finish(s);
5172	return sbuf_data(s);
5173}
5174
5175static int
5176ixl_sysctl_switch_config(SYSCTL_HANDLER_ARGS)
5177{
5178	struct ixl_pf *pf = (struct ixl_pf *)arg1;
5179	struct i40e_hw *hw = &pf->hw;
5180	device_t dev = pf->dev;
5181	struct sbuf *buf;
5182	struct sbuf *nmbuf;
5183	int error = 0;
5184	u8 aq_buf[I40E_AQ_LARGE_BUF];
5185
5186	u16 next = 0;
5187	struct i40e_aqc_get_switch_config_resp *sw_config;
5188	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
5189
5190	buf = sbuf_new_for_sysctl(NULL, NULL, 0, req);
5191	if (!buf) {
5192		device_printf(dev, "Could not allocate sbuf for sysctl output.\n");
5193		return (ENOMEM);
5194	}
5195
5196	error = i40e_aq_get_switch_config(hw, sw_config,
5197	    sizeof(aq_buf), &next, NULL);
5198	if (error) {
5199		device_printf(dev,
5200		    "%s: aq_get_switch_config() error %d, aq error %d\n",
5201		    __func__, error, hw->aq.asq_last_status);
5202		sbuf_delete(buf);
5203		return error;
5204	}
5205
5206	nmbuf = sbuf_new_auto();
5207	if (!nmbuf) {
5208		device_printf(dev, "Could not allocate sbuf for name output.\n");
5209		return (ENOMEM);
5210	}
5211
5212	sbuf_cat(buf, "\n");
5213	// Assuming <= 255 elements in switch
5214	sbuf_printf(buf, "# of elements: %d\n", sw_config->header.num_reported);
5215	/* Exclude:
5216	** Revision -- all elements are revision 1 for now
5217	*/
5218	sbuf_printf(buf,
5219	    "SEID (  Name  ) |  Uplink  | Downlink | Conn Type\n"
5220	    "                |          |          | (uplink)\n");
5221	for (int i = 0; i < sw_config->header.num_reported; i++) {
5222		// "%4d (%8s) | %8s   %8s   %#8x",
5223		sbuf_printf(buf, "%4d", sw_config->element[i].seid);
5224		sbuf_cat(buf, " ");
5225		sbuf_printf(buf, "(%8s)", ixl_switch_element_string(nmbuf,
5226		    sw_config->element[i].seid, false));
5227		sbuf_cat(buf, " | ");
5228		sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5229		    sw_config->element[i].uplink_seid, true));
5230		sbuf_cat(buf, "   ");
5231		sbuf_printf(buf, "%8s", ixl_switch_element_string(nmbuf,
5232		    sw_config->element[i].downlink_seid, false));
5233		sbuf_cat(buf, "   ");
5234		sbuf_printf(buf, "%#8x", sw_config->element[i].connection_type);
5235		if (i < sw_config->header.num_reported - 1)
5236			sbuf_cat(buf, "\n");
5237	}
5238	sbuf_delete(nmbuf);
5239
5240	error = sbuf_finish(buf);
5241	sbuf_delete(buf);
5242
5243	return (error);
5244}
5245#endif /* IXL_DEBUG_SYSCTL */
5246
5247
5248#ifdef PCI_IOV
5249static int
5250ixl_vf_alloc_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5251{
5252	struct i40e_hw *hw;
5253	struct ixl_vsi *vsi;
5254	struct i40e_vsi_context vsi_ctx;
5255	int i;
5256	uint16_t first_queue;
5257	enum i40e_status_code code;
5258
5259	hw = &pf->hw;
5260	vsi = &pf->vsi;
5261
5262	vsi_ctx.pf_num = hw->pf_id;
5263	vsi_ctx.uplink_seid = pf->veb_seid;
5264	vsi_ctx.connection_type = IXL_VSI_DATA_PORT;
5265	vsi_ctx.vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5266	vsi_ctx.flags = I40E_AQ_VSI_TYPE_VF;
5267
5268	bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
5269
5270	vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5271	vsi_ctx.info.switch_id = htole16(0);
5272
5273	vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_SECURITY_VALID);
5274	vsi_ctx.info.sec_flags = 0;
5275	if (vf->vf_flags & VF_FLAG_MAC_ANTI_SPOOF)
5276		vsi_ctx.info.sec_flags |= I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK;
5277
5278	vsi_ctx.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
5279	vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
5280	    I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
5281
5282	vsi_ctx.info.valid_sections |=
5283	    htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID);
5284	vsi_ctx.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
5285	first_queue = vsi->num_queues + vf->vf_num * IXLV_MAX_QUEUES;
5286	for (i = 0; i < IXLV_MAX_QUEUES; i++)
5287		vsi_ctx.info.queue_mapping[i] = htole16(first_queue + i);
5288	for (; i < nitems(vsi_ctx.info.queue_mapping); i++)
5289		vsi_ctx.info.queue_mapping[i] = htole16(I40E_AQ_VSI_QUEUE_MASK);
5290
5291	vsi_ctx.info.tc_mapping[0] = htole16(
5292	    (0 << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5293	    (1 << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT));
5294
5295	code = i40e_aq_add_vsi(hw, &vsi_ctx, NULL);
5296	if (code != I40E_SUCCESS)
5297		return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5298	vf->vsi.seid = vsi_ctx.seid;
5299	vf->vsi.vsi_num = vsi_ctx.vsi_number;
5300	vf->vsi.first_queue = first_queue;
5301	vf->vsi.num_queues = IXLV_MAX_QUEUES;
5302
5303	code = i40e_aq_get_vsi_params(hw, &vsi_ctx, NULL);
5304	if (code != I40E_SUCCESS)
5305		return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5306
5307	code = i40e_aq_config_vsi_bw_limit(hw, vf->vsi.seid, 0, 0, NULL);
5308	if (code != I40E_SUCCESS) {
5309		device_printf(pf->dev, "Failed to disable BW limit: %d\n",
5310		    ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5311		return (ixl_adminq_err_to_errno(hw->aq.asq_last_status));
5312	}
5313
5314	memcpy(&vf->vsi.info, &vsi_ctx.info, sizeof(vf->vsi.info));
5315	return (0);
5316}
5317
5318static int
5319ixl_vf_setup_vsi(struct ixl_pf *pf, struct ixl_vf *vf)
5320{
5321	struct i40e_hw *hw;
5322	int error;
5323
5324	hw = &pf->hw;
5325
5326	error = ixl_vf_alloc_vsi(pf, vf);
5327	if (error != 0)
5328		return (error);
5329
5330	vf->vsi.hw_filters_add = 0;
5331	vf->vsi.hw_filters_del = 0;
5332	ixl_add_filter(&vf->vsi, ixl_bcast_addr, IXL_VLAN_ANY);
5333	ixl_reconfigure_filters(&vf->vsi);
5334
5335	return (0);
5336}
5337
5338static void
5339ixl_vf_map_vsi_queue(struct i40e_hw *hw, struct ixl_vf *vf, int qnum,
5340    uint32_t val)
5341{
5342	uint32_t qtable;
5343	int index, shift;
5344
5345	/*
5346	 * Two queues are mapped in a single register, so we have to do some
5347	 * gymnastics to convert the queue number into a register index and
5348	 * shift.
5349	 */
5350	index = qnum / 2;
5351	shift = (qnum % 2) * I40E_VSILAN_QTABLE_QINDEX_1_SHIFT;
5352
5353	qtable = rd32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num));
5354	qtable &= ~(I40E_VSILAN_QTABLE_QINDEX_0_MASK << shift);
5355	qtable |= val << shift;
5356	wr32(hw, I40E_VSILAN_QTABLE(index, vf->vsi.vsi_num), qtable);
5357}
5358
5359static void
5360ixl_vf_map_queues(struct ixl_pf *pf, struct ixl_vf *vf)
5361{
5362	struct i40e_hw *hw;
5363	uint32_t qtable;
5364	int i;
5365
5366	hw = &pf->hw;
5367
5368	/*
5369	 * Contiguous mappings aren't actually supported by the hardware,
5370	 * so we have to use non-contiguous mappings.
5371	 */
5372	wr32(hw, I40E_VSILAN_QBASE(vf->vsi.vsi_num),
5373	     I40E_VSILAN_QBASE_VSIQTABLE_ENA_MASK);
5374
5375	wr32(hw, I40E_VPLAN_MAPENA(vf->vf_num),
5376	    I40E_VPLAN_MAPENA_TXRX_ENA_MASK);
5377
5378	for (i = 0; i < vf->vsi.num_queues; i++) {
5379		qtable = (vf->vsi.first_queue + i) <<
5380		    I40E_VPLAN_QTABLE_QINDEX_SHIFT;
5381
5382		wr32(hw, I40E_VPLAN_QTABLE(i, vf->vf_num), qtable);
5383	}
5384
5385	/* Map queues allocated to VF to its VSI. */
5386	for (i = 0; i < vf->vsi.num_queues; i++)
5387		ixl_vf_map_vsi_queue(hw, vf, i, vf->vsi.first_queue + i);
5388
5389	/* Set rest of VSI queues as unused. */
5390	for (; i < IXL_MAX_VSI_QUEUES; i++)
5391		ixl_vf_map_vsi_queue(hw, vf, i,
5392		    I40E_VSILAN_QTABLE_QINDEX_0_MASK);
5393
5394	ixl_flush(hw);
5395}
5396
5397static void
5398ixl_vf_vsi_release(struct ixl_pf *pf, struct ixl_vsi *vsi)
5399{
5400	struct i40e_hw *hw;
5401
5402	hw = &pf->hw;
5403
5404	if (vsi->seid == 0)
5405		return;
5406
5407	i40e_aq_delete_element(hw, vsi->seid, NULL);
5408}
5409
5410static void
5411ixl_vf_disable_queue_intr(struct i40e_hw *hw, uint32_t vfint_reg)
5412{
5413
5414	wr32(hw, vfint_reg, I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
5415	ixl_flush(hw);
5416}
5417
5418static void
5419ixl_vf_unregister_intr(struct i40e_hw *hw, uint32_t vpint_reg)
5420{
5421
5422	wr32(hw, vpint_reg, I40E_VPINT_LNKLSTN_FIRSTQ_TYPE_MASK |
5423	    I40E_VPINT_LNKLSTN_FIRSTQ_INDX_MASK);
5424	ixl_flush(hw);
5425}
5426
5427static void
5428ixl_vf_release_resources(struct ixl_pf *pf, struct ixl_vf *vf)
5429{
5430	struct i40e_hw *hw;
5431	uint32_t vfint_reg, vpint_reg;
5432	int i;
5433
5434	hw = &pf->hw;
5435
5436	ixl_vf_vsi_release(pf, &vf->vsi);
5437
5438	/* Index 0 has a special register. */
5439	ixl_vf_disable_queue_intr(hw, I40E_VFINT_DYN_CTL0(vf->vf_num));
5440
5441	for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5442		vfint_reg = IXL_VFINT_DYN_CTLN_REG(hw, i , vf->vf_num);
5443		ixl_vf_disable_queue_intr(hw, vfint_reg);
5444	}
5445
5446	/* Index 0 has a special register. */
5447	ixl_vf_unregister_intr(hw, I40E_VPINT_LNKLST0(vf->vf_num));
5448
5449	for (i = 1; i < hw->func_caps.num_msix_vectors_vf; i++) {
5450		vpint_reg = IXL_VPINT_LNKLSTN_REG(hw, i, vf->vf_num);
5451		ixl_vf_unregister_intr(hw, vpint_reg);
5452	}
5453
5454	vf->vsi.num_queues = 0;
5455}
5456
5457static int
5458ixl_flush_pcie(struct ixl_pf *pf, struct ixl_vf *vf)
5459{
5460	struct i40e_hw *hw;
5461	int i;
5462	uint16_t global_vf_num;
5463	uint32_t ciad;
5464
5465	hw = &pf->hw;
5466	global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5467
5468	wr32(hw, I40E_PF_PCI_CIAA, IXL_PF_PCI_CIAA_VF_DEVICE_STATUS |
5469	     (global_vf_num << I40E_PF_PCI_CIAA_VF_NUM_SHIFT));
5470	for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5471		ciad = rd32(hw, I40E_PF_PCI_CIAD);
5472		if ((ciad & IXL_PF_PCI_CIAD_VF_TRANS_PENDING_MASK) == 0)
5473			return (0);
5474		DELAY(1);
5475	}
5476
5477	return (ETIMEDOUT);
5478}
5479
5480static void
5481ixl_reset_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5482{
5483	struct i40e_hw *hw;
5484	uint32_t vfrtrig;
5485
5486	hw = &pf->hw;
5487
5488	vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5489	vfrtrig |= I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5490	wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5491	ixl_flush(hw);
5492
5493	ixl_reinit_vf(pf, vf);
5494}
5495
5496static void
5497ixl_reinit_vf(struct ixl_pf *pf, struct ixl_vf *vf)
5498{
5499	struct i40e_hw *hw;
5500	uint32_t vfrstat, vfrtrig;
5501	int i, error;
5502
5503	hw = &pf->hw;
5504
5505	error = ixl_flush_pcie(pf, vf);
5506	if (error != 0)
5507		device_printf(pf->dev,
5508		    "Timed out waiting for PCIe activity to stop on VF-%d\n",
5509		    vf->vf_num);
5510
5511	for (i = 0; i < IXL_VF_RESET_TIMEOUT; i++) {
5512		DELAY(10);
5513
5514		vfrstat = rd32(hw, I40E_VPGEN_VFRSTAT(vf->vf_num));
5515		if (vfrstat & I40E_VPGEN_VFRSTAT_VFRD_MASK)
5516			break;
5517	}
5518
5519	if (i == IXL_VF_RESET_TIMEOUT)
5520		device_printf(pf->dev, "VF %d failed to reset\n", vf->vf_num);
5521
5522	wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_COMPLETED);
5523
5524	vfrtrig = rd32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num));
5525	vfrtrig &= ~I40E_VPGEN_VFRTRIG_VFSWR_MASK;
5526	wr32(hw, I40E_VPGEN_VFRTRIG(vf->vf_num), vfrtrig);
5527
5528	if (vf->vsi.seid != 0)
5529		ixl_disable_rings(&vf->vsi);
5530
5531	ixl_vf_release_resources(pf, vf);
5532	ixl_vf_setup_vsi(pf, vf);
5533	ixl_vf_map_queues(pf, vf);
5534
5535	wr32(hw, I40E_VFGEN_RSTAT1(vf->vf_num), I40E_VFR_VFACTIVE);
5536	ixl_flush(hw);
5537}
5538
5539static const char *
5540ixl_vc_opcode_str(uint16_t op)
5541{
5542
5543	switch (op) {
5544	case I40E_VIRTCHNL_OP_VERSION:
5545		return ("VERSION");
5546	case I40E_VIRTCHNL_OP_RESET_VF:
5547		return ("RESET_VF");
5548	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
5549		return ("GET_VF_RESOURCES");
5550	case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
5551		return ("CONFIG_TX_QUEUE");
5552	case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
5553		return ("CONFIG_RX_QUEUE");
5554	case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
5555		return ("CONFIG_VSI_QUEUES");
5556	case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
5557		return ("CONFIG_IRQ_MAP");
5558	case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
5559		return ("ENABLE_QUEUES");
5560	case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
5561		return ("DISABLE_QUEUES");
5562	case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
5563		return ("ADD_ETHER_ADDRESS");
5564	case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
5565		return ("DEL_ETHER_ADDRESS");
5566	case I40E_VIRTCHNL_OP_ADD_VLAN:
5567		return ("ADD_VLAN");
5568	case I40E_VIRTCHNL_OP_DEL_VLAN:
5569		return ("DEL_VLAN");
5570	case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
5571		return ("CONFIG_PROMISCUOUS_MODE");
5572	case I40E_VIRTCHNL_OP_GET_STATS:
5573		return ("GET_STATS");
5574	case I40E_VIRTCHNL_OP_FCOE:
5575		return ("FCOE");
5576	case I40E_VIRTCHNL_OP_EVENT:
5577		return ("EVENT");
5578	default:
5579		return ("UNKNOWN");
5580	}
5581}
5582
5583static int
5584ixl_vc_opcode_level(uint16_t opcode)
5585{
5586
5587	switch (opcode) {
5588	case I40E_VIRTCHNL_OP_GET_STATS:
5589		return (10);
5590	default:
5591		return (5);
5592	}
5593}
5594
5595static void
5596ixl_send_vf_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5597    enum i40e_status_code status, void *msg, uint16_t len)
5598{
5599	struct i40e_hw *hw;
5600	int global_vf_id;
5601
5602	hw = &pf->hw;
5603	global_vf_id = hw->func_caps.vf_base_id + vf->vf_num;
5604
5605	I40E_VC_DEBUG(pf, ixl_vc_opcode_level(op),
5606	    "Sending msg (op=%s[%d], status=%d) to VF-%d\n",
5607	    ixl_vc_opcode_str(op), op, status, vf->vf_num);
5608
5609	i40e_aq_send_msg_to_vf(hw, global_vf_id, op, status, msg, len, NULL);
5610}
5611
5612static void
5613ixl_send_vf_ack(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op)
5614{
5615
5616	ixl_send_vf_msg(pf, vf, op, I40E_SUCCESS, NULL, 0);
5617}
5618
5619static void
5620ixl_send_vf_nack_msg(struct ixl_pf *pf, struct ixl_vf *vf, uint16_t op,
5621    enum i40e_status_code status, const char *file, int line)
5622{
5623
5624	I40E_VC_DEBUG(pf, 1,
5625	    "Sending NACK (op=%s[%d], err=%d) to VF-%d from %s:%d\n",
5626	    ixl_vc_opcode_str(op), op, status, vf->vf_num, file, line);
5627	ixl_send_vf_msg(pf, vf, op, status, NULL, 0);
5628}
5629
5630static void
5631ixl_vf_version_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5632    uint16_t msg_size)
5633{
5634	struct i40e_virtchnl_version_info reply;
5635
5636	if (msg_size != sizeof(struct i40e_virtchnl_version_info)) {
5637		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_VERSION,
5638		    I40E_ERR_PARAM);
5639		return;
5640	}
5641
5642	reply.major = I40E_VIRTCHNL_VERSION_MAJOR;
5643	reply.minor = I40E_VIRTCHNL_VERSION_MINOR;
5644	ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_VERSION, I40E_SUCCESS, &reply,
5645	    sizeof(reply));
5646}
5647
5648static void
5649ixl_vf_reset_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5650    uint16_t msg_size)
5651{
5652
5653	if (msg_size != 0) {
5654		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_RESET_VF,
5655		    I40E_ERR_PARAM);
5656		return;
5657	}
5658
5659	ixl_reset_vf(pf, vf);
5660
5661	/* No response to a reset message. */
5662}
5663
5664static void
5665ixl_vf_get_resources_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5666    uint16_t msg_size)
5667{
5668	struct i40e_virtchnl_vf_resource reply;
5669
5670	if (msg_size != 0) {
5671		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5672		    I40E_ERR_PARAM);
5673		return;
5674	}
5675
5676	bzero(&reply, sizeof(reply));
5677
5678	reply.vf_offload_flags = I40E_VIRTCHNL_VF_OFFLOAD_L2;
5679
5680	reply.num_vsis = 1;
5681	reply.num_queue_pairs = vf->vsi.num_queues;
5682	reply.max_vectors = pf->hw.func_caps.num_msix_vectors_vf;
5683	reply.vsi_res[0].vsi_id = vf->vsi.vsi_num;
5684	reply.vsi_res[0].vsi_type = I40E_VSI_SRIOV;
5685	reply.vsi_res[0].num_queue_pairs = vf->vsi.num_queues;
5686	memcpy(reply.vsi_res[0].default_mac_addr, vf->mac, ETHER_ADDR_LEN);
5687
5688	ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_VF_RESOURCES,
5689	    I40E_SUCCESS, &reply, sizeof(reply));
5690}
5691
5692static int
5693ixl_vf_config_tx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5694    struct i40e_virtchnl_txq_info *info)
5695{
5696	struct i40e_hw *hw;
5697	struct i40e_hmc_obj_txq txq;
5698	uint16_t global_queue_num, global_vf_num;
5699	enum i40e_status_code status;
5700	uint32_t qtx_ctl;
5701
5702	hw = &pf->hw;
5703	global_queue_num = vf->vsi.first_queue + info->queue_id;
5704	global_vf_num = hw->func_caps.vf_base_id + vf->vf_num;
5705	bzero(&txq, sizeof(txq));
5706
5707	status = i40e_clear_lan_tx_queue_context(hw, global_queue_num);
5708	if (status != I40E_SUCCESS)
5709		return (EINVAL);
5710
5711	txq.base = info->dma_ring_addr / IXL_TX_CTX_BASE_UNITS;
5712
5713	txq.head_wb_ena = info->headwb_enabled;
5714	txq.head_wb_addr = info->dma_headwb_addr;
5715	txq.qlen = info->ring_len;
5716	txq.rdylist = le16_to_cpu(vf->vsi.info.qs_handle[0]);
5717	txq.rdylist_act = 0;
5718
5719	status = i40e_set_lan_tx_queue_context(hw, global_queue_num, &txq);
5720	if (status != I40E_SUCCESS)
5721		return (EINVAL);
5722
5723	qtx_ctl = I40E_QTX_CTL_VF_QUEUE |
5724	    (hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) |
5725	    (global_vf_num << I40E_QTX_CTL_VFVM_INDX_SHIFT);
5726	wr32(hw, I40E_QTX_CTL(global_queue_num), qtx_ctl);
5727	ixl_flush(hw);
5728
5729	return (0);
5730}
5731
5732static int
5733ixl_vf_config_rx_queue(struct ixl_pf *pf, struct ixl_vf *vf,
5734    struct i40e_virtchnl_rxq_info *info)
5735{
5736	struct i40e_hw *hw;
5737	struct i40e_hmc_obj_rxq rxq;
5738	uint16_t global_queue_num;
5739	enum i40e_status_code status;
5740
5741	hw = &pf->hw;
5742	global_queue_num = vf->vsi.first_queue + info->queue_id;
5743	bzero(&rxq, sizeof(rxq));
5744
5745	if (info->databuffer_size > IXL_VF_MAX_BUFFER)
5746		return (EINVAL);
5747
5748	if (info->max_pkt_size > IXL_VF_MAX_FRAME ||
5749	    info->max_pkt_size < ETHER_MIN_LEN)
5750		return (EINVAL);
5751
5752	if (info->splithdr_enabled) {
5753		if (info->hdr_size > IXL_VF_MAX_HDR_BUFFER)
5754			return (EINVAL);
5755
5756		rxq.hsplit_0 = info->rx_split_pos &
5757		    (I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_L2 |
5758		     I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_IP |
5759		     I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_TCP_UDP |
5760		     I40E_HMC_OBJ_RX_HSPLIT_0_SPLIT_SCTP);
5761		rxq.hbuff = info->hdr_size >> I40E_RXQ_CTX_HBUFF_SHIFT;
5762
5763		rxq.dtype = 2;
5764	}
5765
5766	status = i40e_clear_lan_rx_queue_context(hw, global_queue_num);
5767	if (status != I40E_SUCCESS)
5768		return (EINVAL);
5769
5770	rxq.base = info->dma_ring_addr / IXL_RX_CTX_BASE_UNITS;
5771	rxq.qlen = info->ring_len;
5772
5773	rxq.dbuff = info->databuffer_size >> I40E_RXQ_CTX_DBUFF_SHIFT;
5774
5775	rxq.dsize = 1;
5776	rxq.crcstrip = 1;
5777	rxq.l2tsel = 1;
5778
5779	rxq.rxmax = info->max_pkt_size;
5780	rxq.tphrdesc_ena = 1;
5781	rxq.tphwdesc_ena = 1;
5782	rxq.tphdata_ena = 1;
5783	rxq.tphhead_ena = 1;
5784	rxq.lrxqthresh = 2;
5785	rxq.prefena = 1;
5786
5787	status = i40e_set_lan_rx_queue_context(hw, global_queue_num, &rxq);
5788	if (status != I40E_SUCCESS)
5789		return (EINVAL);
5790
5791	return (0);
5792}
5793
5794static void
5795ixl_vf_config_vsi_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5796    uint16_t msg_size)
5797{
5798	struct i40e_virtchnl_vsi_queue_config_info *info;
5799	struct i40e_virtchnl_queue_pair_info *pair;
5800	int i;
5801
5802	if (msg_size < sizeof(*info)) {
5803		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5804		    I40E_ERR_PARAM);
5805		return;
5806	}
5807
5808	info = msg;
5809	if (info->num_queue_pairs == 0) {
5810		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5811		    I40E_ERR_PARAM);
5812		return;
5813	}
5814
5815	if (msg_size != sizeof(*info) + info->num_queue_pairs * sizeof(*pair)) {
5816		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5817		    I40E_ERR_PARAM);
5818		return;
5819	}
5820
5821	if (info->vsi_id != vf->vsi.vsi_num) {
5822		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES,
5823		    I40E_ERR_PARAM);
5824		return;
5825	}
5826
5827	for (i = 0; i < info->num_queue_pairs; i++) {
5828		pair = &info->qpair[i];
5829
5830		if (pair->txq.vsi_id != vf->vsi.vsi_num ||
5831		    pair->rxq.vsi_id != vf->vsi.vsi_num ||
5832		    pair->txq.queue_id != pair->rxq.queue_id ||
5833		    pair->txq.queue_id >= vf->vsi.num_queues) {
5834
5835			i40e_send_vf_nack(pf, vf,
5836			    I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5837			return;
5838		}
5839
5840		if (ixl_vf_config_tx_queue(pf, vf, &pair->txq) != 0) {
5841			i40e_send_vf_nack(pf, vf,
5842			    I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5843			return;
5844		}
5845
5846		if (ixl_vf_config_rx_queue(pf, vf, &pair->rxq) != 0) {
5847			i40e_send_vf_nack(pf, vf,
5848			    I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES, I40E_ERR_PARAM);
5849			return;
5850		}
5851	}
5852
5853	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES);
5854}
5855
5856static void
5857ixl_vf_set_qctl(struct ixl_pf *pf,
5858    const struct i40e_virtchnl_vector_map *vector,
5859    enum i40e_queue_type cur_type, uint16_t cur_queue,
5860    enum i40e_queue_type *last_type, uint16_t *last_queue)
5861{
5862	uint32_t offset, qctl;
5863	uint16_t itr_indx;
5864
5865	if (cur_type == I40E_QUEUE_TYPE_RX) {
5866		offset = I40E_QINT_RQCTL(cur_queue);
5867		itr_indx = vector->rxitr_idx;
5868	} else {
5869		offset = I40E_QINT_TQCTL(cur_queue);
5870		itr_indx = vector->txitr_idx;
5871	}
5872
5873	qctl = htole32((vector->vector_id << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
5874	    (*last_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
5875	    (*last_queue << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
5876	    I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5877	    (itr_indx << I40E_QINT_RQCTL_ITR_INDX_SHIFT));
5878
5879	wr32(&pf->hw, offset, qctl);
5880
5881	*last_type = cur_type;
5882	*last_queue = cur_queue;
5883}
5884
5885static void
5886ixl_vf_config_vector(struct ixl_pf *pf, struct ixl_vf *vf,
5887    const struct i40e_virtchnl_vector_map *vector)
5888{
5889	struct i40e_hw *hw;
5890	u_int qindex;
5891	enum i40e_queue_type type, last_type;
5892	uint32_t lnklst_reg;
5893	uint16_t rxq_map, txq_map, cur_queue, last_queue;
5894
5895	hw = &pf->hw;
5896
5897	rxq_map = vector->rxq_map;
5898	txq_map = vector->txq_map;
5899
5900	last_queue = IXL_END_OF_INTR_LNKLST;
5901	last_type = I40E_QUEUE_TYPE_RX;
5902
5903	/*
5904	 * The datasheet says to optimize performance, RX queues and TX queues
5905	 * should be interleaved in the interrupt linked list, so we process
5906	 * both at once here.
5907	 */
5908	while ((rxq_map != 0) || (txq_map != 0)) {
5909		if (txq_map != 0) {
5910			qindex = ffs(txq_map) - 1;
5911			type = I40E_QUEUE_TYPE_TX;
5912			cur_queue = vf->vsi.first_queue + qindex;
5913			ixl_vf_set_qctl(pf, vector, type, cur_queue,
5914			    &last_type, &last_queue);
5915			txq_map &= ~(1 << qindex);
5916		}
5917
5918		if (rxq_map != 0) {
5919			qindex = ffs(rxq_map) - 1;
5920			type = I40E_QUEUE_TYPE_RX;
5921			cur_queue = vf->vsi.first_queue + qindex;
5922			ixl_vf_set_qctl(pf, vector, type, cur_queue,
5923			    &last_type, &last_queue);
5924			rxq_map &= ~(1 << qindex);
5925		}
5926	}
5927
5928	if (vector->vector_id == 0)
5929		lnklst_reg = I40E_VPINT_LNKLST0(vf->vf_num);
5930	else
5931		lnklst_reg = IXL_VPINT_LNKLSTN_REG(hw, vector->vector_id,
5932		    vf->vf_num);
5933	wr32(hw, lnklst_reg,
5934	    (last_queue << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
5935	    (last_type << I40E_VPINT_LNKLST0_FIRSTQ_TYPE_SHIFT));
5936
5937	ixl_flush(hw);
5938}
5939
5940static void
5941ixl_vf_config_irq_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
5942    uint16_t msg_size)
5943{
5944	struct i40e_virtchnl_irq_map_info *map;
5945	struct i40e_virtchnl_vector_map *vector;
5946	struct i40e_hw *hw;
5947	int i, largest_txq, largest_rxq;
5948
5949	hw = &pf->hw;
5950
5951	if (msg_size < sizeof(*map)) {
5952		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5953		    I40E_ERR_PARAM);
5954		return;
5955	}
5956
5957	map = msg;
5958	if (map->num_vectors == 0) {
5959		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5960		    I40E_ERR_PARAM);
5961		return;
5962	}
5963
5964	if (msg_size != sizeof(*map) + map->num_vectors * sizeof(*vector)) {
5965		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5966		    I40E_ERR_PARAM);
5967		return;
5968	}
5969
5970	for (i = 0; i < map->num_vectors; i++) {
5971		vector = &map->vecmap[i];
5972
5973		if ((vector->vector_id >= hw->func_caps.num_msix_vectors_vf) ||
5974		    vector->vsi_id != vf->vsi.vsi_num) {
5975			i40e_send_vf_nack(pf, vf,
5976			    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP, I40E_ERR_PARAM);
5977			return;
5978		}
5979
5980		if (vector->rxq_map != 0) {
5981			largest_rxq = fls(vector->rxq_map) - 1;
5982			if (largest_rxq >= vf->vsi.num_queues) {
5983				i40e_send_vf_nack(pf, vf,
5984				    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5985				    I40E_ERR_PARAM);
5986				return;
5987			}
5988		}
5989
5990		if (vector->txq_map != 0) {
5991			largest_txq = fls(vector->txq_map) - 1;
5992			if (largest_txq >= vf->vsi.num_queues) {
5993				i40e_send_vf_nack(pf, vf,
5994				    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
5995				    I40E_ERR_PARAM);
5996				return;
5997			}
5998		}
5999
6000		if (vector->rxitr_idx > IXL_MAX_ITR_IDX ||
6001		    vector->txitr_idx > IXL_MAX_ITR_IDX) {
6002			i40e_send_vf_nack(pf, vf,
6003			    I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP,
6004			    I40E_ERR_PARAM);
6005			return;
6006		}
6007
6008		ixl_vf_config_vector(pf, vf, vector);
6009	}
6010
6011	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP);
6012}
6013
6014static void
6015ixl_vf_enable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6016    uint16_t msg_size)
6017{
6018	struct i40e_virtchnl_queue_select *select;
6019	int error;
6020
6021	if (msg_size != sizeof(*select)) {
6022		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6023		    I40E_ERR_PARAM);
6024		return;
6025	}
6026
6027	select = msg;
6028	if (select->vsi_id != vf->vsi.vsi_num ||
6029	    select->rx_queues == 0 || select->tx_queues == 0) {
6030		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6031		    I40E_ERR_PARAM);
6032		return;
6033	}
6034
6035	error = ixl_enable_rings(&vf->vsi);
6036	if (error) {
6037		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES,
6038		    I40E_ERR_TIMEOUT);
6039		return;
6040	}
6041
6042	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ENABLE_QUEUES);
6043}
6044
6045static void
6046ixl_vf_disable_queues_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6047    void *msg, uint16_t msg_size)
6048{
6049	struct i40e_virtchnl_queue_select *select;
6050	int error;
6051
6052	if (msg_size != sizeof(*select)) {
6053		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6054		    I40E_ERR_PARAM);
6055		return;
6056	}
6057
6058	select = msg;
6059	if (select->vsi_id != vf->vsi.vsi_num ||
6060	    select->rx_queues == 0 || select->tx_queues == 0) {
6061		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6062		    I40E_ERR_PARAM);
6063		return;
6064	}
6065
6066	error = ixl_disable_rings(&vf->vsi);
6067	if (error) {
6068		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES,
6069		    I40E_ERR_TIMEOUT);
6070		return;
6071	}
6072
6073	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DISABLE_QUEUES);
6074}
6075
6076static boolean_t
6077ixl_zero_mac(const uint8_t *addr)
6078{
6079	uint8_t zero[ETHER_ADDR_LEN] = {0, 0, 0, 0, 0, 0};
6080
6081	return (cmp_etheraddr(addr, zero));
6082}
6083
6084static boolean_t
6085ixl_bcast_mac(const uint8_t *addr)
6086{
6087
6088	return (cmp_etheraddr(addr, ixl_bcast_addr));
6089}
6090
6091static int
6092ixl_vf_mac_valid(struct ixl_vf *vf, const uint8_t *addr)
6093{
6094
6095	if (ixl_zero_mac(addr) || ixl_bcast_mac(addr))
6096		return (EINVAL);
6097
6098	/*
6099	 * If the VF is not allowed to change its MAC address, don't let it
6100	 * set a MAC filter for an address that is not a multicast address and
6101	 * is not its assigned MAC.
6102	 */
6103	if (!(vf->vf_flags & VF_FLAG_SET_MAC_CAP) &&
6104	    !(ETHER_IS_MULTICAST(addr) || cmp_etheraddr(addr, vf->mac)))
6105		return (EPERM);
6106
6107	return (0);
6108}
6109
6110static void
6111ixl_vf_add_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6112    uint16_t msg_size)
6113{
6114	struct i40e_virtchnl_ether_addr_list *addr_list;
6115	struct i40e_virtchnl_ether_addr *addr;
6116	struct ixl_vsi *vsi;
6117	int i;
6118	size_t expected_size;
6119
6120	vsi = &vf->vsi;
6121
6122	if (msg_size < sizeof(*addr_list)) {
6123		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6124		    I40E_ERR_PARAM);
6125		return;
6126	}
6127
6128	addr_list = msg;
6129	expected_size = sizeof(*addr_list) +
6130	    addr_list->num_elements * sizeof(*addr);
6131
6132	if (addr_list->num_elements == 0 ||
6133	    addr_list->vsi_id != vsi->vsi_num ||
6134	    msg_size != expected_size) {
6135		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6136		    I40E_ERR_PARAM);
6137		return;
6138	}
6139
6140	for (i = 0; i < addr_list->num_elements; i++) {
6141		if (ixl_vf_mac_valid(vf, addr_list->list[i].addr) != 0) {
6142			i40e_send_vf_nack(pf, vf,
6143			    I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6144			return;
6145		}
6146	}
6147
6148	for (i = 0; i < addr_list->num_elements; i++) {
6149		addr = &addr_list->list[i];
6150		ixl_add_filter(vsi, addr->addr, IXL_VLAN_ANY);
6151	}
6152
6153	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS);
6154}
6155
6156static void
6157ixl_vf_del_mac_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6158    uint16_t msg_size)
6159{
6160	struct i40e_virtchnl_ether_addr_list *addr_list;
6161	struct i40e_virtchnl_ether_addr *addr;
6162	size_t expected_size;
6163	int i;
6164
6165	if (msg_size < sizeof(*addr_list)) {
6166		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6167		    I40E_ERR_PARAM);
6168		return;
6169	}
6170
6171	addr_list = msg;
6172	expected_size = sizeof(*addr_list) +
6173	    addr_list->num_elements * sizeof(*addr);
6174
6175	if (addr_list->num_elements == 0 ||
6176	    addr_list->vsi_id != vf->vsi.vsi_num ||
6177	    msg_size != expected_size) {
6178		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS,
6179		    I40E_ERR_PARAM);
6180		return;
6181	}
6182
6183	for (i = 0; i < addr_list->num_elements; i++) {
6184		addr = &addr_list->list[i];
6185		if (ixl_zero_mac(addr->addr) || ixl_bcast_mac(addr->addr)) {
6186			i40e_send_vf_nack(pf, vf,
6187			    I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS, I40E_ERR_PARAM);
6188			return;
6189		}
6190	}
6191
6192	for (i = 0; i < addr_list->num_elements; i++) {
6193		addr = &addr_list->list[i];
6194		ixl_del_filter(&vf->vsi, addr->addr, IXL_VLAN_ANY);
6195	}
6196
6197	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS);
6198}
6199
6200static enum i40e_status_code
6201ixl_vf_enable_vlan_strip(struct ixl_pf *pf, struct ixl_vf *vf)
6202{
6203	struct i40e_vsi_context vsi_ctx;
6204
6205	vsi_ctx.seid = vf->vsi.seid;
6206
6207	bzero(&vsi_ctx.info, sizeof(vsi_ctx.info));
6208	vsi_ctx.info.valid_sections = htole16(I40E_AQ_VSI_PROP_VLAN_VALID);
6209	vsi_ctx.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
6210	    I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
6211	return (i40e_aq_update_vsi_params(&pf->hw, &vsi_ctx, NULL));
6212}
6213
6214static void
6215ixl_vf_add_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6216    uint16_t msg_size)
6217{
6218	struct i40e_virtchnl_vlan_filter_list *filter_list;
6219	enum i40e_status_code code;
6220	size_t expected_size;
6221	int i;
6222
6223	if (msg_size < sizeof(*filter_list)) {
6224		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6225		    I40E_ERR_PARAM);
6226		return;
6227	}
6228
6229	filter_list = msg;
6230	expected_size = sizeof(*filter_list) +
6231	    filter_list->num_elements * sizeof(uint16_t);
6232	if (filter_list->num_elements == 0 ||
6233	    filter_list->vsi_id != vf->vsi.vsi_num ||
6234	    msg_size != expected_size) {
6235		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6236		    I40E_ERR_PARAM);
6237		return;
6238	}
6239
6240	if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6241		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6242		    I40E_ERR_PARAM);
6243		return;
6244	}
6245
6246	for (i = 0; i < filter_list->num_elements; i++) {
6247		if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6248			i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6249			    I40E_ERR_PARAM);
6250			return;
6251		}
6252	}
6253
6254	code = ixl_vf_enable_vlan_strip(pf, vf);
6255	if (code != I40E_SUCCESS) {
6256		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6257		    I40E_ERR_PARAM);
6258	}
6259
6260	for (i = 0; i < filter_list->num_elements; i++)
6261		ixl_add_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6262
6263	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN);
6264}
6265
6266static void
6267ixl_vf_del_vlan_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6268    uint16_t msg_size)
6269{
6270	struct i40e_virtchnl_vlan_filter_list *filter_list;
6271	int i;
6272	size_t expected_size;
6273
6274	if (msg_size < sizeof(*filter_list)) {
6275		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6276		    I40E_ERR_PARAM);
6277		return;
6278	}
6279
6280	filter_list = msg;
6281	expected_size = sizeof(*filter_list) +
6282	    filter_list->num_elements * sizeof(uint16_t);
6283	if (filter_list->num_elements == 0 ||
6284	    filter_list->vsi_id != vf->vsi.vsi_num ||
6285	    msg_size != expected_size) {
6286		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN,
6287		    I40E_ERR_PARAM);
6288		return;
6289	}
6290
6291	for (i = 0; i < filter_list->num_elements; i++) {
6292		if (filter_list->vlan_id[i] > EVL_VLID_MASK) {
6293			i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6294			    I40E_ERR_PARAM);
6295			return;
6296		}
6297	}
6298
6299	if (!(vf->vf_flags & VF_FLAG_VLAN_CAP)) {
6300		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_ADD_VLAN,
6301		    I40E_ERR_PARAM);
6302		return;
6303	}
6304
6305	for (i = 0; i < filter_list->num_elements; i++)
6306		ixl_del_filter(&vf->vsi, vf->mac, filter_list->vlan_id[i]);
6307
6308	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_DEL_VLAN);
6309}
6310
6311static void
6312ixl_vf_config_promisc_msg(struct ixl_pf *pf, struct ixl_vf *vf,
6313    void *msg, uint16_t msg_size)
6314{
6315	struct i40e_virtchnl_promisc_info *info;
6316	enum i40e_status_code code;
6317
6318	if (msg_size != sizeof(*info)) {
6319		i40e_send_vf_nack(pf, vf,
6320		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6321		return;
6322	}
6323
6324	if (!(vf->vf_flags & VF_FLAG_PROMISC_CAP)) {
6325		i40e_send_vf_nack(pf, vf,
6326		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6327		return;
6328	}
6329
6330	info = msg;
6331	if (info->vsi_id != vf->vsi.vsi_num) {
6332		i40e_send_vf_nack(pf, vf,
6333		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, I40E_ERR_PARAM);
6334		return;
6335	}
6336
6337	code = i40e_aq_set_vsi_unicast_promiscuous(&pf->hw, info->vsi_id,
6338	    info->flags & I40E_FLAG_VF_UNICAST_PROMISC, NULL);
6339	if (code != I40E_SUCCESS) {
6340		i40e_send_vf_nack(pf, vf,
6341		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6342		return;
6343	}
6344
6345	code = i40e_aq_set_vsi_multicast_promiscuous(&pf->hw, info->vsi_id,
6346	    info->flags & I40E_FLAG_VF_MULTICAST_PROMISC, NULL);
6347	if (code != I40E_SUCCESS) {
6348		i40e_send_vf_nack(pf, vf,
6349		    I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, code);
6350		return;
6351	}
6352
6353	ixl_send_vf_ack(pf, vf, I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE);
6354}
6355
6356static void
6357ixl_vf_get_stats_msg(struct ixl_pf *pf, struct ixl_vf *vf, void *msg,
6358    uint16_t msg_size)
6359{
6360	struct i40e_virtchnl_queue_select *queue;
6361
6362	if (msg_size != sizeof(*queue)) {
6363		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6364		    I40E_ERR_PARAM);
6365		return;
6366	}
6367
6368	queue = msg;
6369	if (queue->vsi_id != vf->vsi.vsi_num) {
6370		i40e_send_vf_nack(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6371		    I40E_ERR_PARAM);
6372		return;
6373	}
6374
6375	ixl_update_eth_stats(&vf->vsi);
6376
6377	ixl_send_vf_msg(pf, vf, I40E_VIRTCHNL_OP_GET_STATS,
6378	    I40E_SUCCESS, &vf->vsi.eth_stats, sizeof(vf->vsi.eth_stats));
6379}
6380
6381static void
6382ixl_handle_vf_msg(struct ixl_pf *pf, struct i40e_arq_event_info *event)
6383{
6384	struct ixl_vf *vf;
6385	void *msg;
6386	uint16_t vf_num, msg_size;
6387	uint32_t opcode;
6388
6389	vf_num = le16toh(event->desc.retval) - pf->hw.func_caps.vf_base_id;
6390	opcode = le32toh(event->desc.cookie_high);
6391
6392	if (vf_num >= pf->num_vfs) {
6393		device_printf(pf->dev, "Got msg from illegal VF: %d\n", vf_num);
6394		return;
6395	}
6396
6397	vf = &pf->vfs[vf_num];
6398	msg = event->msg_buf;
6399	msg_size = event->msg_len;
6400
6401	I40E_VC_DEBUG(pf, ixl_vc_opcode_level(opcode),
6402	    "Got msg %s(%d) from VF-%d of size %d\n",
6403	    ixl_vc_opcode_str(opcode), opcode, vf_num, msg_size);
6404
6405	switch (opcode) {
6406	case I40E_VIRTCHNL_OP_VERSION:
6407		ixl_vf_version_msg(pf, vf, msg, msg_size);
6408		break;
6409	case I40E_VIRTCHNL_OP_RESET_VF:
6410		ixl_vf_reset_msg(pf, vf, msg, msg_size);
6411		break;
6412	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
6413		ixl_vf_get_resources_msg(pf, vf, msg, msg_size);
6414		break;
6415	case I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES:
6416		ixl_vf_config_vsi_msg(pf, vf, msg, msg_size);
6417		break;
6418	case I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP:
6419		ixl_vf_config_irq_msg(pf, vf, msg, msg_size);
6420		break;
6421	case I40E_VIRTCHNL_OP_ENABLE_QUEUES:
6422		ixl_vf_enable_queues_msg(pf, vf, msg, msg_size);
6423		break;
6424	case I40E_VIRTCHNL_OP_DISABLE_QUEUES:
6425		ixl_vf_disable_queues_msg(pf, vf, msg, msg_size);
6426		break;
6427	case I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS:
6428		ixl_vf_add_mac_msg(pf, vf, msg, msg_size);
6429		break;
6430	case I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS:
6431		ixl_vf_del_mac_msg(pf, vf, msg, msg_size);
6432		break;
6433	case I40E_VIRTCHNL_OP_ADD_VLAN:
6434		ixl_vf_add_vlan_msg(pf, vf, msg, msg_size);
6435		break;
6436	case I40E_VIRTCHNL_OP_DEL_VLAN:
6437		ixl_vf_del_vlan_msg(pf, vf, msg, msg_size);
6438		break;
6439	case I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE:
6440		ixl_vf_config_promisc_msg(pf, vf, msg, msg_size);
6441		break;
6442	case I40E_VIRTCHNL_OP_GET_STATS:
6443		ixl_vf_get_stats_msg(pf, vf, msg, msg_size);
6444		break;
6445
6446	/* These two opcodes have been superseded by CONFIG_VSI_QUEUES. */
6447	case I40E_VIRTCHNL_OP_CONFIG_TX_QUEUE:
6448	case I40E_VIRTCHNL_OP_CONFIG_RX_QUEUE:
6449	default:
6450		i40e_send_vf_nack(pf, vf, opcode, I40E_ERR_NOT_IMPLEMENTED);
6451		break;
6452	}
6453}
6454
6455/* Handle any VFs that have reset themselves via a Function Level Reset(FLR). */
6456static void
6457ixl_handle_vflr(void *arg, int pending)
6458{
6459	struct ixl_pf *pf;
6460	struct i40e_hw *hw;
6461	uint16_t global_vf_num;
6462	uint32_t vflrstat_index, vflrstat_mask, vflrstat, icr0;
6463	int i;
6464
6465	pf = arg;
6466	hw = &pf->hw;
6467
6468	IXL_PF_LOCK(pf);
6469	for (i = 0; i < pf->num_vfs; i++) {
6470		global_vf_num = hw->func_caps.vf_base_id + i;
6471
6472		vflrstat_index = IXL_GLGEN_VFLRSTAT_INDEX(global_vf_num);
6473		vflrstat_mask = IXL_GLGEN_VFLRSTAT_MASK(global_vf_num);
6474		vflrstat = rd32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index));
6475		if (vflrstat & vflrstat_mask) {
6476			wr32(hw, I40E_GLGEN_VFLRSTAT(vflrstat_index),
6477			    vflrstat_mask);
6478
6479			ixl_reinit_vf(pf, &pf->vfs[i]);
6480		}
6481	}
6482
6483	icr0 = rd32(hw, I40E_PFINT_ICR0_ENA);
6484	icr0 |= I40E_PFINT_ICR0_ENA_VFLR_MASK;
6485	wr32(hw, I40E_PFINT_ICR0_ENA, icr0);
6486	ixl_flush(hw);
6487
6488	IXL_PF_UNLOCK(pf);
6489}
6490
6491static int
6492ixl_adminq_err_to_errno(enum i40e_admin_queue_err err)
6493{
6494
6495	switch (err) {
6496	case I40E_AQ_RC_EPERM:
6497		return (EPERM);
6498	case I40E_AQ_RC_ENOENT:
6499		return (ENOENT);
6500	case I40E_AQ_RC_ESRCH:
6501		return (ESRCH);
6502	case I40E_AQ_RC_EINTR:
6503		return (EINTR);
6504	case I40E_AQ_RC_EIO:
6505		return (EIO);
6506	case I40E_AQ_RC_ENXIO:
6507		return (ENXIO);
6508	case I40E_AQ_RC_E2BIG:
6509		return (E2BIG);
6510	case I40E_AQ_RC_EAGAIN:
6511		return (EAGAIN);
6512	case I40E_AQ_RC_ENOMEM:
6513		return (ENOMEM);
6514	case I40E_AQ_RC_EACCES:
6515		return (EACCES);
6516	case I40E_AQ_RC_EFAULT:
6517		return (EFAULT);
6518	case I40E_AQ_RC_EBUSY:
6519		return (EBUSY);
6520	case I40E_AQ_RC_EEXIST:
6521		return (EEXIST);
6522	case I40E_AQ_RC_EINVAL:
6523		return (EINVAL);
6524	case I40E_AQ_RC_ENOTTY:
6525		return (ENOTTY);
6526	case I40E_AQ_RC_ENOSPC:
6527		return (ENOSPC);
6528	case I40E_AQ_RC_ENOSYS:
6529		return (ENOSYS);
6530	case I40E_AQ_RC_ERANGE:
6531		return (ERANGE);
6532	case I40E_AQ_RC_EFLUSHED:
6533		return (EINVAL);	/* No exact equivalent in errno.h */
6534	case I40E_AQ_RC_BAD_ADDR:
6535		return (EFAULT);
6536	case I40E_AQ_RC_EMODE:
6537		return (EPERM);
6538	case I40E_AQ_RC_EFBIG:
6539		return (EFBIG);
6540	default:
6541		return (EINVAL);
6542	}
6543}
6544
6545static int
6546ixl_iov_init(device_t dev, uint16_t num_vfs, const nvlist_t *params)
6547{
6548	struct ixl_pf *pf;
6549	struct i40e_hw *hw;
6550	struct ixl_vsi *pf_vsi;
6551	enum i40e_status_code ret;
6552	int i, error;
6553
6554	pf = device_get_softc(dev);
6555	hw = &pf->hw;
6556	pf_vsi = &pf->vsi;
6557
6558	IXL_PF_LOCK(pf);
6559	pf->vfs = malloc(sizeof(struct ixl_vf) * num_vfs, M_IXL, M_NOWAIT |
6560	    M_ZERO);
6561
6562	if (pf->vfs == NULL) {
6563		error = ENOMEM;
6564		goto fail;
6565	}
6566
6567	for (i = 0; i < num_vfs; i++)
6568		sysctl_ctx_init(&pf->vfs[i].ctx);
6569
6570	ret = i40e_aq_add_veb(hw, pf_vsi->uplink_seid, pf_vsi->seid,
6571	    1, FALSE, FALSE, &pf->veb_seid, NULL);
6572	if (ret != I40E_SUCCESS) {
6573		error = ixl_adminq_err_to_errno(hw->aq.asq_last_status);
6574		device_printf(dev, "add_veb failed; code=%d error=%d", ret,
6575		    error);
6576		goto fail;
6577	}
6578
6579	ixl_configure_msix(pf);
6580	ixl_enable_adminq(hw);
6581
6582	pf->num_vfs = num_vfs;
6583	IXL_PF_UNLOCK(pf);
6584	return (0);
6585
6586fail:
6587	free(pf->vfs, M_IXL);
6588	pf->vfs = NULL;
6589	IXL_PF_UNLOCK(pf);
6590	return (error);
6591}
6592
6593static void
6594ixl_iov_uninit(device_t dev)
6595{
6596	struct ixl_pf *pf;
6597	struct i40e_hw *hw;
6598	struct ixl_vsi *vsi;
6599	struct ifnet *ifp;
6600	struct ixl_vf *vfs;
6601	int i, num_vfs;
6602
6603	pf = device_get_softc(dev);
6604	hw = &pf->hw;
6605	vsi = &pf->vsi;
6606	ifp = vsi->ifp;
6607
6608	IXL_PF_LOCK(pf);
6609	for (i = 0; i < pf->num_vfs; i++) {
6610		if (pf->vfs[i].vsi.seid != 0)
6611			i40e_aq_delete_element(hw, pf->vfs[i].vsi.seid, NULL);
6612	}
6613
6614	if (pf->veb_seid != 0) {
6615		i40e_aq_delete_element(hw, pf->veb_seid, NULL);
6616		pf->veb_seid = 0;
6617	}
6618
6619#if __FreeBSD_version > 1100022
6620	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
6621#else
6622	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
6623#endif
6624		ixl_disable_intr(vsi);
6625
6626	vfs = pf->vfs;
6627	num_vfs = pf->num_vfs;
6628
6629	pf->vfs = NULL;
6630	pf->num_vfs = 0;
6631	IXL_PF_UNLOCK(pf);
6632
6633	/* Do this after the unlock as sysctl_ctx_free might sleep. */
6634	for (i = 0; i < num_vfs; i++)
6635		sysctl_ctx_free(&vfs[i].ctx);
6636	free(vfs, M_IXL);
6637}
6638
6639static int
6640ixl_add_vf(device_t dev, uint16_t vfnum, const nvlist_t *params)
6641{
6642	char sysctl_name[QUEUE_NAME_LEN];
6643	struct ixl_pf *pf;
6644	struct ixl_vf *vf;
6645	const void *mac;
6646	size_t size;
6647	int error;
6648
6649	pf = device_get_softc(dev);
6650	vf = &pf->vfs[vfnum];
6651
6652	IXL_PF_LOCK(pf);
6653	vf->vf_num = vfnum;
6654
6655	vf->vsi.back = pf;
6656	vf->vf_flags = VF_FLAG_ENABLED;
6657	SLIST_INIT(&vf->vsi.ftl);
6658
6659	error = ixl_vf_setup_vsi(pf, vf);
6660	if (error != 0)
6661		goto out;
6662
6663	if (nvlist_exists_binary(params, "mac-addr")) {
6664		mac = nvlist_get_binary(params, "mac-addr", &size);
6665		bcopy(mac, vf->mac, ETHER_ADDR_LEN);
6666
6667		if (nvlist_get_bool(params, "allow-set-mac"))
6668			vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6669	} else
6670		/*
6671		 * If the administrator has not specified a MAC address then
6672		 * we must allow the VF to choose one.
6673		 */
6674		vf->vf_flags |= VF_FLAG_SET_MAC_CAP;
6675
6676	if (nvlist_get_bool(params, "mac-anti-spoof"))
6677		vf->vf_flags |= VF_FLAG_MAC_ANTI_SPOOF;
6678
6679	if (nvlist_get_bool(params, "allow-promisc"))
6680		vf->vf_flags |= VF_FLAG_PROMISC_CAP;
6681
6682	vf->vf_flags |= VF_FLAG_VLAN_CAP;
6683
6684	ixl_reset_vf(pf, vf);
6685out:
6686	IXL_PF_UNLOCK(pf);
6687	if (error == 0) {
6688		snprintf(sysctl_name, sizeof(sysctl_name), "vf%d", vfnum);
6689		ixl_add_vsi_sysctls(pf, &vf->vsi, &vf->ctx, sysctl_name);
6690	}
6691
6692	return (error);
6693}
6694#endif /* PCI_IOV */
6695