1/*	$OpenBSD: if_ix.c,v 1.215 2024/05/21 11:19:39 bluhm Exp $	*/
2
3/******************************************************************************
4
5  Copyright (c) 2001-2013, Intel Corporation
6  All rights reserved.
7
8  Redistribution and use in source and binary forms, with or without
9  modification, are permitted provided that the following conditions are met:
10
11   1. Redistributions of source code must retain the above copyright notice,
12      this list of conditions and the following disclaimer.
13
14   2. Redistributions in binary form must reproduce the above copyright
15      notice, this list of conditions and the following disclaimer in the
16      documentation and/or other materials provided with the distribution.
17
18   3. Neither the name of the Intel Corporation nor the names of its
19      contributors may be used to endorse or promote products derived from
20      this software without specific prior written permission.
21
22  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32  POSSIBILITY OF SUCH DAMAGE.
33
34******************************************************************************/
35/* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */
36
37#include <dev/pci/if_ix.h>
38#include <dev/pci/ixgbe_type.h>
39
40/*
41 * Our TCP/IP Stack is unable to handle packets greater than MAXMCLBYTES.
42 * This interface is unable to handle packets greater than IXGBE_TSO_SIZE.
43 */
44CTASSERT(MAXMCLBYTES <= IXGBE_TSO_SIZE);
45
46/*********************************************************************
47 *  Driver version
48 *********************************************************************/
49/* char ixgbe_driver_version[] = "2.5.13"; */
50
51/*********************************************************************
52 *  PCI Device ID Table
53 *
54 *  Used by probe to select devices to load on
55 *********************************************************************/
56
57const struct pci_matchid ixgbe_devices[] = {
58	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 },
59	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX },
60	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL },
61	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF },
62	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT },
63	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT2 },
64	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL },
65	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 },
66	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL },
67	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR },
68	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP },
69	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM },
70	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL },
71	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 },
72	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ },
73	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI },
74	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BP },
75	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE },
76	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 },
77	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_T3_LOM },
78	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP },
79	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM },
80	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF_QP },
81	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF2 },
82	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_FCOE },
83	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599EN_SFP },
84	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_QSFP_SF_QP },
85	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T },
86	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T1 },
87	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T },
88	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T1 },
89	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KX4 },
90	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KR },
91	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_SFP },
92	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_10G_T },
93	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_1G_T },
94	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR },
95	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR_L },
96	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP_N },
97	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP },
98	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII },
99	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII_L },
100	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_10G_T },
101	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T },
102	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T_L }
103};
104
105/*********************************************************************
106 *  Function prototypes
107 *********************************************************************/
108int	ixgbe_probe(struct device *, void *, void *);
109void	ixgbe_attach(struct device *, struct device *, void *);
110int	ixgbe_detach(struct device *, int);
111int	ixgbe_activate(struct device *, int);
112void	ixgbe_start(struct ifqueue *);
113int	ixgbe_ioctl(struct ifnet *, u_long, caddr_t);
114int	ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *);
115int	ixgbe_get_sffpage(struct ix_softc *, struct if_sffpage *);
116void	ixgbe_watchdog(struct ifnet *);
117void	ixgbe_init(void *);
118void	ixgbe_stop(void *);
119void	ixgbe_media_status(struct ifnet *, struct ifmediareq *);
120int	ixgbe_media_change(struct ifnet *);
121void	ixgbe_identify_hardware(struct ix_softc *);
122int	ixgbe_allocate_pci_resources(struct ix_softc *);
123int	ixgbe_allocate_legacy(struct ix_softc *);
124int	ixgbe_allocate_msix(struct ix_softc *);
125void	ixgbe_setup_msix(struct ix_softc *);
126int	ixgbe_allocate_queues(struct ix_softc *);
127void	ixgbe_free_pci_resources(struct ix_softc *);
128void	ixgbe_setup_interface(struct ix_softc *);
129void	ixgbe_config_gpie(struct ix_softc *);
130void	ixgbe_config_delay_values(struct ix_softc *);
131void	ixgbe_add_media_types(struct ix_softc *);
132void	ixgbe_config_link(struct ix_softc *);
133
134int	ixgbe_allocate_transmit_buffers(struct ix_txring *);
135int	ixgbe_setup_transmit_structures(struct ix_softc *);
136int	ixgbe_setup_transmit_ring(struct ix_txring *);
137void	ixgbe_initialize_transmit_units(struct ix_softc *);
138void	ixgbe_free_transmit_structures(struct ix_softc *);
139void	ixgbe_free_transmit_buffers(struct ix_txring *);
140
141int	ixgbe_allocate_receive_buffers(struct ix_rxring *);
142int	ixgbe_setup_receive_structures(struct ix_softc *);
143int	ixgbe_setup_receive_ring(struct ix_rxring *);
144void	ixgbe_initialize_receive_units(struct ix_softc *);
145void	ixgbe_free_receive_structures(struct ix_softc *);
146void	ixgbe_free_receive_buffers(struct ix_rxring *);
147void	ixgbe_initialize_rss_mapping(struct ix_softc *);
148int	ixgbe_rxfill(struct ix_rxring *);
149void	ixgbe_rxrefill(void *);
150
151int	ixgbe_intr(struct ix_softc *sc);
152void	ixgbe_enable_intr(struct ix_softc *);
153void	ixgbe_disable_intr(struct ix_softc *);
154int	ixgbe_txeof(struct ix_txring *);
155int	ixgbe_rxeof(struct ix_rxring *);
156void	ixgbe_rx_offload(uint32_t, uint16_t, struct mbuf *);
157void	ixgbe_iff(struct ix_softc *);
158void	ixgbe_map_queue_statistics(struct ix_softc *);
159void	ixgbe_update_link_status(struct ix_softc *);
160int	ixgbe_get_buf(struct ix_rxring *, int);
161int	ixgbe_encap(struct ix_txring *, struct mbuf *);
162int	ixgbe_dma_malloc(struct ix_softc *, bus_size_t,
163		    struct ixgbe_dma_alloc *, int);
164void	ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *);
165static int
166	ixgbe_tx_ctx_setup(struct ix_txring *, struct mbuf *, uint32_t *,
167	    uint32_t *);
168void	ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t);
169void	ixgbe_configure_ivars(struct ix_softc *);
170uint8_t	*ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *);
171
172void	ixgbe_setup_vlan_hw_support(struct ix_softc *);
173
174/* Support for pluggable optic modules */
175void	ixgbe_handle_mod(struct ix_softc *);
176void	ixgbe_handle_msf(struct ix_softc *);
177void	ixgbe_handle_phy(struct ix_softc *);
178
179/* Legacy (single vector interrupt handler */
180int	ixgbe_legacy_intr(void *);
181void	ixgbe_enable_queue(struct ix_softc *, uint32_t);
182void	ixgbe_enable_queues(struct ix_softc *);
183void	ixgbe_disable_queue(struct ix_softc *, uint32_t);
184
185/* MSI-X (multiple vectors interrupt handlers)  */
186int	ixgbe_link_intr(void *);
187int	ixgbe_queue_intr(void *);
188
189#if NKSTAT > 0
190static void	ix_kstats(struct ix_softc *);
191static void	ix_rxq_kstats(struct ix_softc *, struct ix_rxring *);
192static void	ix_txq_kstats(struct ix_softc *, struct ix_txring *);
193static void	ix_kstats_tick(void *);
194#endif
195
196/*********************************************************************
197 *  OpenBSD Device Interface Entry Points
198 *********************************************************************/
199
200struct cfdriver ix_cd = {
201	NULL, "ix", DV_IFNET
202};
203
204const struct cfattach ix_ca = {
205	sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach,
206	ixgbe_activate
207};
208
209int ixgbe_smart_speed = ixgbe_smart_speed_on;
210int ixgbe_enable_msix = 1;
211
212/*********************************************************************
213 *  Device identification routine
214 *
215 *  ixgbe_probe determines if the driver should be loaded on
216 *  adapter based on PCI vendor/device id of the adapter.
217 *
218 *  return 0 on success, positive on failure
219 *********************************************************************/
220
221int
222ixgbe_probe(struct device *parent, void *match, void *aux)
223{
224	INIT_DEBUGOUT("ixgbe_probe: begin");
225
226	return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices,
227	    nitems(ixgbe_devices)));
228}
229
230/*********************************************************************
231 *  Device initialization routine
232 *
233 *  The attach entry point is called when the driver is being loaded.
234 *  This routine identifies the type of hardware, allocates all resources
235 *  and initializes the hardware.
236 *
237 *  return 0 on success, positive on failure
238 *********************************************************************/
239
240void
241ixgbe_attach(struct device *parent, struct device *self, void *aux)
242{
243	struct pci_attach_args	*pa = (struct pci_attach_args *)aux;
244	struct ix_softc		*sc = (struct ix_softc *)self;
245	int			 error = 0;
246	uint16_t		 csum;
247	uint32_t			 ctrl_ext;
248	struct ixgbe_hw		*hw = &sc->hw;
249
250	INIT_DEBUGOUT("ixgbe_attach: begin");
251
252	sc->osdep.os_sc = sc;
253	sc->osdep.os_pa = *pa;
254
255	rw_init(&sc->sfflock, "ixsff");
256
257#if NKSTAT > 0
258	ix_kstats(sc);
259#endif
260
261	/* Determine hardware revision */
262	ixgbe_identify_hardware(sc);
263
264	/* Indicate to RX setup to use Jumbo Clusters */
265	sc->num_tx_desc = DEFAULT_TXD;
266	sc->num_rx_desc = DEFAULT_RXD;
267
268	/* Do base PCI setup - map BAR0 */
269	if (ixgbe_allocate_pci_resources(sc))
270		goto err_out;
271
272	/* Allocate our TX/RX Queues */
273	if (ixgbe_allocate_queues(sc))
274		goto err_out;
275
276	/* Allocate multicast array memory. */
277	sc->mta = mallocarray(IXGBE_ETH_LENGTH_OF_ADDRESS,
278	    MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT);
279	if (sc->mta == NULL) {
280		printf(": Can not allocate multicast setup array\n");
281		goto err_late;
282	}
283
284	/* Initialize the shared code */
285	error = ixgbe_init_shared_code(hw);
286	if (error) {
287		printf(": Unable to initialize the shared code\n");
288		goto err_late;
289	}
290
291	/* Make sure we have a good EEPROM before we read from it */
292	if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) {
293		printf(": The EEPROM Checksum Is Not Valid\n");
294		goto err_late;
295	}
296
297	error = ixgbe_init_hw(hw);
298	if (error == IXGBE_ERR_EEPROM_VERSION) {
299		printf(": This device is a pre-production adapter/"
300		    "LOM.  Please be aware there may be issues associated "
301		    "with your hardware.\nIf you are experiencing problems "
302		    "please contact your Intel or hardware representative "
303		    "who provided you with this hardware.\n");
304	} else if (error && (error != IXGBE_ERR_SFP_NOT_PRESENT &&
305	    error != IXGBE_ERR_SFP_NOT_SUPPORTED)) {
306		printf(": Hardware Initialization Failure\n");
307		goto err_late;
308	}
309
310	bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr,
311	    IXGBE_ETH_LENGTH_OF_ADDRESS);
312
313	if (sc->sc_intrmap)
314		error = ixgbe_allocate_msix(sc);
315	else
316		error = ixgbe_allocate_legacy(sc);
317	if (error)
318		goto err_late;
319
320	/* Enable the optics for 82599 SFP+ fiber */
321	if (sc->hw.mac.ops.enable_tx_laser)
322		sc->hw.mac.ops.enable_tx_laser(&sc->hw);
323
324	/* Enable power to the phy */
325	if (hw->phy.ops.set_phy_power)
326		hw->phy.ops.set_phy_power(&sc->hw, TRUE);
327
328	/* Setup OS specific network interface */
329	ixgbe_setup_interface(sc);
330
331	/* Get the PCI-E bus info and determine LAN ID */
332	hw->mac.ops.get_bus_info(hw);
333
334	/* Set an initial default flow control value */
335	sc->fc = ixgbe_fc_full;
336
337	/* let hardware know driver is loaded */
338	ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);
339	ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
340	IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);
341
342	printf(", address %s\n", ether_sprintf(sc->hw.mac.addr));
343
344	INIT_DEBUGOUT("ixgbe_attach: end");
345	return;
346
347err_late:
348	ixgbe_free_transmit_structures(sc);
349	ixgbe_free_receive_structures(sc);
350err_out:
351	ixgbe_free_pci_resources(sc);
352	free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS *
353	    MAX_NUM_MULTICAST_ADDRESSES);
354}
355
356/*********************************************************************
357 *  Device removal routine
358 *
359 *  The detach entry point is called when the driver is being removed.
360 *  This routine stops the adapter and deallocates all the resources
361 *  that were allocated for driver operation.
362 *
363 *  return 0 on success, positive on failure
364 *********************************************************************/
365
366int
367ixgbe_detach(struct device *self, int flags)
368{
369	struct ix_softc *sc = (struct ix_softc *)self;
370	struct ifnet *ifp = &sc->arpcom.ac_if;
371	uint32_t	ctrl_ext;
372
373	INIT_DEBUGOUT("ixgbe_detach: begin");
374
375	ixgbe_stop(sc);
376
377	/* let hardware know driver is unloading */
378	ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);
379	ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD;
380	IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);
381
382	ether_ifdetach(ifp);
383	if_detach(ifp);
384
385	ixgbe_free_pci_resources(sc);
386
387	ixgbe_free_transmit_structures(sc);
388	ixgbe_free_receive_structures(sc);
389	free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS *
390	    MAX_NUM_MULTICAST_ADDRESSES);
391
392	/* XXX kstat */
393
394	return (0);
395}
396
397int
398ixgbe_activate(struct device *self, int act)
399{
400	struct ix_softc *sc = (struct ix_softc *)self;
401	struct ifnet *ifp = &sc->arpcom.ac_if;
402	struct ixgbe_hw		*hw = &sc->hw;
403	uint32_t			 ctrl_ext;
404	int rv = 0;
405
406	switch (act) {
407	case DVACT_QUIESCE:
408		if (ifp->if_flags & IFF_RUNNING)
409			ixgbe_stop(sc);
410		break;
411	case DVACT_RESUME:
412		ixgbe_init_hw(hw);
413
414		/* Enable the optics for 82599 SFP+ fiber */
415		if (sc->hw.mac.ops.enable_tx_laser)
416			sc->hw.mac.ops.enable_tx_laser(&sc->hw);
417
418		/* Enable power to the phy */
419		if (hw->phy.ops.set_phy_power)
420			hw->phy.ops.set_phy_power(&sc->hw, TRUE);
421
422		/* Get the PCI-E bus info and determine LAN ID */
423		hw->mac.ops.get_bus_info(hw);
424
425		/* let hardware know driver is loaded */
426		ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT);
427		ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
428		IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext);
429
430		if (ifp->if_flags & IFF_UP)
431			ixgbe_init(sc);
432		break;
433	default:
434		break;
435	}
436	return (rv);
437}
438
439/*********************************************************************
440 *  Transmit entry point
441 *
442 *  ixgbe_start is called by the stack to initiate a transmit.
443 *  The driver will remain in this routine as long as there are
444 *  packets to transmit and transmit resources are available.
445 *  In case resources are not available stack is notified and
446 *  the packet is requeued.
447 **********************************************************************/
448
449void
450ixgbe_start(struct ifqueue *ifq)
451{
452	struct ifnet		*ifp = ifq->ifq_if;
453	struct ix_softc		*sc = ifp->if_softc;
454	struct ix_txring	*txr = ifq->ifq_softc;
455	struct mbuf  		*m_head;
456	unsigned int		 head, free, used;
457	int			 post = 0;
458
459	if (!sc->link_up)
460		return;
461
462	head = txr->next_avail_desc;
463	free = txr->next_to_clean;
464	if (free <= head)
465		free += sc->num_tx_desc;
466	free -= head;
467
468	membar_consumer();
469
470	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
471	    0, txr->txdma.dma_map->dm_mapsize,
472	    BUS_DMASYNC_POSTWRITE);
473
474	for (;;) {
475		/* Check that we have the minimal number of TX descriptors. */
476		if (free <= IXGBE_TX_OP_THRESHOLD) {
477			ifq_set_oactive(ifq);
478			break;
479		}
480
481		m_head = ifq_dequeue(ifq);
482		if (m_head == NULL)
483			break;
484
485		used = ixgbe_encap(txr, m_head);
486		if (used == 0) {
487			m_freem(m_head);
488			continue;
489		}
490
491		free -= used;
492
493#if NBPFILTER > 0
494		if (ifp->if_bpf)
495			bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT);
496#endif
497
498		/* Set timeout in case hardware has problems transmitting */
499		txr->watchdog_timer = IXGBE_TX_TIMEOUT;
500		ifp->if_timer = IXGBE_TX_TIMEOUT;
501
502		post = 1;
503	}
504
505	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
506	    0, txr->txdma.dma_map->dm_mapsize,
507	    BUS_DMASYNC_PREWRITE);
508
509	/*
510	 * Advance the Transmit Descriptor Tail (Tdt), this tells the
511	 * hardware that this frame is available to transmit.
512	 */
513	if (post)
514		IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me),
515		    txr->next_avail_desc);
516}
517
518/*********************************************************************
519 *  Ioctl entry point
520 *
521 *  ixgbe_ioctl is called when the user wants to configure the
522 *  interface.
523 *
524 *  return 0 on success, positive on failure
525 **********************************************************************/
526
527int
528ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data)
529{
530	struct ix_softc	*sc = ifp->if_softc;
531	struct ifreq	*ifr = (struct ifreq *) data;
532	int		s, error = 0;
533
534	s = splnet();
535
536	switch (command) {
537	case SIOCSIFADDR:
538		IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)");
539		ifp->if_flags |= IFF_UP;
540		if (!(ifp->if_flags & IFF_RUNNING))
541			ixgbe_init(sc);
542		break;
543
544	case SIOCSIFFLAGS:
545		IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)");
546		if (ifp->if_flags & IFF_UP) {
547			if (ifp->if_flags & IFF_RUNNING)
548				error = ENETRESET;
549			else
550				ixgbe_init(sc);
551		} else {
552			if (ifp->if_flags & IFF_RUNNING)
553				ixgbe_stop(sc);
554		}
555		break;
556
557	case SIOCSIFMEDIA:
558	case SIOCGIFMEDIA:
559		IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)");
560		error = ifmedia_ioctl(ifp, ifr, &sc->media, command);
561		break;
562
563	case SIOCGIFRXR:
564		error = ixgbe_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data);
565		break;
566
567	case SIOCGIFSFFPAGE:
568		error = rw_enter(&sc->sfflock, RW_WRITE|RW_INTR);
569		if (error != 0)
570			break;
571
572		error = ixgbe_get_sffpage(sc, (struct if_sffpage *)data);
573		rw_exit(&sc->sfflock);
574		break;
575
576	default:
577		error = ether_ioctl(ifp, &sc->arpcom, command, data);
578	}
579
580	if (error == ENETRESET) {
581		if (ifp->if_flags & IFF_RUNNING) {
582			ixgbe_disable_intr(sc);
583			ixgbe_iff(sc);
584			ixgbe_enable_intr(sc);
585			ixgbe_enable_queues(sc);
586		}
587		error = 0;
588	}
589
590	splx(s);
591	return (error);
592}
593
594int
595ixgbe_get_sffpage(struct ix_softc *sc, struct if_sffpage *sff)
596{
597	struct ixgbe_hw *hw = &sc->hw;
598	uint32_t swfw_mask = hw->phy.phy_semaphore_mask;
599	uint8_t page;
600	size_t i;
601	int error = EIO;
602
603	if (hw->phy.type == ixgbe_phy_fw)
604		return (ENODEV);
605
606	if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
607		return (EBUSY); /* XXX */
608
609	if (sff->sff_addr == IFSFF_ADDR_EEPROM) {
610		if (hw->phy.ops.read_i2c_byte_unlocked(hw, 127,
611		    IFSFF_ADDR_EEPROM, &page))
612			goto error;
613		if (page != sff->sff_page &&
614		    hw->phy.ops.write_i2c_byte_unlocked(hw, 127,
615		    IFSFF_ADDR_EEPROM, sff->sff_page))
616			goto error;
617	}
618
619	for (i = 0; i < sizeof(sff->sff_data); i++) {
620		if (hw->phy.ops.read_i2c_byte_unlocked(hw, i,
621		    sff->sff_addr, &sff->sff_data[i]))
622			goto error;
623	}
624
625	if (sff->sff_addr == IFSFF_ADDR_EEPROM) {
626		if (page != sff->sff_page &&
627		    hw->phy.ops.write_i2c_byte_unlocked(hw, 127,
628		    IFSFF_ADDR_EEPROM, page))
629			goto error;
630	}
631
632	error = 0;
633error:
634	hw->mac.ops.release_swfw_sync(hw, swfw_mask);
635	return (error);
636}
637
638int
639ixgbe_rxrinfo(struct ix_softc *sc, struct if_rxrinfo *ifri)
640{
641	struct if_rxring_info *ifr, ifr1;
642	struct ix_rxring *rxr;
643	int error, i;
644	u_int n = 0;
645
646	if (sc->num_queues > 1) {
647		ifr = mallocarray(sc->num_queues, sizeof(*ifr), M_DEVBUF,
648		    M_WAITOK | M_ZERO);
649	} else
650		ifr = &ifr1;
651
652	for (i = 0; i < sc->num_queues; i++) {
653		rxr = &sc->rx_rings[i];
654		ifr[n].ifr_size = MCLBYTES;
655		snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i);
656		ifr[n].ifr_info = rxr->rx_ring;
657		n++;
658	}
659
660	error = if_rxr_info_ioctl(ifri, sc->num_queues, ifr);
661
662	if (sc->num_queues > 1)
663		free(ifr, M_DEVBUF, sc->num_queues * sizeof(*ifr));
664	return (error);
665}
666
667/*********************************************************************
668 *  Watchdog entry point
669 *
670 **********************************************************************/
671
672void
673ixgbe_watchdog(struct ifnet * ifp)
674{
675	struct ix_softc *sc = (struct ix_softc *)ifp->if_softc;
676	struct ix_txring *txr = sc->tx_rings;
677	struct ixgbe_hw *hw = &sc->hw;
678	int		tx_hang = FALSE;
679	int		i;
680
681	/*
682	 * The timer is set to 5 every time ixgbe_start() queues a packet.
683	 * Anytime all descriptors are clean the timer is set to 0.
684	 */
685	for (i = 0; i < sc->num_queues; i++, txr++) {
686		if (txr->watchdog_timer == 0 || --txr->watchdog_timer)
687			continue;
688		else {
689			tx_hang = TRUE;
690			break;
691		}
692	}
693	if (tx_hang == FALSE)
694		return;
695
696	/*
697	 * If we are in this routine because of pause frames, then don't
698	 * reset the hardware.
699	 */
700	if (!(IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXON)) {
701		for (i = 0; i < sc->num_queues; i++, txr++)
702			txr->watchdog_timer = IXGBE_TX_TIMEOUT;
703		ifp->if_timer = IXGBE_TX_TIMEOUT;
704		return;
705	}
706
707
708	printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname);
709	for (i = 0; i < sc->num_queues; i++, txr++) {
710		printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i,
711		    IXGBE_READ_REG(hw, IXGBE_TDH(i)),
712		    IXGBE_READ_REG(hw, IXGBE_TDT(i)));
713		printf("%s: TX(%d) Next TX to Clean = %d\n", ifp->if_xname,
714		    i, txr->next_to_clean);
715	}
716	ifp->if_flags &= ~IFF_RUNNING;
717
718	ixgbe_init(sc);
719}
720
721/*********************************************************************
722 *  Init entry point
723 *
724 *  This routine is used in two ways. It is used by the stack as
725 *  init entry point in network interface structure. It is also used
726 *  by the driver as a hw/sw initialization routine to get to a
727 *  consistent state.
728 *
729 *  return 0 on success, positive on failure
730 **********************************************************************/
731#define IXGBE_MHADD_MFS_SHIFT 16
732
733void
734ixgbe_init(void *arg)
735{
736	struct ix_softc	*sc = (struct ix_softc *)arg;
737	struct ifnet	*ifp = &sc->arpcom.ac_if;
738	struct ix_rxring	*rxr = sc->rx_rings;
739	uint32_t	 k, txdctl, rxdctl, rxctrl, mhadd, itr;
740	int		 i, s, err;
741
742	INIT_DEBUGOUT("ixgbe_init: begin");
743
744	s = splnet();
745
746	ixgbe_stop(sc);
747
748	/* reprogram the RAR[0] in case user changed it. */
749	ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV);
750
751	/* Get the latest mac address, User can use a LAA */
752	bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr,
753	      IXGBE_ETH_LENGTH_OF_ADDRESS);
754	ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1);
755	sc->hw.addr_ctrl.rar_used_count = 1;
756
757	/* Prepare transmit descriptors and buffers */
758	if (ixgbe_setup_transmit_structures(sc)) {
759		printf("%s: Could not setup transmit structures\n",
760		    ifp->if_xname);
761		ixgbe_stop(sc);
762		splx(s);
763		return;
764	}
765
766	ixgbe_init_hw(&sc->hw);
767	ixgbe_initialize_transmit_units(sc);
768
769	/* Use 2k clusters, even for jumbo frames */
770	sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN;
771
772	/* Prepare receive descriptors and buffers */
773	if (ixgbe_setup_receive_structures(sc)) {
774		printf("%s: Could not setup receive structures\n",
775		    ifp->if_xname);
776		ixgbe_stop(sc);
777		splx(s);
778		return;
779	}
780
781	/* Configure RX settings */
782	ixgbe_initialize_receive_units(sc);
783
784	/* Enable SDP & MSIX interrupts based on adapter */
785	ixgbe_config_gpie(sc);
786
787	/* Program promiscuous mode and multicast filters. */
788	ixgbe_iff(sc);
789
790	/* Set MRU size */
791	mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD);
792	mhadd &= ~IXGBE_MHADD_MFS_MASK;
793	mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT;
794	IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd);
795
796	/* Now enable all the queues */
797	for (i = 0; i < sc->num_queues; i++) {
798		txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i));
799		txdctl |= IXGBE_TXDCTL_ENABLE;
800		/* Set WTHRESH to 8, burst writeback */
801		txdctl |= (8 << 16);
802		/*
803		 * When the internal queue falls below PTHRESH (16),
804		 * start prefetching as long as there are at least
805		 * HTHRESH (1) buffers ready.
806		 */
807		txdctl |= (16 << 0) | (1 << 8);
808		IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl);
809	}
810
811	for (i = 0; i < sc->num_queues; i++) {
812		rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i));
813		if (sc->hw.mac.type == ixgbe_mac_82598EB) {
814			/*
815			 * PTHRESH = 21
816			 * HTHRESH = 4
817			 * WTHRESH = 8
818			 */
819			rxdctl &= ~0x3FFFFF;
820			rxdctl |= 0x080420;
821		}
822		rxdctl |= IXGBE_RXDCTL_ENABLE;
823		IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl);
824		for (k = 0; k < 10; k++) {
825			if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) &
826			    IXGBE_RXDCTL_ENABLE)
827				break;
828			else
829				msec_delay(1);
830		}
831		IXGBE_WRITE_FLUSH(&sc->hw);
832		IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled);
833	}
834
835	/* Set up VLAN support and filter */
836	ixgbe_setup_vlan_hw_support(sc);
837
838	/* Enable Receive engine */
839	rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL);
840	if (sc->hw.mac.type == ixgbe_mac_82598EB)
841		rxctrl |= IXGBE_RXCTRL_DMBYPS;
842	rxctrl |= IXGBE_RXCTRL_RXEN;
843	sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl);
844
845	/* Set up MSI/X routing */
846	if (sc->sc_intrmap) {
847		ixgbe_configure_ivars(sc);
848		/* Set up auto-mask */
849		if (sc->hw.mac.type == ixgbe_mac_82598EB)
850			IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
851		else {
852			IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
853			IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
854		}
855	} else {  /* Simple settings for Legacy/MSI */
856		ixgbe_set_ivar(sc, 0, 0, 0);
857		ixgbe_set_ivar(sc, 0, 0, 1);
858		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
859	}
860
861	/* Check on any SFP devices that need to be kick-started */
862	if (sc->hw.phy.type == ixgbe_phy_none) {
863		err = sc->hw.phy.ops.identify(&sc->hw);
864		if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
865			printf("Unsupported SFP+ module type was detected.\n");
866			splx(s);
867			return;
868		}
869	}
870
871	/* Setup interrupt moderation */
872	itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8;
873	if (sc->hw.mac.type != ixgbe_mac_82598EB)
874		itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS;
875	IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr);
876
877	if (sc->sc_intrmap) {
878		/* Set moderation on the Link interrupt */
879		IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(sc->linkvec),
880		    IXGBE_LINK_ITR);
881	}
882
883	/* Enable power to the phy */
884	if (sc->hw.phy.ops.set_phy_power)
885		sc->hw.phy.ops.set_phy_power(&sc->hw, TRUE);
886
887	/* Config/Enable Link */
888	ixgbe_config_link(sc);
889
890	/* Hardware Packet Buffer & Flow Control setup */
891	ixgbe_config_delay_values(sc);
892
893	/* Initialize the FC settings */
894	sc->hw.mac.ops.start_hw(&sc->hw);
895
896	/* And now turn on interrupts */
897	ixgbe_enable_intr(sc);
898	ixgbe_enable_queues(sc);
899
900	/* Now inform the stack we're ready */
901	ifp->if_flags |= IFF_RUNNING;
902	for (i = 0; i < sc->num_queues; i++)
903		ifq_clr_oactive(ifp->if_ifqs[i]);
904
905#if NKSTAT > 0
906	ix_kstats_tick(sc);
907#endif
908
909	splx(s);
910}
911
912void
913ixgbe_config_gpie(struct ix_softc *sc)
914{
915	struct ixgbe_hw	*hw = &sc->hw;
916	uint32_t gpie;
917
918	gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE);
919
920	/* Fan Failure Interrupt */
921	if (hw->device_id == IXGBE_DEV_ID_82598AT)
922		gpie |= IXGBE_SDP1_GPIEN;
923
924	if (sc->hw.mac.type == ixgbe_mac_82599EB) {
925		/* Add for Module detection */
926		gpie |= IXGBE_SDP2_GPIEN;
927
928		/* Media ready */
929		if (hw->device_id != IXGBE_DEV_ID_82599_QSFP_SF_QP)
930			gpie |= IXGBE_SDP1_GPIEN;
931
932		/*
933		 * Set LL interval to max to reduce the number of low latency
934		 * interrupts hitting the card when the ring is getting full.
935		 */
936		gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT;
937	}
938
939	if (sc->hw.mac.type == ixgbe_mac_X540 ||
940	    sc->hw.mac.type == ixgbe_mac_X550EM_x ||
941	    sc->hw.mac.type == ixgbe_mac_X550EM_a) {
942		/*
943		 * Thermal Failure Detection (X540)
944		 * Link Detection (X552 SFP+, X552/X557-AT)
945		 */
946		gpie |= IXGBE_SDP0_GPIEN_X540;
947
948		/*
949		 * Set LL interval to max to reduce the number of low latency
950		 * interrupts hitting the card when the ring is getting full.
951		 */
952		gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT;
953	}
954
955	if (sc->sc_intrmap) {
956		/* Enable Enhanced MSIX mode */
957		gpie |= IXGBE_GPIE_MSIX_MODE;
958		gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT |
959		    IXGBE_GPIE_OCD;
960	}
961
962	IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie);
963}
964
965/*
966 * Requires sc->max_frame_size to be set.
967 */
968void
969ixgbe_config_delay_values(struct ix_softc *sc)
970{
971	struct ixgbe_hw *hw = &sc->hw;
972	uint32_t rxpb, frame, size, tmp;
973
974	frame = sc->max_frame_size;
975
976	/* Calculate High Water */
977	switch (hw->mac.type) {
978	case ixgbe_mac_X540:
979	case ixgbe_mac_X550:
980	case ixgbe_mac_X550EM_x:
981	case ixgbe_mac_X550EM_a:
982		tmp = IXGBE_DV_X540(frame, frame);
983		break;
984	default:
985		tmp = IXGBE_DV(frame, frame);
986		break;
987	}
988	size = IXGBE_BT2KB(tmp);
989	rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10;
990	hw->fc.high_water[0] = rxpb - size;
991
992	/* Now calculate Low Water */
993	switch (hw->mac.type) {
994	case ixgbe_mac_X540:
995	case ixgbe_mac_X550:
996	case ixgbe_mac_X550EM_x:
997	case ixgbe_mac_X550EM_a:
998		tmp = IXGBE_LOW_DV_X540(frame);
999		break;
1000	default:
1001		tmp = IXGBE_LOW_DV(frame);
1002		break;
1003	}
1004	hw->fc.low_water[0] = IXGBE_BT2KB(tmp);
1005
1006	hw->fc.requested_mode = sc->fc;
1007	hw->fc.pause_time = IXGBE_FC_PAUSE;
1008	hw->fc.send_xon = TRUE;
1009}
1010
1011/*
1012 * MSIX Interrupt Handlers
1013 */
1014void
1015ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector)
1016{
1017	uint64_t queue = 1ULL << vector;
1018	uint32_t mask;
1019
1020	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
1021		mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1022		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask);
1023	} else {
1024		mask = (queue & 0xFFFFFFFF);
1025		if (mask)
1026			IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask);
1027		mask = (queue >> 32);
1028		if (mask)
1029			IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask);
1030	}
1031}
1032
1033void
1034ixgbe_enable_queues(struct ix_softc *sc)
1035{
1036	struct ix_queue *que;
1037	int i;
1038
1039	for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++)
1040		ixgbe_enable_queue(sc, que->msix);
1041}
1042
1043void
1044ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector)
1045{
1046	uint64_t queue = 1ULL << vector;
1047	uint32_t mask;
1048
1049	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
1050		mask = (IXGBE_EIMS_RTX_QUEUE & queue);
1051		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask);
1052	} else {
1053		mask = (queue & 0xFFFFFFFF);
1054		if (mask)
1055			IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask);
1056		mask = (queue >> 32);
1057		if (mask)
1058			IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask);
1059	}
1060}
1061
1062/*
1063 * MSIX Interrupt Handlers
1064 */
1065int
1066ixgbe_link_intr(void *vsc)
1067{
1068	struct ix_softc	*sc = (struct ix_softc *)vsc;
1069
1070	return ixgbe_intr(sc);
1071}
1072
1073int
1074ixgbe_queue_intr(void *vque)
1075{
1076	struct ix_queue *que = vque;
1077	struct ix_softc	*sc = que->sc;
1078	struct ifnet	*ifp = &sc->arpcom.ac_if;
1079	struct ix_rxring	*rxr = que->rxr;
1080	struct ix_txring	*txr = que->txr;
1081
1082	if (ISSET(ifp->if_flags, IFF_RUNNING)) {
1083		ixgbe_rxeof(rxr);
1084		ixgbe_txeof(txr);
1085		ixgbe_rxrefill(rxr);
1086	}
1087
1088	ixgbe_enable_queue(sc, que->msix);
1089
1090	return (1);
1091}
1092
1093/*********************************************************************
1094 *
1095 *  Legacy Interrupt Service routine
1096 *
1097 **********************************************************************/
1098
1099int
1100ixgbe_legacy_intr(void *arg)
1101{
1102	struct ix_softc	*sc = (struct ix_softc *)arg;
1103	struct ifnet	*ifp = &sc->arpcom.ac_if;
1104	struct ix_rxring	*rxr = sc->rx_rings;
1105	struct ix_txring	*txr = sc->tx_rings;
1106	int rv;
1107
1108	rv = ixgbe_intr(sc);
1109	if (rv == 0) {
1110		return (0);
1111	}
1112
1113	if (ISSET(ifp->if_flags, IFF_RUNNING)) {
1114		ixgbe_rxeof(rxr);
1115		ixgbe_txeof(txr);
1116		ixgbe_rxrefill(rxr);
1117	}
1118
1119	ixgbe_enable_queues(sc);
1120	return (rv);
1121}
1122
1123int
1124ixgbe_intr(struct ix_softc *sc)
1125{
1126	struct ifnet	*ifp = &sc->arpcom.ac_if;
1127	struct ixgbe_hw	*hw = &sc->hw;
1128	uint32_t	 reg_eicr, mod_mask, msf_mask;
1129
1130	if (sc->sc_intrmap) {
1131		/* Pause other interrupts */
1132		IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER);
1133		/* First get the cause */
1134		reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1135		/* Be sure the queue bits are not cleared */
1136		reg_eicr &= ~IXGBE_EICR_RTX_QUEUE;
1137		/* Clear interrupt with write */
1138		IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr);
1139	} else {
1140		reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
1141		if (reg_eicr == 0) {
1142			ixgbe_enable_intr(sc);
1143			ixgbe_enable_queues(sc);
1144			return (0);
1145		}
1146	}
1147
1148	/* Link status change */
1149	if (reg_eicr & IXGBE_EICR_LSC) {
1150		IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
1151		KERNEL_LOCK();
1152		ixgbe_update_link_status(sc);
1153		KERNEL_UNLOCK();
1154	}
1155
1156	if (hw->mac.type != ixgbe_mac_82598EB) {
1157		if (reg_eicr & IXGBE_EICR_ECC) {
1158			printf("%s: CRITICAL: ECC ERROR!! "
1159			    "Please Reboot!!\n", sc->dev.dv_xname);
1160			IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);
1161		}
1162		/* Check for over temp condition */
1163		if (reg_eicr & IXGBE_EICR_TS) {
1164			printf("%s: CRITICAL: OVER TEMP!! "
1165			    "PHY IS SHUT DOWN!!\n", ifp->if_xname);
1166			IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS);
1167		}
1168	}
1169
1170	/* Pluggable optics-related interrupt */
1171	if (ixgbe_is_sfp(hw)) {
1172		if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP) {
1173			mod_mask = IXGBE_EICR_GPI_SDP0_X540;
1174			msf_mask = IXGBE_EICR_GPI_SDP1_X540;
1175		} else if (hw->mac.type == ixgbe_mac_X540 ||
1176		    hw->mac.type == ixgbe_mac_X550 ||
1177		    hw->mac.type == ixgbe_mac_X550EM_x) {
1178			mod_mask = IXGBE_EICR_GPI_SDP2_X540;
1179			msf_mask = IXGBE_EICR_GPI_SDP1_X540;
1180		} else {
1181			mod_mask = IXGBE_EICR_GPI_SDP2;
1182			msf_mask = IXGBE_EICR_GPI_SDP1;
1183		}
1184		if (reg_eicr & mod_mask) {
1185			/* Clear the interrupt */
1186			IXGBE_WRITE_REG(hw, IXGBE_EICR, mod_mask);
1187			KERNEL_LOCK();
1188			ixgbe_handle_mod(sc);
1189			KERNEL_UNLOCK();
1190		} else if ((hw->phy.media_type != ixgbe_media_type_copper) &&
1191		    (reg_eicr & msf_mask)) {
1192			/* Clear the interrupt */
1193			IXGBE_WRITE_REG(hw, IXGBE_EICR, msf_mask);
1194			KERNEL_LOCK();
1195			ixgbe_handle_msf(sc);
1196			KERNEL_UNLOCK();
1197		}
1198	}
1199
1200	/* Check for fan failure */
1201	if ((hw->device_id == IXGBE_DEV_ID_82598AT) &&
1202	    (reg_eicr & IXGBE_EICR_GPI_SDP1)) {
1203		printf("%s: CRITICAL: FAN FAILURE!! "
1204		    "REPLACE IMMEDIATELY!!\n", ifp->if_xname);
1205		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1206	}
1207
1208	/* External PHY interrupt */
1209	if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T &&
1210	    (reg_eicr & IXGBE_EICR_GPI_SDP0_X540)) {
1211		/* Clear the interrupt */
1212		IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540);
1213		KERNEL_LOCK();
1214		ixgbe_handle_phy(sc);
1215		KERNEL_UNLOCK();
1216	}
1217
1218	IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
1219
1220	return (1);
1221}
1222
1223/*********************************************************************
1224 *
1225 *  Media Ioctl callback
1226 *
1227 *  This routine is called whenever the user queries the status of
1228 *  the interface using ifconfig.
1229 *
1230 **********************************************************************/
1231void
1232ixgbe_media_status(struct ifnet * ifp, struct ifmediareq *ifmr)
1233{
1234	struct ix_softc *sc = ifp->if_softc;
1235	uint64_t layer;
1236
1237	ifmr->ifm_active = IFM_ETHER;
1238	ifmr->ifm_status = IFM_AVALID;
1239
1240	INIT_DEBUGOUT("ixgbe_media_status: begin");
1241	ixgbe_update_link_status(sc);
1242
1243	if (!LINK_STATE_IS_UP(ifp->if_link_state))
1244		return;
1245
1246	ifmr->ifm_status |= IFM_ACTIVE;
1247	layer = sc->phy_layer;
1248
1249	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T ||
1250	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_T ||
1251	    layer & IXGBE_PHYSICAL_LAYER_100BASE_TX ||
1252	    layer & IXGBE_PHYSICAL_LAYER_10BASE_T) {
1253		switch (sc->link_speed) {
1254		case IXGBE_LINK_SPEED_10GB_FULL:
1255			ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
1256			break;
1257		case IXGBE_LINK_SPEED_1GB_FULL:
1258			ifmr->ifm_active |= IFM_1000_T | IFM_FDX;
1259			break;
1260		case IXGBE_LINK_SPEED_100_FULL:
1261			ifmr->ifm_active |= IFM_100_TX | IFM_FDX;
1262			break;
1263		case IXGBE_LINK_SPEED_10_FULL:
1264			ifmr->ifm_active |= IFM_10_T | IFM_FDX;
1265			break;
1266		}
1267	}
1268	if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
1269	    layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) {
1270		switch (sc->link_speed) {
1271		case IXGBE_LINK_SPEED_10GB_FULL:
1272			ifmr->ifm_active |= IFM_10G_SFP_CU | IFM_FDX;
1273			break;
1274		}
1275	}
1276	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) {
1277		switch (sc->link_speed) {
1278		case IXGBE_LINK_SPEED_10GB_FULL:
1279			ifmr->ifm_active |= IFM_10G_LR | IFM_FDX;
1280			break;
1281		case IXGBE_LINK_SPEED_1GB_FULL:
1282			ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
1283			break;
1284		}
1285	}
1286	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR ||
1287	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) {
1288		switch (sc->link_speed) {
1289		case IXGBE_LINK_SPEED_10GB_FULL:
1290			ifmr->ifm_active |= IFM_10G_SR | IFM_FDX;
1291			break;
1292		case IXGBE_LINK_SPEED_1GB_FULL:
1293			ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1294			break;
1295		}
1296	}
1297	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) {
1298		switch (sc->link_speed) {
1299		case IXGBE_LINK_SPEED_10GB_FULL:
1300			ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX;
1301			break;
1302		}
1303	}
1304	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) {
1305		switch (sc->link_speed) {
1306		case IXGBE_LINK_SPEED_10GB_FULL:
1307			ifmr->ifm_active |= IFM_10G_KR | IFM_FDX;
1308			break;
1309		case IXGBE_LINK_SPEED_2_5GB_FULL:
1310			ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;
1311			break;
1312		case IXGBE_LINK_SPEED_1GB_FULL:
1313			ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;
1314			break;
1315		}
1316	} else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 ||
1317	    layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX ||
1318	    layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) {
1319		switch (sc->link_speed) {
1320		case IXGBE_LINK_SPEED_10GB_FULL:
1321			ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX;
1322			break;
1323		case IXGBE_LINK_SPEED_2_5GB_FULL:
1324			ifmr->ifm_active |= IFM_2500_KX | IFM_FDX;
1325			break;
1326		case IXGBE_LINK_SPEED_1GB_FULL:
1327			ifmr->ifm_active |= IFM_1000_KX | IFM_FDX;
1328			break;
1329		}
1330	}
1331
1332	switch (sc->hw.fc.current_mode) {
1333	case ixgbe_fc_tx_pause:
1334		ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE;
1335		break;
1336	case ixgbe_fc_rx_pause:
1337		ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE;
1338		break;
1339	case ixgbe_fc_full:
1340		ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE |
1341		    IFM_ETH_TXPAUSE;
1342		break;
1343	default:
1344		ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE |
1345		    IFM_ETH_TXPAUSE);
1346		break;
1347	}
1348}
1349
1350/*********************************************************************
1351 *
1352 *  Media Ioctl callback
1353 *
1354 *  This routine is called when the user changes speed/duplex using
1355 *  media/mediopt option with ifconfig.
1356 *
1357 **********************************************************************/
1358int
1359ixgbe_media_change(struct ifnet *ifp)
1360{
1361	struct ix_softc	*sc = ifp->if_softc;
1362	struct ixgbe_hw	*hw = &sc->hw;
1363	struct ifmedia	*ifm = &sc->media;
1364	ixgbe_link_speed speed = 0;
1365
1366	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1367		return (EINVAL);
1368
1369	if (hw->phy.media_type == ixgbe_media_type_backplane)
1370		return (ENODEV);
1371
1372	switch (IFM_SUBTYPE(ifm->ifm_media)) {
1373		case IFM_AUTO:
1374		case IFM_10G_T:
1375			speed |= IXGBE_LINK_SPEED_100_FULL;
1376			speed |= IXGBE_LINK_SPEED_1GB_FULL;
1377			speed |= IXGBE_LINK_SPEED_10GB_FULL;
1378			break;
1379		case IFM_10G_SR:
1380		case IFM_10G_KR:
1381		case IFM_10G_LR:
1382		case IFM_10G_LRM:
1383		case IFM_10G_CX4:
1384		case IFM_10G_KX4:
1385			speed |= IXGBE_LINK_SPEED_1GB_FULL;
1386			speed |= IXGBE_LINK_SPEED_10GB_FULL;
1387			break;
1388		case IFM_10G_SFP_CU:
1389			speed |= IXGBE_LINK_SPEED_10GB_FULL;
1390			break;
1391		case IFM_1000_T:
1392			speed |= IXGBE_LINK_SPEED_100_FULL;
1393			speed |= IXGBE_LINK_SPEED_1GB_FULL;
1394			break;
1395		case IFM_1000_LX:
1396		case IFM_1000_SX:
1397		case IFM_1000_CX:
1398		case IFM_1000_KX:
1399			speed |= IXGBE_LINK_SPEED_1GB_FULL;
1400			break;
1401		case IFM_100_TX:
1402			speed |= IXGBE_LINK_SPEED_100_FULL;
1403			break;
1404		case IFM_10_T:
1405			speed |= IXGBE_LINK_SPEED_10_FULL;
1406			break;
1407		default:
1408			return (EINVAL);
1409	}
1410
1411	hw->mac.autotry_restart = TRUE;
1412	hw->mac.ops.setup_link(hw, speed, TRUE);
1413
1414	return (0);
1415}
1416
1417/*********************************************************************
1418 *
1419 *  This routine maps the mbufs to tx descriptors, allowing the
1420 *  TX engine to transmit the packets.
1421 *  	- return 0 on success, positive on failure
1422 *
1423 **********************************************************************/
1424
1425int
1426ixgbe_encap(struct ix_txring *txr, struct mbuf *m_head)
1427{
1428	struct ix_softc *sc = txr->sc;
1429	uint32_t	olinfo_status = 0, cmd_type_len;
1430	int             i, j, ntxc;
1431	int		first, last = 0;
1432	bus_dmamap_t	map;
1433	struct ixgbe_tx_buf *txbuf;
1434	union ixgbe_adv_tx_desc *txd = NULL;
1435
1436	/* Basic descriptor defines */
1437	cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA |
1438	    IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
1439
1440	/*
1441	 * Important to capture the first descriptor
1442	 * used because it will contain the index of
1443	 * the one we tell the hardware to report back
1444	 */
1445	first = txr->next_avail_desc;
1446	txbuf = &txr->tx_buffers[first];
1447	map = txbuf->map;
1448
1449	/*
1450	 * Set the appropriate offload context
1451	 * this will becomes the first descriptor.
1452	 */
1453	ntxc = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status);
1454	if (ntxc == -1)
1455		goto xmit_fail;
1456
1457	/*
1458	 * Map the packet for DMA.
1459	 */
1460	switch (bus_dmamap_load_mbuf(txr->txdma.dma_tag, map,
1461	    m_head, BUS_DMA_NOWAIT)) {
1462	case 0:
1463		break;
1464	case EFBIG:
1465		if (m_defrag(m_head, M_NOWAIT) == 0 &&
1466		    bus_dmamap_load_mbuf(txr->txdma.dma_tag, map,
1467		     m_head, BUS_DMA_NOWAIT) == 0)
1468			break;
1469		/* FALLTHROUGH */
1470	default:
1471		return (0);
1472	}
1473
1474	i = txr->next_avail_desc + ntxc;
1475	if (i >= sc->num_tx_desc)
1476		i -= sc->num_tx_desc;
1477
1478	for (j = 0; j < map->dm_nsegs; j++) {
1479		txd = &txr->tx_base[i];
1480
1481		txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr);
1482		txd->read.cmd_type_len = htole32(txr->txd_cmd |
1483		    cmd_type_len | map->dm_segs[j].ds_len);
1484		txd->read.olinfo_status = htole32(olinfo_status);
1485		last = i; /* descriptor that will get completion IRQ */
1486
1487		if (++i == sc->num_tx_desc)
1488			i = 0;
1489	}
1490
1491	txd->read.cmd_type_len |=
1492	    htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS);
1493
1494	bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize,
1495	    BUS_DMASYNC_PREWRITE);
1496
1497	/* Set the index of the descriptor that will be marked done */
1498	txbuf->m_head = m_head;
1499	txbuf->eop_index = last;
1500
1501	membar_producer();
1502
1503	txr->next_avail_desc = i;
1504
1505	return (ntxc + j);
1506
1507xmit_fail:
1508	bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map);
1509	return (0);
1510}
1511
1512void
1513ixgbe_iff(struct ix_softc *sc)
1514{
1515	struct ifnet *ifp = &sc->arpcom.ac_if;
1516	struct arpcom *ac = &sc->arpcom;
1517	uint32_t	fctrl;
1518	uint8_t	*mta;
1519	uint8_t	*update_ptr;
1520	struct ether_multi *enm;
1521	struct ether_multistep step;
1522	int	mcnt = 0;
1523
1524	IOCTL_DEBUGOUT("ixgbe_iff: begin");
1525
1526	mta = sc->mta;
1527	bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS *
1528	    MAX_NUM_MULTICAST_ADDRESSES);
1529
1530	fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL);
1531	fctrl &= ~(IXGBE_FCTRL_MPE | IXGBE_FCTRL_UPE);
1532	ifp->if_flags &= ~IFF_ALLMULTI;
1533
1534	if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 ||
1535	    ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) {
1536		ifp->if_flags |= IFF_ALLMULTI;
1537		fctrl |= IXGBE_FCTRL_MPE;
1538		if (ifp->if_flags & IFF_PROMISC)
1539			fctrl |= IXGBE_FCTRL_UPE;
1540	} else {
1541		ETHER_FIRST_MULTI(step, &sc->arpcom, enm);
1542		while (enm != NULL) {
1543			bcopy(enm->enm_addrlo,
1544			    &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS],
1545			    IXGBE_ETH_LENGTH_OF_ADDRESS);
1546			mcnt++;
1547
1548			ETHER_NEXT_MULTI(step, enm);
1549		}
1550
1551		update_ptr = mta;
1552		sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt,
1553		    ixgbe_mc_array_itr, TRUE);
1554	}
1555
1556	IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl);
1557}
1558
1559/*
1560 * This is an iterator function now needed by the multicast
1561 * shared code. It simply feeds the shared code routine the
1562 * addresses in the array of ixgbe_iff() one by one.
1563 */
1564uint8_t *
1565ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq)
1566{
1567	uint8_t *addr = *update_ptr;
1568	uint8_t *newptr;
1569	*vmdq = 0;
1570
1571	newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS;
1572	*update_ptr = newptr;
1573	return addr;
1574}
1575
1576void
1577ixgbe_update_link_status(struct ix_softc *sc)
1578{
1579	struct ifnet	*ifp = &sc->arpcom.ac_if;
1580	int		link_state = LINK_STATE_DOWN;
1581
1582	splassert(IPL_NET);
1583	KERNEL_ASSERT_LOCKED();
1584
1585	ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0);
1586
1587	ifp->if_baudrate = 0;
1588	if (sc->link_up) {
1589		link_state = LINK_STATE_FULL_DUPLEX;
1590
1591		switch (sc->link_speed) {
1592		case IXGBE_LINK_SPEED_UNKNOWN:
1593			ifp->if_baudrate = 0;
1594			break;
1595		case IXGBE_LINK_SPEED_100_FULL:
1596			ifp->if_baudrate = IF_Mbps(100);
1597			break;
1598		case IXGBE_LINK_SPEED_1GB_FULL:
1599			ifp->if_baudrate = IF_Gbps(1);
1600			break;
1601		case IXGBE_LINK_SPEED_10GB_FULL:
1602			ifp->if_baudrate = IF_Gbps(10);
1603			break;
1604		}
1605
1606		/* Update any Flow Control changes */
1607		sc->hw.mac.ops.fc_enable(&sc->hw);
1608	}
1609	if (ifp->if_link_state != link_state) {
1610		ifp->if_link_state = link_state;
1611		if_link_state_change(ifp);
1612	}
1613}
1614
1615
1616/*********************************************************************
1617 *
1618 *  This routine disables all traffic on the adapter by issuing a
1619 *  global reset on the MAC and deallocates TX/RX buffers.
1620 *
1621 **********************************************************************/
1622
1623void
1624ixgbe_stop(void *arg)
1625{
1626	struct ix_softc *sc = arg;
1627	struct ifnet   *ifp = &sc->arpcom.ac_if;
1628	int i;
1629
1630	/* Tell the stack that the interface is no longer active */
1631	ifp->if_flags &= ~IFF_RUNNING;
1632
1633#if NKSTAT > 0
1634	timeout_del(&sc->sc_kstat_tmo);
1635#endif
1636	ifp->if_timer = 0;
1637
1638	INIT_DEBUGOUT("ixgbe_stop: begin\n");
1639	ixgbe_disable_intr(sc);
1640
1641	sc->hw.mac.ops.reset_hw(&sc->hw);
1642	sc->hw.adapter_stopped = FALSE;
1643	sc->hw.mac.ops.stop_adapter(&sc->hw);
1644	if (sc->hw.mac.type == ixgbe_mac_82599EB)
1645		sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw);
1646	/* Turn off the laser */
1647	if (sc->hw.mac.ops.disable_tx_laser)
1648		sc->hw.mac.ops.disable_tx_laser(&sc->hw);
1649
1650	/* reprogram the RAR[0] in case user changed it. */
1651	ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV);
1652
1653	intr_barrier(sc->tag);
1654	for (i = 0; i < sc->num_queues; i++) {
1655		struct ifqueue *ifq = ifp->if_ifqs[i];
1656		ifq_barrier(ifq);
1657		ifq_clr_oactive(ifq);
1658
1659		if (sc->queues[i].tag != NULL)
1660			intr_barrier(sc->queues[i].tag);
1661		timeout_del(&sc->rx_rings[i].rx_refill);
1662	}
1663
1664	KASSERT((ifp->if_flags & IFF_RUNNING) == 0);
1665
1666	/* Should we really clear all structures on stop? */
1667	ixgbe_free_transmit_structures(sc);
1668	ixgbe_free_receive_structures(sc);
1669
1670	ixgbe_update_link_status(sc);
1671}
1672
1673
1674/*********************************************************************
1675 *
1676 *  Determine hardware revision.
1677 *
1678 **********************************************************************/
1679void
1680ixgbe_identify_hardware(struct ix_softc *sc)
1681{
1682	struct ixgbe_osdep	*os = &sc->osdep;
1683	struct pci_attach_args	*pa = &os->os_pa;
1684	uint32_t		 reg;
1685
1686	/* Save off the information about this board */
1687	sc->hw.vendor_id = PCI_VENDOR(pa->pa_id);
1688	sc->hw.device_id = PCI_PRODUCT(pa->pa_id);
1689
1690	reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG);
1691	sc->hw.revision_id = PCI_REVISION(reg);
1692
1693	reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG);
1694	sc->hw.subsystem_vendor_id = PCI_VENDOR(reg);
1695	sc->hw.subsystem_device_id = PCI_PRODUCT(reg);
1696
1697	/* We need this here to set the num_segs below */
1698	ixgbe_set_mac_type(&sc->hw);
1699
1700	/* Pick up the 82599 and VF settings */
1701	if (sc->hw.mac.type != ixgbe_mac_82598EB)
1702		sc->hw.phy.smart_speed = ixgbe_smart_speed;
1703	sc->num_segs = IXGBE_82599_SCATTER;
1704}
1705
1706/*********************************************************************
1707 *
1708 *  Setup the Legacy or MSI Interrupt handler
1709 *
1710 **********************************************************************/
1711int
1712ixgbe_allocate_legacy(struct ix_softc *sc)
1713{
1714	struct ixgbe_osdep	*os = &sc->osdep;
1715	struct pci_attach_args	*pa = &os->os_pa;
1716	const char		*intrstr = NULL;
1717	pci_chipset_tag_t	pc = pa->pa_pc;
1718	pci_intr_handle_t	ih;
1719
1720	/* We allocate a single interrupt resource */
1721	if (pci_intr_map_msi(pa, &ih) != 0 &&
1722	    pci_intr_map(pa, &ih) != 0) {
1723		printf(": couldn't map interrupt\n");
1724		return (ENXIO);
1725	}
1726
1727#if 0
1728	/* XXX */
1729	/* Tasklets for Link, SFP and Multispeed Fiber */
1730	TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc);
1731	TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc);
1732	TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc);
1733#endif
1734
1735	intrstr = pci_intr_string(pc, ih);
1736	sc->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE,
1737	    ixgbe_legacy_intr, sc, sc->dev.dv_xname);
1738	if (sc->tag == NULL) {
1739		printf(": couldn't establish interrupt");
1740		if (intrstr != NULL)
1741			printf(" at %s", intrstr);
1742		printf("\n");
1743		return (ENXIO);
1744	}
1745	printf(": %s", intrstr);
1746
1747	/* For simplicity in the handlers */
1748	sc->que_mask = IXGBE_EIMS_ENABLE_MASK;
1749
1750	return (0);
1751}
1752
1753/*********************************************************************
1754 *
1755 *  Setup the MSI-X Interrupt handlers
1756 *
1757 **********************************************************************/
1758int
1759ixgbe_allocate_msix(struct ix_softc *sc)
1760{
1761	struct ixgbe_osdep	*os = &sc->osdep;
1762	struct pci_attach_args	*pa  = &os->os_pa;
1763	int                      i = 0, error = 0;
1764	struct ix_queue         *que;
1765	pci_intr_handle_t	ih;
1766
1767	for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) {
1768		if (pci_intr_map_msix(pa, i, &ih)) {
1769			printf("ixgbe_allocate_msix: "
1770			    "pci_intr_map_msix vec %d failed\n", i);
1771			error = ENOMEM;
1772			goto fail;
1773		}
1774
1775		que->tag = pci_intr_establish_cpu(pa->pa_pc, ih,
1776		    IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i),
1777		    ixgbe_queue_intr, que, que->name);
1778		if (que->tag == NULL) {
1779			printf("ixgbe_allocate_msix: "
1780			    "pci_intr_establish vec %d failed\n", i);
1781			error = ENOMEM;
1782			goto fail;
1783		}
1784
1785		que->msix = i;
1786	}
1787
1788	/* Now the link status/control last MSI-X vector */
1789	if (pci_intr_map_msix(pa, i, &ih)) {
1790		printf("ixgbe_allocate_msix: "
1791		    "pci_intr_map_msix link vector failed\n");
1792		error = ENOMEM;
1793		goto fail;
1794	}
1795
1796	sc->tag = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE,
1797	    ixgbe_link_intr, sc, sc->dev.dv_xname);
1798	if (sc->tag == NULL) {
1799		printf("ixgbe_allocate_msix: "
1800		    "pci_intr_establish link vector failed\n");
1801		error = ENOMEM;
1802		goto fail;
1803	}
1804	sc->linkvec = i;
1805	printf(", %s, %d queue%s", pci_intr_string(pa->pa_pc, ih),
1806	    i, (i > 1) ? "s" : "");
1807
1808	return (0);
1809fail:
1810	for (que = sc->queues; i > 0; i--, que++) {
1811		if (que->tag == NULL)
1812			continue;
1813		pci_intr_disestablish(pa->pa_pc, que->tag);
1814		que->tag = NULL;
1815	}
1816
1817	return (error);
1818}
1819
1820void
1821ixgbe_setup_msix(struct ix_softc *sc)
1822{
1823	struct ixgbe_osdep	*os = &sc->osdep;
1824	struct pci_attach_args	*pa = &os->os_pa;
1825	int			 nmsix;
1826	unsigned int		 maxq;
1827
1828	if (!ixgbe_enable_msix)
1829		return;
1830
1831	nmsix = pci_intr_msix_count(pa);
1832	if (nmsix <= 1)
1833		return;
1834
1835	/* give one vector to events */
1836	nmsix--;
1837
1838	/* XXX the number of queues is limited to what we can keep stats on */
1839	maxq = (sc->hw.mac.type == ixgbe_mac_82598EB) ? 8 : 16;
1840
1841	sc->sc_intrmap = intrmap_create(&sc->dev, nmsix, maxq, 0);
1842	sc->num_queues = intrmap_count(sc->sc_intrmap);
1843}
1844
1845int
1846ixgbe_allocate_pci_resources(struct ix_softc *sc)
1847{
1848	struct ixgbe_osdep	*os = &sc->osdep;
1849	struct pci_attach_args	*pa = &os->os_pa;
1850	int			 val;
1851
1852	val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0));
1853	if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) {
1854		printf(": mmba is not mem space\n");
1855		return (ENXIO);
1856	}
1857
1858	if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0,
1859	    &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) {
1860		printf(": cannot find mem space\n");
1861		return (ENXIO);
1862	}
1863	sc->hw.hw_addr = (uint8_t *)os->os_membase;
1864
1865	/* Legacy defaults */
1866	sc->num_queues = 1;
1867	sc->hw.back = os;
1868
1869	/* Now setup MSI or MSI/X, return us the number of supported vectors. */
1870	ixgbe_setup_msix(sc);
1871
1872	return (0);
1873}
1874
1875void
1876ixgbe_free_pci_resources(struct ix_softc * sc)
1877{
1878	struct ixgbe_osdep	*os = &sc->osdep;
1879	struct pci_attach_args	*pa = &os->os_pa;
1880	struct ix_queue *que = sc->queues;
1881	int i;
1882
1883	/* Release all msix queue resources: */
1884	for (i = 0; i < sc->num_queues; i++, que++) {
1885		if (que->tag)
1886			pci_intr_disestablish(pa->pa_pc, que->tag);
1887		que->tag = NULL;
1888	}
1889
1890	if (sc->tag)
1891		pci_intr_disestablish(pa->pa_pc, sc->tag);
1892	sc->tag = NULL;
1893	if (os->os_membase != 0)
1894		bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize);
1895	os->os_membase = 0;
1896}
1897
1898/*********************************************************************
1899 *
1900 *  Setup networking device structure and register an interface.
1901 *
1902 **********************************************************************/
1903void
1904ixgbe_setup_interface(struct ix_softc *sc)
1905{
1906	struct ifnet   *ifp = &sc->arpcom.ac_if;
1907	int i;
1908
1909	strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ);
1910	ifp->if_softc = sc;
1911	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1912	ifp->if_xflags = IFXF_MPSAFE;
1913	ifp->if_ioctl = ixgbe_ioctl;
1914	ifp->if_qstart = ixgbe_start;
1915	ifp->if_timer = 0;
1916	ifp->if_watchdog = ixgbe_watchdog;
1917	ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE -
1918	    ETHER_HDR_LEN - ETHER_CRC_LEN;
1919	ifq_init_maxlen(&ifp->if_snd, sc->num_tx_desc - 1);
1920
1921	ifp->if_capabilities = IFCAP_VLAN_MTU;
1922
1923#if NVLAN > 0
1924	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING;
1925#endif
1926
1927	ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4;
1928	ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6;
1929	ifp->if_capabilities |= IFCAP_CSUM_IPv4;
1930
1931	ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
1932	if (sc->hw.mac.type != ixgbe_mac_82598EB) {
1933#ifndef __sparc64__
1934		ifp->if_xflags |= IFXF_LRO;
1935#endif
1936		ifp->if_capabilities |= IFCAP_LRO;
1937	}
1938
1939	/*
1940	 * Specify the media types supported by this sc and register
1941	 * callbacks to update media and link information
1942	 */
1943	ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change,
1944	    ixgbe_media_status);
1945	ixgbe_add_media_types(sc);
1946	ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
1947
1948	if_attach(ifp);
1949	ether_ifattach(ifp);
1950
1951	if_attach_queues(ifp, sc->num_queues);
1952	if_attach_iqueues(ifp, sc->num_queues);
1953	for (i = 0; i < sc->num_queues; i++) {
1954		struct ifqueue *ifq = ifp->if_ifqs[i];
1955		struct ifiqueue *ifiq = ifp->if_iqs[i];
1956		struct ix_txring *txr = &sc->tx_rings[i];
1957		struct ix_rxring *rxr = &sc->rx_rings[i];
1958
1959		ifq->ifq_softc = txr;
1960		txr->ifq = ifq;
1961
1962		ifiq->ifiq_softc = rxr;
1963		rxr->ifiq = ifiq;
1964
1965#if NKSTAT > 0
1966		ix_txq_kstats(sc, txr);
1967		ix_rxq_kstats(sc, rxr);
1968#endif
1969	}
1970
1971	sc->max_frame_size = IXGBE_MAX_FRAME_SIZE;
1972}
1973
1974void
1975ixgbe_add_media_types(struct ix_softc *sc)
1976{
1977	struct ixgbe_hw	*hw = &sc->hw;
1978	uint64_t layer;
1979
1980	sc->phy_layer = hw->mac.ops.get_supported_physical_layer(hw);
1981	layer = sc->phy_layer;
1982
1983	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T)
1984		ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL);
1985	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T)
1986		ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL);
1987	if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX)
1988		ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL);
1989	if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU ||
1990	    layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA)
1991		ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SFP_CU, 0, NULL);
1992	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) {
1993		ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL);
1994		if (hw->phy.multispeed_fiber)
1995			ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_LX, 0,
1996			    NULL);
1997	}
1998	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) {
1999		ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL);
2000		if (hw->phy.multispeed_fiber)
2001			ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0,
2002			    NULL);
2003	} else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX)
2004		ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL);
2005	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4)
2006		ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL);
2007	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR)
2008		ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL);
2009	if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4)
2010		ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL);
2011	if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX)
2012		ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL);
2013	if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX)
2014		ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL);
2015
2016	if (hw->device_id == IXGBE_DEV_ID_82598AT) {
2017		ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0,
2018		    NULL);
2019		ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL);
2020	}
2021
2022	ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2023}
2024
2025void
2026ixgbe_config_link(struct ix_softc *sc)
2027{
2028	uint32_t	autoneg, err = 0;
2029	bool		negotiate;
2030
2031	if (ixgbe_is_sfp(&sc->hw)) {
2032		if (sc->hw.phy.multispeed_fiber) {
2033			sc->hw.mac.ops.setup_sfp(&sc->hw);
2034			if (sc->hw.mac.ops.enable_tx_laser)
2035				sc->hw.mac.ops.enable_tx_laser(&sc->hw);
2036			ixgbe_handle_msf(sc);
2037		} else
2038			ixgbe_handle_mod(sc);
2039	} else {
2040		if (sc->hw.mac.ops.check_link)
2041			err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg,
2042			    &sc->link_up, FALSE);
2043		if (err)
2044			return;
2045		autoneg = sc->hw.phy.autoneg_advertised;
2046		if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities))
2047			err = sc->hw.mac.ops.get_link_capabilities(&sc->hw,
2048			    &autoneg, &negotiate);
2049		if (err)
2050			return;
2051		if (sc->hw.mac.ops.setup_link)
2052			sc->hw.mac.ops.setup_link(&sc->hw,
2053			    autoneg, sc->link_up);
2054	}
2055}
2056
2057/********************************************************************
2058 * Manage DMA'able memory.
2059  *******************************************************************/
2060int
2061ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size,
2062		struct ixgbe_dma_alloc *dma, int mapflags)
2063{
2064	struct ifnet		*ifp = &sc->arpcom.ac_if;
2065	struct ixgbe_osdep	*os = &sc->osdep;
2066	int			 r;
2067
2068	dma->dma_tag = os->os_pa.pa_dmat;
2069	r = bus_dmamap_create(dma->dma_tag, size, 1,
2070	    size, 0, BUS_DMA_NOWAIT, &dma->dma_map);
2071	if (r != 0) {
2072		printf("%s: ixgbe_dma_malloc: bus_dmamap_create failed; "
2073		       "error %u\n", ifp->if_xname, r);
2074		goto fail_0;
2075	}
2076
2077	r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg,
2078	    1, &dma->dma_nseg, BUS_DMA_NOWAIT);
2079	if (r != 0) {
2080		printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; "
2081		       "error %u\n", ifp->if_xname, r);
2082		goto fail_1;
2083	}
2084
2085	r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size,
2086	    &dma->dma_vaddr, BUS_DMA_NOWAIT);
2087	if (r != 0) {
2088		printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; "
2089		       "error %u\n", ifp->if_xname, r);
2090		goto fail_2;
2091	}
2092
2093	r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2094	    size, NULL, mapflags | BUS_DMA_NOWAIT);
2095	if (r != 0) {
2096		printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; "
2097		       "error %u\n", ifp->if_xname, r);
2098		goto fail_3;
2099	}
2100
2101	dma->dma_size = size;
2102	return (0);
2103fail_3:
2104	bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size);
2105fail_2:
2106	bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg);
2107fail_1:
2108	bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
2109fail_0:
2110	dma->dma_map = NULL;
2111	dma->dma_tag = NULL;
2112	return (r);
2113}
2114
2115void
2116ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma)
2117{
2118	if (dma->dma_tag == NULL)
2119		return;
2120
2121	if (dma->dma_map != NULL) {
2122		bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0,
2123		    dma->dma_map->dm_mapsize,
2124		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2125		bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2126		bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size);
2127		bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg);
2128		bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
2129		dma->dma_map = NULL;
2130	}
2131}
2132
2133
2134/*********************************************************************
2135 *
2136 *  Allocate memory for the transmit and receive rings, and then
2137 *  the descriptors associated with each, called only once at attach.
2138 *
2139 **********************************************************************/
2140int
2141ixgbe_allocate_queues(struct ix_softc *sc)
2142{
2143	struct ifnet	*ifp = &sc->arpcom.ac_if;
2144	struct ix_queue *que;
2145	struct ix_txring *txr;
2146	struct ix_rxring *rxr;
2147	int rsize, tsize;
2148	int txconf = 0, rxconf = 0, i;
2149
2150	/* First allocate the top level queue structs */
2151	if (!(sc->queues = mallocarray(sc->num_queues,
2152	    sizeof(struct ix_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) {
2153		printf("%s: Unable to allocate queue memory\n", ifp->if_xname);
2154		goto fail;
2155	}
2156
2157	/* Then allocate the TX ring struct memory */
2158	if (!(sc->tx_rings = mallocarray(sc->num_queues,
2159	    sizeof(struct ix_txring), M_DEVBUF, M_NOWAIT | M_ZERO))) {
2160		printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname);
2161		goto fail;
2162	}
2163
2164	/* Next allocate the RX */
2165	if (!(sc->rx_rings = mallocarray(sc->num_queues,
2166	    sizeof(struct ix_rxring), M_DEVBUF, M_NOWAIT | M_ZERO))) {
2167		printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname);
2168		goto rx_fail;
2169	}
2170
2171	/* For the ring itself */
2172	tsize = roundup2(sc->num_tx_desc *
2173	    sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN);
2174
2175	/*
2176	 * Now set up the TX queues, txconf is needed to handle the
2177	 * possibility that things fail midcourse and we need to
2178	 * undo memory gracefully
2179	 */
2180	for (i = 0; i < sc->num_queues; i++, txconf++) {
2181		/* Set up some basics */
2182		txr = &sc->tx_rings[i];
2183		txr->sc = sc;
2184		txr->me = i;
2185
2186		if (ixgbe_dma_malloc(sc, tsize,
2187		    &txr->txdma, BUS_DMA_NOWAIT)) {
2188			printf("%s: Unable to allocate TX Descriptor memory\n",
2189			    ifp->if_xname);
2190			goto err_tx_desc;
2191		}
2192		txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr;
2193		bzero((void *)txr->tx_base, tsize);
2194	}
2195
2196	/*
2197	 * Next the RX queues...
2198	 */
2199	rsize = roundup2(sc->num_rx_desc *
2200	    sizeof(union ixgbe_adv_rx_desc), 4096);
2201	for (i = 0; i < sc->num_queues; i++, rxconf++) {
2202		rxr = &sc->rx_rings[i];
2203		/* Set up some basics */
2204		rxr->sc = sc;
2205		rxr->me = i;
2206		timeout_set(&rxr->rx_refill, ixgbe_rxrefill, rxr);
2207
2208		if (ixgbe_dma_malloc(sc, rsize,
2209			&rxr->rxdma, BUS_DMA_NOWAIT)) {
2210			printf("%s: Unable to allocate RxDescriptor memory\n",
2211			    ifp->if_xname);
2212			goto err_rx_desc;
2213		}
2214		rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr;
2215		bzero((void *)rxr->rx_base, rsize);
2216	}
2217
2218	/*
2219	 * Finally set up the queue holding structs
2220	 */
2221	for (i = 0; i < sc->num_queues; i++) {
2222		que = &sc->queues[i];
2223		que->sc = sc;
2224		que->txr = &sc->tx_rings[i];
2225		que->rxr = &sc->rx_rings[i];
2226		snprintf(que->name, sizeof(que->name), "%s:%d",
2227		    sc->dev.dv_xname, i);
2228	}
2229
2230	return (0);
2231
2232err_rx_desc:
2233	for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--)
2234		ixgbe_dma_free(sc, &rxr->rxdma);
2235err_tx_desc:
2236	for (txr = sc->tx_rings; txconf > 0; txr++, txconf--)
2237		ixgbe_dma_free(sc, &txr->txdma);
2238	free(sc->rx_rings, M_DEVBUF, sc->num_queues * sizeof(struct ix_rxring));
2239	sc->rx_rings = NULL;
2240rx_fail:
2241	free(sc->tx_rings, M_DEVBUF, sc->num_queues * sizeof(struct ix_txring));
2242	sc->tx_rings = NULL;
2243fail:
2244	return (ENOMEM);
2245}
2246
2247/*********************************************************************
2248 *
2249 *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2250 *  the information needed to transmit a packet on the wire. This is
2251 *  called only once at attach, setup is done every reset.
2252 *
2253 **********************************************************************/
2254int
2255ixgbe_allocate_transmit_buffers(struct ix_txring *txr)
2256{
2257	struct ix_softc 	*sc = txr->sc;
2258	struct ifnet		*ifp = &sc->arpcom.ac_if;
2259	struct ixgbe_tx_buf	*txbuf;
2260	int			 error, i;
2261
2262	if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc,
2263	    sizeof(struct ixgbe_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) {
2264		printf("%s: Unable to allocate tx_buffer memory\n",
2265		    ifp->if_xname);
2266		error = ENOMEM;
2267		goto fail;
2268	}
2269	txr->txtag = txr->txdma.dma_tag;
2270
2271	/* Create the descriptor buffer dma maps */
2272	for (i = 0; i < sc->num_tx_desc; i++) {
2273		txbuf = &txr->tx_buffers[i];
2274		error = bus_dmamap_create(txr->txdma.dma_tag, MAXMCLBYTES,
2275			    sc->num_segs, PAGE_SIZE, 0,
2276			    BUS_DMA_NOWAIT, &txbuf->map);
2277
2278		if (error != 0) {
2279			printf("%s: Unable to create TX DMA map\n",
2280			    ifp->if_xname);
2281			goto fail;
2282		}
2283	}
2284
2285	return 0;
2286fail:
2287	return (error);
2288}
2289
2290/*********************************************************************
2291 *
2292 *  Initialize a transmit ring.
2293 *
2294 **********************************************************************/
2295int
2296ixgbe_setup_transmit_ring(struct ix_txring *txr)
2297{
2298	struct ix_softc		*sc = txr->sc;
2299	int			 error;
2300
2301	/* Now allocate transmit buffers for the ring */
2302	if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0)
2303		return (error);
2304
2305	/* Clear the old ring contents */
2306	bzero((void *)txr->tx_base,
2307	      (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc);
2308
2309	/* Reset indices */
2310	txr->next_avail_desc = 0;
2311	txr->next_to_clean = 0;
2312
2313	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2314	    0, txr->txdma.dma_map->dm_mapsize,
2315	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2316
2317	return (0);
2318}
2319
2320/*********************************************************************
2321 *
2322 *  Initialize all transmit rings.
2323 *
2324 **********************************************************************/
2325int
2326ixgbe_setup_transmit_structures(struct ix_softc *sc)
2327{
2328	struct ix_txring *txr = sc->tx_rings;
2329	int		i, error;
2330
2331	for (i = 0; i < sc->num_queues; i++, txr++) {
2332		if ((error = ixgbe_setup_transmit_ring(txr)) != 0)
2333			goto fail;
2334	}
2335
2336	return (0);
2337fail:
2338	ixgbe_free_transmit_structures(sc);
2339	return (error);
2340}
2341
2342/*********************************************************************
2343 *
2344 *  Enable transmit unit.
2345 *
2346 **********************************************************************/
2347void
2348ixgbe_initialize_transmit_units(struct ix_softc *sc)
2349{
2350	struct ifnet	*ifp = &sc->arpcom.ac_if;
2351	struct ix_txring	*txr;
2352	struct ixgbe_hw	*hw = &sc->hw;
2353	int		 i;
2354	uint64_t	 tdba;
2355	uint32_t	 txctrl;
2356	uint32_t	 hlreg;
2357
2358	/* Setup the Base and Length of the Tx Descriptor Ring */
2359
2360	for (i = 0; i < sc->num_queues; i++) {
2361		txr = &sc->tx_rings[i];
2362
2363		/* Setup descriptor base address */
2364		tdba = txr->txdma.dma_map->dm_segs[0].ds_addr;
2365		IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i),
2366		       (tdba & 0x00000000ffffffffULL));
2367		IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32));
2368		IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i),
2369		    sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc));
2370
2371		/* Setup the HW Tx Head and Tail descriptor pointers */
2372		IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0);
2373		IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0);
2374
2375		/* Setup Transmit Descriptor Cmd Settings */
2376		txr->txd_cmd = IXGBE_TXD_CMD_IFCS;
2377		txr->queue_status = IXGBE_QUEUE_IDLE;
2378		txr->watchdog_timer = 0;
2379
2380		/* Disable Head Writeback */
2381		switch (hw->mac.type) {
2382		case ixgbe_mac_82598EB:
2383			txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i));
2384			break;
2385		case ixgbe_mac_82599EB:
2386		case ixgbe_mac_X540:
2387		default:
2388			txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i));
2389			break;
2390		}
2391		txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN;
2392		switch (hw->mac.type) {
2393		case ixgbe_mac_82598EB:
2394			IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl);
2395			break;
2396		case ixgbe_mac_82599EB:
2397		case ixgbe_mac_X540:
2398		default:
2399			IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl);
2400			break;
2401		}
2402	}
2403	ifp->if_timer = 0;
2404
2405	if (hw->mac.type != ixgbe_mac_82598EB) {
2406		uint32_t dmatxctl, rttdcs;
2407		dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2408		dmatxctl |= IXGBE_DMATXCTL_TE;
2409		IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2410		/* Disable arbiter to set MTQC */
2411		rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2412		rttdcs |= IXGBE_RTTDCS_ARBDIS;
2413		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2414		IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB);
2415		rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2416		IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2417	}
2418
2419	/* Enable TCP/UDP padding when using TSO */
2420	hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2421	hlreg |= IXGBE_HLREG0_TXPADEN;
2422	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
2423}
2424
2425/*********************************************************************
2426 *
2427 *  Free all transmit rings.
2428 *
2429 **********************************************************************/
2430void
2431ixgbe_free_transmit_structures(struct ix_softc *sc)
2432{
2433	struct ix_txring *txr = sc->tx_rings;
2434	int		i;
2435
2436	for (i = 0; i < sc->num_queues; i++, txr++)
2437		ixgbe_free_transmit_buffers(txr);
2438}
2439
2440/*********************************************************************
2441 *
2442 *  Free transmit ring related data structures.
2443 *
2444 **********************************************************************/
2445void
2446ixgbe_free_transmit_buffers(struct ix_txring *txr)
2447{
2448	struct ix_softc *sc = txr->sc;
2449	struct ixgbe_tx_buf *tx_buffer;
2450	int             i;
2451
2452	INIT_DEBUGOUT("free_transmit_ring: begin");
2453
2454	if (txr->tx_buffers == NULL)
2455		return;
2456
2457	tx_buffer = txr->tx_buffers;
2458	for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) {
2459		if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) {
2460			bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map,
2461			    0, tx_buffer->map->dm_mapsize,
2462			    BUS_DMASYNC_POSTWRITE);
2463			bus_dmamap_unload(txr->txdma.dma_tag,
2464			    tx_buffer->map);
2465		}
2466		if (tx_buffer->m_head != NULL) {
2467			m_freem(tx_buffer->m_head);
2468			tx_buffer->m_head = NULL;
2469		}
2470		if (tx_buffer->map != NULL) {
2471			bus_dmamap_destroy(txr->txdma.dma_tag,
2472			    tx_buffer->map);
2473			tx_buffer->map = NULL;
2474		}
2475	}
2476
2477	if (txr->tx_buffers != NULL)
2478		free(txr->tx_buffers, M_DEVBUF,
2479		    sc->num_tx_desc * sizeof(struct ixgbe_tx_buf));
2480	txr->tx_buffers = NULL;
2481	txr->txtag = NULL;
2482}
2483
2484/*********************************************************************
2485 *
2486 *  Advanced Context Descriptor setup for VLAN or CSUM
2487 *
2488 **********************************************************************/
2489
2490static inline int
2491ixgbe_tx_offload(struct mbuf *mp, uint32_t *vlan_macip_lens,
2492    uint32_t *type_tucmd_mlhl, uint32_t *olinfo_status, uint32_t *cmd_type_len,
2493    uint32_t *mss_l4len_idx)
2494{
2495	struct ether_extracted ext;
2496	int offload = 0;
2497
2498	ether_extract_headers(mp, &ext);
2499
2500	*vlan_macip_lens |= (sizeof(*ext.eh) << IXGBE_ADVTXD_MACLEN_SHIFT);
2501
2502	if (ext.ip4) {
2503		if (ISSET(mp->m_pkthdr.csum_flags, M_IPV4_CSUM_OUT)) {
2504			*olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
2505			offload = 1;
2506		}
2507
2508		*type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
2509#ifdef INET6
2510	} else if (ext.ip6) {
2511		*type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
2512#endif
2513	} else {
2514		if (mp->m_pkthdr.csum_flags & M_TCP_TSO)
2515			tcpstat_inc(tcps_outbadtso);
2516		return offload;
2517	}
2518
2519	*vlan_macip_lens |= ext.iphlen;
2520
2521	if (ext.tcp) {
2522		*type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
2523		if (ISSET(mp->m_pkthdr.csum_flags, M_TCP_CSUM_OUT)) {
2524			*olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
2525			offload = 1;
2526		}
2527	} else if (ext.udp) {
2528		*type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
2529		if (ISSET(mp->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) {
2530			*olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
2531			offload = 1;
2532		}
2533	}
2534
2535	if (mp->m_pkthdr.csum_flags & M_TCP_TSO) {
2536		if (ext.tcp && mp->m_pkthdr.ph_mss > 0) {
2537			uint32_t hdrlen, thlen, paylen, outlen;
2538
2539			thlen = ext.tcphlen;
2540
2541			outlen = mp->m_pkthdr.ph_mss;
2542			*mss_l4len_idx |= outlen << IXGBE_ADVTXD_MSS_SHIFT;
2543			*mss_l4len_idx |= thlen << IXGBE_ADVTXD_L4LEN_SHIFT;
2544
2545			hdrlen = sizeof(*ext.eh) + ext.iphlen + thlen;
2546			paylen = mp->m_pkthdr.len - hdrlen;
2547			CLR(*olinfo_status, IXGBE_ADVTXD_PAYLEN_MASK
2548			    << IXGBE_ADVTXD_PAYLEN_SHIFT);
2549			*olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;
2550
2551			*cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE;
2552			offload = 1;
2553
2554			tcpstat_add(tcps_outpkttso,
2555			    (paylen + outlen - 1) / outlen);
2556		} else
2557			tcpstat_inc(tcps_outbadtso);
2558	}
2559
2560	return offload;
2561}
2562
2563static int
2564ixgbe_tx_ctx_setup(struct ix_txring *txr, struct mbuf *mp,
2565    uint32_t *cmd_type_len, uint32_t *olinfo_status)
2566{
2567	struct ixgbe_adv_tx_context_desc *TXD;
2568	struct ixgbe_tx_buf *tx_buffer;
2569	uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0;
2570	uint32_t mss_l4len_idx = 0;
2571	int	ctxd = txr->next_avail_desc;
2572	int	offload = 0;
2573
2574	/* Indicate the whole packet as payload when not doing TSO */
2575	*olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT;
2576
2577#if NVLAN > 0
2578	if (ISSET(mp->m_flags, M_VLANTAG)) {
2579		uint32_t vtag = mp->m_pkthdr.ether_vtag;
2580		vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT);
2581		*cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE;
2582		offload |= 1;
2583	}
2584#endif
2585
2586	offload |= ixgbe_tx_offload(mp, &vlan_macip_lens, &type_tucmd_mlhl,
2587	    olinfo_status, cmd_type_len, &mss_l4len_idx);
2588
2589	if (!offload)
2590		return (0);
2591
2592	TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd];
2593	tx_buffer = &txr->tx_buffers[ctxd];
2594
2595	type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
2596
2597	/* Now copy bits into descriptor */
2598	TXD->vlan_macip_lens = htole32(vlan_macip_lens);
2599	TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
2600	TXD->seqnum_seed = htole32(0);
2601	TXD->mss_l4len_idx = htole32(mss_l4len_idx);
2602
2603	tx_buffer->m_head = NULL;
2604	tx_buffer->eop_index = -1;
2605
2606	return (1);
2607}
2608
2609/**********************************************************************
2610 *
2611 *  Examine each tx_buffer in the used queue. If the hardware is done
2612 *  processing the packet then free associated resources. The
2613 *  tx_buffer is put back on the free queue.
2614 *
2615 **********************************************************************/
2616int
2617ixgbe_txeof(struct ix_txring *txr)
2618{
2619	struct ix_softc			*sc = txr->sc;
2620	struct ifqueue			*ifq = txr->ifq;
2621	struct ifnet			*ifp = &sc->arpcom.ac_if;
2622	unsigned int			 head, tail, last;
2623	struct ixgbe_tx_buf		*tx_buffer;
2624	struct ixgbe_legacy_tx_desc	*tx_desc;
2625
2626	if (!ISSET(ifp->if_flags, IFF_RUNNING))
2627		return FALSE;
2628
2629	head = txr->next_avail_desc;
2630	tail = txr->next_to_clean;
2631
2632	membar_consumer();
2633
2634	if (head == tail)
2635		return (FALSE);
2636
2637	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2638	    0, txr->txdma.dma_map->dm_mapsize,
2639	    BUS_DMASYNC_POSTREAD);
2640
2641	for (;;) {
2642		tx_buffer = &txr->tx_buffers[tail];
2643		last = tx_buffer->eop_index;
2644		tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last];
2645
2646		if (!ISSET(tx_desc->upper.fields.status, IXGBE_TXD_STAT_DD))
2647			break;
2648
2649		bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map,
2650		    0, tx_buffer->map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
2651		bus_dmamap_unload(txr->txdma.dma_tag, tx_buffer->map);
2652		m_freem(tx_buffer->m_head);
2653
2654		tx_buffer->m_head = NULL;
2655		tx_buffer->eop_index = -1;
2656
2657		tail = last + 1;
2658		if (tail == sc->num_tx_desc)
2659			tail = 0;
2660		if (head == tail) {
2661			/* All clean, turn off the timer */
2662			ifp->if_timer = 0;
2663			break;
2664		}
2665	}
2666
2667	bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map,
2668	    0, txr->txdma.dma_map->dm_mapsize,
2669	    BUS_DMASYNC_PREREAD);
2670
2671	membar_producer();
2672
2673	txr->next_to_clean = tail;
2674
2675	if (ifq_is_oactive(ifq))
2676		ifq_restart(ifq);
2677
2678	return TRUE;
2679}
2680
2681/*********************************************************************
2682 *
2683 *  Get a buffer from system mbuf buffer pool.
2684 *
2685 **********************************************************************/
2686int
2687ixgbe_get_buf(struct ix_rxring *rxr, int i)
2688{
2689	struct ix_softc		*sc = rxr->sc;
2690	struct ixgbe_rx_buf	*rxbuf;
2691	struct mbuf		*mp;
2692	int			error;
2693	union ixgbe_adv_rx_desc	*rxdesc;
2694
2695	rxbuf = &rxr->rx_buffers[i];
2696	rxdesc = &rxr->rx_base[i];
2697	if (rxbuf->buf) {
2698		printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n",
2699		    sc->dev.dv_xname, i);
2700		return (ENOBUFS);
2701	}
2702
2703	/* needed in any case so prealocate since this one will fail for sure */
2704	mp = MCLGETL(NULL, M_DONTWAIT, sc->rx_mbuf_sz);
2705	if (!mp)
2706		return (ENOBUFS);
2707
2708	mp->m_data += (mp->m_ext.ext_size - sc->rx_mbuf_sz);
2709	mp->m_len = mp->m_pkthdr.len = sc->rx_mbuf_sz;
2710
2711	error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map,
2712	    mp, BUS_DMA_NOWAIT);
2713	if (error) {
2714		m_freem(mp);
2715		return (error);
2716	}
2717
2718	bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map,
2719	    0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD);
2720	rxbuf->buf = mp;
2721
2722	rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr);
2723
2724	return (0);
2725}
2726
2727/*********************************************************************
2728 *
2729 *  Allocate memory for rx_buffer structures. Since we use one
2730 *  rx_buffer per received packet, the maximum number of rx_buffer's
2731 *  that we'll need is equal to the number of receive descriptors
2732 *  that we've allocated.
2733 *
2734 **********************************************************************/
2735int
2736ixgbe_allocate_receive_buffers(struct ix_rxring *rxr)
2737{
2738	struct ix_softc		*sc = rxr->sc;
2739	struct ifnet		*ifp = &sc->arpcom.ac_if;
2740	struct ixgbe_rx_buf 	*rxbuf;
2741	int			i, error;
2742
2743	if (!(rxr->rx_buffers = mallocarray(sc->num_rx_desc,
2744	    sizeof(struct ixgbe_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) {
2745		printf("%s: Unable to allocate rx_buffer memory\n",
2746		    ifp->if_xname);
2747		error = ENOMEM;
2748		goto fail;
2749	}
2750
2751	rxbuf = rxr->rx_buffers;
2752	for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) {
2753		error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1,
2754		    16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map);
2755		if (error) {
2756			printf("%s: Unable to create Pack DMA map\n",
2757			    ifp->if_xname);
2758			goto fail;
2759		}
2760	}
2761	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0,
2762	    rxr->rxdma.dma_map->dm_mapsize,
2763	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2764
2765	return (0);
2766
2767fail:
2768	return (error);
2769}
2770
2771/*********************************************************************
2772 *
2773 *  Initialize a receive ring and its buffers.
2774 *
2775 **********************************************************************/
2776int
2777ixgbe_setup_receive_ring(struct ix_rxring *rxr)
2778{
2779	struct ix_softc		*sc = rxr->sc;
2780	struct ifnet		*ifp = &sc->arpcom.ac_if;
2781	int			 rsize, error;
2782
2783	rsize = roundup2(sc->num_rx_desc *
2784	    sizeof(union ixgbe_adv_rx_desc), 4096);
2785	/* Clear the ring contents */
2786	bzero((void *)rxr->rx_base, rsize);
2787
2788	if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0)
2789		return (error);
2790
2791	/* Setup our descriptor indices */
2792	rxr->next_to_check = 0;
2793	rxr->last_desc_filled = sc->num_rx_desc - 1;
2794
2795	if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1),
2796	    sc->num_rx_desc - 1);
2797
2798	ixgbe_rxfill(rxr);
2799	if (if_rxr_inuse(&rxr->rx_ring) == 0) {
2800		printf("%s: unable to fill any rx descriptors\n",
2801		    sc->dev.dv_xname);
2802		return (ENOBUFS);
2803	}
2804
2805	return (0);
2806}
2807
2808int
2809ixgbe_rxfill(struct ix_rxring *rxr)
2810{
2811	struct ix_softc *sc = rxr->sc;
2812	int		 post = 0;
2813	u_int		 slots;
2814	int		 i;
2815
2816	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
2817	    0, rxr->rxdma.dma_map->dm_mapsize,
2818	    BUS_DMASYNC_POSTWRITE);
2819
2820	i = rxr->last_desc_filled;
2821	for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc);
2822	    slots > 0; slots--) {
2823		if (++i == sc->num_rx_desc)
2824			i = 0;
2825
2826		if (ixgbe_get_buf(rxr, i) != 0)
2827			break;
2828
2829		rxr->last_desc_filled = i;
2830		post = 1;
2831	}
2832
2833	bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
2834	    0, rxr->rxdma.dma_map->dm_mapsize,
2835	    BUS_DMASYNC_PREWRITE);
2836
2837	if_rxr_put(&rxr->rx_ring, slots);
2838
2839	return (post);
2840}
2841
2842void
2843ixgbe_rxrefill(void *xrxr)
2844{
2845	struct ix_rxring *rxr = xrxr;
2846	struct ix_softc *sc = rxr->sc;
2847
2848	if (ixgbe_rxfill(rxr)) {
2849		/* Advance the Rx Queue "Tail Pointer" */
2850		IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me),
2851		    rxr->last_desc_filled);
2852	} else if (if_rxr_inuse(&rxr->rx_ring) == 0)
2853		timeout_add(&rxr->rx_refill, 1);
2854
2855}
2856
2857/*********************************************************************
2858 *
2859 *  Initialize all receive rings.
2860 *
2861 **********************************************************************/
2862int
2863ixgbe_setup_receive_structures(struct ix_softc *sc)
2864{
2865	struct ix_rxring *rxr = sc->rx_rings;
2866	int i;
2867
2868	for (i = 0; i < sc->num_queues; i++, rxr++)
2869		if (ixgbe_setup_receive_ring(rxr))
2870			goto fail;
2871
2872	return (0);
2873fail:
2874	ixgbe_free_receive_structures(sc);
2875	return (ENOBUFS);
2876}
2877
2878/*********************************************************************
2879 *
2880 *  Setup receive registers and features.
2881 *
2882 **********************************************************************/
2883#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2884
2885void
2886ixgbe_initialize_receive_units(struct ix_softc *sc)
2887{
2888	struct ifnet	*ifp = &sc->arpcom.ac_if;
2889	struct ix_rxring	*rxr = sc->rx_rings;
2890	struct ixgbe_hw	*hw = &sc->hw;
2891	uint32_t	bufsz, fctrl, srrctl, rxcsum, rdrxctl;
2892	uint32_t	hlreg;
2893	int		i;
2894
2895	/*
2896	 * Make sure receives are disabled while
2897	 * setting up the descriptor ring
2898	 */
2899	ixgbe_disable_rx(hw);
2900
2901	/* Enable broadcasts */
2902	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
2903	fctrl |= IXGBE_FCTRL_BAM;
2904	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
2905		fctrl |= IXGBE_FCTRL_DPF;
2906		fctrl |= IXGBE_FCTRL_PMCF;
2907	}
2908	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
2909
2910	hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2911	/* Always enable jumbo frame reception */
2912	hlreg |= IXGBE_HLREG0_JUMBOEN;
2913	/* Always enable CRC stripping */
2914	hlreg |= IXGBE_HLREG0_RXCRCSTRP;
2915	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg);
2916
2917	if (ISSET(ifp->if_xflags, IFXF_LRO)) {
2918		rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);
2919
2920		/* This field has to be set to zero. */
2921		rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;
2922
2923		/* RSC Coalescing on ACK Change */
2924		rdrxctl |= IXGBE_RDRXCTL_RSCACKC;
2925		rdrxctl |= IXGBE_RDRXCTL_FCOE_WRFIX;
2926
2927		IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);
2928	}
2929
2930	bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2931
2932	for (i = 0; i < sc->num_queues; i++, rxr++) {
2933		uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr;
2934
2935		/* Setup the Base and Length of the Rx Descriptor Ring */
2936		IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i),
2937			       (rdba & 0x00000000ffffffffULL));
2938		IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32));
2939		IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i),
2940		    sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc));
2941
2942		/* Set up the SRRCTL register */
2943		srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2944		IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl);
2945
2946		if (ISSET(ifp->if_xflags, IFXF_LRO)) {
2947			rdrxctl = IXGBE_READ_REG(&sc->hw, IXGBE_RSCCTL(i));
2948
2949			/* Enable Receive Side Coalescing */
2950			rdrxctl |= IXGBE_RSCCTL_RSCEN;
2951			rdrxctl |= IXGBE_RSCCTL_MAXDESC_16;
2952
2953			IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), rdrxctl);
2954		}
2955
2956		/* Setup the HW Rx Head and Tail Descriptor Pointers */
2957		IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0);
2958		IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0);
2959	}
2960
2961	if (sc->hw.mac.type != ixgbe_mac_82598EB) {
2962		uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR |
2963			      IXGBE_PSRTYPE_UDPHDR |
2964			      IXGBE_PSRTYPE_IPV4HDR |
2965			      IXGBE_PSRTYPE_IPV6HDR;
2966		IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype);
2967	}
2968
2969	rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2970	rxcsum &= ~IXGBE_RXCSUM_PCSD;
2971
2972	ixgbe_initialize_rss_mapping(sc);
2973
2974	/* Setup RSS */
2975	if (sc->num_queues > 1) {
2976		/* RSS and RX IPP Checksum are mutually exclusive */
2977		rxcsum |= IXGBE_RXCSUM_PCSD;
2978	}
2979
2980	/* Map QPRC/QPRDC/QPTC on a per queue basis */
2981	ixgbe_map_queue_statistics(sc);
2982
2983	/* This is useful for calculating UDP/IP fragment checksums */
2984	if (!(rxcsum & IXGBE_RXCSUM_PCSD))
2985		rxcsum |= IXGBE_RXCSUM_IPPCSE;
2986
2987	IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2988}
2989
2990void
2991ixgbe_initialize_rss_mapping(struct ix_softc *sc)
2992{
2993	struct ixgbe_hw	*hw = &sc->hw;
2994	uint32_t reta = 0, mrqc, rss_key[10];
2995	int i, j, queue_id, table_size, index_mult;
2996
2997	/* set up random bits */
2998	stoeplitz_to_key(&rss_key, sizeof(rss_key));
2999
3000	/* Set multiplier for RETA setup and table size based on MAC */
3001	index_mult = 0x1;
3002	table_size = 128;
3003	switch (sc->hw.mac.type) {
3004	case ixgbe_mac_82598EB:
3005		index_mult = 0x11;
3006		break;
3007	case ixgbe_mac_X550:
3008	case ixgbe_mac_X550EM_x:
3009	case ixgbe_mac_X550EM_a:
3010		table_size = 512;
3011		break;
3012	default:
3013		break;
3014	}
3015
3016	/* Set up the redirection table */
3017	for (i = 0, j = 0; i < table_size; i++, j++) {
3018		if (j == sc->num_queues) j = 0;
3019		queue_id = (j * index_mult);
3020		/*
3021		 * The low 8 bits are for hash value (n+0);
3022		 * The next 8 bits are for hash value (n+1), etc.
3023		 */
3024		reta = reta >> 8;
3025		reta = reta | ( ((uint32_t) queue_id) << 24);
3026		if ((i & 3) == 3) {
3027			if (i < 128)
3028				IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
3029			else
3030				IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),
3031				    reta);
3032			reta = 0;
3033		}
3034	}
3035
3036	/* Now fill our hash function seeds */
3037	for (i = 0; i < 10; i++)
3038		IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]);
3039
3040	/*
3041	 * Disable UDP - IP fragments aren't currently being handled
3042	 * and so we end up with a mix of 2-tuple and 4-tuple
3043	 * traffic.
3044	 */
3045	mrqc = IXGBE_MRQC_RSSEN
3046	     | IXGBE_MRQC_RSS_FIELD_IPV4
3047	     | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
3048	     | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP
3049	     | IXGBE_MRQC_RSS_FIELD_IPV6_EX
3050	     | IXGBE_MRQC_RSS_FIELD_IPV6
3051	     | IXGBE_MRQC_RSS_FIELD_IPV6_TCP
3052	;
3053	IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
3054}
3055
3056/*********************************************************************
3057 *
3058 *  Free all receive rings.
3059 *
3060 **********************************************************************/
3061void
3062ixgbe_free_receive_structures(struct ix_softc *sc)
3063{
3064	struct ix_rxring *rxr;
3065	int		i;
3066
3067	for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++)
3068		if_rxr_init(&rxr->rx_ring, 0, 0);
3069
3070	for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++)
3071		ixgbe_free_receive_buffers(rxr);
3072}
3073
3074/*********************************************************************
3075 *
3076 *  Free receive ring data structures
3077 *
3078 **********************************************************************/
3079void
3080ixgbe_free_receive_buffers(struct ix_rxring *rxr)
3081{
3082	struct ix_softc		*sc;
3083	struct ixgbe_rx_buf	*rxbuf;
3084	int			 i;
3085
3086	sc = rxr->sc;
3087	if (rxr->rx_buffers != NULL) {
3088		for (i = 0; i < sc->num_rx_desc; i++) {
3089			rxbuf = &rxr->rx_buffers[i];
3090			if (rxbuf->buf != NULL) {
3091				bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map,
3092				    0, rxbuf->map->dm_mapsize,
3093				    BUS_DMASYNC_POSTREAD);
3094				bus_dmamap_unload(rxr->rxdma.dma_tag,
3095				    rxbuf->map);
3096				m_freem(rxbuf->buf);
3097				rxbuf->buf = NULL;
3098			}
3099			if (rxbuf->map != NULL) {
3100				bus_dmamap_destroy(rxr->rxdma.dma_tag,
3101				    rxbuf->map);
3102				rxbuf->map = NULL;
3103			}
3104		}
3105		free(rxr->rx_buffers, M_DEVBUF,
3106		    sc->num_rx_desc * sizeof(struct ixgbe_rx_buf));
3107		rxr->rx_buffers = NULL;
3108	}
3109}
3110
3111/*********************************************************************
3112 *
3113 *  This routine executes in interrupt context. It replenishes
3114 *  the mbufs in the descriptor and sends data which has been
3115 *  dma'ed into host memory to upper layer.
3116 *
3117 *********************************************************************/
3118int
3119ixgbe_rxeof(struct ix_rxring *rxr)
3120{
3121	struct ix_softc 	*sc = rxr->sc;
3122	struct ifnet   		*ifp = &sc->arpcom.ac_if;
3123	struct mbuf_list	 ml = MBUF_LIST_INITIALIZER();
3124	struct mbuf    		*mp, *sendmp;
3125	uint8_t		    	 eop = 0;
3126	uint16_t		 len, vtag;
3127	uint32_t		 staterr = 0;
3128	struct ixgbe_rx_buf	*rxbuf, *nxbuf;
3129	union ixgbe_adv_rx_desc	*rxdesc;
3130	size_t			 dsize = sizeof(union ixgbe_adv_rx_desc);
3131	int			 i, nextp, rsccnt;
3132
3133	if (!ISSET(ifp->if_flags, IFF_RUNNING))
3134		return FALSE;
3135
3136	i = rxr->next_to_check;
3137	while (if_rxr_inuse(&rxr->rx_ring) > 0) {
3138		uint32_t hash;
3139		uint16_t hashtype;
3140
3141		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3142		    dsize * i, dsize, BUS_DMASYNC_POSTREAD);
3143
3144		rxdesc = &rxr->rx_base[i];
3145		staterr = letoh32(rxdesc->wb.upper.status_error);
3146		if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) {
3147			bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3148			    dsize * i, dsize,
3149			    BUS_DMASYNC_PREREAD);
3150			break;
3151		}
3152
3153		/* Zero out the receive descriptors status  */
3154		rxdesc->wb.upper.status_error = 0;
3155		rxbuf = &rxr->rx_buffers[i];
3156
3157		/* pull the mbuf off the ring */
3158		bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0,
3159		    rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD);
3160		bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map);
3161
3162		mp = rxbuf->buf;
3163		len = letoh16(rxdesc->wb.upper.length);
3164		vtag = letoh16(rxdesc->wb.upper.vlan);
3165		eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
3166		hash = lemtoh32(&rxdesc->wb.lower.hi_dword.rss);
3167		hashtype =
3168		    lemtoh16(&rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) &
3169		    IXGBE_RXDADV_RSSTYPE_MASK;
3170		rsccnt = lemtoh32(&rxdesc->wb.lower.lo_dword.data) &
3171		    IXGBE_RXDADV_RSCCNT_MASK;
3172		rsccnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;
3173
3174		if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) {
3175			if (rxbuf->fmp) {
3176				m_freem(rxbuf->fmp);
3177			} else {
3178				m_freem(mp);
3179			}
3180			rxbuf->fmp = NULL;
3181			rxbuf->buf = NULL;
3182			goto next_desc;
3183		}
3184
3185		if (mp == NULL) {
3186			panic("%s: ixgbe_rxeof: NULL mbuf in slot %d "
3187			    "(nrx %d, filled %d)", sc->dev.dv_xname,
3188			    i, if_rxr_inuse(&rxr->rx_ring),
3189			    rxr->last_desc_filled);
3190		}
3191
3192		if (!eop) {
3193			/*
3194			 * Figure out the next descriptor of this frame.
3195			 */
3196			if (rsccnt) {
3197				nextp = staterr & IXGBE_RXDADV_NEXTP_MASK;
3198				nextp >>= IXGBE_RXDADV_NEXTP_SHIFT;
3199			} else {
3200				nextp = i + 1;
3201			}
3202			if (nextp == sc->num_rx_desc)
3203				nextp = 0;
3204			nxbuf = &rxr->rx_buffers[nextp];
3205			/* prefetch(nxbuf); */
3206		}
3207
3208		/*
3209		 * Rather than using the fmp/lmp global pointers
3210		 * we now keep the head of a packet chain in the
3211		 * buffer struct and pass this along from one
3212		 * descriptor to the next, until we get EOP.
3213		 */
3214		mp->m_len = len;
3215		/*
3216		 * See if there is a stored head
3217		 * that determines what we are
3218		 */
3219		sendmp = rxbuf->fmp;
3220		rxbuf->buf = rxbuf->fmp = NULL;
3221
3222		if (sendmp == NULL) {
3223			/* first desc of a non-ps chain */
3224			sendmp = mp;
3225			sendmp->m_pkthdr.len = 0;
3226			sendmp->m_pkthdr.ph_mss = 0;
3227		} else {
3228			mp->m_flags &= ~M_PKTHDR;
3229		}
3230		sendmp->m_pkthdr.len += mp->m_len;
3231		/*
3232		 * This function iterates over interleaved descriptors.
3233		 * Thus, we reuse ph_mss as global segment counter per
3234		 * TCP connection, instead of introducing a new variable
3235		 * in m_pkthdr.
3236		 */
3237		if (rsccnt)
3238			sendmp->m_pkthdr.ph_mss += rsccnt - 1;
3239
3240		/* Pass the head pointer on */
3241		if (eop == 0) {
3242			nxbuf->fmp = sendmp;
3243			sendmp = NULL;
3244			mp->m_next = nxbuf->buf;
3245		} else { /* Sending this frame? */
3246			ixgbe_rx_offload(staterr, vtag, sendmp);
3247
3248			if (hashtype != IXGBE_RXDADV_RSSTYPE_NONE) {
3249				sendmp->m_pkthdr.ph_flowid = hash;
3250				SET(sendmp->m_pkthdr.csum_flags, M_FLOWID);
3251			}
3252
3253			ml_enqueue(&ml, sendmp);
3254		}
3255next_desc:
3256		if_rxr_put(&rxr->rx_ring, 1);
3257		bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map,
3258		    dsize * i, dsize,
3259		    BUS_DMASYNC_PREREAD);
3260
3261		/* Advance our pointers to the next descriptor. */
3262		if (++i == sc->num_rx_desc)
3263			i = 0;
3264	}
3265	rxr->next_to_check = i;
3266
3267	if (ifiq_input(rxr->ifiq, &ml))
3268		if_rxr_livelocked(&rxr->rx_ring);
3269
3270	if (!(staterr & IXGBE_RXD_STAT_DD))
3271		return FALSE;
3272
3273	return TRUE;
3274}
3275
3276/*********************************************************************
3277 *
3278 *  Check VLAN indication from hardware and inform the stack about the
3279 *  annotated TAG.
3280 *
3281 *  Verify that the hardware indicated that the checksum is valid.
3282 *  Inform the stack about the status of checksum so that stack
3283 *  doesn't spend time verifying the checksum.
3284 *
3285 *  Propagate TCP LRO packet from hardware to the stack with MSS annotation.
3286 *
3287 *********************************************************************/
3288void
3289ixgbe_rx_offload(uint32_t staterr, uint16_t vtag, struct mbuf *m)
3290{
3291	uint16_t status = (uint16_t) staterr;
3292	uint8_t  errors = (uint8_t) (staterr >> 24);
3293	int16_t  pkts;
3294
3295	/*
3296	 * VLAN Offload
3297	 */
3298
3299#if NVLAN > 0
3300	if (ISSET(staterr, IXGBE_RXD_STAT_VP)) {
3301		m->m_pkthdr.ether_vtag = vtag;
3302		SET(m->m_flags, M_VLANTAG);
3303	}
3304#endif
3305
3306	/*
3307	 * Checksum Offload
3308	 */
3309
3310	if (ISSET(status, IXGBE_RXD_STAT_IPCS)) {
3311		if (ISSET(errors, IXGBE_RXD_ERR_IPE))
3312			SET(m->m_pkthdr.csum_flags, M_IPV4_CSUM_IN_BAD);
3313		else
3314			SET(m->m_pkthdr.csum_flags, M_IPV4_CSUM_IN_OK);
3315	}
3316	if (ISSET(status, IXGBE_RXD_STAT_L4CS) &&
3317	    !ISSET(status, IXGBE_RXD_STAT_UDPCS)) {
3318		if (ISSET(errors, IXGBE_RXD_ERR_TCPE)) {
3319			/* on some hardware IPv6 + TCP + Bad is broken */
3320			if (ISSET(status, IXGBE_RXD_STAT_IPCS))
3321				SET(m->m_pkthdr.csum_flags, M_TCP_CSUM_IN_BAD);
3322		} else
3323			SET(m->m_pkthdr.csum_flags, M_TCP_CSUM_IN_OK);
3324	}
3325	if (ISSET(status, IXGBE_RXD_STAT_L4CS) &&
3326	    ISSET(status, IXGBE_RXD_STAT_UDPCS)) {
3327		if (ISSET(errors, IXGBE_RXD_ERR_TCPE))
3328			SET(m->m_pkthdr.csum_flags, M_UDP_CSUM_IN_BAD);
3329		else
3330			SET(m->m_pkthdr.csum_flags, M_UDP_CSUM_IN_OK);
3331	}
3332
3333	/*
3334	 * TCP Large Receive Offload
3335	 */
3336
3337	pkts = m->m_pkthdr.ph_mss;
3338	m->m_pkthdr.ph_mss = 0;
3339
3340	if (pkts > 1) {
3341		struct ether_extracted ext;
3342		uint32_t paylen;
3343
3344		/*
3345		 * Calculate the payload size:
3346		 *
3347		 * The packet length returned by the NIC (m->m_pkthdr.len)
3348		 * can contain padding, which we don't want to count in to the
3349		 * payload size.  Therefore, we calculate the real payload size
3350		 * based on the total ip length field (ext.iplen).
3351		 */
3352		ether_extract_headers(m, &ext);
3353		paylen = ext.iplen;
3354		if (ext.ip4 || ext.ip6)
3355			paylen -= ext.iphlen;
3356		if (ext.tcp) {
3357			paylen -= ext.tcphlen;
3358			tcpstat_inc(tcps_inhwlro);
3359			tcpstat_add(tcps_inpktlro, pkts);
3360		} else {
3361			tcpstat_inc(tcps_inbadlro);
3362		}
3363
3364		/*
3365		 * If we gonna forward this packet, we have to mark it as TSO,
3366		 * set a correct mss, and recalculate the TCP checksum.
3367		 */
3368		if (ext.tcp && paylen >= pkts) {
3369			SET(m->m_pkthdr.csum_flags, M_TCP_TSO);
3370			m->m_pkthdr.ph_mss = paylen / pkts;
3371		}
3372		if (ext.tcp && ISSET(m->m_pkthdr.csum_flags, M_TCP_CSUM_IN_OK))
3373			SET(m->m_pkthdr.csum_flags, M_TCP_CSUM_OUT);
3374	}
3375}
3376
3377void
3378ixgbe_setup_vlan_hw_support(struct ix_softc *sc)
3379{
3380	uint32_t	ctrl;
3381	int		i;
3382
3383	/*
3384	 * A soft reset zero's out the VFTA, so
3385	 * we need to repopulate it now.
3386	 */
3387	for (i = 0; i < IXGBE_VFTA_SIZE; i++) {
3388		if (sc->shadow_vfta[i] != 0)
3389			IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i),
3390			    sc->shadow_vfta[i]);
3391	}
3392
3393	ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL);
3394#if 0
3395	/* Enable the Filter Table if enabled */
3396	if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) {
3397		ctrl &= ~IXGBE_VLNCTRL_CFIEN;
3398		ctrl |= IXGBE_VLNCTRL_VFE;
3399	}
3400#endif
3401	if (sc->hw.mac.type == ixgbe_mac_82598EB)
3402		ctrl |= IXGBE_VLNCTRL_VME;
3403	IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl);
3404
3405	/* On 82599 the VLAN enable is per/queue in RXDCTL */
3406	if (sc->hw.mac.type != ixgbe_mac_82598EB) {
3407		for (i = 0; i < sc->num_queues; i++) {
3408			ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i));
3409			ctrl |= IXGBE_RXDCTL_VME;
3410			IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl);
3411		}
3412	}
3413}
3414
3415void
3416ixgbe_enable_intr(struct ix_softc *sc)
3417{
3418	struct ixgbe_hw *hw = &sc->hw;
3419	uint32_t	mask, fwsm;
3420
3421	mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
3422	/* Enable Fan Failure detection */
3423	if (hw->device_id == IXGBE_DEV_ID_82598AT)
3424		    mask |= IXGBE_EIMS_GPI_SDP1;
3425
3426	switch (sc->hw.mac.type) {
3427	case ixgbe_mac_82599EB:
3428		mask |= IXGBE_EIMS_ECC;
3429		/* Temperature sensor on some adapters */
3430		mask |= IXGBE_EIMS_GPI_SDP0;
3431		/* SFP+ (RX_LOS_N & MOD_ABS_N) */
3432		mask |= IXGBE_EIMS_GPI_SDP1;
3433		mask |= IXGBE_EIMS_GPI_SDP2;
3434		break;
3435	case ixgbe_mac_X540:
3436		mask |= IXGBE_EIMS_ECC;
3437		/* Detect if Thermal Sensor is enabled */
3438		fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM);
3439		if (fwsm & IXGBE_FWSM_TS_ENABLED)
3440			mask |= IXGBE_EIMS_TS;
3441		break;
3442	case ixgbe_mac_X550:
3443	case ixgbe_mac_X550EM_x:
3444	case ixgbe_mac_X550EM_a:
3445		mask |= IXGBE_EIMS_ECC;
3446		/* MAC thermal sensor is automatically enabled */
3447		mask |= IXGBE_EIMS_TS;
3448		/* Some devices use SDP0 for important information */
3449		if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP ||
3450		    hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T)
3451			mask |= IXGBE_EIMS_GPI_SDP0_X540;
3452	default:
3453		break;
3454	}
3455
3456	IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
3457
3458	/* With MSI-X we use auto clear */
3459	if (sc->sc_intrmap) {
3460		mask = IXGBE_EIMS_ENABLE_MASK;
3461		/* Don't autoclear Link */
3462		mask &= ~IXGBE_EIMS_OTHER;
3463		mask &= ~IXGBE_EIMS_LSC;
3464		IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
3465	}
3466
3467	IXGBE_WRITE_FLUSH(hw);
3468}
3469
3470void
3471ixgbe_disable_intr(struct ix_softc *sc)
3472{
3473	if (sc->sc_intrmap)
3474		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0);
3475	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
3476		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0);
3477	} else {
3478		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000);
3479		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0);
3480		IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0);
3481	}
3482	IXGBE_WRITE_FLUSH(&sc->hw);
3483}
3484
3485uint16_t
3486ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg)
3487{
3488	struct pci_attach_args	*pa;
3489	uint32_t value;
3490	int high = 0;
3491
3492	if (reg & 0x2) {
3493		high = 1;
3494		reg &= ~0x2;
3495	}
3496	pa = &((struct ixgbe_osdep *)hw->back)->os_pa;
3497	value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg);
3498
3499	if (high)
3500		value >>= 16;
3501
3502	return (value & 0xffff);
3503}
3504
3505void
3506ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value)
3507{
3508	struct pci_attach_args	*pa;
3509	uint32_t rv;
3510	int high = 0;
3511
3512	/* Need to do read/mask/write... because 16 vs 32 bit!!! */
3513	if (reg & 0x2) {
3514		high = 1;
3515		reg &= ~0x2;
3516	}
3517	pa = &((struct ixgbe_osdep *)hw->back)->os_pa;
3518	rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg);
3519	if (!high)
3520		rv = (rv & 0xffff0000) | value;
3521	else
3522		rv = (rv & 0xffff) | ((uint32_t)value << 16);
3523	pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv);
3524}
3525
3526/*
3527 * Setup the correct IVAR register for a particular MSIX interrupt
3528 *   (yes this is all very magic and confusing :)
3529 *  - entry is the register array entry
3530 *  - vector is the MSIX vector for this queue
3531 *  - type is RX/TX/MISC
3532 */
3533void
3534ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type)
3535{
3536	struct ixgbe_hw *hw = &sc->hw;
3537	uint32_t ivar, index;
3538
3539	vector |= IXGBE_IVAR_ALLOC_VAL;
3540
3541	switch (hw->mac.type) {
3542
3543	case ixgbe_mac_82598EB:
3544		if (type == -1)
3545			entry = IXGBE_IVAR_OTHER_CAUSES_INDEX;
3546		else
3547			entry += (type * 64);
3548		index = (entry >> 2) & 0x1F;
3549		ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
3550		ivar &= ~((uint32_t)0xFF << (8 * (entry & 0x3)));
3551		ivar |= ((uint32_t)vector << (8 * (entry & 0x3)));
3552		IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar);
3553		break;
3554
3555	case ixgbe_mac_82599EB:
3556	case ixgbe_mac_X540:
3557	case ixgbe_mac_X550:
3558	case ixgbe_mac_X550EM_x:
3559	case ixgbe_mac_X550EM_a:
3560		if (type == -1) { /* MISC IVAR */
3561			index = (entry & 1) * 8;
3562			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
3563			ivar &= ~((uint32_t)0xFF << index);
3564			ivar |= ((uint32_t)vector << index);
3565			IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar);
3566		} else {	/* RX/TX IVARS */
3567			index = (16 * (entry & 1)) + (8 * type);
3568			ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1));
3569			ivar &= ~((uint32_t)0xFF << index);
3570			ivar |= ((uint32_t)vector << index);
3571			IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar);
3572		}
3573
3574	default:
3575		break;
3576	}
3577}
3578
3579void
3580ixgbe_configure_ivars(struct ix_softc *sc)
3581{
3582	struct ix_queue *que = sc->queues;
3583	uint32_t newitr;
3584	int i;
3585
3586	newitr = (4000000 / IXGBE_INTS_PER_SEC) & 0x0FF8;
3587
3588	for (i = 0; i < sc->num_queues; i++, que++) {
3589		/* First the RX queue entry */
3590		ixgbe_set_ivar(sc, i, que->msix, 0);
3591		/* ... and the TX */
3592		ixgbe_set_ivar(sc, i, que->msix, 1);
3593		/* Set an Initial EITR value */
3594		IXGBE_WRITE_REG(&sc->hw,
3595		    IXGBE_EITR(que->msix), newitr);
3596	}
3597
3598	/* For the Link interrupt */
3599	ixgbe_set_ivar(sc, 1, sc->linkvec, -1);
3600}
3601
3602/*
3603 * SFP module interrupts handler
3604 */
3605void
3606ixgbe_handle_mod(struct ix_softc *sc)
3607{
3608	struct ixgbe_hw *hw = &sc->hw;
3609	uint32_t err;
3610
3611	err = hw->phy.ops.identify_sfp(hw);
3612	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3613		printf("%s: Unsupported SFP+ module type was detected!\n",
3614		    sc->dev.dv_xname);
3615		return;
3616	}
3617	err = hw->mac.ops.setup_sfp(hw);
3618	if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) {
3619		printf("%s: Setup failure - unsupported SFP+ module type!\n",
3620		    sc->dev.dv_xname);
3621		return;
3622	}
3623
3624	ixgbe_handle_msf(sc);
3625}
3626
3627
3628/*
3629 * MSF (multispeed fiber) interrupts handler
3630 */
3631void
3632ixgbe_handle_msf(struct ix_softc *sc)
3633{
3634	struct ixgbe_hw *hw = &sc->hw;
3635	uint32_t autoneg;
3636	bool negotiate;
3637
3638	autoneg = hw->phy.autoneg_advertised;
3639	if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) {
3640		if (hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate))
3641			return;
3642	}
3643	if (hw->mac.ops.setup_link)
3644		hw->mac.ops.setup_link(hw, autoneg, TRUE);
3645
3646	ifmedia_delete_instance(&sc->media, IFM_INST_ANY);
3647	ixgbe_add_media_types(sc);
3648	ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
3649}
3650
3651/*
3652 * External PHY interrupts handler
3653 */
3654void
3655ixgbe_handle_phy(struct ix_softc *sc)
3656{
3657	struct ixgbe_hw *hw = &sc->hw;
3658	int error;
3659
3660	error = hw->phy.ops.handle_lasi(hw);
3661	if (error == IXGBE_ERR_OVERTEMP)
3662		printf("%s: CRITICAL: EXTERNAL PHY OVER TEMP!! "
3663		    " PHY will downshift to lower power state!\n",
3664		    sc->dev.dv_xname);
3665	else if (error)
3666		printf("%s: Error handling LASI interrupt: %d\n",
3667		    sc->dev.dv_xname, error);
3668
3669}
3670
3671#if NKSTAT > 0
3672enum ix_counter_idx {
3673	ix_counter_crcerrs,
3674	ix_counter_lxontxc,
3675	ix_counter_lxonrxc,
3676	ix_counter_lxofftxc,
3677	ix_counter_lxoffrxc,
3678	ix_counter_prc64,
3679	ix_counter_prc127,
3680	ix_counter_prc255,
3681	ix_counter_prc511,
3682	ix_counter_prc1023,
3683	ix_counter_prc1522,
3684	ix_counter_gptc,
3685	ix_counter_gorc,
3686	ix_counter_gotc,
3687	ix_counter_ruc,
3688	ix_counter_rfc,
3689	ix_counter_roc,
3690	ix_counter_rjc,
3691	ix_counter_tor,
3692	ix_counter_tpr,
3693	ix_counter_tpt,
3694	ix_counter_gprc,
3695	ix_counter_bprc,
3696	ix_counter_mprc,
3697	ix_counter_ptc64,
3698	ix_counter_ptc127,
3699	ix_counter_ptc255,
3700	ix_counter_ptc511,
3701	ix_counter_ptc1023,
3702	ix_counter_ptc1522,
3703	ix_counter_mptc,
3704	ix_counter_bptc,
3705
3706	ix_counter_num,
3707};
3708
3709CTASSERT(KSTAT_KV_U_PACKETS <= 0xff);
3710CTASSERT(KSTAT_KV_U_BYTES <= 0xff);
3711
3712struct ix_counter {
3713	char			 name[KSTAT_KV_NAMELEN];
3714	uint32_t		 reg;
3715	uint8_t			 width;
3716	uint8_t			 unit;
3717};
3718
3719static const struct ix_counter ix_counters[ix_counter_num] = {
3720	[ix_counter_crcerrs] = {	"crc errs",	IXGBE_CRCERRS,	32,
3721					    KSTAT_KV_U_PACKETS },
3722	[ix_counter_lxontxc] = {	"tx link xon",	IXGBE_LXONTXC,	32,
3723					    KSTAT_KV_U_PACKETS },
3724	[ix_counter_lxonrxc] = {	"rx link xon",	0,		32,
3725					    KSTAT_KV_U_PACKETS },
3726	[ix_counter_lxofftxc] = {	"tx link xoff",	IXGBE_LXOFFTXC,	32,
3727					    KSTAT_KV_U_PACKETS },
3728	[ix_counter_lxoffrxc] = {	"rx link xoff",	0,		32,
3729					    KSTAT_KV_U_PACKETS },
3730	[ix_counter_prc64] = {		"rx 64B",	IXGBE_PRC64,	32,
3731					    KSTAT_KV_U_PACKETS },
3732	[ix_counter_prc127] = {		"rx 65-127B",	IXGBE_PRC127,	32,
3733					    KSTAT_KV_U_PACKETS },
3734	[ix_counter_prc255] = {		"rx 128-255B",	IXGBE_PRC255,	32,
3735					    KSTAT_KV_U_PACKETS },
3736	[ix_counter_prc511] = {		"rx 256-511B",	IXGBE_PRC511,	32,
3737					    KSTAT_KV_U_PACKETS },
3738	[ix_counter_prc1023] = {	"rx 512-1023B",	IXGBE_PRC1023,	32,
3739					    KSTAT_KV_U_PACKETS },
3740	[ix_counter_prc1522] = {	"rx 1024-maxB",	IXGBE_PRC1522,	32,
3741					    KSTAT_KV_U_PACKETS },
3742	[ix_counter_gptc] = {		"tx good",	IXGBE_GPTC,	32,
3743					    KSTAT_KV_U_PACKETS },
3744	[ix_counter_gorc] = {		"rx good",	IXGBE_GORCL,	36,
3745					    KSTAT_KV_U_BYTES },
3746	[ix_counter_gotc] = {		"tx good",	IXGBE_GOTCL,	36,
3747					    KSTAT_KV_U_BYTES },
3748	[ix_counter_ruc] = {		"rx undersize",	IXGBE_RUC,	32,
3749					    KSTAT_KV_U_PACKETS },
3750	[ix_counter_rfc] = {		"rx fragment",	IXGBE_RFC,	32,
3751					    KSTAT_KV_U_PACKETS },
3752	[ix_counter_roc] = {		"rx oversize",	IXGBE_ROC,	32,
3753					    KSTAT_KV_U_PACKETS },
3754	[ix_counter_rjc] = {		"rx jabber",	IXGBE_RJC,	32,
3755					    KSTAT_KV_U_PACKETS },
3756	[ix_counter_tor] = {		"rx total",	IXGBE_TORL,	36,
3757					    KSTAT_KV_U_BYTES },
3758	[ix_counter_tpr] = {		"rx total",	IXGBE_TPR,	32,
3759					    KSTAT_KV_U_PACKETS },
3760	[ix_counter_tpt] = {		"tx total",	IXGBE_TPT,	32,
3761					    KSTAT_KV_U_PACKETS },
3762	[ix_counter_gprc] = {		"rx good",	IXGBE_GPRC,	32,
3763					    KSTAT_KV_U_PACKETS },
3764	[ix_counter_bprc] = {		"rx bcast",	IXGBE_BPRC,	32,
3765					    KSTAT_KV_U_PACKETS },
3766	[ix_counter_mprc] = {		"rx mcast",	IXGBE_MPRC,	32,
3767					    KSTAT_KV_U_PACKETS },
3768	[ix_counter_ptc64] = {		"tx 64B",	IXGBE_PTC64,	32,
3769					    KSTAT_KV_U_PACKETS },
3770	[ix_counter_ptc127] = {		"tx 65-127B",	IXGBE_PTC127,	32,
3771					    KSTAT_KV_U_PACKETS },
3772	[ix_counter_ptc255] = {		"tx 128-255B",	IXGBE_PTC255,	32,
3773					    KSTAT_KV_U_PACKETS },
3774	[ix_counter_ptc511] = {		"tx 256-511B",	IXGBE_PTC511,	32,
3775					    KSTAT_KV_U_PACKETS },
3776	[ix_counter_ptc1023] = {	"tx 512-1023B",	IXGBE_PTC1023,	32,
3777					    KSTAT_KV_U_PACKETS },
3778	[ix_counter_ptc1522] = {	"tx 1024-maxB",	IXGBE_PTC1522,	32,
3779					    KSTAT_KV_U_PACKETS },
3780	[ix_counter_mptc] = {		"tx mcast",	IXGBE_MPTC,	32,
3781					    KSTAT_KV_U_PACKETS },
3782	[ix_counter_bptc] = {		"tx bcast",	IXGBE_BPTC,	32,
3783					    KSTAT_KV_U_PACKETS },
3784};
3785
3786struct ix_rxq_kstats {
3787	struct kstat_kv	qprc;
3788	struct kstat_kv	qbrc;
3789	struct kstat_kv	qprdc;
3790};
3791
3792static const struct ix_rxq_kstats ix_rxq_kstats_tpl = {
3793	KSTAT_KV_UNIT_INITIALIZER("packets",
3794	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
3795	KSTAT_KV_UNIT_INITIALIZER("bytes",
3796	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES),
3797	KSTAT_KV_UNIT_INITIALIZER("qdrops",
3798	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
3799};
3800
3801struct ix_txq_kstats {
3802	struct kstat_kv	qptc;
3803	struct kstat_kv	qbtc;
3804};
3805
3806static const struct ix_txq_kstats ix_txq_kstats_tpl = {
3807	KSTAT_KV_UNIT_INITIALIZER("packets",
3808	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS),
3809	KSTAT_KV_UNIT_INITIALIZER("bytes",
3810	    KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES),
3811};
3812
3813static int	ix_kstats_read(struct kstat *ks);
3814static int	ix_rxq_kstats_read(struct kstat *ks);
3815static int	ix_txq_kstats_read(struct kstat *ks);
3816
3817static void
3818ix_kstats(struct ix_softc *sc)
3819{
3820	struct kstat *ks;
3821	struct kstat_kv *kvs;
3822	unsigned int i;
3823
3824	mtx_init(&sc->sc_kstat_mtx, IPL_SOFTCLOCK);
3825	timeout_set(&sc->sc_kstat_tmo, ix_kstats_tick, sc);
3826
3827	ks = kstat_create(sc->dev.dv_xname, 0, "ix-stats", 0,
3828	    KSTAT_T_KV, 0);
3829	if (ks == NULL)
3830		return;
3831
3832	kvs = mallocarray(nitems(ix_counters), sizeof(*kvs),
3833	    M_DEVBUF, M_WAITOK|M_ZERO);
3834
3835	for (i = 0; i < nitems(ix_counters); i++) {
3836		const struct ix_counter *ixc = &ix_counters[i];
3837
3838		kstat_kv_unit_init(&kvs[i], ixc->name,
3839		    KSTAT_KV_T_COUNTER64, ixc->unit);
3840	}
3841
3842	kstat_set_mutex(ks, &sc->sc_kstat_mtx);
3843	ks->ks_softc = sc;
3844	ks->ks_data = kvs;
3845	ks->ks_datalen = nitems(ix_counters) * sizeof(*kvs);
3846	ks->ks_read = ix_kstats_read;
3847
3848	sc->sc_kstat = ks;
3849	kstat_install(ks);
3850}
3851
3852static void
3853ix_rxq_kstats(struct ix_softc *sc, struct ix_rxring *rxr)
3854{
3855	struct ix_rxq_kstats *stats;
3856	struct kstat *ks;
3857
3858	ks = kstat_create(sc->dev.dv_xname, 0, "ix-rxq", rxr->me,
3859	    KSTAT_T_KV, 0);
3860	if (ks == NULL)
3861		return;
3862
3863	stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO);
3864	*stats = ix_rxq_kstats_tpl;
3865
3866	kstat_set_mutex(ks, &sc->sc_kstat_mtx);
3867	ks->ks_softc = rxr;
3868	ks->ks_data = stats;
3869	ks->ks_datalen = sizeof(*stats);
3870	ks->ks_read = ix_rxq_kstats_read;
3871
3872	rxr->kstat = ks;
3873	kstat_install(ks);
3874}
3875
3876static void
3877ix_txq_kstats(struct ix_softc *sc, struct ix_txring *txr)
3878{
3879	struct ix_txq_kstats *stats;
3880	struct kstat *ks;
3881
3882	ks = kstat_create(sc->dev.dv_xname, 0, "ix-txq", txr->me,
3883	    KSTAT_T_KV, 0);
3884	if (ks == NULL)
3885		return;
3886
3887	stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO);
3888	*stats = ix_txq_kstats_tpl;
3889
3890	kstat_set_mutex(ks, &sc->sc_kstat_mtx);
3891	ks->ks_softc = txr;
3892	ks->ks_data = stats;
3893	ks->ks_datalen = sizeof(*stats);
3894	ks->ks_read = ix_txq_kstats_read;
3895
3896	txr->kstat = ks;
3897	kstat_install(ks);
3898}
3899
3900/**********************************************************************
3901 *
3902 *  Update the board statistics counters.
3903 *
3904 **********************************************************************/
3905
3906static void
3907ix_kstats_tick(void *arg)
3908{
3909	struct ix_softc *sc = arg;
3910	int i;
3911
3912	timeout_add_sec(&sc->sc_kstat_tmo, 1);
3913
3914	mtx_enter(&sc->sc_kstat_mtx);
3915	ix_kstats_read(sc->sc_kstat);
3916	for (i = 0; i < sc->num_queues; i++) {
3917		ix_rxq_kstats_read(sc->rx_rings[i].kstat);
3918		ix_txq_kstats_read(sc->tx_rings[i].kstat);
3919	}
3920	mtx_leave(&sc->sc_kstat_mtx);
3921}
3922
3923static uint64_t
3924ix_read36(struct ixgbe_hw *hw, bus_size_t loreg, bus_size_t hireg)
3925{
3926	uint64_t lo, hi;
3927
3928	lo = IXGBE_READ_REG(hw, loreg);
3929	hi = IXGBE_READ_REG(hw, hireg);
3930
3931	return (((hi & 0xf) << 32) | lo);
3932}
3933
3934static int
3935ix_kstats_read(struct kstat *ks)
3936{
3937	struct ix_softc *sc = ks->ks_softc;
3938	struct kstat_kv *kvs = ks->ks_data;
3939	struct ixgbe_hw	*hw = &sc->hw;
3940	unsigned int i;
3941
3942	for (i = 0; i < nitems(ix_counters); i++) {
3943		const struct ix_counter *ixc = &ix_counters[i];
3944		uint32_t reg = ixc->reg;
3945		uint64_t v;
3946
3947		if (reg == 0)
3948			continue;
3949
3950		if (ixc->width > 32) {
3951			if (sc->hw.mac.type == ixgbe_mac_82598EB)
3952				v = IXGBE_READ_REG(hw, reg + 4);
3953			else
3954				v = ix_read36(hw, reg, reg + 4);
3955		} else
3956			v = IXGBE_READ_REG(hw, reg);
3957
3958		kstat_kv_u64(&kvs[i]) += v;
3959	}
3960
3961	/* handle the exceptions */
3962	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
3963		kstat_kv_u64(&kvs[ix_counter_lxonrxc]) +=
3964		    IXGBE_READ_REG(hw, IXGBE_LXONRXC);
3965		kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) +=
3966		    IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
3967	} else {
3968		kstat_kv_u64(&kvs[ix_counter_lxonrxc]) +=
3969		    IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
3970		kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) +=
3971		    IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
3972	}
3973
3974	getnanouptime(&ks->ks_updated);
3975
3976	return (0);
3977}
3978
3979int
3980ix_rxq_kstats_read(struct kstat *ks)
3981{
3982	struct ix_rxq_kstats *stats = ks->ks_data;
3983	struct ix_rxring *rxr = ks->ks_softc;
3984	struct ix_softc *sc = rxr->sc;
3985	struct ixgbe_hw	*hw = &sc->hw;
3986	uint32_t i = rxr->me;
3987
3988	kstat_kv_u64(&stats->qprc) += IXGBE_READ_REG(hw, IXGBE_QPRC(i));
3989	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
3990		kstat_kv_u64(&stats->qprdc) +=
3991		    IXGBE_READ_REG(hw, IXGBE_RNBC(i));
3992		kstat_kv_u64(&stats->qbrc) +=
3993		    IXGBE_READ_REG(hw, IXGBE_QBRC(i));
3994	} else {
3995		kstat_kv_u64(&stats->qprdc) +=
3996		    IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
3997		kstat_kv_u64(&stats->qbrc) +=
3998		    ix_read36(hw, IXGBE_QBRC_L(i), IXGBE_QBRC_H(i));
3999	}
4000
4001	getnanouptime(&ks->ks_updated);
4002
4003	return (0);
4004}
4005
4006int
4007ix_txq_kstats_read(struct kstat *ks)
4008{
4009	struct ix_txq_kstats *stats = ks->ks_data;
4010	struct ix_txring *txr = ks->ks_softc;
4011	struct ix_softc *sc = txr->sc;
4012	struct ixgbe_hw	*hw = &sc->hw;
4013	uint32_t i = txr->me;
4014
4015	kstat_kv_u64(&stats->qptc) += IXGBE_READ_REG(hw, IXGBE_QPTC(i));
4016	if (sc->hw.mac.type == ixgbe_mac_82598EB) {
4017		kstat_kv_u64(&stats->qbtc) +=
4018		    IXGBE_READ_REG(hw, IXGBE_QBTC(i));
4019	} else {
4020		kstat_kv_u64(&stats->qbtc) +=
4021		    ix_read36(hw, IXGBE_QBTC_L(i), IXGBE_QBTC_H(i));
4022	}
4023
4024	getnanouptime(&ks->ks_updated);
4025
4026	return (0);
4027}
4028#endif /* NKVSTAT > 0 */
4029
4030void
4031ixgbe_map_queue_statistics(struct ix_softc *sc)
4032{
4033	int i;
4034	uint32_t r;
4035
4036	for (i = 0; i < 32; i++) {
4037		/*
4038		 * Queues 0-15 are mapped 1:1
4039		 * Queue 0 -> Counter 0
4040		 * Queue 1 -> Counter 1
4041		 * Queue 2 -> Counter 2....
4042		 * Queues 16-127 are mapped to Counter 0
4043		 */
4044		if (i < 4) {
4045			r = (i * 4 + 0);
4046			r |= (i * 4 + 1) << 8;
4047			r |= (i * 4 + 2) << 16;
4048			r |= (i * 4 + 3) << 24;
4049		} else
4050			r = 0;
4051
4052		IXGBE_WRITE_REG(&sc->hw, IXGBE_RQSMR(i), r);
4053		IXGBE_WRITE_REG(&sc->hw, IXGBE_TQSM(i), r);
4054	}
4055}
4056