if_em.c revision 185353 
1/******************************************************************************
2
3  Copyright (c) 2001-2008, Intel Corporation
4  All rights reserved.
5
6  Redistribution and use in source and binary forms, with or without
7  modification, are permitted provided that the following conditions are met:
8
9   1. Redistributions of source code must retain the above copyright notice,
10      this list of conditions and the following disclaimer.
11
12   2. Redistributions in binary form must reproduce the above copyright
13      notice, this list of conditions and the following disclaimer in the
14      documentation and/or other materials provided with the distribution.
15
16   3. Neither the name of the Intel Corporation nor the names of its
17      contributors may be used to endorse or promote products derived from
18      this software without specific prior written permission.
19
20  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  POSSIBILITY OF SUCH DAMAGE.
31
32******************************************************************************/
33/*$FreeBSD: head/sys/dev/e1000/if_em.c 185353 2008-11-26 23:57:23Z jfv $*/
34
35#ifdef HAVE_KERNEL_OPTION_HEADERS
36#include "opt_device_polling.h"
37#endif
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/bus.h>
42#include <sys/endian.h>
43#include <sys/kernel.h>
44#include <sys/kthread.h>
45#include <sys/malloc.h>
46#include <sys/mbuf.h>
47#include <sys/module.h>
48#include <sys/rman.h>
49#include <sys/socket.h>
50#include <sys/sockio.h>
51#include <sys/sysctl.h>
52#include <sys/taskqueue.h>
53#include <sys/eventhandler.h>
54#ifdef EM_TIMESYNC
55#include <sys/ioccom.h>
56#include <sys/time.h>
57#endif
58#include <machine/bus.h>
59#include <machine/resource.h>
60
61#include <net/bpf.h>
62#include <net/ethernet.h>
63#include <net/if.h>
64#include <net/if_arp.h>
65#include <net/if_dl.h>
66#include <net/if_media.h>
67
68#include <net/if_types.h>
69#include <net/if_vlan_var.h>
70
71#include <netinet/in_systm.h>
72#include <netinet/in.h>
73#include <netinet/if_ether.h>
74#include <netinet/ip.h>
75#include <netinet/ip6.h>
76#include <netinet/tcp.h>
77#include <netinet/udp.h>
78
79#include <machine/in_cksum.h>
80#include <dev/pci/pcivar.h>
81#include <dev/pci/pcireg.h>
82
83#include "e1000_api.h"
84#include "e1000_82571.h"
85#include "if_em.h"
86
87/*********************************************************************
88 *  Set this to one to display debug statistics
89 *********************************************************************/
90int	em_display_debug_stats = 0;
91
92/*********************************************************************
93 *  Driver version:
94 *********************************************************************/
95char em_driver_version[] = "6.9.6";
96
97
98/*********************************************************************
99 *  PCI Device ID Table
100 *
101 *  Used by probe to select devices to load on
102 *  Last field stores an index into e1000_strings
103 *  Last entry must be all 0s
104 *
105 *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
106 *********************************************************************/
107
108static em_vendor_info_t em_vendor_info_array[] =
109{
110	/* Intel(R) PRO/1000 Network Connection */
111	{ 0x8086, E1000_DEV_ID_82540EM,		PCI_ANY_ID, PCI_ANY_ID, 0},
112	{ 0x8086, E1000_DEV_ID_82540EM_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
113	{ 0x8086, E1000_DEV_ID_82540EP,		PCI_ANY_ID, PCI_ANY_ID, 0},
114	{ 0x8086, E1000_DEV_ID_82540EP_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
115	{ 0x8086, E1000_DEV_ID_82540EP_LP,	PCI_ANY_ID, PCI_ANY_ID, 0},
116
117	{ 0x8086, E1000_DEV_ID_82541EI,		PCI_ANY_ID, PCI_ANY_ID, 0},
118	{ 0x8086, E1000_DEV_ID_82541ER,		PCI_ANY_ID, PCI_ANY_ID, 0},
119	{ 0x8086, E1000_DEV_ID_82541ER_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
120	{ 0x8086, E1000_DEV_ID_82541EI_MOBILE,	PCI_ANY_ID, PCI_ANY_ID, 0},
121	{ 0x8086, E1000_DEV_ID_82541GI,		PCI_ANY_ID, PCI_ANY_ID, 0},
122	{ 0x8086, E1000_DEV_ID_82541GI_LF,	PCI_ANY_ID, PCI_ANY_ID, 0},
123	{ 0x8086, E1000_DEV_ID_82541GI_MOBILE,	PCI_ANY_ID, PCI_ANY_ID, 0},
124
125	{ 0x8086, E1000_DEV_ID_82542,		PCI_ANY_ID, PCI_ANY_ID, 0},
126
127	{ 0x8086, E1000_DEV_ID_82543GC_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
128	{ 0x8086, E1000_DEV_ID_82543GC_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
129
130	{ 0x8086, E1000_DEV_ID_82544EI_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
131	{ 0x8086, E1000_DEV_ID_82544EI_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
132	{ 0x8086, E1000_DEV_ID_82544GC_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
133	{ 0x8086, E1000_DEV_ID_82544GC_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
134
135	{ 0x8086, E1000_DEV_ID_82545EM_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
136	{ 0x8086, E1000_DEV_ID_82545EM_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
137	{ 0x8086, E1000_DEV_ID_82545GM_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
138	{ 0x8086, E1000_DEV_ID_82545GM_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
139	{ 0x8086, E1000_DEV_ID_82545GM_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
140
141	{ 0x8086, E1000_DEV_ID_82546EB_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
142	{ 0x8086, E1000_DEV_ID_82546EB_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
143	{ 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
144	{ 0x8086, E1000_DEV_ID_82546GB_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
145	{ 0x8086, E1000_DEV_ID_82546GB_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
146	{ 0x8086, E1000_DEV_ID_82546GB_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
147	{ 0x8086, E1000_DEV_ID_82546GB_PCIE,	PCI_ANY_ID, PCI_ANY_ID, 0},
148	{ 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
149	{ 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
150						PCI_ANY_ID, PCI_ANY_ID, 0},
151
152	{ 0x8086, E1000_DEV_ID_82547EI,		PCI_ANY_ID, PCI_ANY_ID, 0},
153	{ 0x8086, E1000_DEV_ID_82547EI_MOBILE,	PCI_ANY_ID, PCI_ANY_ID, 0},
154	{ 0x8086, E1000_DEV_ID_82547GI,		PCI_ANY_ID, PCI_ANY_ID, 0},
155
156	{ 0x8086, E1000_DEV_ID_82571EB_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
157	{ 0x8086, E1000_DEV_ID_82571EB_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
158	{ 0x8086, E1000_DEV_ID_82571EB_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
159	{ 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL,
160						PCI_ANY_ID, PCI_ANY_ID, 0},
161	{ 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD,
162						PCI_ANY_ID, PCI_ANY_ID, 0},
163	{ 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
164						PCI_ANY_ID, PCI_ANY_ID, 0},
165	{ 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP,
166						PCI_ANY_ID, PCI_ANY_ID, 0},
167	{ 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER,
168						PCI_ANY_ID, PCI_ANY_ID, 0},
169	{ 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER,
170						PCI_ANY_ID, PCI_ANY_ID, 0},
171	{ 0x8086, E1000_DEV_ID_82572EI_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
172	{ 0x8086, E1000_DEV_ID_82572EI_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
173	{ 0x8086, E1000_DEV_ID_82572EI_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
174	{ 0x8086, E1000_DEV_ID_82572EI,		PCI_ANY_ID, PCI_ANY_ID, 0},
175
176	{ 0x8086, E1000_DEV_ID_82573E,		PCI_ANY_ID, PCI_ANY_ID, 0},
177	{ 0x8086, E1000_DEV_ID_82573E_IAMT,	PCI_ANY_ID, PCI_ANY_ID, 0},
178	{ 0x8086, E1000_DEV_ID_82573L,		PCI_ANY_ID, PCI_ANY_ID, 0},
179	{ 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
180						PCI_ANY_ID, PCI_ANY_ID, 0},
181	{ 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
182						PCI_ANY_ID, PCI_ANY_ID, 0},
183	{ 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
184						PCI_ANY_ID, PCI_ANY_ID, 0},
185	{ 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
186						PCI_ANY_ID, PCI_ANY_ID, 0},
187	{ 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,	PCI_ANY_ID, PCI_ANY_ID, 0},
188	{ 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,	PCI_ANY_ID, PCI_ANY_ID, 0},
189	{ 0x8086, E1000_DEV_ID_ICH8_IGP_C,	PCI_ANY_ID, PCI_ANY_ID, 0},
190	{ 0x8086, E1000_DEV_ID_ICH8_IFE,	PCI_ANY_ID, PCI_ANY_ID, 0},
191	{ 0x8086, E1000_DEV_ID_ICH8_IFE_GT,	PCI_ANY_ID, PCI_ANY_ID, 0},
192	{ 0x8086, E1000_DEV_ID_ICH8_IFE_G,	PCI_ANY_ID, PCI_ANY_ID, 0},
193	{ 0x8086, E1000_DEV_ID_ICH8_IGP_M,	PCI_ANY_ID, PCI_ANY_ID, 0},
194
195	{ 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT,	PCI_ANY_ID, PCI_ANY_ID, 0},
196	{ 0x8086, E1000_DEV_ID_ICH9_IGP_AMT,	PCI_ANY_ID, PCI_ANY_ID, 0},
197	{ 0x8086, E1000_DEV_ID_ICH9_IGP_C,	PCI_ANY_ID, PCI_ANY_ID, 0},
198	{ 0x8086, E1000_DEV_ID_ICH9_IGP_M,	PCI_ANY_ID, PCI_ANY_ID, 0},
199	{ 0x8086, E1000_DEV_ID_ICH9_IGP_M_V,	PCI_ANY_ID, PCI_ANY_ID, 0},
200	{ 0x8086, E1000_DEV_ID_ICH9_IFE,	PCI_ANY_ID, PCI_ANY_ID, 0},
201	{ 0x8086, E1000_DEV_ID_ICH9_IFE_GT,	PCI_ANY_ID, PCI_ANY_ID, 0},
202	{ 0x8086, E1000_DEV_ID_ICH9_IFE_G,	PCI_ANY_ID, PCI_ANY_ID, 0},
203	{ 0x8086, E1000_DEV_ID_ICH9_BM,		PCI_ANY_ID, PCI_ANY_ID, 0},
204	{ 0x8086, E1000_DEV_ID_82574L,		PCI_ANY_ID, PCI_ANY_ID, 0},
205	{ 0x8086, E1000_DEV_ID_ICH10_R_BM_LM,	PCI_ANY_ID, PCI_ANY_ID, 0},
206	{ 0x8086, E1000_DEV_ID_ICH10_R_BM_LF,	PCI_ANY_ID, PCI_ANY_ID, 0},
207	{ 0x8086, E1000_DEV_ID_ICH10_R_BM_V,	PCI_ANY_ID, PCI_ANY_ID, 0},
208	{ 0x8086, E1000_DEV_ID_ICH10_D_BM_LM,	PCI_ANY_ID, PCI_ANY_ID, 0},
209	{ 0x8086, E1000_DEV_ID_ICH10_D_BM_LF,	PCI_ANY_ID, PCI_ANY_ID, 0},
210	/* required last entry */
211	{ 0, 0, 0, 0, 0}
212};
213
214/*********************************************************************
215 *  Table of branding strings for all supported NICs.
216 *********************************************************************/
217
218static char *em_strings[] = {
219	"Intel(R) PRO/1000 Network Connection"
220};
221
222/*********************************************************************
223 *  Function prototypes
224 *********************************************************************/
225static int	em_probe(device_t);
226static int	em_attach(device_t);
227static int	em_detach(device_t);
228static int	em_shutdown(device_t);
229static int	em_suspend(device_t);
230static int	em_resume(device_t);
231static void	em_start(struct ifnet *);
232static void	em_start_locked(struct ifnet *ifp);
233static int	em_ioctl(struct ifnet *, u_long, caddr_t);
234static void	em_watchdog(struct adapter *);
235static void	em_init(void *);
236static void	em_init_locked(struct adapter *);
237static void	em_stop(void *);
238static void	em_media_status(struct ifnet *, struct ifmediareq *);
239static int	em_media_change(struct ifnet *);
240static void	em_identify_hardware(struct adapter *);
241static int	em_allocate_pci_resources(struct adapter *);
242static int	em_allocate_legacy(struct adapter *adapter);
243static int	em_allocate_msix(struct adapter *adapter);
244static int	em_setup_msix(struct adapter *);
245static void	em_free_pci_resources(struct adapter *);
246static void	em_local_timer(void *);
247static int	em_hardware_init(struct adapter *);
248static void	em_setup_interface(device_t, struct adapter *);
249static void	em_setup_transmit_structures(struct adapter *);
250static void	em_initialize_transmit_unit(struct adapter *);
251static int	em_setup_receive_structures(struct adapter *);
252static void	em_initialize_receive_unit(struct adapter *);
253static void	em_enable_intr(struct adapter *);
254static void	em_disable_intr(struct adapter *);
255static void	em_free_transmit_structures(struct adapter *);
256static void	em_free_receive_structures(struct adapter *);
257static void	em_update_stats_counters(struct adapter *);
258static void	em_txeof(struct adapter *);
259static void	em_tx_purge(struct adapter *);
260static int	em_allocate_receive_structures(struct adapter *);
261static int	em_allocate_transmit_structures(struct adapter *);
262static int	em_rxeof(struct adapter *, int);
263#ifndef __NO_STRICT_ALIGNMENT
264static int	em_fixup_rx(struct adapter *);
265#endif
266static void	em_receive_checksum(struct adapter *, struct e1000_rx_desc *,
267		    struct mbuf *);
268static void	em_transmit_checksum_setup(struct adapter *, struct mbuf *,
269		    u32 *, u32 *);
270#if __FreeBSD_version >= 700000
271static bool	em_tso_setup(struct adapter *, struct mbuf *,
272		    u32 *, u32 *);
273#endif /* FreeBSD_version >= 700000 */
274static void	em_set_promisc(struct adapter *);
275static void	em_disable_promisc(struct adapter *);
276static void	em_set_multi(struct adapter *);
277static void	em_print_hw_stats(struct adapter *);
278static void	em_update_link_status(struct adapter *);
279static int	em_get_buf(struct adapter *, int);
280
281static void	em_register_vlan(void *, struct ifnet *, u16);
282static void	em_unregister_vlan(void *, struct ifnet *, u16);
283
284static int	em_xmit(struct adapter *, struct mbuf **);
285static void	em_smartspeed(struct adapter *);
286static int	em_82547_fifo_workaround(struct adapter *, int);
287static void	em_82547_update_fifo_head(struct adapter *, int);
288static int	em_82547_tx_fifo_reset(struct adapter *);
289static void	em_82547_move_tail(void *);
290static int	em_dma_malloc(struct adapter *, bus_size_t,
291		    struct em_dma_alloc *, int);
292static void	em_dma_free(struct adapter *, struct em_dma_alloc *);
293static void	em_print_debug_info(struct adapter *);
294static void	em_print_nvm_info(struct adapter *);
295static int 	em_is_valid_ether_addr(u8 *);
296static int	em_sysctl_stats(SYSCTL_HANDLER_ARGS);
297static int	em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
298static u32	em_fill_descriptors (bus_addr_t address, u32 length,
299		    PDESC_ARRAY desc_array);
300static int	em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
301static void	em_add_int_delay_sysctl(struct adapter *, const char *,
302		    const char *, struct em_int_delay_info *, int, int);
303/* Management and WOL Support */
304static void	em_init_manageability(struct adapter *);
305static void	em_release_manageability(struct adapter *);
306static void     em_get_hw_control(struct adapter *);
307static void     em_release_hw_control(struct adapter *);
308static void     em_enable_wakeup(device_t);
309
310#ifdef EM_TIMESYNC
311/* Precision Time sync support */
312static int	em_tsync_init(struct adapter *);
313static void	em_tsync_disable(struct adapter *);
314#endif
315
316#ifdef EM_LEGACY_IRQ
317static void	em_intr(void *);
318#else /* FAST IRQ */
319#if __FreeBSD_version < 700000
320static void	em_irq_fast(void *);
321#else
322static int	em_irq_fast(void *);
323#endif
324/* MSIX handlers */
325static void	em_msix_tx(void *);
326static void	em_msix_rx(void *);
327static void	em_msix_link(void *);
328static void	em_add_rx_process_limit(struct adapter *, const char *,
329		    const char *, int *, int);
330static void	em_handle_rxtx(void *context, int pending);
331static void	em_handle_rx(void *context, int pending);
332static void	em_handle_tx(void *context, int pending);
333static void	em_handle_link(void *context, int pending);
334#endif /* EM_LEGACY_IRQ */
335
336#ifdef DEVICE_POLLING
337static poll_handler_t em_poll;
338#endif /* POLLING */
339
340/*********************************************************************
341 *  FreeBSD Device Interface Entry Points
342 *********************************************************************/
343
344static device_method_t em_methods[] = {
345	/* Device interface */
346	DEVMETHOD(device_probe, em_probe),
347	DEVMETHOD(device_attach, em_attach),
348	DEVMETHOD(device_detach, em_detach),
349	DEVMETHOD(device_shutdown, em_shutdown),
350	DEVMETHOD(device_suspend, em_suspend),
351	DEVMETHOD(device_resume, em_resume),
352	{0, 0}
353};
354
355static driver_t em_driver = {
356	"em", em_methods, sizeof(struct adapter),
357};
358
359static devclass_t em_devclass;
360DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
361MODULE_DEPEND(em, pci, 1, 1, 1);
362MODULE_DEPEND(em, ether, 1, 1, 1);
363
364/*********************************************************************
365 *  Tunable default values.
366 *********************************************************************/
367
368#define EM_TICKS_TO_USECS(ticks)	((1024 * (ticks) + 500) / 1000)
369#define EM_USECS_TO_TICKS(usecs)	((1000 * (usecs) + 512) / 1024)
370#define M_TSO_LEN			66
371
372/* Allow common code without TSO */
373#ifndef CSUM_TSO
374#define CSUM_TSO	0
375#endif
376
377static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV);
378static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR);
379static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV);
380static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV);
381static int em_rxd = EM_DEFAULT_RXD;
382static int em_txd = EM_DEFAULT_TXD;
383static int em_smart_pwr_down = FALSE;
384/* Controls whether promiscuous also shows bad packets */
385static int em_debug_sbp = FALSE;
386/* Local switch for MSI/MSIX */
387static int em_enable_msi = TRUE;
388
389TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
390TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
391TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
392TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
393TUNABLE_INT("hw.em.rxd", &em_rxd);
394TUNABLE_INT("hw.em.txd", &em_txd);
395TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
396TUNABLE_INT("hw.em.sbp", &em_debug_sbp);
397TUNABLE_INT("hw.em.enable_msi", &em_enable_msi);
398
399#ifndef EM_LEGACY_IRQ
400/* How many packets rxeof tries to clean at a time */
401static int em_rx_process_limit = 100;
402TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
403#endif
404
405/* Global used in WOL setup with multiport cards */
406static int global_quad_port_a = 0;
407
408/*********************************************************************
409 *  Device identification routine
410 *
411 *  em_probe determines if the driver should be loaded on
412 *  adapter based on PCI vendor/device id of the adapter.
413 *
414 *  return BUS_PROBE_DEFAULT on success, positive on failure
415 *********************************************************************/
416
417static int
418em_probe(device_t dev)
419{
420	char		adapter_name[60];
421	u16		pci_vendor_id = 0;
422	u16		pci_device_id = 0;
423	u16		pci_subvendor_id = 0;
424	u16		pci_subdevice_id = 0;
425	em_vendor_info_t *ent;
426
427	INIT_DEBUGOUT("em_probe: begin");
428
429	pci_vendor_id = pci_get_vendor(dev);
430	if (pci_vendor_id != EM_VENDOR_ID)
431		return (ENXIO);
432
433	pci_device_id = pci_get_device(dev);
434	pci_subvendor_id = pci_get_subvendor(dev);
435	pci_subdevice_id = pci_get_subdevice(dev);
436
437	ent = em_vendor_info_array;
438	while (ent->vendor_id != 0) {
439		if ((pci_vendor_id == ent->vendor_id) &&
440		    (pci_device_id == ent->device_id) &&
441
442		    ((pci_subvendor_id == ent->subvendor_id) ||
443		    (ent->subvendor_id == PCI_ANY_ID)) &&
444
445		    ((pci_subdevice_id == ent->subdevice_id) ||
446		    (ent->subdevice_id == PCI_ANY_ID))) {
447			sprintf(adapter_name, "%s %s",
448				em_strings[ent->index],
449				em_driver_version);
450			device_set_desc_copy(dev, adapter_name);
451			return (BUS_PROBE_DEFAULT);
452		}
453		ent++;
454	}
455
456	return (ENXIO);
457}
458
459/*********************************************************************
460 *  Device initialization routine
461 *
462 *  The attach entry point is called when the driver is being loaded.
463 *  This routine identifies the type of hardware, allocates all resources
464 *  and initializes the hardware.
465 *
466 *  return 0 on success, positive on failure
467 *********************************************************************/
468
469static int
470em_attach(device_t dev)
471{
472	struct adapter	*adapter;
473	int		tsize, rsize;
474	int		error = 0;
475	u16		eeprom_data, device_id;
476
477	INIT_DEBUGOUT("em_attach: begin");
478
479	adapter = device_get_softc(dev);
480	adapter->dev = adapter->osdep.dev = dev;
481	EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev));
482	EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev));
483	EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev));
484
485	/* SYSCTL stuff */
486	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
487	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
488	    OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
489	    em_sysctl_debug_info, "I", "Debug Information");
490
491	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
492	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
493	    OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
494	    em_sysctl_stats, "I", "Statistics");
495
496	callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0);
497	callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0);
498
499	/* Determine hardware and mac info */
500	em_identify_hardware(adapter);
501
502	/* Setup PCI resources */
503	if (em_allocate_pci_resources(adapter)) {
504		device_printf(dev, "Allocation of PCI resources failed\n");
505		error = ENXIO;
506		goto err_pci;
507	}
508
509	/*
510	** For ICH8 and family we need to
511	** map the flash memory, and this
512	** must happen after the MAC is
513	** identified
514	*/
515	if ((adapter->hw.mac.type == e1000_ich8lan) ||
516	    (adapter->hw.mac.type == e1000_ich10lan) ||
517	    (adapter->hw.mac.type == e1000_ich9lan)) {
518		int rid = EM_BAR_TYPE_FLASH;
519		adapter->flash = bus_alloc_resource_any(dev,
520		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
521		if (adapter->flash == NULL) {
522			device_printf(dev, "Mapping of Flash failed\n");
523			error = ENXIO;
524			goto err_pci;
525		}
526		/* This is used in the shared code */
527		adapter->hw.flash_address = (u8 *)adapter->flash;
528		adapter->osdep.flash_bus_space_tag =
529		    rman_get_bustag(adapter->flash);
530		adapter->osdep.flash_bus_space_handle =
531		    rman_get_bushandle(adapter->flash);
532	}
533
534	/* Do Shared Code initialization */
535	if (e1000_setup_init_funcs(&adapter->hw, TRUE)) {
536		device_printf(dev, "Setup of Shared code failed\n");
537		error = ENXIO;
538		goto err_pci;
539	}
540
541	e1000_get_bus_info(&adapter->hw);
542
543	/* Set up some sysctls for the tunable interrupt delays */
544	em_add_int_delay_sysctl(adapter, "rx_int_delay",
545	    "receive interrupt delay in usecs", &adapter->rx_int_delay,
546	    E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt);
547	em_add_int_delay_sysctl(adapter, "tx_int_delay",
548	    "transmit interrupt delay in usecs", &adapter->tx_int_delay,
549	    E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt);
550	if (adapter->hw.mac.type >= e1000_82540) {
551		em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
552		    "receive interrupt delay limit in usecs",
553		    &adapter->rx_abs_int_delay,
554		    E1000_REGISTER(&adapter->hw, E1000_RADV),
555		    em_rx_abs_int_delay_dflt);
556		em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
557		    "transmit interrupt delay limit in usecs",
558		    &adapter->tx_abs_int_delay,
559		    E1000_REGISTER(&adapter->hw, E1000_TADV),
560		    em_tx_abs_int_delay_dflt);
561	}
562
563#ifndef EM_LEGACY_IRQ
564	/* Sysctls for limiting the amount of work done in the taskqueue */
565	em_add_rx_process_limit(adapter, "rx_processing_limit",
566	    "max number of rx packets to process", &adapter->rx_process_limit,
567	    em_rx_process_limit);
568#endif
569
570	/*
571	 * Validate number of transmit and receive descriptors. It
572	 * must not exceed hardware maximum, and must be multiple
573	 * of E1000_DBA_ALIGN.
574	 */
575	if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 ||
576	    (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) ||
577	    (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) ||
578	    (em_txd < EM_MIN_TXD)) {
579		device_printf(dev, "Using %d TX descriptors instead of %d!\n",
580		    EM_DEFAULT_TXD, em_txd);
581		adapter->num_tx_desc = EM_DEFAULT_TXD;
582	} else
583		adapter->num_tx_desc = em_txd;
584	if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 ||
585	    (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) ||
586	    (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) ||
587	    (em_rxd < EM_MIN_RXD)) {
588		device_printf(dev, "Using %d RX descriptors instead of %d!\n",
589		    EM_DEFAULT_RXD, em_rxd);
590		adapter->num_rx_desc = EM_DEFAULT_RXD;
591	} else
592		adapter->num_rx_desc = em_rxd;
593
594	adapter->hw.mac.autoneg = DO_AUTO_NEG;
595	adapter->hw.phy.autoneg_wait_to_complete = FALSE;
596	adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
597	adapter->rx_buffer_len = 2048;
598
599	e1000_init_script_state_82541(&adapter->hw, TRUE);
600	e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE);
601
602	/* Copper options */
603	if (adapter->hw.phy.media_type == e1000_media_type_copper) {
604		adapter->hw.phy.mdix = AUTO_ALL_MODES;
605		adapter->hw.phy.disable_polarity_correction = FALSE;
606		adapter->hw.phy.ms_type = EM_MASTER_SLAVE;
607	}
608
609	/*
610	 * Set the frame limits assuming
611	 * standard ethernet sized frames.
612	 */
613	adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE;
614	adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE;
615
616	/*
617	 * This controls when hardware reports transmit completion
618	 * status.
619	 */
620	adapter->hw.mac.report_tx_early = 1;
621
622	tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc),
623	    EM_DBA_ALIGN);
624
625	/* Allocate Transmit Descriptor ring */
626	if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
627		device_printf(dev, "Unable to allocate tx_desc memory\n");
628		error = ENOMEM;
629		goto err_tx_desc;
630	}
631	adapter->tx_desc_base =
632	    (struct e1000_tx_desc *)adapter->txdma.dma_vaddr;
633
634	rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc),
635	    EM_DBA_ALIGN);
636
637	/* Allocate Receive Descriptor ring */
638	if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
639		device_printf(dev, "Unable to allocate rx_desc memory\n");
640		error = ENOMEM;
641		goto err_rx_desc;
642	}
643	adapter->rx_desc_base =
644	    (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr;
645
646	/* Make sure we have a good EEPROM before we read from it */
647	if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
648		/*
649		** Some PCI-E parts fail the first check due to
650		** the link being in sleep state, call it again,
651		** if it fails a second time its a real issue.
652		*/
653		if (e1000_validate_nvm_checksum(&adapter->hw) < 0) {
654			device_printf(dev,
655			    "The EEPROM Checksum Is Not Valid\n");
656			error = EIO;
657			goto err_hw_init;
658		}
659	}
660
661	/* Initialize the hardware */
662	if (em_hardware_init(adapter)) {
663		device_printf(dev, "Unable to initialize the hardware\n");
664		error = EIO;
665		goto err_hw_init;
666	}
667
668	/* Copy the permanent MAC address out of the EEPROM */
669	if (e1000_read_mac_addr(&adapter->hw) < 0) {
670		device_printf(dev, "EEPROM read error while reading MAC"
671		    " address\n");
672		error = EIO;
673		goto err_hw_init;
674	}
675
676	if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) {
677		device_printf(dev, "Invalid MAC address\n");
678		error = EIO;
679		goto err_hw_init;
680	}
681
682	/* Allocate transmit descriptors and buffers */
683	if (em_allocate_transmit_structures(adapter)) {
684		device_printf(dev, "Could not setup transmit structures\n");
685		error = ENOMEM;
686		goto err_tx_struct;
687	}
688
689	/* Allocate receive descriptors and buffers */
690	if (em_allocate_receive_structures(adapter)) {
691		device_printf(dev, "Could not setup receive structures\n");
692		error = ENOMEM;
693		goto err_rx_struct;
694	}
695
696	/*
697	**  Do interrupt configuration
698	*/
699	if (adapter->msi > 1) /* Do MSI/X */
700		error = em_allocate_msix(adapter);
701	else  /* MSI or Legacy */
702		error = em_allocate_legacy(adapter);
703	if (error)
704		goto err_rx_struct;
705
706	/* Setup OS specific network interface */
707	em_setup_interface(dev, adapter);
708
709	/* Initialize statistics */
710	em_update_stats_counters(adapter);
711
712	adapter->hw.mac.get_link_status = 1;
713	em_update_link_status(adapter);
714
715	/* Indicate SOL/IDER usage */
716	if (e1000_check_reset_block(&adapter->hw))
717		device_printf(dev,
718		    "PHY reset is blocked due to SOL/IDER session.\n");
719
720	/* Determine if we have to control management hardware */
721	adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw);
722
723	/*
724	 * Setup Wake-on-Lan
725	 */
726	switch (adapter->hw.mac.type) {
727
728	case e1000_82542:
729	case e1000_82543:
730		break;
731	case e1000_82546:
732	case e1000_82546_rev_3:
733	case e1000_82571:
734	case e1000_80003es2lan:
735		if (adapter->hw.bus.func == 1)
736			e1000_read_nvm(&adapter->hw,
737			    NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
738		else
739			e1000_read_nvm(&adapter->hw,
740			    NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
741		eeprom_data &= EM_EEPROM_APME;
742		break;
743	default:
744		/* APME bit in EEPROM is mapped to WUC.APME */
745		eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) &
746		    E1000_WUC_APME;
747		break;
748	}
749	if (eeprom_data)
750		adapter->wol = E1000_WUFC_MAG;
751	/*
752         * We have the eeprom settings, now apply the special cases
753         * where the eeprom may be wrong or the board won't support
754         * wake on lan on a particular port
755	 */
756	device_id = pci_get_device(dev);
757        switch (device_id) {
758	case E1000_DEV_ID_82546GB_PCIE:
759		adapter->wol = 0;
760		break;
761	case E1000_DEV_ID_82546EB_FIBER:
762	case E1000_DEV_ID_82546GB_FIBER:
763	case E1000_DEV_ID_82571EB_FIBER:
764		/* Wake events only supported on port A for dual fiber
765		 * regardless of eeprom setting */
766		if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
767		    E1000_STATUS_FUNC_1)
768			adapter->wol = 0;
769		break;
770	case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3:
771	case E1000_DEV_ID_82571EB_QUAD_COPPER:
772	case E1000_DEV_ID_82571EB_QUAD_FIBER:
773	case E1000_DEV_ID_82571EB_QUAD_COPPER_LP:
774                /* if quad port adapter, disable WoL on all but port A */
775		if (global_quad_port_a != 0)
776			adapter->wol = 0;
777		/* Reset for multiple quad port adapters */
778		if (++global_quad_port_a == 4)
779			global_quad_port_a = 0;
780                break;
781	}
782
783	/* Do we need workaround for 82544 PCI-X adapter? */
784	if (adapter->hw.bus.type == e1000_bus_type_pcix &&
785	    adapter->hw.mac.type == e1000_82544)
786		adapter->pcix_82544 = TRUE;
787	else
788		adapter->pcix_82544 = FALSE;
789
790	/* Register for VLAN events */
791	adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
792	    em_register_vlan, 0, EVENTHANDLER_PRI_FIRST);
793	adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
794	    em_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST);
795
796	/* Tell the stack that the interface is not active */
797	adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
798
799	INIT_DEBUGOUT("em_attach: end");
800
801	return (0);
802
803err_rx_struct:
804	em_free_transmit_structures(adapter);
805err_tx_struct:
806err_hw_init:
807	em_release_hw_control(adapter);
808	em_dma_free(adapter, &adapter->rxdma);
809err_rx_desc:
810	em_dma_free(adapter, &adapter->txdma);
811err_tx_desc:
812err_pci:
813	em_free_pci_resources(adapter);
814	EM_TX_LOCK_DESTROY(adapter);
815	EM_RX_LOCK_DESTROY(adapter);
816	EM_CORE_LOCK_DESTROY(adapter);
817
818	return (error);
819}
820
821/*********************************************************************
822 *  Device removal routine
823 *
824 *  The detach entry point is called when the driver is being removed.
825 *  This routine stops the adapter and deallocates all the resources
826 *  that were allocated for driver operation.
827 *
828 *  return 0 on success, positive on failure
829 *********************************************************************/
830
831static int
832em_detach(device_t dev)
833{
834	struct adapter	*adapter = device_get_softc(dev);
835	struct ifnet	*ifp = adapter->ifp;
836
837	INIT_DEBUGOUT("em_detach: begin");
838
839	/* Make sure VLANS are not using driver */
840#if __FreeBSD_version >= 700000
841	if (adapter->ifp->if_vlantrunk != NULL) {
842#else
843	if (adapter->ifp->if_nvlans != 0) {
844#endif
845		device_printf(dev,"Vlan in use, detach first\n");
846		return (EBUSY);
847	}
848
849#ifdef DEVICE_POLLING
850	if (ifp->if_capenable & IFCAP_POLLING)
851		ether_poll_deregister(ifp);
852#endif
853
854	EM_CORE_LOCK(adapter);
855	EM_TX_LOCK(adapter);
856	adapter->in_detach = 1;
857	em_stop(adapter);
858	e1000_phy_hw_reset(&adapter->hw);
859
860	em_release_manageability(adapter);
861
862	if (((adapter->hw.mac.type == e1000_82573) ||
863	    (adapter->hw.mac.type == e1000_ich8lan) ||
864	    (adapter->hw.mac.type == e1000_ich10lan) ||
865	    (adapter->hw.mac.type == e1000_ich9lan)) &&
866	    e1000_check_mng_mode(&adapter->hw))
867		em_release_hw_control(adapter);
868
869	if (adapter->wol) {
870		E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
871		E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
872		em_enable_wakeup(dev);
873	}
874
875	EM_TX_UNLOCK(adapter);
876	EM_CORE_UNLOCK(adapter);
877
878	/* Unregister VLAN events */
879	if (adapter->vlan_attach != NULL)
880		EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach);
881	if (adapter->vlan_detach != NULL)
882		EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach);
883
884	ether_ifdetach(adapter->ifp);
885	callout_drain(&adapter->timer);
886	callout_drain(&adapter->tx_fifo_timer);
887
888	em_free_pci_resources(adapter);
889	bus_generic_detach(dev);
890	if_free(ifp);
891
892	em_free_transmit_structures(adapter);
893	em_free_receive_structures(adapter);
894
895	/* Free Transmit Descriptor ring */
896	if (adapter->tx_desc_base) {
897		em_dma_free(adapter, &adapter->txdma);
898		adapter->tx_desc_base = NULL;
899	}
900
901	/* Free Receive Descriptor ring */
902	if (adapter->rx_desc_base) {
903		em_dma_free(adapter, &adapter->rxdma);
904		adapter->rx_desc_base = NULL;
905	}
906
907	EM_TX_LOCK_DESTROY(adapter);
908	EM_RX_LOCK_DESTROY(adapter);
909	EM_CORE_LOCK_DESTROY(adapter);
910
911	return (0);
912}
913
914/*********************************************************************
915 *
916 *  Shutdown entry point
917 *
918 **********************************************************************/
919
920static int
921em_shutdown(device_t dev)
922{
923	return em_suspend(dev);
924}
925
926/*
927 * Suspend/resume device methods.
928 */
929static int
930em_suspend(device_t dev)
931{
932	struct adapter *adapter = device_get_softc(dev);
933
934	EM_CORE_LOCK(adapter);
935
936	EM_TX_LOCK(adapter);
937	em_stop(adapter);
938	EM_TX_UNLOCK(adapter);
939
940        em_release_manageability(adapter);
941
942        if (((adapter->hw.mac.type == e1000_82573) ||
943            (adapter->hw.mac.type == e1000_ich8lan) ||
944            (adapter->hw.mac.type == e1000_ich10lan) ||
945            (adapter->hw.mac.type == e1000_ich9lan)) &&
946            e1000_check_mng_mode(&adapter->hw))
947                em_release_hw_control(adapter);
948
949        if (adapter->wol) {
950                E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN);
951                E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol);
952                em_enable_wakeup(dev);
953        }
954
955	EM_CORE_UNLOCK(adapter);
956
957	return bus_generic_suspend(dev);
958}
959
960static int
961em_resume(device_t dev)
962{
963	struct adapter *adapter = device_get_softc(dev);
964	struct ifnet *ifp = adapter->ifp;
965
966	EM_CORE_LOCK(adapter);
967	em_init_locked(adapter);
968	em_init_manageability(adapter);
969	EM_CORE_UNLOCK(adapter);
970	em_start(ifp);
971
972	return bus_generic_resume(dev);
973}
974
975
976/*********************************************************************
977 *  Transmit entry point
978 *
979 *  em_start is called by the stack to initiate a transmit.
980 *  The driver will remain in this routine as long as there are
981 *  packets to transmit and transmit resources are available.
982 *  In case resources are not available stack is notified and
983 *  the packet is requeued.
984 **********************************************************************/
985
986static void
987em_start_locked(struct ifnet *ifp)
988{
989	struct adapter	*adapter = ifp->if_softc;
990	struct mbuf	*m_head;
991
992	EM_TX_LOCK_ASSERT(adapter);
993
994	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
995	    IFF_DRV_RUNNING)
996		return;
997	if (!adapter->link_active)
998		return;
999
1000	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
1001
1002		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
1003		if (m_head == NULL)
1004			break;
1005		/*
1006		 *  Encapsulation can modify our pointer, and or make it
1007		 *  NULL on failure.  In that event, we can't requeue.
1008		 */
1009		if (em_xmit(adapter, &m_head)) {
1010			if (m_head == NULL)
1011				break;
1012			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
1013			IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
1014			break;
1015		}
1016
1017		/* Send a copy of the frame to the BPF listener */
1018		ETHER_BPF_MTAP(ifp, m_head);
1019
1020		/* Set timeout in case hardware has problems transmitting. */
1021		adapter->watchdog_timer = EM_TX_TIMEOUT;
1022	}
1023}
1024
1025static void
1026em_start(struct ifnet *ifp)
1027{
1028	struct adapter *adapter = ifp->if_softc;
1029
1030	EM_TX_LOCK(adapter);
1031	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1032		em_start_locked(ifp);
1033	EM_TX_UNLOCK(adapter);
1034}
1035
1036/*********************************************************************
1037 *  Ioctl entry point
1038 *
1039 *  em_ioctl is called when the user wants to configure the
1040 *  interface.
1041 *
1042 *  return 0 on success, positive on failure
1043 **********************************************************************/
1044
1045static int
1046em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
1047{
1048	struct adapter	*adapter = ifp->if_softc;
1049	struct ifreq *ifr = (struct ifreq *)data;
1050#ifdef INET
1051	struct ifaddr *ifa = (struct ifaddr *)data;
1052#endif
1053	int error = 0;
1054
1055	if (adapter->in_detach)
1056		return (error);
1057
1058	switch (command) {
1059	case SIOCSIFADDR:
1060#ifdef INET
1061		if (ifa->ifa_addr->sa_family == AF_INET) {
1062			/*
1063			 * XXX
1064			 * Since resetting hardware takes a very long time
1065			 * and results in link renegotiation we only
1066			 * initialize the hardware only when it is absolutely
1067			 * required.
1068			 */
1069			ifp->if_flags |= IFF_UP;
1070			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1071				EM_CORE_LOCK(adapter);
1072				em_init_locked(adapter);
1073				EM_CORE_UNLOCK(adapter);
1074			}
1075			arp_ifinit(ifp, ifa);
1076		} else
1077#endif
1078			error = ether_ioctl(ifp, command, data);
1079		break;
1080	case SIOCSIFMTU:
1081	    {
1082		int max_frame_size;
1083		u16 eeprom_data = 0;
1084
1085		IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
1086
1087		EM_CORE_LOCK(adapter);
1088		switch (adapter->hw.mac.type) {
1089		case e1000_82573:
1090			/*
1091			 * 82573 only supports jumbo frames
1092			 * if ASPM is disabled.
1093			 */
1094			e1000_read_nvm(&adapter->hw,
1095			    NVM_INIT_3GIO_3, 1, &eeprom_data);
1096			if (eeprom_data & NVM_WORD1A_ASPM_MASK) {
1097				max_frame_size = ETHER_MAX_LEN;
1098				break;
1099			}
1100			/* Allow Jumbo frames - fall thru */
1101		case e1000_82571:
1102		case e1000_82572:
1103		case e1000_ich9lan:
1104		case e1000_ich10lan:
1105		case e1000_82574:
1106		case e1000_80003es2lan:	/* Limit Jumbo Frame size */
1107			max_frame_size = 9234;
1108			break;
1109			/* Adapters that do not support jumbo frames */
1110		case e1000_82542:
1111		case e1000_ich8lan:
1112			max_frame_size = ETHER_MAX_LEN;
1113			break;
1114		default:
1115			max_frame_size = MAX_JUMBO_FRAME_SIZE;
1116		}
1117		if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
1118		    ETHER_CRC_LEN) {
1119			EM_CORE_UNLOCK(adapter);
1120			error = EINVAL;
1121			break;
1122		}
1123
1124		ifp->if_mtu = ifr->ifr_mtu;
1125		adapter->max_frame_size =
1126		    ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
1127		em_init_locked(adapter);
1128		EM_CORE_UNLOCK(adapter);
1129		break;
1130	    }
1131	case SIOCSIFFLAGS:
1132		IOCTL_DEBUGOUT("ioctl rcv'd:\
1133		    SIOCSIFFLAGS (Set Interface Flags)");
1134		EM_CORE_LOCK(adapter);
1135		if (ifp->if_flags & IFF_UP) {
1136			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1137				if ((ifp->if_flags ^ adapter->if_flags) &
1138				    (IFF_PROMISC | IFF_ALLMULTI)) {
1139					em_disable_promisc(adapter);
1140					em_set_promisc(adapter);
1141				}
1142			} else
1143				em_init_locked(adapter);
1144		} else
1145			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1146				EM_TX_LOCK(adapter);
1147				em_stop(adapter);
1148				EM_TX_UNLOCK(adapter);
1149			}
1150		adapter->if_flags = ifp->if_flags;
1151		EM_CORE_UNLOCK(adapter);
1152		break;
1153	case SIOCADDMULTI:
1154	case SIOCDELMULTI:
1155		IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
1156		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1157			EM_CORE_LOCK(adapter);
1158			em_disable_intr(adapter);
1159			em_set_multi(adapter);
1160			if (adapter->hw.mac.type == e1000_82542 &&
1161	    		    adapter->hw.revision_id == E1000_REVISION_2) {
1162				em_initialize_receive_unit(adapter);
1163			}
1164#ifdef DEVICE_POLLING
1165			if (!(ifp->if_capenable & IFCAP_POLLING))
1166#endif
1167				em_enable_intr(adapter);
1168			EM_CORE_UNLOCK(adapter);
1169		}
1170		break;
1171	case SIOCSIFMEDIA:
1172		/* Check SOL/IDER usage */
1173		EM_CORE_LOCK(adapter);
1174		if (e1000_check_reset_block(&adapter->hw)) {
1175			EM_CORE_UNLOCK(adapter);
1176			device_printf(adapter->dev, "Media change is"
1177			    " blocked due to SOL/IDER session.\n");
1178			break;
1179		}
1180		EM_CORE_UNLOCK(adapter);
1181	case SIOCGIFMEDIA:
1182		IOCTL_DEBUGOUT("ioctl rcv'd: \
1183		    SIOCxIFMEDIA (Get/Set Interface Media)");
1184		error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
1185		break;
1186	case SIOCSIFCAP:
1187	    {
1188		int mask, reinit;
1189
1190		IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
1191		reinit = 0;
1192		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1193#ifdef DEVICE_POLLING
1194		if (mask & IFCAP_POLLING) {
1195			if (ifr->ifr_reqcap & IFCAP_POLLING) {
1196				error = ether_poll_register(em_poll, ifp);
1197				if (error)
1198					return (error);
1199				EM_CORE_LOCK(adapter);
1200				em_disable_intr(adapter);
1201				ifp->if_capenable |= IFCAP_POLLING;
1202				EM_CORE_UNLOCK(adapter);
1203			} else {
1204				error = ether_poll_deregister(ifp);
1205				/* Enable interrupt even in error case */
1206				EM_CORE_LOCK(adapter);
1207				em_enable_intr(adapter);
1208				ifp->if_capenable &= ~IFCAP_POLLING;
1209				EM_CORE_UNLOCK(adapter);
1210			}
1211		}
1212#endif
1213		if (mask & IFCAP_HWCSUM) {
1214			ifp->if_capenable ^= IFCAP_HWCSUM;
1215			reinit = 1;
1216		}
1217#if __FreeBSD_version >= 700000
1218		if (mask & IFCAP_TSO4) {
1219			ifp->if_capenable ^= IFCAP_TSO4;
1220			reinit = 1;
1221		}
1222#endif
1223
1224		if (mask & IFCAP_VLAN_HWTAGGING) {
1225			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
1226			reinit = 1;
1227		}
1228		if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
1229			em_init(adapter);
1230#if __FreeBSD_version >= 700000
1231		VLAN_CAPABILITIES(ifp);
1232#endif
1233		break;
1234	    }
1235
1236#ifdef EM_TIMESYNC
1237	/*
1238	** IOCTL support for Precision Time (IEEE 1588) Support
1239	*/
1240	case EM_TIMESYNC_READTS:
1241	    {
1242		u32 rx_ctl, tx_ctl;
1243		struct em_tsync_read *tdata;
1244
1245		tdata = (struct em_tsync_read *) ifr->ifr_data;
1246
1247		IOCTL_DEBUGOUT("Reading Timestamp\n");
1248
1249		if (tdata->read_current_time) {
1250			getnanotime(&tdata->system_time);
1251			tdata->network_time = E1000_READ_REG(&adapter->hw, E1000_SYSTIML);
1252			tdata->network_time |=
1253			    (u64)E1000_READ_REG(&adapter->hw, E1000_SYSTIMH ) << 32;
1254		}
1255
1256		rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL);
1257		tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL);
1258
1259		IOCTL_DEBUGOUT1("RX_CTL value = %u\n", rx_ctl);
1260		IOCTL_DEBUGOUT1("TX_CTL value = %u\n", tx_ctl);
1261
1262		if (rx_ctl & 0x1) {
1263			IOCTL_DEBUGOUT("RX timestamp is valid\n");
1264			u32 tmp;
1265			unsigned char *tmp_cp;
1266
1267			tdata->rx_valid = 1;
1268			tdata->rx_stamp = E1000_READ_REG(&adapter->hw, E1000_RXSTMPL);
1269			tdata->rx_stamp |= (u64)E1000_READ_REG(&adapter->hw,
1270			    E1000_RXSTMPH) << 32;
1271
1272			tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRL);
1273			tmp_cp = (unsigned char *) &tmp;
1274			tdata->srcid[0] = tmp_cp[0];
1275			tdata->srcid[1] = tmp_cp[1];
1276			tdata->srcid[2] = tmp_cp[2];
1277			tdata->srcid[3] = tmp_cp[3];
1278			tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRH);
1279			tmp_cp = (unsigned char *) &tmp;
1280			tdata->srcid[4] = tmp_cp[0];
1281			tdata->srcid[5] = tmp_cp[1];
1282			tdata->seqid = tmp >> 16;
1283			tdata->seqid = htons(tdata->seqid);
1284		} else
1285			tdata->rx_valid = 0;
1286
1287		if (tx_ctl & 0x1) {
1288			IOCTL_DEBUGOUT("TX timestamp is valid\n");
1289			tdata->tx_valid = 1;
1290			tdata->tx_stamp = E1000_READ_REG(&adapter->hw, E1000_TXSTMPL);
1291			tdata->tx_stamp |= (u64) E1000_READ_REG(&adapter->hw,
1292			    E1000_TXSTMPH) << 32;
1293		} else
1294			tdata->tx_valid = 0;
1295
1296		return (0);
1297	    }
1298#endif	/* EM_TIMESYNC */
1299
1300	default:
1301		error = ether_ioctl(ifp, command, data);
1302		break;
1303	}
1304
1305	return (error);
1306}
1307
1308/*********************************************************************
1309 *  Watchdog timer:
1310 *
1311 *  This routine is called from the local timer every second.
1312 *  As long as transmit descriptors are being cleaned the value
1313 *  is non-zero and we do nothing. Reaching 0 indicates a tx hang
1314 *  and we then reset the device.
1315 *
1316 **********************************************************************/
1317
1318static void
1319em_watchdog(struct adapter *adapter)
1320{
1321
1322	EM_CORE_LOCK_ASSERT(adapter);
1323
1324	/*
1325	** The timer is set to 5 every time start queues a packet.
1326	** Then txeof keeps resetting it as long as it cleans at
1327	** least one descriptor.
1328	** Finally, anytime all descriptors are clean the timer is
1329	** set to 0.
1330	*/
1331	EM_TX_LOCK(adapter);
1332	if ((adapter->watchdog_timer == 0) || (--adapter->watchdog_timer)) {
1333		EM_TX_UNLOCK(adapter);
1334		return;
1335	}
1336
1337	/* If we are in this routine because of pause frames, then
1338	 * don't reset the hardware.
1339	 */
1340	if (E1000_READ_REG(&adapter->hw, E1000_STATUS) &
1341	    E1000_STATUS_TXOFF) {
1342		adapter->watchdog_timer = EM_TX_TIMEOUT;
1343		EM_TX_UNLOCK(adapter);
1344		return;
1345	}
1346
1347	if (e1000_check_for_link(&adapter->hw) == 0)
1348		device_printf(adapter->dev, "watchdog timeout -- resetting\n");
1349	adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
1350	adapter->watchdog_events++;
1351	EM_TX_UNLOCK(adapter);
1352
1353	em_init_locked(adapter);
1354}
1355
1356/*********************************************************************
1357 *  Init entry point
1358 *
1359 *  This routine is used in two ways. It is used by the stack as
1360 *  init entry point in network interface structure. It is also used
1361 *  by the driver as a hw/sw initialization routine to get to a
1362 *  consistent state.
1363 *
1364 *  return 0 on success, positive on failure
1365 **********************************************************************/
1366
1367static void
1368em_init_locked(struct adapter *adapter)
1369{
1370	struct ifnet	*ifp = adapter->ifp;
1371	device_t	dev = adapter->dev;
1372	u32		pba;
1373
1374	INIT_DEBUGOUT("em_init: begin");
1375
1376	EM_CORE_LOCK_ASSERT(adapter);
1377
1378	EM_TX_LOCK(adapter);
1379	em_stop(adapter);
1380	EM_TX_UNLOCK(adapter);
1381
1382	/*
1383	 * Packet Buffer Allocation (PBA)
1384	 * Writing PBA sets the receive portion of the buffer
1385	 * the remainder is used for the transmit buffer.
1386	 *
1387	 * Devices before the 82547 had a Packet Buffer of 64K.
1388	 *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1389	 * After the 82547 the buffer was reduced to 40K.
1390	 *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1391	 *   Note: default does not leave enough room for Jumbo Frame >10k.
1392	 */
1393	switch (adapter->hw.mac.type) {
1394	case e1000_82547:
1395	case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1396		if (adapter->max_frame_size > 8192)
1397			pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1398		else
1399			pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1400		adapter->tx_fifo_head = 0;
1401		adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1402		adapter->tx_fifo_size =
1403		    (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1404		break;
1405	/* Total Packet Buffer on these is 48K */
1406	case e1000_82571:
1407	case e1000_82572:
1408	case e1000_80003es2lan:
1409			pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1410		break;
1411	case e1000_82573: /* 82573: Total Packet Buffer is 32K */
1412			pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1413		break;
1414	case e1000_82574:
1415			pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */
1416		break;
1417	case e1000_ich9lan:
1418	case e1000_ich10lan:
1419#define E1000_PBA_10K	0x000A
1420		pba = E1000_PBA_10K;
1421		break;
1422	case e1000_ich8lan:
1423		pba = E1000_PBA_8K;
1424		break;
1425	default:
1426		/* Devices before 82547 had a Packet Buffer of 64K.   */
1427		if (adapter->max_frame_size > 8192)
1428			pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1429		else
1430			pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1431	}
1432
1433	INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1434	E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba);
1435
1436	/* Get the latest mac address, User can use a LAA */
1437        bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr,
1438              ETHER_ADDR_LEN);
1439
1440	/* Put the address into the Receive Address Array */
1441	e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
1442
1443	/*
1444	 * With the 82571 adapter, RAR[0] may be overwritten
1445	 * when the other port is reset, we make a duplicate
1446	 * in RAR[14] for that eventuality, this assures
1447	 * the interface continues to function.
1448	 */
1449	if (adapter->hw.mac.type == e1000_82571) {
1450		e1000_set_laa_state_82571(&adapter->hw, TRUE);
1451		e1000_rar_set(&adapter->hw, adapter->hw.mac.addr,
1452		    E1000_RAR_ENTRIES - 1);
1453	}
1454
1455	/* Initialize the hardware */
1456	if (em_hardware_init(adapter)) {
1457		device_printf(dev, "Unable to initialize the hardware\n");
1458		return;
1459	}
1460	em_update_link_status(adapter);
1461
1462	/* Setup VLAN support, basic and offload if available */
1463	E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN);
1464
1465	/* New register interface replaces this but
1466	   waiting on kernel support to be added */
1467	if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) &&
1468	    ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) {
1469		u32 ctrl;
1470		ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
1471		ctrl |= E1000_CTRL_VME;
1472		E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
1473	}
1474
1475	/* Set hardware offload abilities */
1476	ifp->if_hwassist = 0;
1477	if (adapter->hw.mac.type >= e1000_82543) {
1478		if (ifp->if_capenable & IFCAP_TXCSUM)
1479			ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP);
1480#if __FreeBSD_version >= 700000
1481		if (ifp->if_capenable & IFCAP_TSO4)
1482			ifp->if_hwassist |= CSUM_TSO;
1483#endif
1484	}
1485
1486	/* Configure for OS presence */
1487	em_init_manageability(adapter);
1488
1489	/* Prepare transmit descriptors and buffers */
1490	em_setup_transmit_structures(adapter);
1491	em_initialize_transmit_unit(adapter);
1492
1493	/* Setup Multicast table */
1494	em_set_multi(adapter);
1495
1496	/* Prepare receive descriptors and buffers */
1497	if (em_setup_receive_structures(adapter)) {
1498		device_printf(dev, "Could not setup receive structures\n");
1499		EM_TX_LOCK(adapter);
1500		em_stop(adapter);
1501		EM_TX_UNLOCK(adapter);
1502		return;
1503	}
1504	em_initialize_receive_unit(adapter);
1505
1506	/* Don't lose promiscuous settings */
1507	em_set_promisc(adapter);
1508
1509	ifp->if_drv_flags |= IFF_DRV_RUNNING;
1510	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1511
1512	callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1513	e1000_clear_hw_cntrs_base_generic(&adapter->hw);
1514
1515	/* MSI/X configuration for 82574 */
1516	if (adapter->hw.mac.type == e1000_82574) {
1517		int tmp;
1518		tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
1519		tmp |= E1000_CTRL_EXT_PBA_CLR;
1520		E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp);
1521		/*
1522		** Set the IVAR - interrupt vector routing.
1523		** Each nibble represents a vector, high bit
1524		** is enable, other 3 bits are the MSIX table
1525		** entry, we map RXQ0 to 0, TXQ0 to 1, and
1526		** Link (other) to 2, hence the magic number.
1527		*/
1528		E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908);
1529	}
1530
1531#ifdef DEVICE_POLLING
1532	/*
1533	 * Only enable interrupts if we are not polling, make sure
1534	 * they are off otherwise.
1535	 */
1536	if (ifp->if_capenable & IFCAP_POLLING)
1537		em_disable_intr(adapter);
1538	else
1539#endif /* DEVICE_POLLING */
1540		em_enable_intr(adapter);
1541
1542#ifdef EM_TIMESYNC
1543	/* Initializae IEEE 1588 Precision Time hardware */
1544	if ((adapter->hw.mac.type == e1000_82574) ||
1545	    (adapter->hw.mac.type == e1000_ich10lan))
1546		em_tsync_init(adapter);
1547#endif
1548
1549	/* Don't reset the phy next time init gets called */
1550	adapter->hw.phy.reset_disable = TRUE;
1551}
1552
1553static void
1554em_init(void *arg)
1555{
1556	struct adapter *adapter = arg;
1557
1558	EM_CORE_LOCK(adapter);
1559	em_init_locked(adapter);
1560	EM_CORE_UNLOCK(adapter);
1561}
1562
1563
1564#ifdef DEVICE_POLLING
1565/*********************************************************************
1566 *
1567 *  Legacy polling routine
1568 *
1569 *********************************************************************/
1570static void
1571em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1572{
1573	struct adapter *adapter = ifp->if_softc;
1574	u32		reg_icr;
1575
1576	EM_CORE_LOCK(adapter);
1577	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1578		EM_CORE_UNLOCK(adapter);
1579		return;
1580	}
1581
1582	if (cmd == POLL_AND_CHECK_STATUS) {
1583		reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1584		if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1585			callout_stop(&adapter->timer);
1586			adapter->hw.mac.get_link_status = 1;
1587			em_update_link_status(adapter);
1588			callout_reset(&adapter->timer, hz,
1589			    em_local_timer, adapter);
1590		}
1591	}
1592	EM_CORE_UNLOCK(adapter);
1593
1594	em_rxeof(adapter, count);
1595
1596	EM_TX_LOCK(adapter);
1597	em_txeof(adapter);
1598
1599	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1600		em_start_locked(ifp);
1601	EM_TX_UNLOCK(adapter);
1602}
1603#endif /* DEVICE_POLLING */
1604
1605#ifdef EM_LEGACY_IRQ
1606/*********************************************************************
1607 *
1608 *  Legacy Interrupt Service routine
1609 *
1610 *********************************************************************/
1611
1612static void
1613em_intr(void *arg)
1614{
1615	struct adapter	*adapter = arg;
1616	struct ifnet	*ifp = adapter->ifp;
1617	u32		reg_icr;
1618
1619
1620	if (ifp->if_capenable & IFCAP_POLLING)
1621		return;
1622
1623	EM_CORE_LOCK(adapter);
1624	for (;;) {
1625		reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1626
1627		if (adapter->hw.mac.type >= e1000_82571 &&
1628	    	    (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1629			break;
1630		else if (reg_icr == 0)
1631			break;
1632
1633		/*
1634		 * XXX: some laptops trigger several spurious interrupts
1635		 * on em(4) when in the resume cycle. The ICR register
1636		 * reports all-ones value in this case. Processing such
1637		 * interrupts would lead to a freeze. I don't know why.
1638		 */
1639		if (reg_icr == 0xffffffff)
1640			break;
1641
1642		EM_CORE_UNLOCK(adapter);
1643		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1644			em_rxeof(adapter, -1);
1645			EM_TX_LOCK(adapter);
1646			em_txeof(adapter);
1647			EM_TX_UNLOCK(adapter);
1648		}
1649		EM_CORE_LOCK(adapter);
1650
1651		/* Link status change */
1652		if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1653			callout_stop(&adapter->timer);
1654			adapter->hw.mac.get_link_status = 1;
1655			em_update_link_status(adapter);
1656			/* Deal with TX cruft when link lost */
1657			em_tx_purge(adapter);
1658			callout_reset(&adapter->timer, hz,
1659			    em_local_timer, adapter);
1660		}
1661
1662		if (reg_icr & E1000_ICR_RXO)
1663			adapter->rx_overruns++;
1664	}
1665	EM_CORE_UNLOCK(adapter);
1666
1667	if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
1668	    !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1669		em_start(ifp);
1670}
1671
1672#else /* EM_FAST_IRQ, then fast interrupt routines only */
1673
1674static void
1675em_handle_link(void *context, int pending)
1676{
1677	struct adapter	*adapter = context;
1678	struct ifnet *ifp = adapter->ifp;
1679
1680	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
1681		return;
1682
1683	EM_CORE_LOCK(adapter);
1684	callout_stop(&adapter->timer);
1685	em_update_link_status(adapter);
1686	/* Deal with TX cruft when link lost */
1687	em_tx_purge(adapter);
1688	callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1689	EM_CORE_UNLOCK(adapter);
1690}
1691
1692
1693/* Combined RX/TX handler, used by Legacy and MSI */
1694static void
1695em_handle_rxtx(void *context, int pending)
1696{
1697	struct adapter	*adapter = context;
1698	struct ifnet	*ifp = adapter->ifp;
1699
1700
1701	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1702		if (em_rxeof(adapter, adapter->rx_process_limit) != 0)
1703			taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1704		EM_TX_LOCK(adapter);
1705		em_txeof(adapter);
1706
1707		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1708			em_start_locked(ifp);
1709		EM_TX_UNLOCK(adapter);
1710	}
1711
1712	em_enable_intr(adapter);
1713}
1714
1715static void
1716em_handle_rx(void *context, int pending)
1717{
1718	struct adapter	*adapter = context;
1719	struct ifnet	*ifp = adapter->ifp;
1720
1721	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1722	    (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1723		taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1724
1725}
1726
1727static void
1728em_handle_tx(void *context, int pending)
1729{
1730	struct adapter	*adapter = context;
1731	struct ifnet	*ifp = adapter->ifp;
1732
1733	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1734		EM_TX_LOCK(adapter);
1735		em_txeof(adapter);
1736		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1737			em_start_locked(ifp);
1738		EM_TX_UNLOCK(adapter);
1739	}
1740}
1741
1742/*********************************************************************
1743 *
1744 *  Fast Legacy/MSI Combined Interrupt Service routine
1745 *
1746 *********************************************************************/
1747#if __FreeBSD_version < 700000
1748#define FILTER_STRAY
1749#define FILTER_HANDLED
1750static void
1751#else
1752static int
1753#endif
1754em_irq_fast(void *arg)
1755{
1756	struct adapter	*adapter = arg;
1757	struct ifnet	*ifp;
1758	u32		reg_icr;
1759
1760	ifp = adapter->ifp;
1761
1762	reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1763
1764	/* Hot eject?  */
1765	if (reg_icr == 0xffffffff)
1766		return FILTER_STRAY;
1767
1768	/* Definitely not our interrupt.  */
1769	if (reg_icr == 0x0)
1770		return FILTER_STRAY;
1771
1772	/*
1773	 * Starting with the 82571 chip, bit 31 should be used to
1774	 * determine whether the interrupt belongs to us.
1775	 */
1776	if (adapter->hw.mac.type >= e1000_82571 &&
1777	    (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1778		return FILTER_STRAY;
1779
1780	/*
1781	 * Mask interrupts until the taskqueue is finished running.  This is
1782	 * cheap, just assume that it is needed.  This also works around the
1783	 * MSI message reordering errata on certain systems.
1784	 */
1785	em_disable_intr(adapter);
1786	taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1787
1788	/* Link status change */
1789	if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1790		adapter->hw.mac.get_link_status = 1;
1791		taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1792	}
1793
1794	if (reg_icr & E1000_ICR_RXO)
1795		adapter->rx_overruns++;
1796	return FILTER_HANDLED;
1797}
1798
1799/*********************************************************************
1800 *
1801 *  MSIX Interrupt Service Routines
1802 *
1803 **********************************************************************/
1804#define EM_MSIX_TX	0x00040000
1805#define EM_MSIX_RX	0x00010000
1806#define EM_MSIX_LINK	0x00100000
1807
1808static void
1809em_msix_tx(void *arg)
1810{
1811	struct adapter *adapter = arg;
1812	struct ifnet	*ifp = adapter->ifp;
1813
1814	++adapter->tx_irq;
1815	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1816		EM_TX_LOCK(adapter);
1817		em_txeof(adapter);
1818		EM_TX_UNLOCK(adapter);
1819		taskqueue_enqueue(adapter->tq, &adapter->tx_task);
1820	}
1821	/* Reenable this interrupt */
1822	E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX);
1823	return;
1824}
1825
1826/*********************************************************************
1827 *
1828 *  MSIX RX Interrupt Service routine
1829 *
1830 **********************************************************************/
1831
1832static void
1833em_msix_rx(void *arg)
1834{
1835	struct adapter *adapter = arg;
1836	struct ifnet	*ifp = adapter->ifp;
1837
1838	++adapter->rx_irq;
1839	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) &&
1840	    (em_rxeof(adapter, adapter->rx_process_limit) != 0))
1841		taskqueue_enqueue(adapter->tq, &adapter->rx_task);
1842	/* Reenable this interrupt */
1843	E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX);
1844	return;
1845}
1846
1847/*********************************************************************
1848 *
1849 *  MSIX Link Fast Interrupt Service routine
1850 *
1851 **********************************************************************/
1852
1853static void
1854em_msix_link(void *arg)
1855{
1856	struct adapter	*adapter = arg;
1857	u32		reg_icr;
1858
1859	++adapter->link_irq;
1860	reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR);
1861
1862	if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1863		adapter->hw.mac.get_link_status = 1;
1864		taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1865	}
1866	E1000_WRITE_REG(&adapter->hw, E1000_IMS,
1867	    EM_MSIX_LINK | E1000_IMS_LSC);
1868	return;
1869}
1870#endif /* EM_FAST_IRQ */
1871
1872/*********************************************************************
1873 *
1874 *  Media Ioctl callback
1875 *
1876 *  This routine is called whenever the user queries the status of
1877 *  the interface using ifconfig.
1878 *
1879 **********************************************************************/
1880static void
1881em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1882{
1883	struct adapter *adapter = ifp->if_softc;
1884	u_char fiber_type = IFM_1000_SX;
1885
1886	INIT_DEBUGOUT("em_media_status: begin");
1887
1888	EM_CORE_LOCK(adapter);
1889	em_update_link_status(adapter);
1890
1891	ifmr->ifm_status = IFM_AVALID;
1892	ifmr->ifm_active = IFM_ETHER;
1893
1894	if (!adapter->link_active) {
1895		EM_CORE_UNLOCK(adapter);
1896		return;
1897	}
1898
1899	ifmr->ifm_status |= IFM_ACTIVE;
1900
1901	if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
1902	    (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
1903		if (adapter->hw.mac.type == e1000_82545)
1904			fiber_type = IFM_1000_LX;
1905		ifmr->ifm_active |= fiber_type | IFM_FDX;
1906	} else {
1907		switch (adapter->link_speed) {
1908		case 10:
1909			ifmr->ifm_active |= IFM_10_T;
1910			break;
1911		case 100:
1912			ifmr->ifm_active |= IFM_100_TX;
1913			break;
1914		case 1000:
1915			ifmr->ifm_active |= IFM_1000_T;
1916			break;
1917		}
1918		if (adapter->link_duplex == FULL_DUPLEX)
1919			ifmr->ifm_active |= IFM_FDX;
1920		else
1921			ifmr->ifm_active |= IFM_HDX;
1922	}
1923	EM_CORE_UNLOCK(adapter);
1924}
1925
1926/*********************************************************************
1927 *
1928 *  Media Ioctl callback
1929 *
1930 *  This routine is called when the user changes speed/duplex using
1931 *  media/mediopt option with ifconfig.
1932 *
1933 **********************************************************************/
1934static int
1935em_media_change(struct ifnet *ifp)
1936{
1937	struct adapter *adapter = ifp->if_softc;
1938	struct ifmedia  *ifm = &adapter->media;
1939
1940	INIT_DEBUGOUT("em_media_change: begin");
1941
1942	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1943		return (EINVAL);
1944
1945	EM_CORE_LOCK(adapter);
1946	switch (IFM_SUBTYPE(ifm->ifm_media)) {
1947	case IFM_AUTO:
1948		adapter->hw.mac.autoneg = DO_AUTO_NEG;
1949		adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1950		break;
1951	case IFM_1000_LX:
1952	case IFM_1000_SX:
1953	case IFM_1000_T:
1954		adapter->hw.mac.autoneg = DO_AUTO_NEG;
1955		adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
1956		break;
1957	case IFM_100_TX:
1958		adapter->hw.mac.autoneg = FALSE;
1959		adapter->hw.phy.autoneg_advertised = 0;
1960		if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1961			adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL;
1962		else
1963			adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF;
1964		break;
1965	case IFM_10_T:
1966		adapter->hw.mac.autoneg = FALSE;
1967		adapter->hw.phy.autoneg_advertised = 0;
1968		if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1969			adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL;
1970		else
1971			adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF;
1972		break;
1973	default:
1974		device_printf(adapter->dev, "Unsupported media type\n");
1975	}
1976
1977	/* As the speed/duplex settings my have changed we need to
1978	 * reset the PHY.
1979	 */
1980	adapter->hw.phy.reset_disable = FALSE;
1981
1982	em_init_locked(adapter);
1983	EM_CORE_UNLOCK(adapter);
1984
1985	return (0);
1986}
1987
1988/*********************************************************************
1989 *
1990 *  This routine maps the mbufs to tx descriptors.
1991 *
1992 *  return 0 on success, positive on failure
1993 **********************************************************************/
1994
1995static int
1996em_xmit(struct adapter *adapter, struct mbuf **m_headp)
1997{
1998	bus_dma_segment_t	segs[EM_MAX_SCATTER];
1999	bus_dmamap_t		map;
2000	struct em_buffer	*tx_buffer, *tx_buffer_mapped;
2001	struct e1000_tx_desc	*ctxd = NULL;
2002	struct mbuf		*m_head;
2003	u32			txd_upper, txd_lower, txd_used, txd_saved;
2004	int			nsegs, i, j, first, last = 0;
2005	int			error, do_tso, tso_desc = 0;
2006#if __FreeBSD_version < 700000
2007	struct m_tag		*mtag;
2008#endif
2009	m_head = *m_headp;
2010	txd_upper = txd_lower = txd_used = txd_saved = 0;
2011
2012#if __FreeBSD_version >= 700000
2013	do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
2014#else
2015	do_tso = 0;
2016#endif
2017
2018        /*
2019         * Force a cleanup if number of TX descriptors
2020         * available hits the threshold
2021         */
2022	if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
2023		em_txeof(adapter);
2024		/* Now do we at least have a minimal? */
2025		if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) {
2026			adapter->no_tx_desc_avail1++;
2027			return (ENOBUFS);
2028		}
2029	}
2030
2031
2032	/*
2033	 * TSO workaround:
2034	 *  If an mbuf is only header we need
2035	 *     to pull 4 bytes of data into it.
2036	 */
2037	if (do_tso && (m_head->m_len <= M_TSO_LEN)) {
2038		m_head = m_pullup(m_head, M_TSO_LEN + 4);
2039		*m_headp = m_head;
2040		if (m_head == NULL)
2041			return (ENOBUFS);
2042	}
2043
2044	/*
2045	 * Map the packet for DMA
2046	 *
2047	 * Capture the first descriptor index,
2048	 * this descriptor will have the index
2049	 * of the EOP which is the only one that
2050	 * now gets a DONE bit writeback.
2051	 */
2052	first = adapter->next_avail_tx_desc;
2053	tx_buffer = &adapter->tx_buffer_area[first];
2054	tx_buffer_mapped = tx_buffer;
2055	map = tx_buffer->map;
2056
2057	error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
2058	    *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2059
2060	/*
2061	 * There are two types of errors we can (try) to handle:
2062	 * - EFBIG means the mbuf chain was too long and bus_dma ran
2063	 *   out of segments.  Defragment the mbuf chain and try again.
2064	 * - ENOMEM means bus_dma could not obtain enough bounce buffers
2065	 *   at this point in time.  Defer sending and try again later.
2066	 * All other errors, in particular EINVAL, are fatal and prevent the
2067	 * mbuf chain from ever going through.  Drop it and report error.
2068	 */
2069	if (error == EFBIG) {
2070		struct mbuf *m;
2071
2072		m = m_defrag(*m_headp, M_DONTWAIT);
2073		if (m == NULL) {
2074			adapter->mbuf_alloc_failed++;
2075			m_freem(*m_headp);
2076			*m_headp = NULL;
2077			return (ENOBUFS);
2078		}
2079		*m_headp = m;
2080
2081		/* Try it again */
2082		error = bus_dmamap_load_mbuf_sg(adapter->txtag, map,
2083		    *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
2084
2085		if (error == ENOMEM) {
2086			adapter->no_tx_dma_setup++;
2087			return (error);
2088		} else if (error != 0) {
2089			adapter->no_tx_dma_setup++;
2090			m_freem(*m_headp);
2091			*m_headp = NULL;
2092			return (error);
2093		}
2094	} else if (error == ENOMEM) {
2095		adapter->no_tx_dma_setup++;
2096		return (error);
2097	} else if (error != 0) {
2098		adapter->no_tx_dma_setup++;
2099		m_freem(*m_headp);
2100		*m_headp = NULL;
2101		return (error);
2102	}
2103
2104	/*
2105	 * TSO Hardware workaround, if this packet is not
2106	 * TSO, and is only a single descriptor long, and
2107	 * it follows a TSO burst, then we need to add a
2108	 * sentinel descriptor to prevent premature writeback.
2109	 */
2110	if ((do_tso == 0) && (adapter->tx_tso == TRUE)) {
2111		if (nsegs == 1)
2112			tso_desc = TRUE;
2113		adapter->tx_tso = FALSE;
2114	}
2115
2116        if (nsegs > (adapter->num_tx_desc_avail - 2)) {
2117                adapter->no_tx_desc_avail2++;
2118		bus_dmamap_unload(adapter->txtag, map);
2119		return (ENOBUFS);
2120        }
2121	m_head = *m_headp;
2122
2123	/* Do hardware assists */
2124#if __FreeBSD_version >= 700000
2125	if (m_head->m_pkthdr.csum_flags & CSUM_TSO) {
2126		error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower);
2127		if (error != TRUE)
2128			return (ENXIO); /* something foobar */
2129		/* we need to make a final sentinel transmit desc */
2130		tso_desc = TRUE;
2131	} else
2132#endif
2133#ifndef EM_TIMESYNC
2134	/*
2135	** Timesync needs to check the packet header
2136	** so call checksum code to do so, but don't
2137	** penalize the code if not defined.
2138	*/
2139	if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD)
2140#endif
2141		em_transmit_checksum_setup(adapter,  m_head,
2142		    &txd_upper, &txd_lower);
2143
2144	i = adapter->next_avail_tx_desc;
2145	if (adapter->pcix_82544)
2146		txd_saved = i;
2147
2148	/* Set up our transmit descriptors */
2149	for (j = 0; j < nsegs; j++) {
2150		bus_size_t seg_len;
2151		bus_addr_t seg_addr;
2152		/* If adapter is 82544 and on PCIX bus */
2153		if(adapter->pcix_82544) {
2154			DESC_ARRAY	desc_array;
2155			u32		array_elements, counter;
2156			/*
2157			 * Check the Address and Length combination and
2158			 * split the data accordingly
2159			 */
2160			array_elements = em_fill_descriptors(segs[j].ds_addr,
2161			    segs[j].ds_len, &desc_array);
2162			for (counter = 0; counter < array_elements; counter++) {
2163				if (txd_used == adapter->num_tx_desc_avail) {
2164					adapter->next_avail_tx_desc = txd_saved;
2165					adapter->no_tx_desc_avail2++;
2166					bus_dmamap_unload(adapter->txtag, map);
2167					return (ENOBUFS);
2168				}
2169				tx_buffer = &adapter->tx_buffer_area[i];
2170				ctxd = &adapter->tx_desc_base[i];
2171				ctxd->buffer_addr = htole64(
2172				    desc_array.descriptor[counter].address);
2173				ctxd->lower.data = htole32(
2174				    (adapter->txd_cmd | txd_lower | (u16)
2175				    desc_array.descriptor[counter].length));
2176				ctxd->upper.data =
2177				    htole32((txd_upper));
2178				last = i;
2179				if (++i == adapter->num_tx_desc)
2180                                         i = 0;
2181				tx_buffer->m_head = NULL;
2182				tx_buffer->next_eop = -1;
2183				txd_used++;
2184                        }
2185		} else {
2186			tx_buffer = &adapter->tx_buffer_area[i];
2187			ctxd = &adapter->tx_desc_base[i];
2188			seg_addr = segs[j].ds_addr;
2189			seg_len  = segs[j].ds_len;
2190			/*
2191			** TSO Workaround:
2192			** If this is the last descriptor, we want to
2193			** split it so we have a small final sentinel
2194			*/
2195			if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
2196				seg_len -= 4;
2197				ctxd->buffer_addr = htole64(seg_addr);
2198				ctxd->lower.data = htole32(
2199				adapter->txd_cmd | txd_lower | seg_len);
2200				ctxd->upper.data =
2201				    htole32(txd_upper);
2202				if (++i == adapter->num_tx_desc)
2203					i = 0;
2204				/* Now make the sentinel */
2205				++txd_used; /* using an extra txd */
2206				ctxd = &adapter->tx_desc_base[i];
2207				tx_buffer = &adapter->tx_buffer_area[i];
2208				ctxd->buffer_addr =
2209				    htole64(seg_addr + seg_len);
2210				ctxd->lower.data = htole32(
2211				adapter->txd_cmd | txd_lower | 4);
2212				ctxd->upper.data =
2213				    htole32(txd_upper);
2214				last = i;
2215				if (++i == adapter->num_tx_desc)
2216					i = 0;
2217			} else {
2218				ctxd->buffer_addr = htole64(seg_addr);
2219				ctxd->lower.data = htole32(
2220				adapter->txd_cmd | txd_lower | seg_len);
2221				ctxd->upper.data =
2222				    htole32(txd_upper);
2223				last = i;
2224				if (++i == adapter->num_tx_desc)
2225					i = 0;
2226			}
2227			tx_buffer->m_head = NULL;
2228			tx_buffer->next_eop = -1;
2229		}
2230	}
2231
2232	adapter->next_avail_tx_desc = i;
2233	if (adapter->pcix_82544)
2234		adapter->num_tx_desc_avail -= txd_used;
2235	else {
2236		adapter->num_tx_desc_avail -= nsegs;
2237		if (tso_desc) /* TSO used an extra for sentinel */
2238			adapter->num_tx_desc_avail -= txd_used;
2239	}
2240
2241        /*
2242	** Handle VLAN tag, this is the
2243	** biggest difference between
2244	** 6.x and 7
2245	*/
2246#if __FreeBSD_version < 700000
2247        /* Find out if we are in vlan mode. */
2248        mtag = VLAN_OUTPUT_TAG(ifp, m_head);
2249        if (mtag != NULL) {
2250                ctxd->upper.fields.special =
2251                    htole16(VLAN_TAG_VALUE(mtag));
2252#else /* FreeBSD 7 */
2253	if (m_head->m_flags & M_VLANTAG) {
2254		/* Set the vlan id. */
2255		ctxd->upper.fields.special =
2256		    htole16(m_head->m_pkthdr.ether_vtag);
2257#endif
2258                /* Tell hardware to add tag */
2259                ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE);
2260        }
2261
2262        tx_buffer->m_head = m_head;
2263	tx_buffer_mapped->map = tx_buffer->map;
2264	tx_buffer->map = map;
2265        bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
2266
2267        /*
2268         * Last Descriptor of Packet
2269	 * needs End Of Packet (EOP)
2270	 * and Report Status (RS)
2271         */
2272        ctxd->lower.data |=
2273	    htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
2274	/*
2275	 * Keep track in the first buffer which
2276	 * descriptor will be written back
2277	 */
2278	tx_buffer = &adapter->tx_buffer_area[first];
2279	tx_buffer->next_eop = last;
2280
2281	/*
2282	 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000
2283	 * that this frame is available to transmit.
2284	 */
2285	bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2286	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2287	if (adapter->hw.mac.type == e1000_82547 &&
2288	    adapter->link_duplex == HALF_DUPLEX)
2289		em_82547_move_tail(adapter);
2290	else {
2291		E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i);
2292		if (adapter->hw.mac.type == e1000_82547)
2293			em_82547_update_fifo_head(adapter,
2294			    m_head->m_pkthdr.len);
2295	}
2296
2297#ifdef EM_TIMESYNC
2298	if (ctxd->upper.data & E1000_TXD_EXTCMD_TSTAMP) {
2299		HW_DEBUGOUT( "@@@ Timestamp bit is set in transmit descriptor\n" );
2300	}
2301#endif
2302	return (0);
2303}
2304
2305/*********************************************************************
2306 *
2307 * 82547 workaround to avoid controller hang in half-duplex environment.
2308 * The workaround is to avoid queuing a large packet that would span
2309 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
2310 * in this case. We do that only when FIFO is quiescent.
2311 *
2312 **********************************************************************/
2313static void
2314em_82547_move_tail(void *arg)
2315{
2316	struct adapter *adapter = arg;
2317	struct e1000_tx_desc *tx_desc;
2318	u16	hw_tdt, sw_tdt, length = 0;
2319	bool	eop = 0;
2320
2321	EM_TX_LOCK_ASSERT(adapter);
2322
2323	hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0));
2324	sw_tdt = adapter->next_avail_tx_desc;
2325
2326	while (hw_tdt != sw_tdt) {
2327		tx_desc = &adapter->tx_desc_base[hw_tdt];
2328		length += tx_desc->lower.flags.length;
2329		eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
2330		if (++hw_tdt == adapter->num_tx_desc)
2331			hw_tdt = 0;
2332
2333		if (eop) {
2334			if (em_82547_fifo_workaround(adapter, length)) {
2335				adapter->tx_fifo_wrk_cnt++;
2336				callout_reset(&adapter->tx_fifo_timer, 1,
2337					em_82547_move_tail, adapter);
2338				break;
2339			}
2340			E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt);
2341			em_82547_update_fifo_head(adapter, length);
2342			length = 0;
2343		}
2344	}
2345}
2346
2347static int
2348em_82547_fifo_workaround(struct adapter *adapter, int len)
2349{
2350	int fifo_space, fifo_pkt_len;
2351
2352	fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2353
2354	if (adapter->link_duplex == HALF_DUPLEX) {
2355		fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
2356
2357		if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
2358			if (em_82547_tx_fifo_reset(adapter))
2359				return (0);
2360			else
2361				return (1);
2362		}
2363	}
2364
2365	return (0);
2366}
2367
2368static void
2369em_82547_update_fifo_head(struct adapter *adapter, int len)
2370{
2371	int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
2372
2373	/* tx_fifo_head is always 16 byte aligned */
2374	adapter->tx_fifo_head += fifo_pkt_len;
2375	if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
2376		adapter->tx_fifo_head -= adapter->tx_fifo_size;
2377	}
2378}
2379
2380
2381static int
2382em_82547_tx_fifo_reset(struct adapter *adapter)
2383{
2384	u32 tctl;
2385
2386	if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) ==
2387	    E1000_READ_REG(&adapter->hw, E1000_TDH(0))) &&
2388	    (E1000_READ_REG(&adapter->hw, E1000_TDFT) ==
2389	    E1000_READ_REG(&adapter->hw, E1000_TDFH)) &&
2390	    (E1000_READ_REG(&adapter->hw, E1000_TDFTS) ==
2391	    E1000_READ_REG(&adapter->hw, E1000_TDFHS)) &&
2392	    (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) {
2393		/* Disable TX unit */
2394		tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
2395		E1000_WRITE_REG(&adapter->hw, E1000_TCTL,
2396		    tctl & ~E1000_TCTL_EN);
2397
2398		/* Reset FIFO pointers */
2399		E1000_WRITE_REG(&adapter->hw, E1000_TDFT,
2400		    adapter->tx_head_addr);
2401		E1000_WRITE_REG(&adapter->hw, E1000_TDFH,
2402		    adapter->tx_head_addr);
2403		E1000_WRITE_REG(&adapter->hw, E1000_TDFTS,
2404		    adapter->tx_head_addr);
2405		E1000_WRITE_REG(&adapter->hw, E1000_TDFHS,
2406		    adapter->tx_head_addr);
2407
2408		/* Re-enable TX unit */
2409		E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
2410		E1000_WRITE_FLUSH(&adapter->hw);
2411
2412		adapter->tx_fifo_head = 0;
2413		adapter->tx_fifo_reset_cnt++;
2414
2415		return (TRUE);
2416	}
2417	else {
2418		return (FALSE);
2419	}
2420}
2421
2422static void
2423em_set_promisc(struct adapter *adapter)
2424{
2425	struct ifnet	*ifp = adapter->ifp;
2426	u32		reg_rctl;
2427
2428	reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2429
2430	if (ifp->if_flags & IFF_PROMISC) {
2431		reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
2432		/* Turn this on if you want to see bad packets */
2433		if (em_debug_sbp)
2434			reg_rctl |= E1000_RCTL_SBP;
2435		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2436	} else if (ifp->if_flags & IFF_ALLMULTI) {
2437		reg_rctl |= E1000_RCTL_MPE;
2438		reg_rctl &= ~E1000_RCTL_UPE;
2439		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2440	}
2441}
2442
2443static void
2444em_disable_promisc(struct adapter *adapter)
2445{
2446	u32	reg_rctl;
2447
2448	reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2449
2450	reg_rctl &=  (~E1000_RCTL_UPE);
2451	reg_rctl &=  (~E1000_RCTL_MPE);
2452	reg_rctl &=  (~E1000_RCTL_SBP);
2453	E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2454}
2455
2456
2457/*********************************************************************
2458 *  Multicast Update
2459 *
2460 *  This routine is called whenever multicast address list is updated.
2461 *
2462 **********************************************************************/
2463
2464static void
2465em_set_multi(struct adapter *adapter)
2466{
2467	struct ifnet	*ifp = adapter->ifp;
2468	struct ifmultiaddr *ifma;
2469	u32 reg_rctl = 0;
2470	u8  mta[512]; /* Largest MTS is 4096 bits */
2471	int mcnt = 0;
2472
2473	IOCTL_DEBUGOUT("em_set_multi: begin");
2474
2475	if (adapter->hw.mac.type == e1000_82542 &&
2476	    adapter->hw.revision_id == E1000_REVISION_2) {
2477		reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2478		if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2479			e1000_pci_clear_mwi(&adapter->hw);
2480		reg_rctl |= E1000_RCTL_RST;
2481		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2482		msec_delay(5);
2483	}
2484
2485	IF_ADDR_LOCK(ifp);
2486	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
2487		if (ifma->ifma_addr->sa_family != AF_LINK)
2488			continue;
2489
2490		if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
2491			break;
2492
2493		bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
2494		    &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN);
2495		mcnt++;
2496	}
2497	IF_ADDR_UNLOCK(ifp);
2498
2499	if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
2500		reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2501		reg_rctl |= E1000_RCTL_MPE;
2502		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2503	} else
2504		e1000_update_mc_addr_list(&adapter->hw, mta,
2505		    mcnt, 1, adapter->hw.mac.rar_entry_count);
2506
2507	if (adapter->hw.mac.type == e1000_82542 &&
2508	    adapter->hw.revision_id == E1000_REVISION_2) {
2509		reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
2510		reg_rctl &= ~E1000_RCTL_RST;
2511		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl);
2512		msec_delay(5);
2513		if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
2514			e1000_pci_set_mwi(&adapter->hw);
2515	}
2516}
2517
2518
2519/*********************************************************************
2520 *  Timer routine
2521 *
2522 *  This routine checks for link status and updates statistics.
2523 *
2524 **********************************************************************/
2525
2526static void
2527em_local_timer(void *arg)
2528{
2529	struct adapter	*adapter = arg;
2530	struct ifnet	*ifp = adapter->ifp;
2531
2532	EM_CORE_LOCK_ASSERT(adapter);
2533
2534	em_update_link_status(adapter);
2535	em_update_stats_counters(adapter);
2536
2537	/* Reset LAA into RAR[0] on 82571 */
2538	if (e1000_get_laa_state_82571(&adapter->hw) == TRUE)
2539		e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
2540
2541	if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
2542		em_print_hw_stats(adapter);
2543
2544	em_smartspeed(adapter);
2545
2546	/*
2547	 * Each second we check the watchdog to
2548	 * protect against hardware hangs.
2549	 */
2550	em_watchdog(adapter);
2551
2552	callout_reset(&adapter->timer, hz, em_local_timer, adapter);
2553
2554}
2555
2556static void
2557em_update_link_status(struct adapter *adapter)
2558{
2559	struct e1000_hw *hw = &adapter->hw;
2560	struct ifnet *ifp = adapter->ifp;
2561	device_t dev = adapter->dev;
2562	u32 link_check = 0;
2563
2564	/* Get the cached link value or read phy for real */
2565	switch (hw->phy.media_type) {
2566	case e1000_media_type_copper:
2567		if (hw->mac.get_link_status) {
2568			/* Do the work to read phy */
2569			e1000_check_for_link(hw);
2570			link_check = !hw->mac.get_link_status;
2571			if (link_check) /* ESB2 fix */
2572				e1000_cfg_on_link_up(hw);
2573		} else
2574			link_check = TRUE;
2575		break;
2576	case e1000_media_type_fiber:
2577		e1000_check_for_link(hw);
2578		link_check = (E1000_READ_REG(hw, E1000_STATUS) &
2579                                 E1000_STATUS_LU);
2580		break;
2581	case e1000_media_type_internal_serdes:
2582		e1000_check_for_link(hw);
2583		link_check = adapter->hw.mac.serdes_has_link;
2584		break;
2585	default:
2586	case e1000_media_type_unknown:
2587		break;
2588	}
2589
2590	/* Now check for a transition */
2591	if (link_check && (adapter->link_active == 0)) {
2592		e1000_get_speed_and_duplex(hw, &adapter->link_speed,
2593		    &adapter->link_duplex);
2594		/* Check if we must disable SPEED_MODE bit on PCI-E */
2595		if ((adapter->link_speed != SPEED_1000) &&
2596		    ((hw->mac.type == e1000_82571) ||
2597		    (hw->mac.type == e1000_82572))) {
2598			int tarc0;
2599			tarc0 = E1000_READ_REG(hw, E1000_TARC(0));
2600			tarc0 &= ~SPEED_MODE_BIT;
2601			E1000_WRITE_REG(hw, E1000_TARC(0), tarc0);
2602		}
2603		if (bootverbose)
2604			device_printf(dev, "Link is up %d Mbps %s\n",
2605			    adapter->link_speed,
2606			    ((adapter->link_duplex == FULL_DUPLEX) ?
2607			    "Full Duplex" : "Half Duplex"));
2608		adapter->link_active = 1;
2609		adapter->smartspeed = 0;
2610		ifp->if_baudrate = adapter->link_speed * 1000000;
2611		if_link_state_change(ifp, LINK_STATE_UP);
2612	} else if (!link_check && (adapter->link_active == 1)) {
2613		ifp->if_baudrate = adapter->link_speed = 0;
2614		adapter->link_duplex = 0;
2615		if (bootverbose)
2616			device_printf(dev, "Link is Down\n");
2617		adapter->link_active = 0;
2618		/* Link down, disable watchdog */
2619		adapter->watchdog_timer = FALSE;
2620		if_link_state_change(ifp, LINK_STATE_DOWN);
2621	}
2622}
2623
2624/*********************************************************************
2625 *
2626 *  This routine disables all traffic on the adapter by issuing a
2627 *  global reset on the MAC and deallocates TX/RX buffers.
2628 *
2629 *  This routine should always be called with BOTH the CORE
2630 *  and TX locks.
2631 **********************************************************************/
2632
2633static void
2634em_stop(void *arg)
2635{
2636	struct adapter	*adapter = arg;
2637	struct ifnet	*ifp = adapter->ifp;
2638
2639	EM_CORE_LOCK_ASSERT(adapter);
2640	EM_TX_LOCK_ASSERT(adapter);
2641
2642	INIT_DEBUGOUT("em_stop: begin");
2643
2644	em_disable_intr(adapter);
2645	callout_stop(&adapter->timer);
2646	callout_stop(&adapter->tx_fifo_timer);
2647
2648	/* Tell the stack that the interface is no longer active */
2649	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
2650
2651#ifdef EM_TIMESYNC
2652	/* Disable IEEE 1588 Time hardware */
2653	if ((adapter->hw.mac.type == e1000_82574) ||
2654	    (adapter->hw.mac.type == e1000_ich10lan))
2655		em_tsync_disable(adapter);
2656#endif
2657
2658	e1000_reset_hw(&adapter->hw);
2659	if (adapter->hw.mac.type >= e1000_82544)
2660		E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0);
2661}
2662
2663
2664/*********************************************************************
2665 *
2666 *  Determine hardware revision.
2667 *
2668 **********************************************************************/
2669static void
2670em_identify_hardware(struct adapter *adapter)
2671{
2672	device_t dev = adapter->dev;
2673
2674	/* Make sure our PCI config space has the necessary stuff set */
2675	adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
2676	if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) &&
2677	    (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) {
2678		device_printf(dev, "Memory Access and/or Bus Master bits "
2679		    "were not set!\n");
2680		adapter->hw.bus.pci_cmd_word |=
2681		(PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
2682		pci_write_config(dev, PCIR_COMMAND,
2683		    adapter->hw.bus.pci_cmd_word, 2);
2684	}
2685
2686	/* Save off the information about this board */
2687	adapter->hw.vendor_id = pci_get_vendor(dev);
2688	adapter->hw.device_id = pci_get_device(dev);
2689	adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2690	adapter->hw.subsystem_vendor_id =
2691	    pci_read_config(dev, PCIR_SUBVEND_0, 2);
2692	adapter->hw.subsystem_device_id =
2693	    pci_read_config(dev, PCIR_SUBDEV_0, 2);
2694
2695	/* Do Shared Code Init and Setup */
2696	if (e1000_set_mac_type(&adapter->hw)) {
2697		device_printf(dev, "Setup init failure\n");
2698		return;
2699	}
2700}
2701
2702static int
2703em_allocate_pci_resources(struct adapter *adapter)
2704{
2705	device_t	dev = adapter->dev;
2706	int		val, rid, error = E1000_SUCCESS;
2707
2708	rid = PCIR_BAR(0);
2709	adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2710	    &rid, RF_ACTIVE);
2711	if (adapter->memory == NULL) {
2712		device_printf(dev, "Unable to allocate bus resource: memory\n");
2713		return (ENXIO);
2714	}
2715	adapter->osdep.mem_bus_space_tag =
2716	    rman_get_bustag(adapter->memory);
2717	adapter->osdep.mem_bus_space_handle =
2718	    rman_get_bushandle(adapter->memory);
2719	adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle;
2720
2721	/* Only older adapters use IO mapping */
2722	if ((adapter->hw.mac.type > e1000_82543) &&
2723	    (adapter->hw.mac.type < e1000_82571)) {
2724		/* Figure our where our IO BAR is ? */
2725		for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2726			val = pci_read_config(dev, rid, 4);
2727			if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2728				adapter->io_rid = rid;
2729				break;
2730			}
2731			rid += 4;
2732			/* check for 64bit BAR */
2733			if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2734				rid += 4;
2735		}
2736		if (rid >= PCIR_CIS) {
2737			device_printf(dev, "Unable to locate IO BAR\n");
2738			return (ENXIO);
2739		}
2740		adapter->ioport = bus_alloc_resource_any(dev,
2741		    SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
2742		if (adapter->ioport == NULL) {
2743			device_printf(dev, "Unable to allocate bus resource: "
2744			    "ioport\n");
2745			return (ENXIO);
2746		}
2747		adapter->hw.io_base = 0;
2748		adapter->osdep.io_bus_space_tag =
2749		    rman_get_bustag(adapter->ioport);
2750		adapter->osdep.io_bus_space_handle =
2751		    rman_get_bushandle(adapter->ioport);
2752	}
2753
2754	/*
2755	** Init the resource arrays
2756	**  used by MSIX setup
2757	*/
2758	for (int i = 0; i < 3; i++) {
2759		adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */
2760		adapter->tag[i] = NULL;
2761		adapter->res[i] = NULL;
2762	}
2763
2764	/*
2765	 * Setup MSI/X or MSI if PCI Express
2766	 */
2767	if (em_enable_msi)
2768		adapter->msi = em_setup_msix(adapter);
2769
2770	adapter->hw.back = &adapter->osdep;
2771
2772	return (error);
2773}
2774
2775/*********************************************************************
2776 *
2777 *  Setup the Legacy or MSI Interrupt handler
2778 *
2779 **********************************************************************/
2780int
2781em_allocate_legacy(struct adapter *adapter)
2782{
2783	device_t dev = adapter->dev;
2784	int error;
2785
2786	/* Manually turn off all interrupts */
2787	E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2788
2789	/* Legacy RID is 0 */
2790	if (adapter->msi == 0)
2791		adapter->rid[0] = 0;
2792
2793	/* We allocate a single interrupt resource */
2794	adapter->res[0] = bus_alloc_resource_any(dev,
2795	    SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE);
2796	if (adapter->res[0] == NULL) {
2797		device_printf(dev, "Unable to allocate bus resource: "
2798		    "interrupt\n");
2799		return (ENXIO);
2800	}
2801
2802#ifdef EM_LEGACY_IRQ
2803	/* We do Legacy setup */
2804	if ((error = bus_setup_intr(dev, adapter->res[0],
2805#if __FreeBSD_version > 700000
2806	    INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter,
2807#else /* 6.X */
2808	    INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter,
2809#endif
2810	    &adapter->tag[0])) != 0) {
2811		device_printf(dev, "Failed to register interrupt handler");
2812		return (error);
2813	}
2814
2815#else /* FAST_IRQ */
2816	/*
2817	 * Try allocating a fast interrupt and the associated deferred
2818	 * processing contexts.
2819	 */
2820	TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter);
2821	TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2822	adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2823	    taskqueue_thread_enqueue, &adapter->tq);
2824	taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2825	    device_get_nameunit(adapter->dev));
2826#if __FreeBSD_version < 700000
2827	if ((error = bus_setup_intr(dev, adapter->res[0],
2828	    INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter,
2829#else
2830	if ((error = bus_setup_intr(dev, adapter->res[0],
2831	    INTR_TYPE_NET, em_irq_fast, NULL, adapter,
2832#endif
2833	    &adapter->tag[0])) != 0) {
2834		device_printf(dev, "Failed to register fast interrupt "
2835			    "handler: %d\n", error);
2836		taskqueue_free(adapter->tq);
2837		adapter->tq = NULL;
2838		return (error);
2839	}
2840#endif  /* EM_LEGACY_IRQ */
2841
2842	return (0);
2843}
2844
2845/*********************************************************************
2846 *
2847 *  Setup the MSIX Interrupt handlers
2848 *   This is not really Multiqueue, rather
2849 *   its just multiple interrupt vectors.
2850 *
2851 **********************************************************************/
2852int
2853em_allocate_msix(struct adapter *adapter)
2854{
2855	device_t dev = adapter->dev;
2856	int error;
2857
2858	/* Make sure all interrupts are disabled */
2859	E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
2860
2861	/* First get the resources */
2862	for (int i = 0; i < adapter->msi; i++) {
2863		adapter->res[i] = bus_alloc_resource_any(dev,
2864		    SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE);
2865		if (adapter->res[i] == NULL) {
2866			device_printf(dev,
2867			    "Unable to allocate bus resource: "
2868			    "MSIX Interrupt\n");
2869			return (ENXIO);
2870		}
2871	}
2872
2873	/*
2874	 * Now allocate deferred processing contexts.
2875	 */
2876	TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter);
2877	TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter);
2878	TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2879	adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2880	    taskqueue_thread_enqueue, &adapter->tq);
2881	taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2882	    device_get_nameunit(adapter->dev));
2883
2884	/*
2885	 * And setup the interrupt handlers
2886	 */
2887
2888	/* First slot to RX */
2889	if ((error = bus_setup_intr(dev, adapter->res[0],
2890#if __FreeBSD_version > 700000
2891	    INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, adapter,
2892#else /* 6.X */
2893	    INTR_TYPE_NET | INTR_MPSAFE, em_msix_rx, adapter,
2894#endif
2895	    &adapter->tag[0])) != 0) {
2896		device_printf(dev, "Failed to register RX handler");
2897		return (error);
2898	}
2899
2900	/* Next TX */
2901	if ((error = bus_setup_intr(dev, adapter->res[1],
2902#if __FreeBSD_version > 700000
2903	    INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, adapter,
2904#else /* 6.X */
2905	    INTR_TYPE_NET | INTR_MPSAFE, em_msix_tx, adapter,
2906#endif
2907	    &adapter->tag[1])) != 0) {
2908		device_printf(dev, "Failed to register TX handler");
2909		return (error);
2910	}
2911
2912	/* And Link */
2913	if ((error = bus_setup_intr(dev, adapter->res[2],
2914#if __FreeBSD_version > 700000
2915	    INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_link, adapter,
2916#else /* 6.X */
2917	    INTR_TYPE_NET | INTR_MPSAFE, em_msix_link, adapter,
2918#endif
2919	    &adapter->tag[2])) != 0) {
2920		device_printf(dev, "Failed to register TX handler");
2921		return (error);
2922	}
2923
2924	return (0);
2925}
2926
2927static void
2928em_free_pci_resources(struct adapter *adapter)
2929{
2930	device_t dev = adapter->dev;
2931
2932	/* Make sure the for loop below runs once */
2933	if (adapter->msi == 0)
2934		adapter->msi = 1;
2935
2936	/*
2937	 * First release all the interrupt resources:
2938	 *      notice that since these are just kept
2939	 *      in an array we can do the same logic
2940	 *      whether its MSIX or just legacy.
2941	 */
2942	for (int i = 0; i < adapter->msi; i++) {
2943		if (adapter->tag[i] != NULL) {
2944			bus_teardown_intr(dev, adapter->res[i],
2945			    adapter->tag[i]);
2946			adapter->tag[i] = NULL;
2947		}
2948		if (adapter->res[i] != NULL) {
2949			bus_release_resource(dev, SYS_RES_IRQ,
2950			    adapter->rid[i], adapter->res[i]);
2951		}
2952	}
2953
2954	if (adapter->msi)
2955		pci_release_msi(dev);
2956
2957	if (adapter->msix != NULL)
2958		bus_release_resource(dev, SYS_RES_MEMORY,
2959		    PCIR_BAR(EM_MSIX_BAR), adapter->msix);
2960
2961	if (adapter->memory != NULL)
2962		bus_release_resource(dev, SYS_RES_MEMORY,
2963		    PCIR_BAR(0), adapter->memory);
2964
2965	if (adapter->flash != NULL)
2966		bus_release_resource(dev, SYS_RES_MEMORY,
2967		    EM_FLASH, adapter->flash);
2968
2969	if (adapter->ioport != NULL)
2970		bus_release_resource(dev, SYS_RES_IOPORT,
2971		    adapter->io_rid, adapter->ioport);
2972}
2973
2974/*
2975 * Setup MSI/X
2976 */
2977static int
2978em_setup_msix(struct adapter *adapter)
2979{
2980	device_t dev = adapter->dev;
2981	int val = 0;
2982
2983	if (adapter->hw.mac.type < e1000_82571)
2984		return (0);
2985
2986	/* Setup MSI/X for Hartwell */
2987	if (adapter->hw.mac.type == e1000_82574) {
2988		/* Map the MSIX BAR */
2989		int rid = PCIR_BAR(EM_MSIX_BAR);
2990		adapter->msix = bus_alloc_resource_any(dev,
2991		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
2992       		if (!adapter->msix) {
2993			/* May not be enabled */
2994               		device_printf(adapter->dev,
2995			    "Unable to map MSIX table \n");
2996			goto msi;
2997       		}
2998		val = pci_msix_count(dev);
2999		/*
3000		** 82574 can be configured for 5 but
3001		** we limit use to 3.
3002		*/
3003		if (val > 3) val = 3;
3004		if ((val) && pci_alloc_msix(dev, &val) == 0) {
3005               		device_printf(adapter->dev,"Using MSIX interrupts\n");
3006			return (val);
3007		}
3008	}
3009msi:
3010       	val = pci_msi_count(dev);
3011       	if (val == 1 && pci_alloc_msi(dev, &val) == 0) {
3012               	adapter->msi = 1;
3013               	device_printf(adapter->dev,"Using MSI interrupt\n");
3014		return (val);
3015	}
3016	return (0);
3017}
3018
3019/*********************************************************************
3020 *
3021 *  Initialize the hardware to a configuration
3022 *  as specified by the adapter structure.
3023 *
3024 **********************************************************************/
3025static int
3026em_hardware_init(struct adapter *adapter)
3027{
3028	device_t dev = adapter->dev;
3029	u16 	rx_buffer_size;
3030
3031	INIT_DEBUGOUT("em_hardware_init: begin");
3032
3033	/* Issue a global reset */
3034	e1000_reset_hw(&adapter->hw);
3035
3036	/* Get control from any management/hw control */
3037	if (((adapter->hw.mac.type == e1000_82573) ||
3038	    (adapter->hw.mac.type == e1000_ich8lan) ||
3039	    (adapter->hw.mac.type == e1000_ich10lan) ||
3040	    (adapter->hw.mac.type == e1000_ich9lan)) &&
3041	    e1000_check_mng_mode(&adapter->hw))
3042		em_get_hw_control(adapter);
3043
3044	/* When hardware is reset, fifo_head is also reset */
3045	adapter->tx_fifo_head = 0;
3046
3047	/* Set up smart power down as default off on newer adapters. */
3048	if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 ||
3049	    adapter->hw.mac.type == e1000_82572)) {
3050		u16 phy_tmp = 0;
3051
3052		/* Speed up time to link by disabling smart power down. */
3053		e1000_read_phy_reg(&adapter->hw,
3054		    IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
3055		phy_tmp &= ~IGP02E1000_PM_SPD;
3056		e1000_write_phy_reg(&adapter->hw,
3057		    IGP02E1000_PHY_POWER_MGMT, phy_tmp);
3058	}
3059
3060	/*
3061	 * These parameters control the automatic generation (Tx) and
3062	 * response (Rx) to Ethernet PAUSE frames.
3063	 * - High water mark should allow for at least two frames to be
3064	 *   received after sending an XOFF.
3065	 * - Low water mark works best when it is very near the high water mark.
3066	 *   This allows the receiver to restart by sending XON when it has
3067	 *   drained a bit. Here we use an arbitary value of 1500 which will
3068	 *   restart after one full frame is pulled from the buffer. There
3069	 *   could be several smaller frames in the buffer and if so they will
3070	 *   not trigger the XON until their total number reduces the buffer
3071	 *   by 1500.
3072	 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
3073	 */
3074	rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) &
3075	    0xffff) << 10 );
3076
3077	adapter->hw.fc.high_water = rx_buffer_size -
3078	    roundup2(adapter->max_frame_size, 1024);
3079	adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500;
3080
3081	if (adapter->hw.mac.type == e1000_80003es2lan)
3082		adapter->hw.fc.pause_time = 0xFFFF;
3083	else
3084		adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME;
3085	adapter->hw.fc.send_xon = TRUE;
3086	adapter->hw.fc.requested_mode = e1000_fc_full;
3087
3088	if (e1000_init_hw(&adapter->hw) < 0) {
3089		device_printf(dev, "Hardware Initialization Failed\n");
3090		return (EIO);
3091	}
3092
3093	e1000_check_for_link(&adapter->hw);
3094
3095	return (0);
3096}
3097
3098/*********************************************************************
3099 *
3100 *  Setup networking device structure and register an interface.
3101 *
3102 **********************************************************************/
3103static void
3104em_setup_interface(device_t dev, struct adapter *adapter)
3105{
3106	struct ifnet   *ifp;
3107
3108	INIT_DEBUGOUT("em_setup_interface: begin");
3109
3110	ifp = adapter->ifp = if_alloc(IFT_ETHER);
3111	if (ifp == NULL)
3112		panic("%s: can not if_alloc()", device_get_nameunit(dev));
3113	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
3114	ifp->if_mtu = ETHERMTU;
3115	ifp->if_init =  em_init;
3116	ifp->if_softc = adapter;
3117	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
3118	ifp->if_ioctl = em_ioctl;
3119	ifp->if_start = em_start;
3120	IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
3121	ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
3122	IFQ_SET_READY(&ifp->if_snd);
3123
3124	ether_ifattach(ifp, adapter->hw.mac.addr);
3125
3126	ifp->if_capabilities = ifp->if_capenable = 0;
3127
3128	if (adapter->hw.mac.type >= e1000_82543) {
3129		int version_cap;
3130#if __FreeBSD_version < 700000
3131		version_cap = IFCAP_HWCSUM;
3132#else
3133		version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
3134#endif
3135		ifp->if_capabilities |= version_cap;
3136		ifp->if_capenable |= version_cap;
3137	}
3138
3139#if __FreeBSD_version >= 700000
3140	/* Identify TSO capable adapters */
3141	if ((adapter->hw.mac.type > e1000_82544) &&
3142	    (adapter->hw.mac.type != e1000_82547))
3143		ifp->if_capabilities |= IFCAP_TSO4;
3144	/*
3145	 * By default only enable on PCI-E, this
3146	 * can be overriden by ifconfig.
3147	 */
3148	if (adapter->hw.mac.type >= e1000_82571)
3149		ifp->if_capenable |= IFCAP_TSO4;
3150#endif
3151
3152	/*
3153	 * Tell the upper layer(s) we support long frames.
3154	 */
3155	ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
3156	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3157	ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
3158
3159#ifdef DEVICE_POLLING
3160	ifp->if_capabilities |= IFCAP_POLLING;
3161#endif
3162
3163	/*
3164	 * Specify the media types supported by this adapter and register
3165	 * callbacks to update media and link information
3166	 */
3167	ifmedia_init(&adapter->media, IFM_IMASK,
3168	    em_media_change, em_media_status);
3169	if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3170	    (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) {
3171		u_char fiber_type = IFM_1000_SX;	/* default type */
3172
3173		if (adapter->hw.mac.type == e1000_82545)
3174			fiber_type = IFM_1000_LX;
3175		ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
3176			    0, NULL);
3177		ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
3178	} else {
3179		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
3180		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
3181			    0, NULL);
3182		ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
3183			    0, NULL);
3184		ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
3185			    0, NULL);
3186		if (adapter->hw.phy.type != e1000_phy_ife) {
3187			ifmedia_add(&adapter->media,
3188				IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
3189			ifmedia_add(&adapter->media,
3190				IFM_ETHER | IFM_1000_T, 0, NULL);
3191		}
3192	}
3193	ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
3194	ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
3195}
3196
3197
3198/*********************************************************************
3199 *
3200 *  Workaround for SmartSpeed on 82541 and 82547 controllers
3201 *
3202 **********************************************************************/
3203static void
3204em_smartspeed(struct adapter *adapter)
3205{
3206	u16 phy_tmp;
3207
3208	if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) ||
3209	    adapter->hw.mac.autoneg == 0 ||
3210	    (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
3211		return;
3212
3213	if (adapter->smartspeed == 0) {
3214		/* If Master/Slave config fault is asserted twice,
3215		 * we assume back-to-back */
3216		e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3217		if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
3218			return;
3219		e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
3220		if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
3221			e1000_read_phy_reg(&adapter->hw,
3222			    PHY_1000T_CTRL, &phy_tmp);
3223			if(phy_tmp & CR_1000T_MS_ENABLE) {
3224				phy_tmp &= ~CR_1000T_MS_ENABLE;
3225				e1000_write_phy_reg(&adapter->hw,
3226				    PHY_1000T_CTRL, phy_tmp);
3227				adapter->smartspeed++;
3228				if(adapter->hw.mac.autoneg &&
3229				   !e1000_phy_setup_autoneg(&adapter->hw) &&
3230				   !e1000_read_phy_reg(&adapter->hw,
3231				    PHY_CONTROL, &phy_tmp)) {
3232					phy_tmp |= (MII_CR_AUTO_NEG_EN |
3233						    MII_CR_RESTART_AUTO_NEG);
3234					e1000_write_phy_reg(&adapter->hw,
3235					    PHY_CONTROL, phy_tmp);
3236				}
3237			}
3238		}
3239		return;
3240	} else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
3241		/* If still no link, perhaps using 2/3 pair cable */
3242		e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
3243		phy_tmp |= CR_1000T_MS_ENABLE;
3244		e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
3245		if(adapter->hw.mac.autoneg &&
3246		   !e1000_phy_setup_autoneg(&adapter->hw) &&
3247		   !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) {
3248			phy_tmp |= (MII_CR_AUTO_NEG_EN |
3249				    MII_CR_RESTART_AUTO_NEG);
3250			e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp);
3251		}
3252	}
3253	/* Restart process after EM_SMARTSPEED_MAX iterations */
3254	if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
3255		adapter->smartspeed = 0;
3256}
3257
3258
3259/*
3260 * Manage DMA'able memory.
3261 */
3262static void
3263em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
3264{
3265	if (error)
3266		return;
3267	*(bus_addr_t *) arg = segs[0].ds_addr;
3268}
3269
3270static int
3271em_dma_malloc(struct adapter *adapter, bus_size_t size,
3272        struct em_dma_alloc *dma, int mapflags)
3273{
3274	int error;
3275
3276#if __FreeBSD_version >= 700000
3277	error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */
3278#else
3279	error = bus_dma_tag_create(NULL,		 /* parent */
3280#endif
3281				EM_DBA_ALIGN, 0,	/* alignment, bounds */
3282				BUS_SPACE_MAXADDR,	/* lowaddr */
3283				BUS_SPACE_MAXADDR,	/* highaddr */
3284				NULL, NULL,		/* filter, filterarg */
3285				size,			/* maxsize */
3286				1,			/* nsegments */
3287				size,			/* maxsegsize */
3288				0,			/* flags */
3289				NULL,			/* lockfunc */
3290				NULL,			/* lockarg */
3291				&dma->dma_tag);
3292	if (error) {
3293		device_printf(adapter->dev,
3294		    "%s: bus_dma_tag_create failed: %d\n",
3295		    __func__, error);
3296		goto fail_0;
3297	}
3298
3299	error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
3300	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map);
3301	if (error) {
3302		device_printf(adapter->dev,
3303		    "%s: bus_dmamem_alloc(%ju) failed: %d\n",
3304		    __func__, (uintmax_t)size, error);
3305		goto fail_2;
3306	}
3307
3308	dma->dma_paddr = 0;
3309	error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
3310	    size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
3311	if (error || dma->dma_paddr == 0) {
3312		device_printf(adapter->dev,
3313		    "%s: bus_dmamap_load failed: %d\n",
3314		    __func__, error);
3315		goto fail_3;
3316	}
3317
3318	return (0);
3319
3320fail_3:
3321	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3322fail_2:
3323	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3324	bus_dma_tag_destroy(dma->dma_tag);
3325fail_0:
3326	dma->dma_map = NULL;
3327	dma->dma_tag = NULL;
3328
3329	return (error);
3330}
3331
3332static void
3333em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
3334{
3335	if (dma->dma_tag == NULL)
3336		return;
3337	if (dma->dma_map != NULL) {
3338		bus_dmamap_sync(dma->dma_tag, dma->dma_map,
3339		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
3340		bus_dmamap_unload(dma->dma_tag, dma->dma_map);
3341		bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
3342		dma->dma_map = NULL;
3343	}
3344	bus_dma_tag_destroy(dma->dma_tag);
3345	dma->dma_tag = NULL;
3346}
3347
3348
3349/*********************************************************************
3350 *
3351 *  Allocate memory for tx_buffer structures. The tx_buffer stores all
3352 *  the information needed to transmit a packet on the wire.
3353 *
3354 **********************************************************************/
3355static int
3356em_allocate_transmit_structures(struct adapter *adapter)
3357{
3358	device_t dev = adapter->dev;
3359	struct em_buffer *tx_buffer;
3360	int error;
3361
3362	/*
3363	 * Create DMA tags for tx descriptors
3364	 */
3365#if __FreeBSD_version >= 700000
3366	if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
3367#else
3368	if ((error = bus_dma_tag_create(NULL,		 /* parent */
3369#endif
3370				1, 0,			/* alignment, bounds */
3371				BUS_SPACE_MAXADDR,	/* lowaddr */
3372				BUS_SPACE_MAXADDR,	/* highaddr */
3373				NULL, NULL,		/* filter, filterarg */
3374				EM_TSO_SIZE,		/* maxsize */
3375				EM_MAX_SCATTER,		/* nsegments */
3376				EM_TSO_SEG_SIZE,	/* maxsegsize */
3377				0,			/* flags */
3378				NULL,		/* lockfunc */
3379				NULL,		/* lockarg */
3380				&adapter->txtag)) != 0) {
3381		device_printf(dev, "Unable to allocate TX DMA tag\n");
3382		goto fail;
3383	}
3384
3385	adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
3386	    adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
3387	if (adapter->tx_buffer_area == NULL) {
3388		device_printf(dev, "Unable to allocate tx_buffer memory\n");
3389		error = ENOMEM;
3390		goto fail;
3391	}
3392
3393	/* Create the descriptor buffer dma maps */
3394	for (int i = 0; i < adapter->num_tx_desc; i++) {
3395		tx_buffer = &adapter->tx_buffer_area[i];
3396		error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
3397		if (error != 0) {
3398			device_printf(dev, "Unable to create TX DMA map\n");
3399			goto fail;
3400		}
3401		tx_buffer->next_eop = -1;
3402	}
3403
3404	return (0);
3405fail:
3406	em_free_transmit_structures(adapter);
3407	return (error);
3408}
3409
3410/*********************************************************************
3411 *
3412 *  (Re)Initialize transmit structures.
3413 *
3414 **********************************************************************/
3415static void
3416em_setup_transmit_structures(struct adapter *adapter)
3417{
3418	struct em_buffer *tx_buffer;
3419
3420	/* Clear the old ring contents */
3421	bzero(adapter->tx_desc_base,
3422	    (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc);
3423
3424	/* Free any existing TX buffers */
3425	for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
3426		tx_buffer = &adapter->tx_buffer_area[i];
3427		bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3428		    BUS_DMASYNC_POSTWRITE);
3429		bus_dmamap_unload(adapter->txtag, tx_buffer->map);
3430		m_freem(tx_buffer->m_head);
3431		tx_buffer->m_head = NULL;
3432		tx_buffer->next_eop = -1;
3433	}
3434
3435	/* Reset state */
3436	adapter->next_avail_tx_desc = 0;
3437	adapter->next_tx_to_clean = 0;
3438	adapter->num_tx_desc_avail = adapter->num_tx_desc;
3439
3440	bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3441	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3442
3443	return;
3444}
3445
3446/*********************************************************************
3447 *
3448 *  Enable transmit unit.
3449 *
3450 **********************************************************************/
3451static void
3452em_initialize_transmit_unit(struct adapter *adapter)
3453{
3454	u32	tctl, tarc, tipg = 0;
3455	u64	bus_addr;
3456
3457	 INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
3458	/* Setup the Base and Length of the Tx Descriptor Ring */
3459	bus_addr = adapter->txdma.dma_paddr;
3460	E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0),
3461	    adapter->num_tx_desc * sizeof(struct e1000_tx_desc));
3462	E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0),
3463	    (u32)(bus_addr >> 32));
3464	E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0),
3465	    (u32)bus_addr);
3466	/* Setup the HW Tx Head and Tail descriptor pointers */
3467	E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0);
3468	E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0);
3469
3470	HW_DEBUGOUT2("Base = %x, Length = %x\n",
3471	    E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)),
3472	    E1000_READ_REG(&adapter->hw, E1000_TDLEN(0)));
3473
3474	/* Set the default values for the Tx Inter Packet Gap timer */
3475	switch (adapter->hw.mac.type) {
3476	case e1000_82542:
3477		tipg = DEFAULT_82542_TIPG_IPGT;
3478		tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3479		tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3480		break;
3481	case e1000_80003es2lan:
3482		tipg = DEFAULT_82543_TIPG_IPGR1;
3483		tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
3484		    E1000_TIPG_IPGR2_SHIFT;
3485		break;
3486	default:
3487		if ((adapter->hw.phy.media_type == e1000_media_type_fiber) ||
3488		    (adapter->hw.phy.media_type ==
3489		    e1000_media_type_internal_serdes))
3490			tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
3491		else
3492			tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
3493		tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
3494		tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
3495	}
3496
3497	E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg);
3498	E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value);
3499	if(adapter->hw.mac.type >= e1000_82540)
3500		E1000_WRITE_REG(&adapter->hw, E1000_TADV,
3501		    adapter->tx_abs_int_delay.value);
3502
3503	if ((adapter->hw.mac.type == e1000_82571) ||
3504	    (adapter->hw.mac.type == e1000_82572)) {
3505		tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3506		tarc |= SPEED_MODE_BIT;
3507		E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3508	} else if (adapter->hw.mac.type == e1000_80003es2lan) {
3509		tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0));
3510		tarc |= 1;
3511		E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc);
3512		tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1));
3513		tarc |= 1;
3514		E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc);
3515	}
3516
3517	/* Program the Transmit Control Register */
3518	tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL);
3519	tctl &= ~E1000_TCTL_CT;
3520	tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN |
3521		   (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT));
3522
3523	if (adapter->hw.mac.type >= e1000_82571)
3524		tctl |= E1000_TCTL_MULR;
3525
3526	/* This write will effectively turn on the transmit unit. */
3527	E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl);
3528
3529	/* Setup Transmit Descriptor Base Settings */
3530	adapter->txd_cmd = E1000_TXD_CMD_IFCS;
3531
3532	if (adapter->tx_int_delay.value > 0)
3533		adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3534}
3535
3536/*********************************************************************
3537 *
3538 *  Free all transmit related data structures.
3539 *
3540 **********************************************************************/
3541static void
3542em_free_transmit_structures(struct adapter *adapter)
3543{
3544	struct em_buffer *tx_buffer;
3545
3546	INIT_DEBUGOUT("free_transmit_structures: begin");
3547
3548	if (adapter->tx_buffer_area != NULL) {
3549		for (int i = 0; i < adapter->num_tx_desc; i++) {
3550			tx_buffer = &adapter->tx_buffer_area[i];
3551			if (tx_buffer->m_head != NULL) {
3552				bus_dmamap_sync(adapter->txtag, tx_buffer->map,
3553				    BUS_DMASYNC_POSTWRITE);
3554				bus_dmamap_unload(adapter->txtag,
3555				    tx_buffer->map);
3556				m_freem(tx_buffer->m_head);
3557				tx_buffer->m_head = NULL;
3558			} else if (tx_buffer->map != NULL)
3559				bus_dmamap_unload(adapter->txtag,
3560				    tx_buffer->map);
3561			if (tx_buffer->map != NULL) {
3562				bus_dmamap_destroy(adapter->txtag,
3563				    tx_buffer->map);
3564				tx_buffer->map = NULL;
3565			}
3566		}
3567	}
3568	if (adapter->tx_buffer_area != NULL) {
3569		free(adapter->tx_buffer_area, M_DEVBUF);
3570		adapter->tx_buffer_area = NULL;
3571	}
3572	if (adapter->txtag != NULL) {
3573		bus_dma_tag_destroy(adapter->txtag);
3574		adapter->txtag = NULL;
3575	}
3576}
3577
3578/*********************************************************************
3579 *
3580 *  The offload context needs to be set when we transfer the first
3581 *  packet of a particular protocol (TCP/UDP). This routine has been
3582 *  enhanced to deal with inserted VLAN headers, and IPV6 (not complete)
3583 *
3584 **********************************************************************/
3585static void
3586em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
3587    u32 *txd_upper, u32 *txd_lower)
3588{
3589	struct e1000_context_desc *TXD;
3590	struct em_buffer *tx_buffer;
3591	struct ether_vlan_header *eh;
3592	struct ip *ip = NULL;
3593	struct ip6_hdr *ip6;
3594	struct tcp_hdr *th;
3595	int curr_txd, ehdrlen;
3596	u32 cmd, hdr_len, ip_hlen;
3597	u16 etype;
3598	u8 ipproto;
3599
3600	cmd = hdr_len = ipproto = 0;
3601	/* Setup checksum offload context. */
3602	curr_txd = adapter->next_avail_tx_desc;
3603	tx_buffer = &adapter->tx_buffer_area[curr_txd];
3604	TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd];
3605
3606	/*
3607	 * Determine where frame payload starts.
3608	 * Jump over vlan headers if already present,
3609	 * helpful for QinQ too.
3610	 */
3611	eh = mtod(mp, struct ether_vlan_header *);
3612	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3613		etype = ntohs(eh->evl_proto);
3614		ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3615	} else {
3616		etype = ntohs(eh->evl_encap_proto);
3617		ehdrlen = ETHER_HDR_LEN;
3618	}
3619
3620	/*
3621	 * We only support TCP/UDP for IPv4 and IPv6 for the moment.
3622	 * TODO: Support SCTP too when it hits the tree.
3623	 */
3624	switch (etype) {
3625	case ETHERTYPE_IP:
3626		ip = (struct ip *)(mp->m_data + ehdrlen);
3627		ip_hlen = ip->ip_hl << 2;
3628
3629		/* Setup of IP header checksum. */
3630		if (mp->m_pkthdr.csum_flags & CSUM_IP) {
3631			/*
3632			 * Start offset for header checksum calculation.
3633			 * End offset for header checksum calculation.
3634			 * Offset of place to put the checksum.
3635			 */
3636			TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3637			TXD->lower_setup.ip_fields.ipcse =
3638			    htole16(ehdrlen + ip_hlen);
3639			TXD->lower_setup.ip_fields.ipcso =
3640			    ehdrlen + offsetof(struct ip, ip_sum);
3641			cmd |= E1000_TXD_CMD_IP;
3642			*txd_upper |= E1000_TXD_POPTS_IXSM << 8;
3643		}
3644
3645		if (mp->m_len < ehdrlen + ip_hlen)
3646			return;	/* failure */
3647
3648		hdr_len = ehdrlen + ip_hlen;
3649		ipproto = ip->ip_p;
3650
3651		break;
3652	case ETHERTYPE_IPV6:
3653		ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3654		ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */
3655
3656		if (mp->m_len < ehdrlen + ip_hlen)
3657			return;	/* failure */
3658
3659		/* IPv6 doesn't have a header checksum. */
3660
3661		hdr_len = ehdrlen + ip_hlen;
3662		ipproto = ip6->ip6_nxt;
3663
3664		break;
3665#ifdef EM_TIMESYNC
3666	case ETHERTYPE_IEEE1588:
3667		*txd_upper |= E1000_TXD_EXTCMD_TSTAMP;
3668		break;
3669#endif
3670	default:
3671		*txd_upper = 0;
3672		*txd_lower = 0;
3673		return;
3674	}
3675
3676	switch (ipproto) {
3677	case IPPROTO_TCP:
3678		if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
3679			/*
3680			 * Start offset for payload checksum calculation.
3681			 * End offset for payload checksum calculation.
3682			 * Offset of place to put the checksum.
3683			 */
3684			th = (struct tcp_hdr *)(mp->m_data + hdr_len);
3685			TXD->upper_setup.tcp_fields.tucss = hdr_len;
3686			TXD->upper_setup.tcp_fields.tucse = htole16(0);
3687			TXD->upper_setup.tcp_fields.tucso =
3688			    hdr_len + offsetof(struct tcphdr, th_sum);
3689			cmd |= E1000_TXD_CMD_TCP;
3690			*txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3691		}
3692		break;
3693	case IPPROTO_UDP:
3694	{
3695#ifdef EM_TIMESYNC
3696		void *hdr = (caddr_t) ip + ip_hlen;
3697		struct udphdr *uh = (struct udphdr *)hdr;
3698
3699		if (uh->uh_dport == htons(TSYNC_PORT)) {
3700			*txd_upper |= E1000_TXD_EXTCMD_TSTAMP;
3701			IOCTL_DEBUGOUT("@@@ Sending Event Packet\n");
3702		}
3703#endif
3704		if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
3705			/*
3706			 * Start offset for header checksum calculation.
3707			 * End offset for header checksum calculation.
3708			 * Offset of place to put the checksum.
3709			 */
3710			TXD->upper_setup.tcp_fields.tucss = hdr_len;
3711			TXD->upper_setup.tcp_fields.tucse = htole16(0);
3712			TXD->upper_setup.tcp_fields.tucso =
3713			    hdr_len + offsetof(struct udphdr, uh_sum);
3714			*txd_upper |= E1000_TXD_POPTS_TXSM << 8;
3715		}
3716		/* Fall Thru */
3717	}
3718	default:
3719		break;
3720	}
3721
3722#ifdef EM_TIMESYNC
3723	/*
3724	** We might be here just for TIMESYNC
3725	** which means we don't need the context
3726	** descriptor.
3727	*/
3728	if (!mp->m_pkthdr.csum_flags & CSUM_OFFLOAD)
3729		return;
3730#endif
3731	*txd_lower = E1000_TXD_CMD_DEXT |	/* Extended descr type */
3732		     E1000_TXD_DTYP_D;		/* Data descr */
3733	TXD->tcp_seg_setup.data = htole32(0);
3734	TXD->cmd_and_length =
3735	    htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd);
3736	tx_buffer->m_head = NULL;
3737	tx_buffer->next_eop = -1;
3738
3739	if (++curr_txd == adapter->num_tx_desc)
3740		curr_txd = 0;
3741
3742	adapter->num_tx_desc_avail--;
3743	adapter->next_avail_tx_desc = curr_txd;
3744}
3745
3746
3747#if __FreeBSD_version >= 700000
3748/**********************************************************************
3749 *
3750 *  Setup work for hardware segmentation offload (TSO)
3751 *
3752 **********************************************************************/
3753static bool
3754em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper,
3755   u32 *txd_lower)
3756{
3757	struct e1000_context_desc *TXD;
3758	struct em_buffer *tx_buffer;
3759	struct ether_vlan_header *eh;
3760	struct ip *ip;
3761	struct ip6_hdr *ip6;
3762	struct tcphdr *th;
3763	int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6;
3764	u16 etype;
3765
3766	/*
3767	 * This function could/should be extended to support IP/IPv6
3768	 * fragmentation as well.  But as they say, one step at a time.
3769	 */
3770
3771	/*
3772	 * Determine where frame payload starts.
3773	 * Jump over vlan headers if already present,
3774	 * helpful for QinQ too.
3775	 */
3776	eh = mtod(mp, struct ether_vlan_header *);
3777	if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
3778		etype = ntohs(eh->evl_proto);
3779		ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
3780	} else {
3781		etype = ntohs(eh->evl_encap_proto);
3782		ehdrlen = ETHER_HDR_LEN;
3783	}
3784
3785	/* Ensure we have at least the IP+TCP header in the first mbuf. */
3786	if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr))
3787		return FALSE;	/* -1 */
3788
3789	/*
3790	 * We only support TCP for IPv4 and IPv6 (notyet) for the moment.
3791	 * TODO: Support SCTP too when it hits the tree.
3792	 */
3793	switch (etype) {
3794	case ETHERTYPE_IP:
3795		isip6 = 0;
3796		ip = (struct ip *)(mp->m_data + ehdrlen);
3797		if (ip->ip_p != IPPROTO_TCP)
3798			return FALSE;	/* 0 */
3799		ip->ip_len = 0;
3800		ip->ip_sum = 0;
3801		ip_hlen = ip->ip_hl << 2;
3802		if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3803			return FALSE;	/* -1 */
3804		th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
3805#if 1
3806		th->th_sum = in_pseudo(ip->ip_src.s_addr,
3807		    ip->ip_dst.s_addr, htons(IPPROTO_TCP));
3808#else
3809		th->th_sum = mp->m_pkthdr.csum_data;
3810#endif
3811		break;
3812	case ETHERTYPE_IPV6:
3813		isip6 = 1;
3814		return FALSE;			/* Not supported yet. */
3815		ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen);
3816		if (ip6->ip6_nxt != IPPROTO_TCP)
3817			return FALSE;	/* 0 */
3818		ip6->ip6_plen = 0;
3819		ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */
3820		if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr))
3821			return FALSE;	/* -1 */
3822		th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen);
3823#if 0
3824		th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst,
3825		    htons(IPPROTO_TCP));	/* XXX: function notyet. */
3826#else
3827		th->th_sum = mp->m_pkthdr.csum_data;
3828#endif
3829		break;
3830	default:
3831		return FALSE;
3832	}
3833	hdr_len = ehdrlen + ip_hlen + (th->th_off << 2);
3834
3835	*txd_lower = (E1000_TXD_CMD_DEXT |	/* Extended descr type */
3836		      E1000_TXD_DTYP_D |	/* Data descr type */
3837		      E1000_TXD_CMD_TSE);	/* Do TSE on this packet */
3838
3839	/* IP and/or TCP header checksum calculation and insertion. */
3840	*txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) |
3841		      E1000_TXD_POPTS_TXSM) << 8;
3842
3843	curr_txd = adapter->next_avail_tx_desc;
3844	tx_buffer = &adapter->tx_buffer_area[curr_txd];
3845	TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd];
3846
3847	/* IPv6 doesn't have a header checksum. */
3848	if (!isip6) {
3849		/*
3850		 * Start offset for header checksum calculation.
3851		 * End offset for header checksum calculation.
3852		 * Offset of place put the checksum.
3853		 */
3854		TXD->lower_setup.ip_fields.ipcss = ehdrlen;
3855		TXD->lower_setup.ip_fields.ipcse =
3856		    htole16(ehdrlen + ip_hlen - 1);
3857		TXD->lower_setup.ip_fields.ipcso =
3858		    ehdrlen + offsetof(struct ip, ip_sum);
3859	}
3860	/*
3861	 * Start offset for payload checksum calculation.
3862	 * End offset for payload checksum calculation.
3863	 * Offset of place to put the checksum.
3864	 */
3865	TXD->upper_setup.tcp_fields.tucss =
3866	    ehdrlen + ip_hlen;
3867	TXD->upper_setup.tcp_fields.tucse = 0;
3868	TXD->upper_setup.tcp_fields.tucso =
3869	    ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum);
3870	/*
3871	 * Payload size per packet w/o any headers.
3872	 * Length of all headers up to payload.
3873	 */
3874	TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
3875	TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
3876
3877	TXD->cmd_and_length = htole32(adapter->txd_cmd |
3878				E1000_TXD_CMD_DEXT |	/* Extended descr */
3879				E1000_TXD_CMD_TSE |	/* TSE context */
3880				(isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */
3881				E1000_TXD_CMD_TCP |	/* Do TCP checksum */
3882				(mp->m_pkthdr.len - (hdr_len))); /* Total len */
3883
3884	tx_buffer->m_head = NULL;
3885	tx_buffer->next_eop = -1;
3886
3887	if (++curr_txd == adapter->num_tx_desc)
3888		curr_txd = 0;
3889
3890	adapter->num_tx_desc_avail--;
3891	adapter->next_avail_tx_desc = curr_txd;
3892	adapter->tx_tso = TRUE;
3893
3894	return TRUE;
3895}
3896
3897#endif /* __FreeBSD_version >= 700000 */
3898
3899/**********************************************************************
3900 *
3901 *  Examine each tx_buffer in the used queue. If the hardware is done
3902 *  processing the packet then free associated resources. The
3903 *  tx_buffer is put back on the free queue.
3904 *
3905 **********************************************************************/
3906static void
3907em_txeof(struct adapter *adapter)
3908{
3909        int first, last, done, num_avail;
3910        struct em_buffer *tx_buffer;
3911        struct e1000_tx_desc   *tx_desc, *eop_desc;
3912	struct ifnet   *ifp = adapter->ifp;
3913
3914	EM_TX_LOCK_ASSERT(adapter);
3915
3916        if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
3917                return;
3918
3919        num_avail = adapter->num_tx_desc_avail;
3920        first = adapter->next_tx_to_clean;
3921        tx_desc = &adapter->tx_desc_base[first];
3922        tx_buffer = &adapter->tx_buffer_area[first];
3923	last = tx_buffer->next_eop;
3924        eop_desc = &adapter->tx_desc_base[last];
3925
3926	/*
3927	 * What this does is get the index of the
3928	 * first descriptor AFTER the EOP of the
3929	 * first packet, that way we can do the
3930	 * simple comparison on the inner while loop.
3931	 */
3932	if (++last == adapter->num_tx_desc)
3933 		last = 0;
3934	done = last;
3935
3936        bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3937            BUS_DMASYNC_POSTREAD);
3938
3939        while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
3940		/* We clean the range of the packet */
3941		while (first != done) {
3942                	tx_desc->upper.data = 0;
3943                	tx_desc->lower.data = 0;
3944                	tx_desc->buffer_addr = 0;
3945                	num_avail++;
3946
3947			if (tx_buffer->m_head) {
3948				ifp->if_opackets++;
3949				bus_dmamap_sync(adapter->txtag,
3950				    tx_buffer->map,
3951				    BUS_DMASYNC_POSTWRITE);
3952				bus_dmamap_unload(adapter->txtag,
3953				    tx_buffer->map);
3954
3955                        	m_freem(tx_buffer->m_head);
3956                        	tx_buffer->m_head = NULL;
3957                	}
3958			tx_buffer->next_eop = -1;
3959
3960	                if (++first == adapter->num_tx_desc)
3961				first = 0;
3962
3963	                tx_buffer = &adapter->tx_buffer_area[first];
3964			tx_desc = &adapter->tx_desc_base[first];
3965		}
3966		/* See if we can continue to the next packet */
3967		last = tx_buffer->next_eop;
3968		if (last != -1) {
3969        		eop_desc = &adapter->tx_desc_base[last];
3970			/* Get new done point */
3971			if (++last == adapter->num_tx_desc) last = 0;
3972			done = last;
3973		} else
3974			break;
3975        }
3976        bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
3977            BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3978
3979        adapter->next_tx_to_clean = first;
3980
3981        /*
3982         * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack
3983         * that it is OK to send packets.
3984         * If there are no pending descriptors, clear the timeout. Otherwise,
3985         * if some descriptors have been freed, restart the timeout.
3986         */
3987        if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
3988                ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
3989		/* All clean, turn off the timer */
3990                if (num_avail == adapter->num_tx_desc) {
3991			adapter->watchdog_timer = 0;
3992		} else
3993		/* Some cleaned, reset the timer */
3994                if (num_avail != adapter->num_tx_desc_avail)
3995			adapter->watchdog_timer = EM_TX_TIMEOUT;
3996        }
3997        adapter->num_tx_desc_avail = num_avail;
3998	return;
3999}
4000
4001/*********************************************************************
4002 *
4003 *  When Link is lost sometimes there is work still in the TX ring
4004 *  which will result in a watchdog, rather than allow that do an
4005 *  attempted cleanup and then reinit here. Note that this has been
4006 *  seens mostly with fiber adapters.
4007 *
4008 **********************************************************************/
4009static void
4010em_tx_purge(struct adapter *adapter)
4011{
4012	if ((!adapter->link_active) && (adapter->watchdog_timer)) {
4013		EM_TX_LOCK(adapter);
4014		em_txeof(adapter);
4015		EM_TX_UNLOCK(adapter);
4016		if (adapter->watchdog_timer) { /* Still not clean? */
4017			adapter->watchdog_timer = 0;
4018			em_init_locked(adapter);
4019		}
4020	}
4021}
4022
4023/*********************************************************************
4024 *
4025 *  Get a buffer from system mbuf buffer pool.
4026 *
4027 **********************************************************************/
4028static int
4029em_get_buf(struct adapter *adapter, int i)
4030{
4031	struct mbuf		*m;
4032	bus_dma_segment_t	segs[1];
4033	bus_dmamap_t		map;
4034	struct em_buffer	*rx_buffer;
4035	int			error, nsegs;
4036
4037	m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
4038	if (m == NULL) {
4039		adapter->mbuf_cluster_failed++;
4040		return (ENOBUFS);
4041	}
4042	m->m_len = m->m_pkthdr.len = MCLBYTES;
4043
4044	if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN))
4045		m_adj(m, ETHER_ALIGN);
4046
4047	/*
4048	 * Using memory from the mbuf cluster pool, invoke the
4049	 * bus_dma machinery to arrange the memory mapping.
4050	 */
4051	error = bus_dmamap_load_mbuf_sg(adapter->rxtag,
4052	    adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT);
4053	if (error != 0) {
4054		m_free(m);
4055		return (error);
4056	}
4057
4058	/* If nsegs is wrong then the stack is corrupt. */
4059	KASSERT(nsegs == 1, ("Too many segments returned!"));
4060
4061	rx_buffer = &adapter->rx_buffer_area[i];
4062	if (rx_buffer->m_head != NULL)
4063		bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
4064
4065	map = rx_buffer->map;
4066	rx_buffer->map = adapter->rx_sparemap;
4067	adapter->rx_sparemap = map;
4068	bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
4069	rx_buffer->m_head = m;
4070
4071	adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr);
4072	return (0);
4073}
4074
4075/*********************************************************************
4076 *
4077 *  Allocate memory for rx_buffer structures. Since we use one
4078 *  rx_buffer per received packet, the maximum number of rx_buffer's
4079 *  that we'll need is equal to the number of receive descriptors
4080 *  that we've allocated.
4081 *
4082 **********************************************************************/
4083static int
4084em_allocate_receive_structures(struct adapter *adapter)
4085{
4086	device_t dev = adapter->dev;
4087	struct em_buffer *rx_buffer;
4088	int i, error;
4089
4090	adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) *
4091	    adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
4092	if (adapter->rx_buffer_area == NULL) {
4093		device_printf(dev, "Unable to allocate rx_buffer memory\n");
4094		return (ENOMEM);
4095	}
4096
4097#if __FreeBSD_version >= 700000
4098	error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
4099#else
4100	error = bus_dma_tag_create(NULL,		 /* parent */
4101#endif
4102				1, 0,			/* alignment, bounds */
4103				BUS_SPACE_MAXADDR,	/* lowaddr */
4104				BUS_SPACE_MAXADDR,	/* highaddr */
4105				NULL, NULL,		/* filter, filterarg */
4106				MCLBYTES,		/* maxsize */
4107				1,			/* nsegments */
4108				MCLBYTES,		/* maxsegsize */
4109				0,			/* flags */
4110				NULL,			/* lockfunc */
4111				NULL,			/* lockarg */
4112				&adapter->rxtag);
4113	if (error) {
4114		device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
4115		    __func__, error);
4116		goto fail;
4117	}
4118
4119	/* Create the spare map (used by getbuf) */
4120	error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
4121	     &adapter->rx_sparemap);
4122	if (error) {
4123		device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4124		    __func__, error);
4125		goto fail;
4126	}
4127
4128	rx_buffer = adapter->rx_buffer_area;
4129	for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4130		error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
4131		    &rx_buffer->map);
4132		if (error) {
4133			device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
4134			    __func__, error);
4135			goto fail;
4136		}
4137	}
4138
4139	return (0);
4140
4141fail:
4142	em_free_receive_structures(adapter);
4143	return (error);
4144}
4145
4146/*********************************************************************
4147 *
4148 *  (Re)initialize receive structures.
4149 *
4150 **********************************************************************/
4151static int
4152em_setup_receive_structures(struct adapter *adapter)
4153{
4154	struct em_buffer *rx_buffer;
4155	int i, error;
4156
4157	/* Reset descriptor ring */
4158	bzero(adapter->rx_desc_base,
4159	    (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc);
4160
4161	/* Free current RX buffers. */
4162	rx_buffer = adapter->rx_buffer_area;
4163	for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4164		if (rx_buffer->m_head != NULL) {
4165			bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
4166			    BUS_DMASYNC_POSTREAD);
4167			bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
4168			m_freem(rx_buffer->m_head);
4169			rx_buffer->m_head = NULL;
4170		}
4171        }
4172
4173	/* Allocate new ones. */
4174	for (i = 0; i < adapter->num_rx_desc; i++) {
4175		error = em_get_buf(adapter, i);
4176		if (error)
4177                        return (error);
4178	}
4179
4180	/* Setup our descriptor pointers */
4181	adapter->next_rx_desc_to_check = 0;
4182	bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4183	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4184
4185	return (0);
4186}
4187
4188/*********************************************************************
4189 *
4190 *  Enable receive unit.
4191 *
4192 **********************************************************************/
4193#define MAX_INTS_PER_SEC	8000
4194#define DEFAULT_ITR	     1000000000/(MAX_INTS_PER_SEC * 256)
4195
4196static void
4197em_initialize_receive_unit(struct adapter *adapter)
4198{
4199	struct ifnet	*ifp = adapter->ifp;
4200	u64	bus_addr;
4201	u32	rctl, rxcsum;
4202
4203	INIT_DEBUGOUT("em_initialize_receive_unit: begin");
4204
4205	/*
4206	 * Make sure receives are disabled while setting
4207	 * up the descriptor ring
4208	 */
4209	rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4210	E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN);
4211
4212	if (adapter->hw.mac.type >= e1000_82540) {
4213		E1000_WRITE_REG(&adapter->hw, E1000_RADV,
4214		    adapter->rx_abs_int_delay.value);
4215		/*
4216		 * Set the interrupt throttling rate. Value is calculated
4217		 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
4218		 */
4219		E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR);
4220	}
4221
4222	/*
4223	** When using MSIX interrupts we need to throttle
4224	** using the EITR register (82574 only)
4225	*/
4226	if (adapter->msix)
4227		for (int i = 0; i < 4; i++)
4228			E1000_WRITE_REG(&adapter->hw,
4229			    E1000_EITR_82574(i), DEFAULT_ITR);
4230
4231	/* Disable accelerated ackknowledge */
4232	if (adapter->hw.mac.type == e1000_82574)
4233		E1000_WRITE_REG(&adapter->hw,
4234		    E1000_RFCTL, E1000_RFCTL_ACK_DIS);
4235
4236	/* Setup the Base and Length of the Rx Descriptor Ring */
4237	bus_addr = adapter->rxdma.dma_paddr;
4238	E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0),
4239	    adapter->num_rx_desc * sizeof(struct e1000_rx_desc));
4240	E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0),
4241	    (u32)(bus_addr >> 32));
4242	E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0),
4243	    (u32)bus_addr);
4244
4245	/* Setup the Receive Control Register */
4246	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4247	rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
4248		   E1000_RCTL_RDMTS_HALF |
4249		   (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4250
4251	/* Make sure VLAN Filters are off */
4252	rctl &= ~E1000_RCTL_VFE;
4253
4254	if (e1000_tbi_sbp_enabled_82543(&adapter->hw))
4255		rctl |= E1000_RCTL_SBP;
4256	else
4257		rctl &= ~E1000_RCTL_SBP;
4258
4259	switch (adapter->rx_buffer_len) {
4260	default:
4261	case 2048:
4262		rctl |= E1000_RCTL_SZ_2048;
4263		break;
4264	case 4096:
4265		rctl |= E1000_RCTL_SZ_4096 |
4266		    E1000_RCTL_BSEX | E1000_RCTL_LPE;
4267		break;
4268	case 8192:
4269		rctl |= E1000_RCTL_SZ_8192 |
4270		    E1000_RCTL_BSEX | E1000_RCTL_LPE;
4271		break;
4272	case 16384:
4273		rctl |= E1000_RCTL_SZ_16384 |
4274		    E1000_RCTL_BSEX | E1000_RCTL_LPE;
4275		break;
4276	}
4277
4278	if (ifp->if_mtu > ETHERMTU)
4279		rctl |= E1000_RCTL_LPE;
4280	else
4281		rctl &= ~E1000_RCTL_LPE;
4282
4283	/* Enable 82543 Receive Checksum Offload for TCP and UDP */
4284	if ((adapter->hw.mac.type >= e1000_82543) &&
4285	    (ifp->if_capenable & IFCAP_RXCSUM)) {
4286		rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM);
4287		rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
4288		E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum);
4289	}
4290
4291	/*
4292	** XXX TEMPORARY WORKAROUND: on some systems with 82573
4293	** long latencies are observed, like Lenovo X60. This
4294	** change eliminates the problem, but since having positive
4295	** values in RDTR is a known source of problems on other
4296	** platforms another solution is being sought.
4297	*/
4298	if (adapter->hw.mac.type == e1000_82573)
4299		E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20);
4300
4301	/* Enable Receives */
4302	E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4303
4304	/*
4305	 * Setup the HW Rx Head and
4306	 * Tail Descriptor Pointers
4307	 */
4308	E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0);
4309	E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1);
4310
4311	return;
4312}
4313
4314/*********************************************************************
4315 *
4316 *  Free receive related data structures.
4317 *
4318 **********************************************************************/
4319static void
4320em_free_receive_structures(struct adapter *adapter)
4321{
4322	struct em_buffer *rx_buffer;
4323	int i;
4324
4325	INIT_DEBUGOUT("free_receive_structures: begin");
4326
4327	if (adapter->rx_sparemap) {
4328		bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
4329		adapter->rx_sparemap = NULL;
4330	}
4331
4332	/* Cleanup any existing buffers */
4333	if (adapter->rx_buffer_area != NULL) {
4334		rx_buffer = adapter->rx_buffer_area;
4335		for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
4336			if (rx_buffer->m_head != NULL) {
4337				bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
4338				    BUS_DMASYNC_POSTREAD);
4339				bus_dmamap_unload(adapter->rxtag,
4340				    rx_buffer->map);
4341				m_freem(rx_buffer->m_head);
4342				rx_buffer->m_head = NULL;
4343			} else if (rx_buffer->map != NULL)
4344				bus_dmamap_unload(adapter->rxtag,
4345				    rx_buffer->map);
4346			if (rx_buffer->map != NULL) {
4347				bus_dmamap_destroy(adapter->rxtag,
4348				    rx_buffer->map);
4349				rx_buffer->map = NULL;
4350			}
4351		}
4352	}
4353
4354	if (adapter->rx_buffer_area != NULL) {
4355		free(adapter->rx_buffer_area, M_DEVBUF);
4356		adapter->rx_buffer_area = NULL;
4357	}
4358
4359	if (adapter->rxtag != NULL) {
4360		bus_dma_tag_destroy(adapter->rxtag);
4361		adapter->rxtag = NULL;
4362	}
4363}
4364
4365/*********************************************************************
4366 *
4367 *  This routine executes in interrupt context. It replenishes
4368 *  the mbufs in the descriptor and sends data which has been
4369 *  dma'ed into host memory to upper layer.
4370 *
4371 *  We loop at most count times if count is > 0, or until done if
4372 *  count < 0.
4373 *
4374 *********************************************************************/
4375static int
4376em_rxeof(struct adapter *adapter, int count)
4377{
4378	struct ifnet	*ifp = adapter->ifp;;
4379	struct mbuf	*mp;
4380	u8		status, accept_frame = 0, eop = 0;
4381	u16 		len, desc_len, prev_len_adj;
4382	int		i;
4383	struct e1000_rx_desc   *current_desc;
4384
4385	EM_RX_LOCK(adapter);
4386	i = adapter->next_rx_desc_to_check;
4387	current_desc = &adapter->rx_desc_base[i];
4388	bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4389	    BUS_DMASYNC_POSTREAD);
4390
4391	if (!((current_desc->status) & E1000_RXD_STAT_DD)) {
4392		EM_RX_UNLOCK(adapter);
4393		return (0);
4394	}
4395
4396	while ((current_desc->status & E1000_RXD_STAT_DD) &&
4397	    (count != 0) &&
4398	    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
4399		struct mbuf *m = NULL;
4400
4401		mp = adapter->rx_buffer_area[i].m_head;
4402		/*
4403		 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
4404		 * needs to access the last received byte in the mbuf.
4405		 */
4406		bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
4407		    BUS_DMASYNC_POSTREAD);
4408
4409		accept_frame = 1;
4410		prev_len_adj = 0;
4411		desc_len = le16toh(current_desc->length);
4412		status = current_desc->status;
4413		if (status & E1000_RXD_STAT_EOP) {
4414			count--;
4415			eop = 1;
4416			if (desc_len < ETHER_CRC_LEN) {
4417				len = 0;
4418				prev_len_adj = ETHER_CRC_LEN - desc_len;
4419			} else
4420				len = desc_len - ETHER_CRC_LEN;
4421		} else {
4422			eop = 0;
4423			len = desc_len;
4424		}
4425
4426		if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
4427			u8	last_byte;
4428			u32	pkt_len = desc_len;
4429
4430			if (adapter->fmp != NULL)
4431				pkt_len += adapter->fmp->m_pkthdr.len;
4432
4433			last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
4434			if (TBI_ACCEPT(&adapter->hw, status,
4435			    current_desc->errors, pkt_len, last_byte,
4436			    adapter->min_frame_size, adapter->max_frame_size)) {
4437				e1000_tbi_adjust_stats_82543(&adapter->hw,
4438				    &adapter->stats, pkt_len,
4439				    adapter->hw.mac.addr,
4440				    adapter->max_frame_size);
4441				if (len > 0)
4442					len--;
4443			} else
4444				accept_frame = 0;
4445		}
4446
4447		if (accept_frame) {
4448			if (em_get_buf(adapter, i) != 0) {
4449				ifp->if_iqdrops++;
4450				goto discard;
4451			}
4452
4453			/* Assign correct length to the current fragment */
4454			mp->m_len = len;
4455
4456			if (adapter->fmp == NULL) {
4457				mp->m_pkthdr.len = len;
4458				adapter->fmp = mp; /* Store the first mbuf */
4459				adapter->lmp = mp;
4460			} else {
4461				/* Chain mbuf's together */
4462				mp->m_flags &= ~M_PKTHDR;
4463				/*
4464				 * Adjust length of previous mbuf in chain if
4465				 * we received less than 4 bytes in the last
4466				 * descriptor.
4467				 */
4468				if (prev_len_adj > 0) {
4469					adapter->lmp->m_len -= prev_len_adj;
4470					adapter->fmp->m_pkthdr.len -=
4471					    prev_len_adj;
4472				}
4473				adapter->lmp->m_next = mp;
4474				adapter->lmp = adapter->lmp->m_next;
4475				adapter->fmp->m_pkthdr.len += len;
4476			}
4477
4478			if (eop) {
4479				adapter->fmp->m_pkthdr.rcvif = ifp;
4480				ifp->if_ipackets++;
4481				em_receive_checksum(adapter, current_desc,
4482				    adapter->fmp);
4483#ifndef __NO_STRICT_ALIGNMENT
4484				if (adapter->max_frame_size >
4485				    (MCLBYTES - ETHER_ALIGN) &&
4486				    em_fixup_rx(adapter) != 0)
4487					goto skip;
4488#endif
4489				if (status & E1000_RXD_STAT_VP) {
4490#if __FreeBSD_version < 700000
4491					VLAN_INPUT_TAG_NEW(ifp, adapter->fmp,
4492					    (le16toh(current_desc->special) &
4493					    E1000_RXD_SPC_VLAN_MASK));
4494#else
4495					adapter->fmp->m_pkthdr.ether_vtag =
4496					    (le16toh(current_desc->special) &
4497					    E1000_RXD_SPC_VLAN_MASK);
4498					adapter->fmp->m_flags |= M_VLANTAG;
4499#endif
4500				}
4501#ifndef __NO_STRICT_ALIGNMENT
4502skip:
4503#endif
4504				m = adapter->fmp;
4505				adapter->fmp = NULL;
4506				adapter->lmp = NULL;
4507			}
4508		} else {
4509			ifp->if_ierrors++;
4510discard:
4511			/* Reuse loaded DMA map and just update mbuf chain */
4512			mp = adapter->rx_buffer_area[i].m_head;
4513			mp->m_len = mp->m_pkthdr.len = MCLBYTES;
4514			mp->m_data = mp->m_ext.ext_buf;
4515			mp->m_next = NULL;
4516			if (adapter->max_frame_size <=
4517			    (MCLBYTES - ETHER_ALIGN))
4518				m_adj(mp, ETHER_ALIGN);
4519			if (adapter->fmp != NULL) {
4520				m_freem(adapter->fmp);
4521				adapter->fmp = NULL;
4522				adapter->lmp = NULL;
4523			}
4524			m = NULL;
4525		}
4526
4527		/* Zero out the receive descriptors status. */
4528		current_desc->status = 0;
4529		bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
4530		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
4531
4532		/* Advance our pointers to the next descriptor. */
4533		if (++i == adapter->num_rx_desc)
4534			i = 0;
4535		if (m != NULL) {
4536			adapter->next_rx_desc_to_check = i;
4537			/* Unlock for call into stack */
4538			EM_RX_UNLOCK(adapter);
4539			(*ifp->if_input)(ifp, m);
4540			EM_RX_LOCK(adapter);
4541			i = adapter->next_rx_desc_to_check;
4542		}
4543		current_desc = &adapter->rx_desc_base[i];
4544	}
4545	adapter->next_rx_desc_to_check = i;
4546
4547	/* Advance the E1000's Receive Queue #0  "Tail Pointer". */
4548	if (--i < 0)
4549		i = adapter->num_rx_desc - 1;
4550	E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i);
4551	EM_RX_UNLOCK(adapter);
4552	if (!((current_desc->status) & E1000_RXD_STAT_DD))
4553		return (0);
4554
4555	return (1);
4556}
4557
4558#ifndef __NO_STRICT_ALIGNMENT
4559/*
4560 * When jumbo frames are enabled we should realign entire payload on
4561 * architecures with strict alignment. This is serious design mistake of 8254x
4562 * as it nullifies DMA operations. 8254x just allows RX buffer size to be
4563 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
4564 * payload. On architecures without strict alignment restrictions 8254x still
4565 * performs unaligned memory access which would reduce the performance too.
4566 * To avoid copying over an entire frame to align, we allocate a new mbuf and
4567 * copy ethernet header to the new mbuf. The new mbuf is prepended into the
4568 * existing mbuf chain.
4569 *
4570 * Be aware, best performance of the 8254x is achived only when jumbo frame is
4571 * not used at all on architectures with strict alignment.
4572 */
4573static int
4574em_fixup_rx(struct adapter *adapter)
4575{
4576	struct mbuf *m, *n;
4577	int error;
4578
4579	error = 0;
4580	m = adapter->fmp;
4581	if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
4582		bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
4583		m->m_data += ETHER_HDR_LEN;
4584	} else {
4585		MGETHDR(n, M_DONTWAIT, MT_DATA);
4586		if (n != NULL) {
4587			bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
4588			m->m_data += ETHER_HDR_LEN;
4589			m->m_len -= ETHER_HDR_LEN;
4590			n->m_len = ETHER_HDR_LEN;
4591			M_MOVE_PKTHDR(n, m);
4592			n->m_next = m;
4593			adapter->fmp = n;
4594		} else {
4595			adapter->dropped_pkts++;
4596			m_freem(adapter->fmp);
4597			adapter->fmp = NULL;
4598			error = ENOMEM;
4599		}
4600	}
4601
4602	return (error);
4603}
4604#endif
4605
4606/*********************************************************************
4607 *
4608 *  Verify that the hardware indicated that the checksum is valid.
4609 *  Inform the stack about the status of checksum so that stack
4610 *  doesn't spend time verifying the checksum.
4611 *
4612 *********************************************************************/
4613static void
4614em_receive_checksum(struct adapter *adapter,
4615	    struct e1000_rx_desc *rx_desc, struct mbuf *mp)
4616{
4617	/* 82543 or newer only */
4618	if ((adapter->hw.mac.type < e1000_82543) ||
4619	    /* Ignore Checksum bit is set */
4620	    (rx_desc->status & E1000_RXD_STAT_IXSM)) {
4621		mp->m_pkthdr.csum_flags = 0;
4622		return;
4623	}
4624
4625	if (rx_desc->status & E1000_RXD_STAT_IPCS) {
4626		/* Did it pass? */
4627		if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
4628			/* IP Checksum Good */
4629			mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
4630			mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
4631
4632		} else {
4633			mp->m_pkthdr.csum_flags = 0;
4634		}
4635	}
4636
4637	if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
4638		/* Did it pass? */
4639		if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
4640			mp->m_pkthdr.csum_flags |=
4641			(CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
4642			mp->m_pkthdr.csum_data = htons(0xffff);
4643		}
4644	}
4645}
4646
4647/*
4648 * This routine is run via an vlan
4649 * config EVENT
4650 */
4651static void
4652em_register_vlan(void *unused, struct ifnet *ifp, u16 vtag)
4653{
4654	struct adapter	*adapter = ifp->if_softc;
4655	u32		ctrl, rctl, index, vfta;
4656
4657	ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL);
4658	ctrl |= E1000_CTRL_VME;
4659	E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl);
4660
4661	/* Setup for Hardware Filter */
4662	rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4663	rctl |= E1000_RCTL_VFE;
4664	rctl &= ~E1000_RCTL_CFIEN;
4665	E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4666
4667	/* Make entry in the hardware filter table */
4668	index = ((vtag >> 5) & 0x7F);
4669	vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index);
4670	vfta |= (1 << (vtag & 0x1F));
4671	E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta);
4672
4673	/* Update the frame size */
4674	E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4675	    adapter->max_frame_size + VLAN_TAG_SIZE);
4676
4677}
4678
4679/*
4680 * This routine is run via an vlan
4681 * unconfig EVENT
4682 */
4683static void
4684em_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag)
4685{
4686	struct adapter	*adapter = ifp->if_softc;
4687	u32		index, vfta;
4688
4689	/* Remove entry in the hardware filter table */
4690	index = ((vtag >> 5) & 0x7F);
4691	vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index);
4692	vfta &= ~(1 << (vtag & 0x1F));
4693	E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta);
4694	/* Have all vlans unregistered? */
4695	if (adapter->ifp->if_vlantrunk == NULL) {
4696		u32 rctl;
4697		/* Turn off the filter table */
4698		rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL);
4699		rctl &= ~E1000_RCTL_VFE;
4700		rctl |= E1000_RCTL_CFIEN;
4701		E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl);
4702		/* Reset the frame size */
4703		E1000_WRITE_REG(&adapter->hw, E1000_RLPML,
4704		    adapter->max_frame_size);
4705	}
4706}
4707
4708static void
4709em_enable_intr(struct adapter *adapter)
4710{
4711	struct e1000_hw *hw = &adapter->hw;
4712	u32 ims_mask = IMS_ENABLE_MASK;
4713
4714	if (adapter->msix) {
4715		E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK);
4716		ims_mask |= EM_MSIX_MASK;
4717	}
4718	E1000_WRITE_REG(hw, E1000_IMS, ims_mask);
4719}
4720
4721static void
4722em_disable_intr(struct adapter *adapter)
4723{
4724	struct e1000_hw *hw = &adapter->hw;
4725
4726	if (adapter->msix)
4727		E1000_WRITE_REG(hw, EM_EIAC, 0);
4728	E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff);
4729}
4730
4731/*
4732 * Bit of a misnomer, what this really means is
4733 * to enable OS management of the system... aka
4734 * to disable special hardware management features
4735 */
4736static void
4737em_init_manageability(struct adapter *adapter)
4738{
4739	/* A shared code workaround */
4740#define E1000_82542_MANC2H E1000_MANC2H
4741	if (adapter->has_manage) {
4742		int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H);
4743		int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4744
4745		/* disable hardware interception of ARP */
4746		manc &= ~(E1000_MANC_ARP_EN);
4747
4748                /* enable receiving management packets to the host */
4749                if (adapter->hw.mac.type >= e1000_82571) {
4750			manc |= E1000_MANC_EN_MNG2HOST;
4751#define E1000_MNG2HOST_PORT_623 (1 << 5)
4752#define E1000_MNG2HOST_PORT_664 (1 << 6)
4753			manc2h |= E1000_MNG2HOST_PORT_623;
4754			manc2h |= E1000_MNG2HOST_PORT_664;
4755			E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h);
4756		}
4757
4758		E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4759	}
4760}
4761
4762/*
4763 * Give control back to hardware management
4764 * controller if there is one.
4765 */
4766static void
4767em_release_manageability(struct adapter *adapter)
4768{
4769	if (adapter->has_manage) {
4770		int manc = E1000_READ_REG(&adapter->hw, E1000_MANC);
4771
4772		/* re-enable hardware interception of ARP */
4773		manc |= E1000_MANC_ARP_EN;
4774
4775		if (adapter->hw.mac.type >= e1000_82571)
4776			manc &= ~E1000_MANC_EN_MNG2HOST;
4777
4778		E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc);
4779	}
4780}
4781
4782/*
4783 * em_get_hw_control sets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4784 * For ASF and Pass Through versions of f/w this means that
4785 * the driver is loaded. For AMT version (only with 82573)
4786 * of the f/w this means that the network i/f is open.
4787 *
4788 */
4789static void
4790em_get_hw_control(struct adapter *adapter)
4791{
4792	u32 ctrl_ext, swsm;
4793
4794	/* Let firmware know the driver has taken over */
4795	switch (adapter->hw.mac.type) {
4796	case e1000_82573:
4797		swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4798		E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4799		    swsm | E1000_SWSM_DRV_LOAD);
4800		break;
4801	case e1000_82571:
4802	case e1000_82572:
4803	case e1000_80003es2lan:
4804	case e1000_ich8lan:
4805	case e1000_ich9lan:
4806	case e1000_ich10lan:
4807		ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4808		E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4809		    ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
4810		break;
4811	default:
4812		break;
4813	}
4814}
4815
4816/*
4817 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit.
4818 * For ASF and Pass Through versions of f/w this means that the
4819 * driver is no longer loaded. For AMT version (only with 82573) i
4820 * of the f/w this means that the network i/f is closed.
4821 *
4822 */
4823static void
4824em_release_hw_control(struct adapter *adapter)
4825{
4826	u32 ctrl_ext, swsm;
4827
4828	/* Let firmware taken over control of h/w */
4829	switch (adapter->hw.mac.type) {
4830	case e1000_82573:
4831		swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM);
4832		E1000_WRITE_REG(&adapter->hw, E1000_SWSM,
4833		    swsm & ~E1000_SWSM_DRV_LOAD);
4834		break;
4835	case e1000_82571:
4836	case e1000_82572:
4837	case e1000_80003es2lan:
4838	case e1000_ich8lan:
4839	case e1000_ich9lan:
4840	case e1000_ich10lan:
4841		ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT);
4842		E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT,
4843		    ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
4844		break;
4845	default:
4846		break;
4847
4848	}
4849}
4850
4851static int
4852em_is_valid_ether_addr(u8 *addr)
4853{
4854	char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
4855
4856	if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
4857		return (FALSE);
4858	}
4859
4860	return (TRUE);
4861}
4862
4863/*
4864 * Enable PCI Wake On Lan capability
4865 */
4866void
4867em_enable_wakeup(device_t dev)
4868{
4869	u16     cap, status;
4870	u8      id;
4871
4872	/* First find the capabilities pointer*/
4873	cap = pci_read_config(dev, PCIR_CAP_PTR, 2);
4874	/* Read the PM Capabilities */
4875	id = pci_read_config(dev, cap, 1);
4876	if (id != PCIY_PMG)     /* Something wrong */
4877		return;
4878	/* OK, we have the power capabilities, so
4879	   now get the status register */
4880	cap += PCIR_POWER_STATUS;
4881	status = pci_read_config(dev, cap, 2);
4882	status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE;
4883	pci_write_config(dev, cap, status, 2);
4884	return;
4885}
4886
4887
4888/*********************************************************************
4889* 82544 Coexistence issue workaround.
4890*    There are 2 issues.
4891*       1. Transmit Hang issue.
4892*    To detect this issue, following equation can be used...
4893*	  SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4894*	  If SUM[3:0] is in between 1 to 4, we will have this issue.
4895*
4896*       2. DAC issue.
4897*    To detect this issue, following equation can be used...
4898*	  SIZE[3:0] + ADDR[2:0] = SUM[3:0].
4899*	  If SUM[3:0] is in between 9 to c, we will have this issue.
4900*
4901*
4902*    WORKAROUND:
4903*	  Make sure we do not have ending address
4904*	  as 1,2,3,4(Hang) or 9,a,b,c (DAC)
4905*
4906*************************************************************************/
4907static u32
4908em_fill_descriptors (bus_addr_t address, u32 length,
4909		PDESC_ARRAY desc_array)
4910{
4911	u32 safe_terminator;
4912
4913	/* Since issue is sensitive to length and address.*/
4914	/* Let us first check the address...*/
4915	if (length <= 4) {
4916		desc_array->descriptor[0].address = address;
4917		desc_array->descriptor[0].length = length;
4918		desc_array->elements = 1;
4919		return (desc_array->elements);
4920	}
4921	safe_terminator = (u32)((((u32)address & 0x7) +
4922	    (length & 0xF)) & 0xF);
4923	/* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
4924	if (safe_terminator == 0   ||
4925	(safe_terminator > 4   &&
4926	safe_terminator < 9)   ||
4927	(safe_terminator > 0xC &&
4928	safe_terminator <= 0xF)) {
4929		desc_array->descriptor[0].address = address;
4930		desc_array->descriptor[0].length = length;
4931		desc_array->elements = 1;
4932		return (desc_array->elements);
4933	}
4934
4935	desc_array->descriptor[0].address = address;
4936	desc_array->descriptor[0].length = length - 4;
4937	desc_array->descriptor[1].address = address + (length - 4);
4938	desc_array->descriptor[1].length = 4;
4939	desc_array->elements = 2;
4940	return (desc_array->elements);
4941}
4942
4943/**********************************************************************
4944 *
4945 *  Update the board statistics counters.
4946 *
4947 **********************************************************************/
4948static void
4949em_update_stats_counters(struct adapter *adapter)
4950{
4951	struct ifnet   *ifp;
4952
4953	if(adapter->hw.phy.media_type == e1000_media_type_copper ||
4954	   (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) {
4955		adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS);
4956		adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC);
4957	}
4958	adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS);
4959	adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC);
4960	adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC);
4961	adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL);
4962
4963	adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC);
4964	adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL);
4965	adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC);
4966	adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC);
4967	adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
4968	adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
4969	adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
4970	adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
4971	adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
4972	adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
4973	adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
4974	adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127);
4975	adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255);
4976	adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511);
4977	adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023);
4978	adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522);
4979	adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC);
4980	adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC);
4981	adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC);
4982	adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC);
4983
4984	/* For the 64-bit byte counters the low dword must be read first. */
4985	/* Both registers clear on the read of the high dword */
4986
4987	adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH);
4988	adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH);
4989
4990	adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC);
4991	adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC);
4992	adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC);
4993	adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC);
4994	adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC);
4995
4996	adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH);
4997	adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH);
4998
4999	adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR);
5000	adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT);
5001	adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64);
5002	adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127);
5003	adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255);
5004	adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511);
5005	adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023);
5006	adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522);
5007	adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC);
5008	adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC);
5009
5010	if (adapter->hw.mac.type >= e1000_82543) {
5011		adapter->stats.algnerrc +=
5012		E1000_READ_REG(&adapter->hw, E1000_ALGNERRC);
5013		adapter->stats.rxerrc +=
5014		E1000_READ_REG(&adapter->hw, E1000_RXERRC);
5015		adapter->stats.tncrs +=
5016		E1000_READ_REG(&adapter->hw, E1000_TNCRS);
5017		adapter->stats.cexterr +=
5018		E1000_READ_REG(&adapter->hw, E1000_CEXTERR);
5019		adapter->stats.tsctc +=
5020		E1000_READ_REG(&adapter->hw, E1000_TSCTC);
5021		adapter->stats.tsctfc +=
5022		E1000_READ_REG(&adapter->hw, E1000_TSCTFC);
5023	}
5024	ifp = adapter->ifp;
5025
5026	ifp->if_collisions = adapter->stats.colc;
5027
5028	/* Rx Errors */
5029	ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
5030	    adapter->stats.crcerrs + adapter->stats.algnerrc +
5031	    adapter->stats.ruc + adapter->stats.roc +
5032	    adapter->stats.mpc + adapter->stats.cexterr;
5033
5034	/* Tx Errors */
5035	ifp->if_oerrors = adapter->stats.ecol +
5036	    adapter->stats.latecol + adapter->watchdog_events;
5037}
5038
5039
5040/**********************************************************************
5041 *
5042 *  This routine is called only when em_display_debug_stats is enabled.
5043 *  This routine provides a way to take a look at important statistics
5044 *  maintained by the driver and hardware.
5045 *
5046 **********************************************************************/
5047static void
5048em_print_debug_info(struct adapter *adapter)
5049{
5050	device_t dev = adapter->dev;
5051	u8 *hw_addr = adapter->hw.hw_addr;
5052
5053	device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
5054	device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
5055	    E1000_READ_REG(&adapter->hw, E1000_CTRL),
5056	    E1000_READ_REG(&adapter->hw, E1000_RCTL));
5057	device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
5058	    ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\
5059	    (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) );
5060	device_printf(dev, "Flow control watermarks high = %d low = %d\n",
5061	    adapter->hw.fc.high_water,
5062	    adapter->hw.fc.low_water);
5063	device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
5064	    E1000_READ_REG(&adapter->hw, E1000_TIDV),
5065	    E1000_READ_REG(&adapter->hw, E1000_TADV));
5066	device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
5067	    E1000_READ_REG(&adapter->hw, E1000_RDTR),
5068	    E1000_READ_REG(&adapter->hw, E1000_RADV));
5069	device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
5070	    (long long)adapter->tx_fifo_wrk_cnt,
5071	    (long long)adapter->tx_fifo_reset_cnt);
5072	device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
5073	    E1000_READ_REG(&adapter->hw, E1000_TDH(0)),
5074	    E1000_READ_REG(&adapter->hw, E1000_TDT(0)));
5075	device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
5076	    E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
5077	    E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
5078	device_printf(dev, "Num Tx descriptors avail = %d\n",
5079	    adapter->num_tx_desc_avail);
5080	device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
5081	    adapter->no_tx_desc_avail1);
5082	device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
5083	    adapter->no_tx_desc_avail2);
5084	device_printf(dev, "Std mbuf failed = %ld\n",
5085	    adapter->mbuf_alloc_failed);
5086	device_printf(dev, "Std mbuf cluster failed = %ld\n",
5087	    adapter->mbuf_cluster_failed);
5088	device_printf(dev, "Driver dropped packets = %ld\n",
5089	    adapter->dropped_pkts);
5090	device_printf(dev, "Driver tx dma failure in encap = %ld\n",
5091		adapter->no_tx_dma_setup);
5092}
5093
5094static void
5095em_print_hw_stats(struct adapter *adapter)
5096{
5097	device_t dev = adapter->dev;
5098
5099	device_printf(dev, "Excessive collisions = %lld\n",
5100	    (long long)adapter->stats.ecol);
5101#if	(DEBUG_HW > 0)  /* Dont output these errors normally */
5102	device_printf(dev, "Symbol errors = %lld\n",
5103	    (long long)adapter->stats.symerrs);
5104#endif
5105	device_printf(dev, "Sequence errors = %lld\n",
5106	    (long long)adapter->stats.sec);
5107	device_printf(dev, "Defer count = %lld\n",
5108	    (long long)adapter->stats.dc);
5109	device_printf(dev, "Missed Packets = %lld\n",
5110	    (long long)adapter->stats.mpc);
5111	device_printf(dev, "Receive No Buffers = %lld\n",
5112	    (long long)adapter->stats.rnbc);
5113	/* RLEC is inaccurate on some hardware, calculate our own. */
5114	device_printf(dev, "Receive Length Errors = %lld\n",
5115	    ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
5116	device_printf(dev, "Receive errors = %lld\n",
5117	    (long long)adapter->stats.rxerrc);
5118	device_printf(dev, "Crc errors = %lld\n",
5119	    (long long)adapter->stats.crcerrs);
5120	device_printf(dev, "Alignment errors = %lld\n",
5121	    (long long)adapter->stats.algnerrc);
5122	device_printf(dev, "Collision/Carrier extension errors = %lld\n",
5123	    (long long)adapter->stats.cexterr);
5124	device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
5125	device_printf(dev, "watchdog timeouts = %ld\n",
5126	    adapter->watchdog_events);
5127	device_printf(dev, "RX MSIX IRQ = %ld TX MSIX IRQ = %ld"
5128	    " LINK MSIX IRQ = %ld\n", adapter->rx_irq,
5129	    adapter->tx_irq , adapter->link_irq);
5130	device_printf(dev, "XON Rcvd = %lld\n",
5131	    (long long)adapter->stats.xonrxc);
5132	device_printf(dev, "XON Xmtd = %lld\n",
5133	    (long long)adapter->stats.xontxc);
5134	device_printf(dev, "XOFF Rcvd = %lld\n",
5135	    (long long)adapter->stats.xoffrxc);
5136	device_printf(dev, "XOFF Xmtd = %lld\n",
5137	    (long long)adapter->stats.xofftxc);
5138	device_printf(dev, "Good Packets Rcvd = %lld\n",
5139	    (long long)adapter->stats.gprc);
5140	device_printf(dev, "Good Packets Xmtd = %lld\n",
5141	    (long long)adapter->stats.gptc);
5142	device_printf(dev, "TSO Contexts Xmtd = %lld\n",
5143	    (long long)adapter->stats.tsctc);
5144	device_printf(dev, "TSO Contexts Failed = %lld\n",
5145	    (long long)adapter->stats.tsctfc);
5146}
5147
5148/**********************************************************************
5149 *
5150 *  This routine provides a way to dump out the adapter eeprom,
5151 *  often a useful debug/service tool. This only dumps the first
5152 *  32 words, stuff that matters is in that extent.
5153 *
5154 **********************************************************************/
5155static void
5156em_print_nvm_info(struct adapter *adapter)
5157{
5158	u16	eeprom_data;
5159	int	i, j, row = 0;
5160
5161	/* Its a bit crude, but it gets the job done */
5162	printf("\nInterface EEPROM Dump:\n");
5163	printf("Offset\n0x0000  ");
5164	for (i = 0, j = 0; i < 32; i++, j++) {
5165		if (j == 8) { /* Make the offset block */
5166			j = 0; ++row;
5167			printf("\n0x00%x0  ",row);
5168		}
5169		e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data);
5170		printf("%04x ", eeprom_data);
5171	}
5172	printf("\n");
5173}
5174
5175static int
5176em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
5177{
5178	struct adapter *adapter;
5179	int error;
5180	int result;
5181
5182	result = -1;
5183	error = sysctl_handle_int(oidp, &result, 0, req);
5184
5185	if (error || !req->newptr)
5186		return (error);
5187
5188	if (result == 1) {
5189		adapter = (struct adapter *)arg1;
5190		em_print_debug_info(adapter);
5191	}
5192	/*
5193	 * This value will cause a hex dump of the
5194	 * first 32 16-bit words of the EEPROM to
5195	 * the screen.
5196	 */
5197	if (result == 2) {
5198		adapter = (struct adapter *)arg1;
5199		em_print_nvm_info(adapter);
5200        }
5201
5202	return (error);
5203}
5204
5205
5206static int
5207em_sysctl_stats(SYSCTL_HANDLER_ARGS)
5208{
5209	struct adapter *adapter;
5210	int error;
5211	int result;
5212
5213	result = -1;
5214	error = sysctl_handle_int(oidp, &result, 0, req);
5215
5216	if (error || !req->newptr)
5217		return (error);
5218
5219	if (result == 1) {
5220		adapter = (struct adapter *)arg1;
5221		em_print_hw_stats(adapter);
5222	}
5223
5224	return (error);
5225}
5226
5227static int
5228em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
5229{
5230	struct em_int_delay_info *info;
5231	struct adapter *adapter;
5232	u32 regval;
5233	int error;
5234	int usecs;
5235	int ticks;
5236
5237	info = (struct em_int_delay_info *)arg1;
5238	usecs = info->value;
5239	error = sysctl_handle_int(oidp, &usecs, 0, req);
5240	if (error != 0 || req->newptr == NULL)
5241		return (error);
5242	if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535))
5243		return (EINVAL);
5244	info->value = usecs;
5245	ticks = EM_USECS_TO_TICKS(usecs);
5246
5247	adapter = info->adapter;
5248
5249	EM_CORE_LOCK(adapter);
5250	regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
5251	regval = (regval & ~0xffff) | (ticks & 0xffff);
5252	/* Handle a few special cases. */
5253	switch (info->offset) {
5254	case E1000_RDTR:
5255		break;
5256	case E1000_TIDV:
5257		if (ticks == 0) {
5258			adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
5259			/* Don't write 0 into the TIDV register. */
5260			regval++;
5261		} else
5262			adapter->txd_cmd |= E1000_TXD_CMD_IDE;
5263		break;
5264	}
5265	E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
5266	EM_CORE_UNLOCK(adapter);
5267	return (0);
5268}
5269
5270static void
5271em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
5272	const char *description, struct em_int_delay_info *info,
5273	int offset, int value)
5274{
5275	info->adapter = adapter;
5276	info->offset = offset;
5277	info->value = value;
5278	SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
5279	    SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5280	    OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
5281	    info, 0, em_sysctl_int_delay, "I", description);
5282}
5283
5284#ifndef EM_LEGACY_IRQ
5285static void
5286em_add_rx_process_limit(struct adapter *adapter, const char *name,
5287	const char *description, int *limit, int value)
5288{
5289	*limit = value;
5290	SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
5291	    SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
5292	    OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
5293}
5294#endif
5295
5296#ifdef EM_TIMESYNC
5297/*
5298 * Initialize the Time Sync Feature
5299 */
5300static int
5301em_tsync_init(struct adapter *adapter)
5302{
5303	device_t	dev = adapter->dev;
5304	u32		tx_ctl, rx_ctl;
5305
5306
5307	E1000_WRITE_REG(&adapter->hw, E1000_TIMINCA, (1<<24) |
5308	    20833/PICOSECS_PER_TICK);
5309
5310	adapter->last_stamp =  E1000_READ_REG(&adapter->hw, E1000_SYSTIML);
5311	adapter->last_stamp |= (u64)E1000_READ_REG(&adapter->hw,
5312	    E1000_SYSTIMH) << 32ULL;
5313
5314	/* Enable the TX side */
5315	tx_ctl =  E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL);
5316	tx_ctl |= 0x10;
5317	E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl);
5318	E1000_WRITE_FLUSH(&adapter->hw);
5319
5320	tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL);
5321	if ((tx_ctl & 0x10) == 0) {
5322     		device_printf(dev, "Failed to enable TX timestamping\n");
5323		return (ENXIO);
5324	}
5325
5326	/* Enable RX */
5327	rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL);
5328	rx_ctl |= 0x10; /* Enable the feature */
5329	rx_ctl |= 0x0a; /* This value turns on Ver 1 and 2 */
5330	E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl);
5331
5332	/*
5333	 * Ethertype Stamping (Ethertype = 0x88F7)
5334	 */
5335	E1000_WRITE_REG(&adapter->hw, E1000_RXMTRL, htonl(0x440088f7));
5336
5337	/*
5338	 * Source Port Queue Filter Setup:
5339	 *  this is for UDP port filtering
5340	 */
5341	E1000_WRITE_REG(&adapter->hw, E1000_RXUDP, htons(TSYNC_PORT));
5342	/* Protocol = UDP, enable Timestamp, and filter on source/protocol */
5343
5344	E1000_WRITE_FLUSH(&adapter->hw);
5345
5346	rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL);
5347	if ((rx_ctl & 0x10) == 0) {
5348     		device_printf(dev, "Failed to enable RX timestamping\n");
5349		return (ENXIO);
5350	}
5351
5352	device_printf(dev, "IEEE 1588 Precision Time Protocol enabled\n");
5353
5354	return (0);
5355}
5356
5357/*
5358 * Disable the Time Sync Feature
5359 */
5360static void
5361em_tsync_disable(struct adapter *adapter)
5362{
5363	u32		tx_ctl, rx_ctl;
5364
5365	tx_ctl =  E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL);
5366	tx_ctl &= ~0x10;
5367	E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl);
5368	E1000_WRITE_FLUSH(&adapter->hw);
5369
5370	/* Invalidate TX Timestamp */
5371	E1000_READ_REG(&adapter->hw, E1000_TXSTMPH);
5372
5373	tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL);
5374	if (tx_ctl & 0x10)
5375     		HW_DEBUGOUT("Failed to disable TX timestamping\n");
5376
5377	rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL);
5378	rx_ctl &= ~0x10;
5379
5380	E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl);
5381	E1000_WRITE_FLUSH(&adapter->hw);
5382
5383	/* Invalidate RX Timestamp */
5384	E1000_READ_REG(&adapter->hw, E1000_RXSATRH);
5385
5386	rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL);
5387	if (rx_ctl & 0x10)
5388		HW_DEBUGOUT("Failed to disable RX timestamping\n");
5389
5390	return;
5391}
5392#endif /* EM_TIMESYNC */
5393