if_em.c revision 161823 
1/**************************************************************************
2
3Copyright (c) 2001-2006, Intel Corporation
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10    this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13    notice, this list of conditions and the following disclaimer in the
14    documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17    contributors may be used to endorse or promote products derived from
18    this software without specific prior written permission.
19
20THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30POSSIBILITY OF SUCH DAMAGE.
31
32***************************************************************************/
33
34/*$FreeBSD: head/sys/dev/em/if_em.c 161823 2006-09-01 16:11:50Z jhb $*/
35
36#ifdef HAVE_KERNEL_OPTION_HEADERS
37#include "opt_device_polling.h"
38#endif
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/bus.h>
43#include <sys/endian.h>
44#include <sys/kernel.h>
45#include <sys/kthread.h>
46#include <sys/malloc.h>
47#include <sys/mbuf.h>
48#include <sys/module.h>
49#include <sys/rman.h>
50#include <sys/socket.h>
51#include <sys/sockio.h>
52#include <sys/sysctl.h>
53#include <sys/taskqueue.h>
54
55#include <machine/bus.h>
56#include <machine/resource.h>
57
58#include <net/bpf.h>
59#include <net/ethernet.h>
60#include <net/if.h>
61#include <net/if_arp.h>
62#include <net/if_dl.h>
63#include <net/if_media.h>
64
65#include <net/if_types.h>
66#include <net/if_vlan_var.h>
67
68#include <netinet/in_systm.h>
69#include <netinet/in.h>
70#include <netinet/if_ether.h>
71#include <netinet/ip.h>
72#include <netinet/tcp.h>
73#include <netinet/udp.h>
74
75#include <dev/pci/pcivar.h>
76#include <dev/pci/pcireg.h>
77#include <dev/em/if_em_hw.h>
78#include <dev/em/if_em.h>
79
80/*********************************************************************
81 *  Set this to one to display debug statistics
82 *********************************************************************/
83int	em_display_debug_stats = 0;
84
85/*********************************************************************
86 *  Driver version
87 *********************************************************************/
88
89char em_driver_version[] = "Version - 6.1.4";
90
91
92/*********************************************************************
93 *  PCI Device ID Table
94 *
95 *  Used by probe to select devices to load on
96 *  Last field stores an index into em_strings
97 *  Last entry must be all 0s
98 *
99 *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
100 *********************************************************************/
101
102static em_vendor_info_t em_vendor_info_array[] =
103{
104	/* Intel(R) PRO/1000 Network Connection */
105	{ 0x8086, E1000_DEV_ID_82540EM,		PCI_ANY_ID, PCI_ANY_ID, 0},
106	{ 0x8086, E1000_DEV_ID_82540EM_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
107	{ 0x8086, E1000_DEV_ID_82540EP,		PCI_ANY_ID, PCI_ANY_ID, 0},
108	{ 0x8086, E1000_DEV_ID_82540EP_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
109	{ 0x8086, E1000_DEV_ID_82540EP_LP,	PCI_ANY_ID, PCI_ANY_ID, 0},
110
111	{ 0x8086, E1000_DEV_ID_82541EI,		PCI_ANY_ID, PCI_ANY_ID, 0},
112	{ 0x8086, E1000_DEV_ID_82541ER,		PCI_ANY_ID, PCI_ANY_ID, 0},
113	{ 0x8086, E1000_DEV_ID_82541ER_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
114	{ 0x8086, E1000_DEV_ID_82541EI_MOBILE,	PCI_ANY_ID, PCI_ANY_ID, 0},
115	{ 0x8086, E1000_DEV_ID_82541GI,		PCI_ANY_ID, PCI_ANY_ID, 0},
116	{ 0x8086, E1000_DEV_ID_82541GI_LF,	PCI_ANY_ID, PCI_ANY_ID, 0},
117	{ 0x8086, E1000_DEV_ID_82541GI_MOBILE,	PCI_ANY_ID, PCI_ANY_ID, 0},
118
119	{ 0x8086, E1000_DEV_ID_82542,		PCI_ANY_ID, PCI_ANY_ID, 0},
120
121	{ 0x8086, E1000_DEV_ID_82543GC_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
122	{ 0x8086, E1000_DEV_ID_82543GC_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
123
124	{ 0x8086, E1000_DEV_ID_82544EI_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
125	{ 0x8086, E1000_DEV_ID_82544EI_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
126	{ 0x8086, E1000_DEV_ID_82544GC_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
127	{ 0x8086, E1000_DEV_ID_82544GC_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
128
129	{ 0x8086, E1000_DEV_ID_82545EM_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
130	{ 0x8086, E1000_DEV_ID_82545EM_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
131	{ 0x8086, E1000_DEV_ID_82545GM_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
132	{ 0x8086, E1000_DEV_ID_82545GM_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
133	{ 0x8086, E1000_DEV_ID_82545GM_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
134
135	{ 0x8086, E1000_DEV_ID_82546EB_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
136	{ 0x8086, E1000_DEV_ID_82546EB_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
137	{ 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
138	{ 0x8086, E1000_DEV_ID_82546GB_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
139	{ 0x8086, E1000_DEV_ID_82546GB_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
140	{ 0x8086, E1000_DEV_ID_82546GB_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
141	{ 0x8086, E1000_DEV_ID_82546GB_PCIE,	PCI_ANY_ID, PCI_ANY_ID, 0},
142	{ 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
143	{ 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
144						PCI_ANY_ID, PCI_ANY_ID, 0},
145
146	{ 0x8086, E1000_DEV_ID_82547EI,		PCI_ANY_ID, PCI_ANY_ID, 0},
147	{ 0x8086, E1000_DEV_ID_82547EI_MOBILE,	PCI_ANY_ID, PCI_ANY_ID, 0},
148	{ 0x8086, E1000_DEV_ID_82547GI,		PCI_ANY_ID, PCI_ANY_ID, 0},
149
150	{ 0x8086, E1000_DEV_ID_82571EB_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
151	{ 0x8086, E1000_DEV_ID_82571EB_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
152	{ 0x8086, E1000_DEV_ID_82571EB_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
153	{ 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
154						PCI_ANY_ID, PCI_ANY_ID, 0},
155
156	{ 0x8086, E1000_DEV_ID_82572EI_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
157	{ 0x8086, E1000_DEV_ID_82572EI_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
158	{ 0x8086, E1000_DEV_ID_82572EI_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
159	{ 0x8086, E1000_DEV_ID_82572EI,		PCI_ANY_ID, PCI_ANY_ID, 0},
160
161	{ 0x8086, E1000_DEV_ID_82573E,		PCI_ANY_ID, PCI_ANY_ID, 0},
162	{ 0x8086, E1000_DEV_ID_82573E_IAMT,	PCI_ANY_ID, PCI_ANY_ID, 0},
163	{ 0x8086, E1000_DEV_ID_82573L,		PCI_ANY_ID, PCI_ANY_ID, 0},
164	{ 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
165						PCI_ANY_ID, PCI_ANY_ID, 0},
166	{ 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
167						PCI_ANY_ID, PCI_ANY_ID, 0},
168	{ 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
169						PCI_ANY_ID, PCI_ANY_ID, 0},
170	{ 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
171						PCI_ANY_ID, PCI_ANY_ID, 0},
172	{ 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,	PCI_ANY_ID, PCI_ANY_ID, 0},
173	{ 0x8086, E1000_DEV_ID_ICH8_IGP_C,	PCI_ANY_ID, PCI_ANY_ID, 0},
174	{ 0x8086, E1000_DEV_ID_ICH8_IFE,	PCI_ANY_ID, PCI_ANY_ID, 0},
175
176	/* required last entry */
177	{ 0, 0, 0, 0, 0}
178};
179
180/*********************************************************************
181 *  Table of branding strings for all supported NICs.
182 *********************************************************************/
183
184static char *em_strings[] = {
185	"Intel(R) PRO/1000 Network Connection"
186};
187
188/*********************************************************************
189 *  Function prototypes
190 *********************************************************************/
191static int	em_probe(device_t);
192static int	em_attach(device_t);
193static int	em_detach(device_t);
194static int	em_shutdown(device_t);
195static int	em_suspend(device_t);
196static int	em_resume(device_t);
197static void	em_start(struct ifnet *);
198static void	em_start_locked(struct ifnet *ifp);
199static int	em_ioctl(struct ifnet *, u_long, caddr_t);
200static void	em_watchdog(struct ifnet *);
201static void	em_init(void *);
202static void	em_init_locked(struct adapter *);
203static void	em_stop(void *);
204static void	em_media_status(struct ifnet *, struct ifmediareq *);
205static int	em_media_change(struct ifnet *);
206static void	em_identify_hardware(struct adapter *);
207static int	em_allocate_pci_resources(struct adapter *);
208static int	em_allocate_intr(struct adapter *);
209static void	em_free_intr(struct adapter *);
210static void	em_free_pci_resources(struct adapter *);
211static void	em_local_timer(void *);
212static int	em_hardware_init(struct adapter *);
213static void	em_setup_interface(device_t, struct adapter *);
214static int	em_setup_transmit_structures(struct adapter *);
215static void	em_initialize_transmit_unit(struct adapter *);
216static int	em_setup_receive_structures(struct adapter *);
217static void	em_initialize_receive_unit(struct adapter *);
218static void	em_enable_intr(struct adapter *);
219static void	em_disable_intr(struct adapter *);
220static void	em_free_transmit_structures(struct adapter *);
221static void	em_free_receive_structures(struct adapter *);
222static void	em_update_stats_counters(struct adapter *);
223static void	em_txeof(struct adapter *);
224static int	em_allocate_receive_structures(struct adapter *);
225static int	em_allocate_transmit_structures(struct adapter *);
226static int	em_rxeof(struct adapter *, int);
227#ifndef __NO_STRICT_ALIGNMENT
228static int	em_fixup_rx(struct adapter *);
229#endif
230static void	em_receive_checksum(struct adapter *, struct em_rx_desc *,
231		    struct mbuf *);
232static void	em_transmit_checksum_setup(struct adapter *, struct mbuf *,
233		    uint32_t *, uint32_t *);
234static void	em_set_promisc(struct adapter *);
235static void	em_disable_promisc(struct adapter *);
236static void	em_set_multi(struct adapter *);
237static void	em_print_hw_stats(struct adapter *);
238static void	em_update_link_status(struct adapter *);
239static int	em_get_buf(struct adapter *, int);
240static void	em_enable_vlans(struct adapter *);
241static void	em_disable_vlans(struct adapter *);
242static int	em_encap(struct adapter *, struct mbuf **);
243static void	em_smartspeed(struct adapter *);
244static int	em_82547_fifo_workaround(struct adapter *, int);
245static void	em_82547_update_fifo_head(struct adapter *, int);
246static int	em_82547_tx_fifo_reset(struct adapter *);
247static void	em_82547_move_tail(void *arg);
248static void	em_82547_move_tail_locked(struct adapter *);
249static int	em_dma_malloc(struct adapter *, bus_size_t,
250		struct em_dma_alloc *, int);
251static void	em_dma_free(struct adapter *, struct em_dma_alloc *);
252static void	em_print_debug_info(struct adapter *);
253static int 	em_is_valid_ether_addr(uint8_t *);
254static int	em_sysctl_stats(SYSCTL_HANDLER_ARGS);
255static int	em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
256static uint32_t	em_fill_descriptors (bus_addr_t address, uint32_t length,
257		    PDESC_ARRAY desc_array);
258static int	em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
259static void	em_add_int_delay_sysctl(struct adapter *, const char *,
260		const char *, struct em_int_delay_info *, int, int);
261
262/*
263 * Fast interrupt handler and legacy ithread/polling modes are
264 * mutually exclusive.
265 */
266#ifdef DEVICE_POLLING
267static poll_handler_t em_poll;
268static void	em_intr(void *);
269#else
270static void	em_intr_fast(void *);
271static void	em_add_int_process_limit(struct adapter *, const char *,
272		const char *, int *, int);
273static void	em_handle_rxtx(void *context, int pending);
274static void	em_handle_link(void *context, int pending);
275#endif
276
277/*********************************************************************
278 *  FreeBSD Device Interface Entry Points
279 *********************************************************************/
280
281static device_method_t em_methods[] = {
282	/* Device interface */
283	DEVMETHOD(device_probe, em_probe),
284	DEVMETHOD(device_attach, em_attach),
285	DEVMETHOD(device_detach, em_detach),
286	DEVMETHOD(device_shutdown, em_shutdown),
287	DEVMETHOD(device_suspend, em_suspend),
288	DEVMETHOD(device_resume, em_resume),
289	{0, 0}
290};
291
292static driver_t em_driver = {
293	"em", em_methods, sizeof(struct adapter),
294};
295
296static devclass_t em_devclass;
297DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
298MODULE_DEPEND(em, pci, 1, 1, 1);
299MODULE_DEPEND(em, ether, 1, 1, 1);
300
301/*********************************************************************
302 *  Tunable default values.
303 *********************************************************************/
304
305#define E1000_TICKS_TO_USECS(ticks)	((1024 * (ticks) + 500) / 1000)
306#define E1000_USECS_TO_TICKS(usecs)	((1000 * (usecs) + 512) / 1024)
307
308static int em_tx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TIDV);
309static int em_rx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RDTR);
310static int em_tx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TADV);
311static int em_rx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RADV);
312static int em_rxd = EM_DEFAULT_RXD;
313static int em_txd = EM_DEFAULT_TXD;
314static int em_smart_pwr_down = FALSE;
315
316TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
317TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
318TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
319TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
320TUNABLE_INT("hw.em.rxd", &em_rxd);
321TUNABLE_INT("hw.em.txd", &em_txd);
322TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
323#ifndef DEVICE_POLLING
324static int em_rx_process_limit = 100;
325TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit);
326#endif
327
328/*********************************************************************
329 *  Device identification routine
330 *
331 *  em_probe determines if the driver should be loaded on
332 *  adapter based on PCI vendor/device id of the adapter.
333 *
334 *  return BUS_PROBE_DEFAULT on success, positive on failure
335 *********************************************************************/
336
337static int
338em_probe(device_t dev)
339{
340	char		adapter_name[60];
341	uint16_t	pci_vendor_id = 0;
342	uint16_t	pci_device_id = 0;
343	uint16_t	pci_subvendor_id = 0;
344	uint16_t	pci_subdevice_id = 0;
345	em_vendor_info_t *ent;
346
347	INIT_DEBUGOUT("em_probe: begin");
348
349	pci_vendor_id = pci_get_vendor(dev);
350	if (pci_vendor_id != EM_VENDOR_ID)
351		return (ENXIO);
352
353	pci_device_id = pci_get_device(dev);
354	pci_subvendor_id = pci_get_subvendor(dev);
355	pci_subdevice_id = pci_get_subdevice(dev);
356
357	ent = em_vendor_info_array;
358	while (ent->vendor_id != 0) {
359		if ((pci_vendor_id == ent->vendor_id) &&
360		    (pci_device_id == ent->device_id) &&
361
362		    ((pci_subvendor_id == ent->subvendor_id) ||
363		    (ent->subvendor_id == PCI_ANY_ID)) &&
364
365		    ((pci_subdevice_id == ent->subdevice_id) ||
366		    (ent->subdevice_id == PCI_ANY_ID))) {
367			sprintf(adapter_name, "%s %s",
368				em_strings[ent->index],
369				em_driver_version);
370			device_set_desc_copy(dev, adapter_name);
371			return (BUS_PROBE_DEFAULT);
372		}
373		ent++;
374	}
375
376	return (ENXIO);
377}
378
379/*********************************************************************
380 *  Device initialization routine
381 *
382 *  The attach entry point is called when the driver is being loaded.
383 *  This routine identifies the type of hardware, allocates all resources
384 *  and initializes the hardware.
385 *
386 *  return 0 on success, positive on failure
387 *********************************************************************/
388
389static int
390em_attach(device_t dev)
391{
392	struct adapter	*adapter;
393	int		tsize, rsize;
394	int		error = 0;
395
396	INIT_DEBUGOUT("em_attach: begin");
397
398	adapter = device_get_softc(dev);
399	adapter->dev = adapter->osdep.dev = dev;
400	EM_LOCK_INIT(adapter, device_get_nameunit(dev));
401
402	/* SYSCTL stuff */
403	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
404	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
405	    OID_AUTO, "debug_info", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
406	    em_sysctl_debug_info, "I", "Debug Information");
407
408	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
409	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
410	    OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
411	    em_sysctl_stats, "I", "Statistics");
412
413	callout_init(&adapter->timer, CALLOUT_MPSAFE);
414	callout_init(&adapter->tx_fifo_timer, CALLOUT_MPSAFE);
415
416	/* Determine hardware revision */
417	em_identify_hardware(adapter);
418
419	/* Set up some sysctls for the tunable interrupt delays */
420	em_add_int_delay_sysctl(adapter, "rx_int_delay",
421	    "receive interrupt delay in usecs", &adapter->rx_int_delay,
422	    E1000_REG_OFFSET(&adapter->hw, RDTR), em_rx_int_delay_dflt);
423	em_add_int_delay_sysctl(adapter, "tx_int_delay",
424	    "transmit interrupt delay in usecs", &adapter->tx_int_delay,
425	    E1000_REG_OFFSET(&adapter->hw, TIDV), em_tx_int_delay_dflt);
426	if (adapter->hw.mac_type >= em_82540) {
427		em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
428		    "receive interrupt delay limit in usecs",
429		    &adapter->rx_abs_int_delay,
430		    E1000_REG_OFFSET(&adapter->hw, RADV),
431		    em_rx_abs_int_delay_dflt);
432		em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
433		    "transmit interrupt delay limit in usecs",
434		    &adapter->tx_abs_int_delay,
435		    E1000_REG_OFFSET(&adapter->hw, TADV),
436		    em_tx_abs_int_delay_dflt);
437	}
438
439#ifndef DEVICE_POLLING
440	/* Sysctls for limiting the amount of work done in the taskqueue */
441	em_add_int_process_limit(adapter, "rx_processing_limit",
442	    "max number of rx packets to process", &adapter->rx_process_limit,
443	    em_rx_process_limit);
444#endif
445
446	/*
447	 * Validate number of transmit and receive descriptors. It
448	 * must not exceed hardware maximum, and must be multiple
449	 * of EM_DBA_ALIGN.
450	 */
451	if (((em_txd * sizeof(struct em_tx_desc)) % EM_DBA_ALIGN) != 0 ||
452	    (adapter->hw.mac_type >= em_82544 && em_txd > EM_MAX_TXD) ||
453	    (adapter->hw.mac_type < em_82544 && em_txd > EM_MAX_TXD_82543) ||
454	    (em_txd < EM_MIN_TXD)) {
455		device_printf(dev, "Using %d TX descriptors instead of %d!\n",
456		    EM_DEFAULT_TXD, em_txd);
457		adapter->num_tx_desc = EM_DEFAULT_TXD;
458	} else
459		adapter->num_tx_desc = em_txd;
460	if (((em_rxd * sizeof(struct em_rx_desc)) % EM_DBA_ALIGN) != 0 ||
461	    (adapter->hw.mac_type >= em_82544 && em_rxd > EM_MAX_RXD) ||
462	    (adapter->hw.mac_type < em_82544 && em_rxd > EM_MAX_RXD_82543) ||
463	    (em_rxd < EM_MIN_RXD)) {
464		device_printf(dev, "Using %d RX descriptors instead of %d!\n",
465		    EM_DEFAULT_RXD, em_rxd);
466		adapter->num_rx_desc = EM_DEFAULT_RXD;
467	} else
468		adapter->num_rx_desc = em_rxd;
469
470	adapter->hw.autoneg = DO_AUTO_NEG;
471	adapter->hw.wait_autoneg_complete = WAIT_FOR_AUTO_NEG_DEFAULT;
472	adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
473	adapter->hw.tbi_compatibility_en = TRUE;
474	adapter->rx_buffer_len = EM_RXBUFFER_2048;
475
476	adapter->hw.phy_init_script = 1;
477	adapter->hw.phy_reset_disable = FALSE;
478
479#ifndef EM_MASTER_SLAVE
480	adapter->hw.master_slave = em_ms_hw_default;
481#else
482	adapter->hw.master_slave = EM_MASTER_SLAVE;
483#endif
484	/*
485	 * Set the max frame size assuming standard ethernet
486	 * sized frames.
487	 */
488	adapter->hw.max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
489
490	adapter->hw.min_frame_size = MINIMUM_ETHERNET_PACKET_SIZE + ETHER_CRC_LEN;
491
492	/*
493	 * This controls when hardware reports transmit completion
494	 * status.
495	 */
496	adapter->hw.report_tx_early = 1;
497	if (em_allocate_pci_resources(adapter)) {
498		device_printf(dev, "Allocation of PCI resources failed\n");
499		error = ENXIO;
500		goto err_pci;
501	}
502
503	/* Initialize eeprom parameters */
504	em_init_eeprom_params(&adapter->hw);
505
506	tsize = roundup2(adapter->num_tx_desc * sizeof(struct em_tx_desc),
507	    EM_DBA_ALIGN);
508
509	/* Allocate Transmit Descriptor ring */
510	if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
511		device_printf(dev, "Unable to allocate tx_desc memory\n");
512		error = ENOMEM;
513		goto err_tx_desc;
514	}
515	adapter->tx_desc_base = (struct em_tx_desc *)adapter->txdma.dma_vaddr;
516
517	rsize = roundup2(adapter->num_rx_desc * sizeof(struct em_rx_desc),
518	    EM_DBA_ALIGN);
519
520	/* Allocate Receive Descriptor ring */
521	if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
522		device_printf(dev, "Unable to allocate rx_desc memory\n");
523		error = ENOMEM;
524		goto err_rx_desc;
525	}
526	adapter->rx_desc_base = (struct em_rx_desc *)adapter->rxdma.dma_vaddr;
527
528	/* Initialize the hardware */
529	if (em_hardware_init(adapter)) {
530		device_printf(dev, "Unable to initialize the hardware\n");
531		error = EIO;
532		goto err_hw_init;
533	}
534
535	/* Copy the permanent MAC address out of the EEPROM */
536	if (em_read_mac_addr(&adapter->hw) < 0) {
537		device_printf(dev, "EEPROM read error while reading MAC"
538		    " address\n");
539		error = EIO;
540		goto err_hw_init;
541	}
542
543	if (!em_is_valid_ether_addr(adapter->hw.mac_addr)) {
544		device_printf(dev, "Invalid MAC address\n");
545		error = EIO;
546		goto err_hw_init;
547	}
548
549	/* Setup OS specific network interface */
550	em_setup_interface(dev, adapter);
551
552	em_allocate_intr(adapter);
553
554	/* Initialize statistics */
555	em_clear_hw_cntrs(&adapter->hw);
556	em_update_stats_counters(adapter);
557	adapter->hw.get_link_status = 1;
558	em_update_link_status(adapter);
559
560	/* Indicate SOL/IDER usage */
561	if (em_check_phy_reset_block(&adapter->hw))
562		device_printf(dev,
563		    "PHY reset is blocked due to SOL/IDER session.\n");
564
565	/* Identify 82544 on PCIX */
566	em_get_bus_info(&adapter->hw);
567	if(adapter->hw.bus_type == em_bus_type_pcix && adapter->hw.mac_type == em_82544)
568		adapter->pcix_82544 = TRUE;
569	else
570		adapter->pcix_82544 = FALSE;
571
572	INIT_DEBUGOUT("em_attach: end");
573
574	return (0);
575
576err_hw_init:
577	em_dma_free(adapter, &adapter->rxdma);
578err_rx_desc:
579	em_dma_free(adapter, &adapter->txdma);
580err_tx_desc:
581err_pci:
582	em_free_intr(adapter);
583	em_free_pci_resources(adapter);
584	EM_LOCK_DESTROY(adapter);
585
586	return (error);
587}
588
589/*********************************************************************
590 *  Device removal routine
591 *
592 *  The detach entry point is called when the driver is being removed.
593 *  This routine stops the adapter and deallocates all the resources
594 *  that were allocated for driver operation.
595 *
596 *  return 0 on success, positive on failure
597 *********************************************************************/
598
599static int
600em_detach(device_t dev)
601{
602	struct adapter	*adapter = device_get_softc(dev);
603	struct ifnet	*ifp = adapter->ifp;
604
605	INIT_DEBUGOUT("em_detach: begin");
606
607#ifdef DEVICE_POLLING
608	if (ifp->if_capenable & IFCAP_POLLING)
609		ether_poll_deregister(ifp);
610#endif
611
612	em_free_intr(adapter);
613	EM_LOCK(adapter);
614	adapter->in_detach = 1;
615	em_stop(adapter);
616	em_phy_hw_reset(&adapter->hw);
617	EM_UNLOCK(adapter);
618	ether_ifdetach(adapter->ifp);
619
620	em_free_pci_resources(adapter);
621	bus_generic_detach(dev);
622	if_free(ifp);
623
624	/* Free Transmit Descriptor ring */
625	if (adapter->tx_desc_base) {
626		em_dma_free(adapter, &adapter->txdma);
627		adapter->tx_desc_base = NULL;
628	}
629
630	/* Free Receive Descriptor ring */
631	if (adapter->rx_desc_base) {
632		em_dma_free(adapter, &adapter->rxdma);
633		adapter->rx_desc_base = NULL;
634	}
635
636	EM_LOCK_DESTROY(adapter);
637
638	return (0);
639}
640
641/*********************************************************************
642 *
643 *  Shutdown entry point
644 *
645 **********************************************************************/
646
647static int
648em_shutdown(device_t dev)
649{
650	struct adapter *adapter = device_get_softc(dev);
651	EM_LOCK(adapter);
652	em_stop(adapter);
653	EM_UNLOCK(adapter);
654	return (0);
655}
656
657/*
658 * Suspend/resume device methods.
659 */
660static int
661em_suspend(device_t dev)
662{
663	struct adapter *adapter = device_get_softc(dev);
664
665	EM_LOCK(adapter);
666	em_stop(adapter);
667	EM_UNLOCK(adapter);
668
669	return bus_generic_suspend(dev);
670}
671
672static int
673em_resume(device_t dev)
674{
675	struct adapter *adapter = device_get_softc(dev);
676	struct ifnet *ifp = adapter->ifp;
677
678	EM_LOCK(adapter);
679	em_init_locked(adapter);
680	if ((ifp->if_flags & IFF_UP) &&
681	    (ifp->if_drv_flags & IFF_DRV_RUNNING))
682		em_start_locked(ifp);
683	EM_UNLOCK(adapter);
684
685	return bus_generic_resume(dev);
686}
687
688
689/*********************************************************************
690 *  Transmit entry point
691 *
692 *  em_start is called by the stack to initiate a transmit.
693 *  The driver will remain in this routine as long as there are
694 *  packets to transmit and transmit resources are available.
695 *  In case resources are not available stack is notified and
696 *  the packet is requeued.
697 **********************************************************************/
698
699static void
700em_start_locked(struct ifnet *ifp)
701{
702	struct adapter	*adapter = ifp->if_softc;
703	struct mbuf	*m_head;
704
705	EM_LOCK_ASSERT(adapter);
706
707	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
708	    IFF_DRV_RUNNING)
709		return;
710	if (!adapter->link_active)
711		return;
712
713	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
714
715		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
716		if (m_head == NULL)
717			break;
718		/*
719		 * em_encap() can modify our pointer, and or make it NULL on
720		 * failure.  In that event, we can't requeue.
721		 */
722		if (em_encap(adapter, &m_head)) {
723			if (m_head == NULL)
724				break;
725			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
726			IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
727			break;
728		}
729
730		/* Send a copy of the frame to the BPF listener */
731		BPF_MTAP(ifp, m_head);
732
733		/* Set timeout in case hardware has problems transmitting. */
734		ifp->if_timer = EM_TX_TIMEOUT;
735	}
736}
737
738static void
739em_start(struct ifnet *ifp)
740{
741	struct adapter *adapter = ifp->if_softc;
742
743	EM_LOCK(adapter);
744	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
745		em_start_locked(ifp);
746	EM_UNLOCK(adapter);
747}
748
749/*********************************************************************
750 *  Ioctl entry point
751 *
752 *  em_ioctl is called when the user wants to configure the
753 *  interface.
754 *
755 *  return 0 on success, positive on failure
756 **********************************************************************/
757
758static int
759em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
760{
761	struct adapter	*adapter = ifp->if_softc;
762	struct ifreq *ifr = (struct ifreq *)data;
763	struct ifaddr *ifa = (struct ifaddr *)data;
764	int error = 0;
765
766	if (adapter->in_detach)
767		return (error);
768
769	switch (command) {
770	case SIOCSIFADDR:
771	case SIOCGIFADDR:
772		if (ifa->ifa_addr->sa_family == AF_INET) {
773			/*
774			 * XXX
775			 * Since resetting hardware takes a very long time
776			 * and results in link renegotiation we only
777			 * initialize the hardware only when it is absolutely
778			 * required.
779			 */
780			ifp->if_flags |= IFF_UP;
781			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
782				EM_LOCK(adapter);
783				em_init_locked(adapter);
784				EM_UNLOCK(adapter);
785			}
786			arp_ifinit(ifp, ifa);
787		} else
788			error = ether_ioctl(ifp, command, data);
789		break;
790	case SIOCSIFMTU:
791	    {
792		int max_frame_size;
793		uint16_t eeprom_data = 0;
794
795		IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
796
797		EM_LOCK(adapter);
798		switch (adapter->hw.mac_type) {
799		case em_82573:
800			/*
801			 * 82573 only supports jumbo frames
802			 * if ASPM is disabled.
803			 */
804			em_read_eeprom(&adapter->hw, EEPROM_INIT_3GIO_3, 1,
805			    &eeprom_data);
806			if (eeprom_data & EEPROM_WORD1A_ASPM_MASK) {
807				max_frame_size = ETHER_MAX_LEN;
808				break;
809			}
810			/* Allow Jumbo frames - fall thru */
811		case em_82571:
812		case em_82572:
813		case em_80003es2lan:	/* Limit Jumbo Frame size */
814			max_frame_size = 9234;
815			break;
816		case em_ich8lan:
817			/* ICH8 does not support jumbo frames */
818			max_frame_size = ETHER_MAX_LEN;
819			break;
820		default:
821			max_frame_size = MAX_JUMBO_FRAME_SIZE;
822		}
823		if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
824		    ETHER_CRC_LEN) {
825			EM_UNLOCK(adapter);
826			error = EINVAL;
827			break;
828		}
829
830		ifp->if_mtu = ifr->ifr_mtu;
831		adapter->hw.max_frame_size =
832		ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
833		em_init_locked(adapter);
834		EM_UNLOCK(adapter);
835		break;
836	    }
837	case SIOCSIFFLAGS:
838		IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)");
839		EM_LOCK(adapter);
840		if (ifp->if_flags & IFF_UP) {
841			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
842				if ((ifp->if_flags ^ adapter->if_flags) &
843				    IFF_PROMISC) {
844					em_disable_promisc(adapter);
845					em_set_promisc(adapter);
846				}
847			} else
848				em_init_locked(adapter);
849		} else {
850			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
851				em_stop(adapter);
852			}
853		}
854		adapter->if_flags = ifp->if_flags;
855		EM_UNLOCK(adapter);
856		break;
857	case SIOCADDMULTI:
858	case SIOCDELMULTI:
859		IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
860		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
861			EM_LOCK(adapter);
862			em_disable_intr(adapter);
863			em_set_multi(adapter);
864			if (adapter->hw.mac_type == em_82542_rev2_0) {
865				em_initialize_receive_unit(adapter);
866			}
867#ifdef DEVICE_POLLING
868			if (!(ifp->if_capenable & IFCAP_POLLING))
869#endif
870				em_enable_intr(adapter);
871			EM_UNLOCK(adapter);
872		}
873		break;
874	case SIOCSIFMEDIA:
875	case SIOCGIFMEDIA:
876		IOCTL_DEBUGOUT("ioctl rcv'd: SIOCxIFMEDIA (Get/Set Interface Media)");
877		error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
878		break;
879	case SIOCSIFCAP:
880	    {
881		int mask, reinit;
882
883		IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
884		reinit = 0;
885		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
886#ifdef DEVICE_POLLING
887		if (mask & IFCAP_POLLING) {
888			if (ifr->ifr_reqcap & IFCAP_POLLING) {
889				error = ether_poll_register(em_poll, ifp);
890				if (error)
891					return (error);
892				EM_LOCK(adapter);
893				em_disable_intr(adapter);
894				ifp->if_capenable |= IFCAP_POLLING;
895				EM_UNLOCK(adapter);
896			} else {
897				error = ether_poll_deregister(ifp);
898				/* Enable interrupt even in error case */
899				EM_LOCK(adapter);
900				em_enable_intr(adapter);
901				ifp->if_capenable &= ~IFCAP_POLLING;
902				EM_UNLOCK(adapter);
903			}
904		}
905#endif
906		if (mask & IFCAP_HWCSUM) {
907			ifp->if_capenable ^= IFCAP_HWCSUM;
908			reinit = 1;
909		}
910		if (mask & IFCAP_VLAN_HWTAGGING) {
911			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
912			reinit = 1;
913		}
914		if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
915			em_init(adapter);
916		VLAN_CAPABILITIES(ifp);
917		break;
918	    }
919	default:
920		error = ether_ioctl(ifp, command, data);
921		break;
922	}
923
924	return (error);
925}
926
927/*********************************************************************
928 *  Watchdog entry point
929 *
930 *  This routine is called whenever hardware quits transmitting.
931 *
932 **********************************************************************/
933
934static void
935em_watchdog(struct ifnet *ifp)
936{
937	struct adapter *adapter = ifp->if_softc;
938
939	EM_LOCK(adapter);
940	/* If we are in this routine because of pause frames, then
941	 * don't reset the hardware.
942	 */
943	if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_TXOFF) {
944		ifp->if_timer = EM_TX_TIMEOUT;
945		EM_UNLOCK(adapter);
946		return;
947	}
948
949	/*
950	 * Reclaim first as there is a possibility of losing Tx completion
951	 * interrupts. Possible cause of missing Tx completion interrupts
952	 * comes from Tx interrupt moderation mechanism(delayed interrupts)
953	 * or chipset bug.
954	 */
955	em_txeof(adapter);
956	if (adapter->num_tx_desc_avail == adapter->num_tx_desc) {
957		EM_UNLOCK(adapter);
958		return;
959	}
960
961	if (em_check_for_link(&adapter->hw) == 0)
962		device_printf(adapter->dev, "watchdog timeout -- resetting\n");
963
964	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
965	adapter->watchdog_events++;
966
967	em_init_locked(adapter);
968	EM_UNLOCK(adapter);
969}
970
971/*********************************************************************
972 *  Init entry point
973 *
974 *  This routine is used in two ways. It is used by the stack as
975 *  init entry point in network interface structure. It is also used
976 *  by the driver as a hw/sw initialization routine to get to a
977 *  consistent state.
978 *
979 *  return 0 on success, positive on failure
980 **********************************************************************/
981
982static void
983em_init_locked(struct adapter *adapter)
984{
985	struct ifnet	*ifp = adapter->ifp;
986	device_t	dev = adapter->dev;
987	uint32_t	pba;
988
989	INIT_DEBUGOUT("em_init: begin");
990
991	EM_LOCK_ASSERT(adapter);
992
993	em_stop(adapter);
994
995	/*
996	 * Packet Buffer Allocation (PBA)
997	 * Writing PBA sets the receive portion of the buffer
998	 * the remainder is used for the transmit buffer.
999	 *
1000	 * Devices before the 82547 had a Packet Buffer of 64K.
1001	 *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1002	 * After the 82547 the buffer was reduced to 40K.
1003	 *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1004	 *   Note: default does not leave enough room for Jumbo Frame >10k.
1005	 */
1006	switch (adapter->hw.mac_type) {
1007	case em_82547:
1008	case em_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1009		if (adapter->hw.max_frame_size > EM_RXBUFFER_8192)
1010			pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1011		else
1012			pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1013		adapter->tx_fifo_head = 0;
1014		adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1015		adapter->tx_fifo_size = (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1016		break;
1017	case em_80003es2lan: /* 80003es2lan: Total Packet Buffer is 48K */
1018	case em_82571: /* 82571: Total Packet Buffer is 48K */
1019	case em_82572: /* 82572: Total Packet Buffer is 48K */
1020			pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1021		break;
1022	case em_82573: /* 82573: Total Packet Buffer is 32K */
1023		/* Jumbo frames not supported */
1024			pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1025		break;
1026	case em_ich8lan:
1027		pba = E1000_PBA_8K;
1028		break;
1029	default:
1030		/* Devices before 82547 had a Packet Buffer of 64K.   */
1031		if(adapter->hw.max_frame_size > EM_RXBUFFER_8192)
1032			pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1033		else
1034			pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1035	}
1036
1037	INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1038	E1000_WRITE_REG(&adapter->hw, PBA, pba);
1039
1040	/* Get the latest mac address, User can use a LAA */
1041	bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac_addr, ETHER_ADDR_LEN);
1042
1043	/* Initialize the hardware */
1044	if (em_hardware_init(adapter)) {
1045		device_printf(dev, "Unable to initialize the hardware\n");
1046		return;
1047	}
1048	em_update_link_status(adapter);
1049
1050	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1051		em_enable_vlans(adapter);
1052
1053	/* Prepare transmit descriptors and buffers */
1054	if (em_setup_transmit_structures(adapter)) {
1055		device_printf(dev, "Could not setup transmit structures\n");
1056		em_stop(adapter);
1057		return;
1058	}
1059	em_initialize_transmit_unit(adapter);
1060
1061	/* Setup Multicast table */
1062	em_set_multi(adapter);
1063
1064	/* Prepare receive descriptors and buffers */
1065	if (em_setup_receive_structures(adapter)) {
1066		device_printf(dev, "Could not setup receive structures\n");
1067		em_stop(adapter);
1068		return;
1069	}
1070	em_initialize_receive_unit(adapter);
1071
1072	/* Don't lose promiscuous settings */
1073	em_set_promisc(adapter);
1074
1075	ifp->if_drv_flags |= IFF_DRV_RUNNING;
1076	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1077
1078	if (adapter->hw.mac_type >= em_82543) {
1079		if (ifp->if_capenable & IFCAP_TXCSUM)
1080			ifp->if_hwassist = EM_CHECKSUM_FEATURES;
1081		else
1082			ifp->if_hwassist = 0;
1083	}
1084
1085	callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1086	em_clear_hw_cntrs(&adapter->hw);
1087#ifdef DEVICE_POLLING
1088	/*
1089	 * Only enable interrupts if we are not polling, make sure
1090	 * they are off otherwise.
1091	 */
1092	if (ifp->if_capenable & IFCAP_POLLING)
1093		em_disable_intr(adapter);
1094	else
1095#endif /* DEVICE_POLLING */
1096		em_enable_intr(adapter);
1097
1098	/* Don't reset the phy next time init gets called */
1099	adapter->hw.phy_reset_disable = TRUE;
1100}
1101
1102static void
1103em_init(void *arg)
1104{
1105	struct adapter *adapter = arg;
1106
1107	EM_LOCK(adapter);
1108	em_init_locked(adapter);
1109	EM_UNLOCK(adapter);
1110}
1111
1112
1113#ifdef DEVICE_POLLING
1114/*********************************************************************
1115 *
1116 *  Legacy polling routine
1117 *
1118 *********************************************************************/
1119static void
1120em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1121{
1122	struct adapter *adapter = ifp->if_softc;
1123	uint32_t reg_icr;
1124
1125	EM_LOCK(adapter);
1126	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1127		EM_UNLOCK(adapter);
1128		return;
1129	}
1130
1131	if (cmd == POLL_AND_CHECK_STATUS) {
1132		reg_icr = E1000_READ_REG(&adapter->hw, ICR);
1133		if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1134			callout_stop(&adapter->timer);
1135			adapter->hw.get_link_status = 1;
1136			em_check_for_link(&adapter->hw);
1137			em_update_link_status(adapter);
1138			callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1139		}
1140	}
1141	em_rxeof(adapter, count);
1142	em_txeof(adapter);
1143
1144	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1145		em_start_locked(ifp);
1146	EM_UNLOCK(adapter);
1147}
1148
1149/*********************************************************************
1150 *
1151 *  Legacy Interrupt Service routine
1152 *
1153 *********************************************************************/
1154static void
1155em_intr(void *arg)
1156{
1157	struct adapter	*adapter = arg;
1158	struct ifnet	*ifp;
1159	uint32_t	reg_icr;
1160
1161	EM_LOCK(adapter);
1162
1163	ifp = adapter->ifp;
1164
1165	if (ifp->if_capenable & IFCAP_POLLING) {
1166		EM_UNLOCK(adapter);
1167		return;
1168	}
1169
1170	for (;;) {
1171		reg_icr = E1000_READ_REG(&adapter->hw, ICR);
1172		if (adapter->hw.mac_type >= em_82571 &&
1173		    (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1174			break;
1175		else if (reg_icr == 0)
1176			break;
1177
1178		/*
1179		 * XXX: some laptops trigger several spurious interrupts
1180		 * on em(4) when in the resume cycle. The ICR register
1181		 * reports all-ones value in this case. Processing such
1182		 * interrupts would lead to a freeze. I don't know why.
1183		 */
1184		if (reg_icr == 0xffffffff)
1185			break;
1186
1187		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1188			em_rxeof(adapter, -1);
1189			em_txeof(adapter);
1190		}
1191
1192		/* Link status change */
1193		if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1194			callout_stop(&adapter->timer);
1195			adapter->hw.get_link_status = 1;
1196			em_check_for_link(&adapter->hw);
1197			em_update_link_status(adapter);
1198			callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1199		}
1200
1201		if (reg_icr & E1000_ICR_RXO)
1202			adapter->rx_overruns++;
1203	}
1204
1205	if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
1206	    !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1207		em_start_locked(ifp);
1208
1209	EM_UNLOCK(adapter);
1210}
1211
1212#else  /* if not DEVICE_POLLING, then fast interrupt routines only */
1213
1214static void
1215em_handle_link(void *context, int pending)
1216{
1217	struct adapter	*adapter = context;
1218	struct ifnet *ifp;
1219
1220	ifp = adapter->ifp;
1221
1222	EM_LOCK(adapter);
1223
1224	callout_stop(&adapter->timer);
1225	adapter->hw.get_link_status = 1;
1226	em_check_for_link(&adapter->hw);
1227	em_update_link_status(adapter);
1228	callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1229	EM_UNLOCK(adapter);
1230}
1231
1232static void
1233em_handle_rxtx(void *context, int pending)
1234{
1235	struct adapter	*adapter = context;
1236	struct ifnet	*ifp;
1237
1238	NET_LOCK_GIANT();
1239	ifp = adapter->ifp;
1240
1241	/*
1242	 * TODO:
1243	 * It should be possible to run the tx clean loop without the lock.
1244	 */
1245	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1246		if (em_rxeof(adapter, adapter->rx_process_limit) != 0)
1247			taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1248		EM_LOCK(adapter);
1249		em_txeof(adapter);
1250
1251		if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1252			em_start_locked(ifp);
1253		EM_UNLOCK(adapter);
1254	}
1255
1256	em_enable_intr(adapter);
1257	NET_UNLOCK_GIANT();
1258}
1259
1260/*********************************************************************
1261 *
1262 *  Fast Interrupt Service routine
1263 *
1264 *********************************************************************/
1265static void
1266em_intr_fast(void *arg)
1267{
1268	struct adapter	*adapter = arg;
1269	struct ifnet	*ifp;
1270	uint32_t	reg_icr;
1271
1272	ifp = adapter->ifp;
1273
1274	reg_icr = E1000_READ_REG(&adapter->hw, ICR);
1275
1276	/* Hot eject?  */
1277	if (reg_icr == 0xffffffff)
1278		return;
1279
1280	/* Definitely not our interrupt.  */
1281	if (reg_icr == 0x0)
1282		return;
1283
1284	/*
1285	 * Starting with the 82571 chip, bit 31 should be used to
1286	 * determine whether the interrupt belongs to us.
1287	 */
1288	if (adapter->hw.mac_type >= em_82571 &&
1289	    (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1290		return;
1291
1292	/*
1293	 * Mask interrupts until the taskqueue is finished running.  This is
1294	 * cheap, just assume that it is needed.  This also works around the
1295	 * MSI message reordering errata on certain systems.
1296	 */
1297	em_disable_intr(adapter);
1298	taskqueue_enqueue(adapter->tq, &adapter->rxtx_task);
1299
1300	/* Link status change */
1301	if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC))
1302		taskqueue_enqueue(taskqueue_fast, &adapter->link_task);
1303
1304	if (reg_icr & E1000_ICR_RXO)
1305		adapter->rx_overruns++;
1306}
1307#endif /* ! DEVICE_POLLING */
1308
1309/*********************************************************************
1310 *
1311 *  Media Ioctl callback
1312 *
1313 *  This routine is called whenever the user queries the status of
1314 *  the interface using ifconfig.
1315 *
1316 **********************************************************************/
1317static void
1318em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1319{
1320	struct adapter *adapter = ifp->if_softc;
1321
1322	INIT_DEBUGOUT("em_media_status: begin");
1323
1324	EM_LOCK(adapter);
1325	em_check_for_link(&adapter->hw);
1326	em_update_link_status(adapter);
1327
1328	ifmr->ifm_status = IFM_AVALID;
1329	ifmr->ifm_active = IFM_ETHER;
1330
1331	if (!adapter->link_active) {
1332		EM_UNLOCK(adapter);
1333		return;
1334	}
1335
1336	ifmr->ifm_status |= IFM_ACTIVE;
1337
1338	if ((adapter->hw.media_type == em_media_type_fiber) ||
1339	    (adapter->hw.media_type == em_media_type_internal_serdes)) {
1340		if (adapter->hw.mac_type == em_82545)
1341			ifmr->ifm_active |= IFM_1000_LX | IFM_FDX;
1342		else
1343			ifmr->ifm_active |= IFM_1000_SX | IFM_FDX;
1344	} else {
1345		switch (adapter->link_speed) {
1346		case 10:
1347			ifmr->ifm_active |= IFM_10_T;
1348			break;
1349		case 100:
1350			ifmr->ifm_active |= IFM_100_TX;
1351			break;
1352		case 1000:
1353			ifmr->ifm_active |= IFM_1000_T;
1354			break;
1355		}
1356		if (adapter->link_duplex == FULL_DUPLEX)
1357			ifmr->ifm_active |= IFM_FDX;
1358		else
1359			ifmr->ifm_active |= IFM_HDX;
1360	}
1361	EM_UNLOCK(adapter);
1362}
1363
1364/*********************************************************************
1365 *
1366 *  Media Ioctl callback
1367 *
1368 *  This routine is called when the user changes speed/duplex using
1369 *  media/mediopt option with ifconfig.
1370 *
1371 **********************************************************************/
1372static int
1373em_media_change(struct ifnet *ifp)
1374{
1375	struct adapter *adapter = ifp->if_softc;
1376	struct ifmedia  *ifm = &adapter->media;
1377
1378	INIT_DEBUGOUT("em_media_change: begin");
1379
1380	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1381		return (EINVAL);
1382
1383	EM_LOCK(adapter);
1384	switch (IFM_SUBTYPE(ifm->ifm_media)) {
1385	case IFM_AUTO:
1386		adapter->hw.autoneg = DO_AUTO_NEG;
1387		adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1388		break;
1389	case IFM_1000_LX:
1390	case IFM_1000_SX:
1391	case IFM_1000_T:
1392		adapter->hw.autoneg = DO_AUTO_NEG;
1393		adapter->hw.autoneg_advertised = ADVERTISE_1000_FULL;
1394		break;
1395	case IFM_100_TX:
1396		adapter->hw.autoneg = FALSE;
1397		adapter->hw.autoneg_advertised = 0;
1398		if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1399			adapter->hw.forced_speed_duplex = em_100_full;
1400		else
1401			adapter->hw.forced_speed_duplex = em_100_half;
1402		break;
1403	case IFM_10_T:
1404		adapter->hw.autoneg = FALSE;
1405		adapter->hw.autoneg_advertised = 0;
1406		if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1407			adapter->hw.forced_speed_duplex = em_10_full;
1408		else
1409			adapter->hw.forced_speed_duplex = em_10_half;
1410		break;
1411	default:
1412		device_printf(adapter->dev, "Unsupported media type\n");
1413	}
1414
1415	/* As the speed/duplex settings my have changed we need to
1416	 * reset the PHY.
1417	 */
1418	adapter->hw.phy_reset_disable = FALSE;
1419
1420	em_init_locked(adapter);
1421	EM_UNLOCK(adapter);
1422
1423	return (0);
1424}
1425
1426/*********************************************************************
1427 *
1428 *  This routine maps the mbufs to tx descriptors.
1429 *
1430 *  return 0 on success, positive on failure
1431 **********************************************************************/
1432static int
1433em_encap(struct adapter *adapter, struct mbuf **m_headp)
1434{
1435	struct ifnet		*ifp = adapter->ifp;
1436	bus_dma_segment_t	segs[EM_MAX_SCATTER];
1437	bus_dmamap_t		map;
1438	struct em_buffer	*tx_buffer, *tx_buffer_last;
1439	struct em_tx_desc	*current_tx_desc;
1440	struct mbuf		*m_head;
1441	struct m_tag		*mtag;
1442	uint32_t		txd_upper, txd_lower, txd_used, txd_saved;
1443	int			nsegs, i, j;
1444	int			error;
1445
1446	m_head = *m_headp;
1447	current_tx_desc = NULL;
1448	txd_used = txd_saved = 0;
1449
1450	/*
1451	 * Force a cleanup if number of TX descriptors
1452	 * available hits the threshold.
1453	 */
1454	if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1455		em_txeof(adapter);
1456		if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1457			adapter->no_tx_desc_avail1++;
1458			return (ENOBUFS);
1459		}
1460	}
1461
1462	/* Find out if we are in vlan mode. */
1463	mtag = VLAN_OUTPUT_TAG(ifp, m_head);
1464
1465	/*
1466	 * When operating in promiscuous mode, hardware encapsulation for
1467	 * packets is disabled.  This means we have to add the vlan
1468	 * encapsulation in the driver, since it will have come down from the
1469	 * VLAN layer with a tag instead of a VLAN header.
1470	 */
1471	if (mtag != NULL && adapter->em_insert_vlan_header) {
1472		struct ether_vlan_header *evl;
1473		struct ether_header eh;
1474
1475		m_head = m_pullup(m_head, sizeof(eh));
1476		if (m_head == NULL) {
1477			*m_headp = NULL;
1478			return (ENOBUFS);
1479		}
1480		eh = *mtod(m_head, struct ether_header *);
1481		M_PREPEND(m_head, sizeof(*evl), M_DONTWAIT);
1482		if (m_head == NULL) {
1483			*m_headp = NULL;
1484			return (ENOBUFS);
1485		}
1486		m_head = m_pullup(m_head, sizeof(*evl));
1487		if (m_head == NULL) {
1488			*m_headp = NULL;
1489			return (ENOBUFS);
1490		}
1491		evl = mtod(m_head, struct ether_vlan_header *);
1492		bcopy(&eh, evl, sizeof(*evl));
1493		evl->evl_proto = evl->evl_encap_proto;
1494		evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
1495		evl->evl_tag = htons(VLAN_TAG_VALUE(mtag));
1496		m_tag_delete(m_head, mtag);
1497		mtag = NULL;
1498		*m_headp = m_head;
1499	}
1500
1501	/*
1502	 * Map the packet for DMA.
1503	 */
1504	tx_buffer = &adapter->tx_buffer_area[adapter->next_avail_tx_desc];
1505	tx_buffer_last = tx_buffer;
1506	map = tx_buffer->map;
1507	error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, *m_headp, segs,
1508	    &nsegs, BUS_DMA_NOWAIT);
1509	if (error == EFBIG) {
1510		struct mbuf *m;
1511
1512		m = m_defrag(*m_headp, M_DONTWAIT);
1513		if (m == NULL) {
1514			/* Assume m_defrag(9) used only m_get(9). */
1515			adapter->mbuf_alloc_failed++;
1516			m_freem(*m_headp);
1517			*m_headp = NULL;
1518			return (ENOBUFS);
1519		}
1520		*m_headp = m;
1521		error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, *m_headp,
1522		    segs, &nsegs, BUS_DMA_NOWAIT);
1523		if (error != 0) {
1524			adapter->no_tx_dma_setup++;
1525			m_freem(*m_headp);
1526			*m_headp = NULL;
1527			return (error);
1528		}
1529	} else if (error != 0) {
1530		adapter->no_tx_dma_setup++;
1531		return (error);
1532	}
1533	if (nsegs == 0) {
1534		m_freem(*m_headp);
1535		*m_headp = NULL;
1536		return (EIO);
1537	}
1538
1539	if (nsegs > adapter->num_tx_desc_avail) {
1540		adapter->no_tx_desc_avail2++;
1541		bus_dmamap_unload(adapter->txtag, map);
1542		return (ENOBUFS);
1543	}
1544
1545	m_head = *m_headp;
1546	if (ifp->if_hwassist > 0)
1547		em_transmit_checksum_setup(adapter,  m_head, &txd_upper, &txd_lower);
1548	else
1549		txd_upper = txd_lower = 0;
1550
1551	i = adapter->next_avail_tx_desc;
1552	if (adapter->pcix_82544) {
1553		txd_saved = i;
1554		txd_used = 0;
1555	}
1556	for (j = 0; j < nsegs; j++) {
1557		/* If adapter is 82544 and on PCIX bus. */
1558		if(adapter->pcix_82544) {
1559			DESC_ARRAY	desc_array;
1560			uint32_t	array_elements, counter;
1561
1562			/*
1563			 * Check the Address and Length combination and
1564			 * split the data accordingly
1565			 */
1566			array_elements = em_fill_descriptors(segs[j].ds_addr,
1567			    segs[j].ds_len, &desc_array);
1568			for (counter = 0; counter < array_elements; counter++) {
1569				if (txd_used == adapter->num_tx_desc_avail) {
1570					adapter->next_avail_tx_desc = txd_saved;
1571					adapter->no_tx_desc_avail2++;
1572					bus_dmamap_unload(adapter->txtag, map);
1573					return (ENOBUFS);
1574				}
1575				tx_buffer = &adapter->tx_buffer_area[i];
1576				current_tx_desc = &adapter->tx_desc_base[i];
1577				current_tx_desc->buffer_addr = htole64(
1578					desc_array.descriptor[counter].address);
1579				current_tx_desc->lower.data = htole32(
1580					(adapter->txd_cmd | txd_lower |
1581					(uint16_t)desc_array.descriptor[counter].length));
1582				current_tx_desc->upper.data = htole32((txd_upper));
1583				if (++i == adapter->num_tx_desc)
1584					i = 0;
1585
1586				tx_buffer->m_head = NULL;
1587				txd_used++;
1588			}
1589		} else {
1590			tx_buffer = &adapter->tx_buffer_area[i];
1591			current_tx_desc = &adapter->tx_desc_base[i];
1592
1593			current_tx_desc->buffer_addr = htole64(segs[j].ds_addr);
1594			current_tx_desc->lower.data = htole32(
1595				adapter->txd_cmd | txd_lower | segs[j].ds_len);
1596			current_tx_desc->upper.data = htole32(txd_upper);
1597
1598			if (++i == adapter->num_tx_desc)
1599				i = 0;
1600
1601			tx_buffer->m_head = NULL;
1602		}
1603	}
1604
1605	adapter->next_avail_tx_desc = i;
1606	if (adapter->pcix_82544)
1607		adapter->num_tx_desc_avail -= txd_used;
1608	else
1609		adapter->num_tx_desc_avail -= nsegs;
1610
1611	if (mtag != NULL) {
1612		/* Set the vlan id. */
1613		current_tx_desc->upper.fields.special =
1614		    htole16(VLAN_TAG_VALUE(mtag));
1615
1616		/* Tell hardware to add tag. */
1617		current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_VLE);
1618	}
1619
1620	tx_buffer->m_head = m_head;
1621	tx_buffer_last->map = tx_buffer->map;
1622	tx_buffer->map = map;
1623	bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
1624
1625	/*
1626	 * Last Descriptor of Packet needs End Of Packet (EOP).
1627	 */
1628	current_tx_desc->lower.data |= htole32(E1000_TXD_CMD_EOP);
1629
1630	/*
1631	 * Advance the Transmit Descriptor Tail (Tdt), this tells the E1000
1632	 * that this frame is available to transmit.
1633	 */
1634	bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
1635	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1636	if (adapter->hw.mac_type == em_82547 && adapter->link_duplex == HALF_DUPLEX)
1637		em_82547_move_tail_locked(adapter);
1638	else {
1639		E1000_WRITE_REG(&adapter->hw, TDT, i);
1640		if (adapter->hw.mac_type == em_82547)
1641			em_82547_update_fifo_head(adapter, m_head->m_pkthdr.len);
1642	}
1643
1644	return (0);
1645}
1646
1647/*********************************************************************
1648 *
1649 * 82547 workaround to avoid controller hang in half-duplex environment.
1650 * The workaround is to avoid queuing a large packet that would span
1651 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1652 * in this case. We do that only when FIFO is quiescent.
1653 *
1654 **********************************************************************/
1655static void
1656em_82547_move_tail_locked(struct adapter *adapter)
1657{
1658	uint16_t hw_tdt;
1659	uint16_t sw_tdt;
1660	struct em_tx_desc *tx_desc;
1661	uint16_t length = 0;
1662	boolean_t eop = 0;
1663
1664	EM_LOCK_ASSERT(adapter);
1665
1666	hw_tdt = E1000_READ_REG(&adapter->hw, TDT);
1667	sw_tdt = adapter->next_avail_tx_desc;
1668
1669	while (hw_tdt != sw_tdt) {
1670		tx_desc = &adapter->tx_desc_base[hw_tdt];
1671		length += tx_desc->lower.flags.length;
1672		eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1673		if(++hw_tdt == adapter->num_tx_desc)
1674			hw_tdt = 0;
1675
1676		if (eop) {
1677			if (em_82547_fifo_workaround(adapter, length)) {
1678				adapter->tx_fifo_wrk_cnt++;
1679				callout_reset(&adapter->tx_fifo_timer, 1,
1680					em_82547_move_tail, adapter);
1681				break;
1682			}
1683			E1000_WRITE_REG(&adapter->hw, TDT, hw_tdt);
1684			em_82547_update_fifo_head(adapter, length);
1685			length = 0;
1686		}
1687	}
1688}
1689
1690static void
1691em_82547_move_tail(void *arg)
1692{
1693	struct adapter *adapter = arg;
1694
1695	EM_LOCK(adapter);
1696	em_82547_move_tail_locked(adapter);
1697	EM_UNLOCK(adapter);
1698}
1699
1700static int
1701em_82547_fifo_workaround(struct adapter *adapter, int len)
1702{
1703	int fifo_space, fifo_pkt_len;
1704
1705	fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1706
1707	if (adapter->link_duplex == HALF_DUPLEX) {
1708		fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
1709
1710		if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1711			if (em_82547_tx_fifo_reset(adapter))
1712				return (0);
1713			else
1714				return (1);
1715		}
1716	}
1717
1718	return (0);
1719}
1720
1721static void
1722em_82547_update_fifo_head(struct adapter *adapter, int len)
1723{
1724	int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1725
1726	/* tx_fifo_head is always 16 byte aligned */
1727	adapter->tx_fifo_head += fifo_pkt_len;
1728	if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
1729		adapter->tx_fifo_head -= adapter->tx_fifo_size;
1730	}
1731}
1732
1733
1734static int
1735em_82547_tx_fifo_reset(struct adapter *adapter)
1736{
1737	uint32_t tctl;
1738
1739	if ((E1000_READ_REG(&adapter->hw, TDT) == E1000_READ_REG(&adapter->hw, TDH)) &&
1740	    (E1000_READ_REG(&adapter->hw, TDFT) == E1000_READ_REG(&adapter->hw, TDFH)) &&
1741	    (E1000_READ_REG(&adapter->hw, TDFTS) == E1000_READ_REG(&adapter->hw, TDFHS))&&
1742	    (E1000_READ_REG(&adapter->hw, TDFPC) == 0)) {
1743
1744		/* Disable TX unit */
1745		tctl = E1000_READ_REG(&adapter->hw, TCTL);
1746		E1000_WRITE_REG(&adapter->hw, TCTL, tctl & ~E1000_TCTL_EN);
1747
1748		/* Reset FIFO pointers */
1749		E1000_WRITE_REG(&adapter->hw, TDFT,  adapter->tx_head_addr);
1750		E1000_WRITE_REG(&adapter->hw, TDFH,  adapter->tx_head_addr);
1751		E1000_WRITE_REG(&adapter->hw, TDFTS, adapter->tx_head_addr);
1752		E1000_WRITE_REG(&adapter->hw, TDFHS, adapter->tx_head_addr);
1753
1754		/* Re-enable TX unit */
1755		E1000_WRITE_REG(&adapter->hw, TCTL, tctl);
1756		E1000_WRITE_FLUSH(&adapter->hw);
1757
1758		adapter->tx_fifo_head = 0;
1759		adapter->tx_fifo_reset_cnt++;
1760
1761		return (TRUE);
1762	}
1763	else {
1764		return (FALSE);
1765	}
1766}
1767
1768static void
1769em_set_promisc(struct adapter *adapter)
1770{
1771	struct ifnet	*ifp = adapter->ifp;
1772	uint32_t	reg_rctl;
1773
1774	reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1775
1776	if (ifp->if_flags & IFF_PROMISC) {
1777		reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1778		E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1779		/* Disable VLAN stripping in promiscous mode
1780		 * This enables bridging of vlan tagged frames to occur
1781		 * and also allows vlan tags to be seen in tcpdump
1782		 */
1783		if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1784			em_disable_vlans(adapter);
1785		adapter->em_insert_vlan_header = 1;
1786	} else if (ifp->if_flags & IFF_ALLMULTI) {
1787		reg_rctl |= E1000_RCTL_MPE;
1788		reg_rctl &= ~E1000_RCTL_UPE;
1789		E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1790		adapter->em_insert_vlan_header = 0;
1791	} else
1792		adapter->em_insert_vlan_header = 0;
1793}
1794
1795static void
1796em_disable_promisc(struct adapter *adapter)
1797{
1798	struct ifnet	*ifp = adapter->ifp;
1799	uint32_t	reg_rctl;
1800
1801	reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1802
1803	reg_rctl &=  (~E1000_RCTL_UPE);
1804	reg_rctl &=  (~E1000_RCTL_MPE);
1805	E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1806
1807	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1808		em_enable_vlans(adapter);
1809	adapter->em_insert_vlan_header = 0;
1810}
1811
1812
1813/*********************************************************************
1814 *  Multicast Update
1815 *
1816 *  This routine is called whenever multicast address list is updated.
1817 *
1818 **********************************************************************/
1819
1820static void
1821em_set_multi(struct adapter *adapter)
1822{
1823	struct ifnet	*ifp = adapter->ifp;
1824	struct ifmultiaddr *ifma;
1825	uint32_t reg_rctl = 0;
1826	uint8_t  mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_LENGTH_OF_ADDRESS];
1827	int mcnt = 0;
1828
1829	IOCTL_DEBUGOUT("em_set_multi: begin");
1830
1831	if (adapter->hw.mac_type == em_82542_rev2_0) {
1832		reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1833		if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1834			em_pci_clear_mwi(&adapter->hw);
1835		reg_rctl |= E1000_RCTL_RST;
1836		E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1837		msec_delay(5);
1838	}
1839
1840	IF_ADDR_LOCK(ifp);
1841	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1842		if (ifma->ifma_addr->sa_family != AF_LINK)
1843			continue;
1844
1845		if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1846			break;
1847
1848		bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1849		    &mta[mcnt*ETH_LENGTH_OF_ADDRESS], ETH_LENGTH_OF_ADDRESS);
1850		mcnt++;
1851	}
1852	IF_ADDR_UNLOCK(ifp);
1853
1854	if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1855		reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1856		reg_rctl |= E1000_RCTL_MPE;
1857		E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1858	} else
1859		em_mc_addr_list_update(&adapter->hw, mta, mcnt, 0, 1);
1860
1861	if (adapter->hw.mac_type == em_82542_rev2_0) {
1862		reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1863		reg_rctl &= ~E1000_RCTL_RST;
1864		E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1865		msec_delay(5);
1866		if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1867			em_pci_set_mwi(&adapter->hw);
1868	}
1869}
1870
1871
1872/*********************************************************************
1873 *  Timer routine
1874 *
1875 *  This routine checks for link status and updates statistics.
1876 *
1877 **********************************************************************/
1878
1879static void
1880em_local_timer(void *arg)
1881{
1882	struct adapter	*adapter = arg;
1883	struct ifnet	*ifp = adapter->ifp;
1884
1885	EM_LOCK(adapter);
1886
1887	em_check_for_link(&adapter->hw);
1888	em_update_link_status(adapter);
1889	em_update_stats_counters(adapter);
1890	if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
1891		em_print_hw_stats(adapter);
1892	em_smartspeed(adapter);
1893
1894	callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1895
1896	EM_UNLOCK(adapter);
1897}
1898
1899static void
1900em_update_link_status(struct adapter *adapter)
1901{
1902	struct ifnet *ifp = adapter->ifp;
1903	device_t dev = adapter->dev;
1904
1905	if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
1906		if (adapter->link_active == 0) {
1907			em_get_speed_and_duplex(&adapter->hw, &adapter->link_speed,
1908			    &adapter->link_duplex);
1909			/* Check if we may set SPEED_MODE bit on PCI-E */
1910			if ((adapter->link_speed == SPEED_1000) &&
1911			    ((adapter->hw.mac_type == em_82571) ||
1912			    (adapter->hw.mac_type == em_82572))) {
1913				int tarc0;
1914
1915				tarc0 = E1000_READ_REG(&adapter->hw, TARC0);
1916				tarc0 |= SPEED_MODE_BIT;
1917				E1000_WRITE_REG(&adapter->hw, TARC0, tarc0);
1918			}
1919			if (bootverbose)
1920				device_printf(dev, "Link is up %d Mbps %s\n",
1921				    adapter->link_speed,
1922				    ((adapter->link_duplex == FULL_DUPLEX) ?
1923				    "Full Duplex" : "Half Duplex"));
1924			adapter->link_active = 1;
1925			adapter->smartspeed = 0;
1926			ifp->if_baudrate = adapter->link_speed * 1000000;
1927			if_link_state_change(ifp, LINK_STATE_UP);
1928		}
1929	} else {
1930		if (adapter->link_active == 1) {
1931			ifp->if_baudrate = adapter->link_speed = 0;
1932			adapter->link_duplex = 0;
1933			if (bootverbose)
1934				device_printf(dev, "Link is Down\n");
1935			adapter->link_active = 0;
1936			if_link_state_change(ifp, LINK_STATE_DOWN);
1937		}
1938	}
1939}
1940
1941/*********************************************************************
1942 *
1943 *  This routine disables all traffic on the adapter by issuing a
1944 *  global reset on the MAC and deallocates TX/RX buffers.
1945 *
1946 **********************************************************************/
1947
1948static void
1949em_stop(void *arg)
1950{
1951	struct adapter	*adapter = arg;
1952	struct ifnet	*ifp = adapter->ifp;
1953
1954	EM_LOCK_ASSERT(adapter);
1955
1956	INIT_DEBUGOUT("em_stop: begin");
1957
1958	em_disable_intr(adapter);
1959	em_reset_hw(&adapter->hw);
1960	callout_stop(&adapter->timer);
1961	callout_stop(&adapter->tx_fifo_timer);
1962	em_free_transmit_structures(adapter);
1963	em_free_receive_structures(adapter);
1964
1965	/* Tell the stack that the interface is no longer active */
1966	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1967}
1968
1969
1970/********************************************************************
1971 *
1972 *  Determine hardware revision.
1973 *
1974 **********************************************************************/
1975static void
1976em_identify_hardware(struct adapter *adapter)
1977{
1978	device_t dev = adapter->dev;
1979
1980	/* Make sure our PCI config space has the necessary stuff set */
1981	pci_enable_busmaster(dev);
1982	pci_enable_io(dev, SYS_RES_MEMORY);
1983	adapter->hw.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1984
1985	/* Save off the information about this board */
1986	adapter->hw.vendor_id = pci_get_vendor(dev);
1987	adapter->hw.device_id = pci_get_device(dev);
1988	adapter->hw.revision_id = pci_get_revid(dev);
1989	adapter->hw.subsystem_vendor_id = pci_get_subvendor(dev);
1990	adapter->hw.subsystem_id = pci_get_subdevice(dev);
1991
1992	/* Identify the MAC */
1993	if (em_set_mac_type(&adapter->hw))
1994		device_printf(dev, "Unknown MAC Type\n");
1995
1996	if(adapter->hw.mac_type == em_82541 || adapter->hw.mac_type == em_82541_rev_2 ||
1997	   adapter->hw.mac_type == em_82547 || adapter->hw.mac_type == em_82547_rev_2)
1998		adapter->hw.phy_init_script = TRUE;
1999}
2000
2001static int
2002em_allocate_pci_resources(struct adapter *adapter)
2003{
2004	device_t	dev = adapter->dev;
2005	int		val, rid;
2006
2007	rid = PCIR_BAR(0);
2008	adapter->res_memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2009	    &rid, RF_ACTIVE);
2010	if (adapter->res_memory == NULL) {
2011		device_printf(dev, "Unable to allocate bus resource: memory\n");
2012		return (ENXIO);
2013	}
2014	adapter->osdep.mem_bus_space_tag =
2015	rman_get_bustag(adapter->res_memory);
2016	adapter->osdep.mem_bus_space_handle = rman_get_bushandle(adapter->res_memory);
2017	adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
2018
2019	if (adapter->hw.mac_type > em_82543) {
2020		/* Figure our where our IO BAR is ? */
2021		for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2022			val = pci_read_config(dev, rid, 4);
2023			if (E1000_BAR_TYPE(val) == E1000_BAR_TYPE_IO) {
2024				adapter->io_rid = rid;
2025				break;
2026			}
2027			rid += 4;
2028			/* check for 64bit BAR */
2029			if (E1000_BAR_MEM_TYPE(val) == E1000_BAR_MEM_TYPE_64BIT)
2030				rid += 4;
2031		}
2032		if (rid >= PCIR_CIS) {
2033			device_printf(dev, "Unable to locate IO BAR\n");
2034			return (ENXIO);
2035		}
2036		adapter->res_ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT,
2037		    &adapter->io_rid, RF_ACTIVE);
2038		if (adapter->res_ioport == NULL) {
2039			device_printf(dev, "Unable to allocate bus resource: "
2040			    "ioport\n");
2041			return (ENXIO);
2042		}
2043		adapter->hw.io_base = 0;
2044		adapter->osdep.io_bus_space_tag = rman_get_bustag(adapter->res_ioport);
2045		adapter->osdep.io_bus_space_handle =
2046		    rman_get_bushandle(adapter->res_ioport);
2047	}
2048
2049	/* For ICH8 we need to find the flash memory. */
2050	if (adapter->hw.mac_type == em_ich8lan) {
2051		rid = EM_FLASH;
2052
2053		adapter->flash_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2054		    &rid, RF_ACTIVE);
2055		adapter->osdep.flash_bus_space_tag = rman_get_bustag(adapter->flash_mem);
2056		adapter->osdep.flash_bus_space_handle =
2057		    rman_get_bushandle(adapter->flash_mem);
2058	}
2059
2060	rid = 0x0;
2061	adapter->res_interrupt = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
2062	    RF_SHAREABLE | RF_ACTIVE);
2063	if (adapter->res_interrupt == NULL) {
2064		device_printf(dev, "Unable to allocate bus resource: "
2065		    "interrupt\n");
2066		return (ENXIO);
2067	}
2068
2069	adapter->hw.back = &adapter->osdep;
2070
2071	return (0);
2072}
2073
2074int
2075em_allocate_intr(struct adapter *adapter)
2076{
2077	device_t dev = adapter->dev;
2078	int error;
2079
2080	/* Manually turn off all interrupts */
2081	E1000_WRITE_REG(&adapter->hw, IMC, 0xffffffff);
2082
2083#ifdef DEVICE_POLLING
2084	if (adapter->int_handler_tag == NULL && (error = bus_setup_intr(dev,
2085	    adapter->res_interrupt, INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter,
2086	    &adapter->int_handler_tag)) != 0) {
2087		device_printf(dev, "Failed to register interrupt handler");
2088		return (error);
2089	}
2090#else
2091	/*
2092	 * Try allocating a fast interrupt and the associated deferred
2093	 * processing contexts.
2094	 */
2095	TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter);
2096	TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter);
2097	adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT,
2098	    taskqueue_thread_enqueue, &adapter->tq);
2099	taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq",
2100	    device_get_nameunit(adapter->dev));
2101	if ((error = bus_setup_intr(dev, adapter->res_interrupt,
2102	    INTR_TYPE_NET | INTR_FAST, em_intr_fast, adapter,
2103	    &adapter->int_handler_tag)) != 0) {
2104		device_printf(dev, "Failed to register fast interrupt "
2105			    "handler: %d\n", error);
2106		taskqueue_free(adapter->tq);
2107		adapter->tq = NULL;
2108		return (error);
2109	}
2110#endif
2111
2112	em_enable_intr(adapter);
2113	return (0);
2114}
2115
2116static void
2117em_free_intr(struct adapter *adapter)
2118{
2119	device_t dev = adapter->dev;
2120
2121	if (adapter->int_handler_tag != NULL) {
2122		bus_teardown_intr(dev, adapter->res_interrupt, adapter->int_handler_tag);
2123		adapter->int_handler_tag = NULL;
2124	}
2125	if (adapter->tq != NULL) {
2126		taskqueue_drain(adapter->tq, &adapter->rxtx_task);
2127		taskqueue_drain(taskqueue_fast, &adapter->link_task);
2128		taskqueue_free(adapter->tq);
2129		adapter->tq = NULL;
2130	}
2131}
2132
2133static void
2134em_free_pci_resources(struct adapter *adapter)
2135{
2136	device_t dev = adapter->dev;
2137
2138	if (adapter->res_interrupt != NULL)
2139		bus_release_resource(dev, SYS_RES_IRQ, 0, adapter->res_interrupt);
2140
2141	if (adapter->res_memory != NULL)
2142		bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(0),
2143		    adapter->res_memory);
2144
2145	if (adapter->flash_mem != NULL)
2146		bus_release_resource(dev, SYS_RES_MEMORY, EM_FLASH,
2147		    adapter->flash_mem);
2148
2149	if (adapter->res_ioport != NULL)
2150		bus_release_resource(dev, SYS_RES_IOPORT, adapter->io_rid,
2151		    adapter->res_ioport);
2152}
2153
2154/*********************************************************************
2155 *
2156 *  Initialize the hardware to a configuration as specified by the
2157 *  adapter structure. The controller is reset, the EEPROM is
2158 *  verified, the MAC address is set, then the shared initialization
2159 *  routines are called.
2160 *
2161 **********************************************************************/
2162static int
2163em_hardware_init(struct adapter *adapter)
2164{
2165	device_t dev = adapter->dev;
2166	uint16_t rx_buffer_size;
2167
2168	INIT_DEBUGOUT("em_hardware_init: begin");
2169	/* Issue a global reset */
2170	em_reset_hw(&adapter->hw);
2171
2172	/* When hardware is reset, fifo_head is also reset */
2173	adapter->tx_fifo_head = 0;
2174
2175	/* Make sure we have a good EEPROM before we read from it */
2176	if (em_validate_eeprom_checksum(&adapter->hw) < 0) {
2177		device_printf(dev, "The EEPROM Checksum Is Not Valid\n");
2178		return (EIO);
2179	}
2180
2181	if (em_read_part_num(&adapter->hw, &(adapter->part_num)) < 0) {
2182		device_printf(dev, "EEPROM read error while reading part "
2183		    "number\n");
2184		return (EIO);
2185	}
2186
2187	/* Set up smart power down as default off on newer adapters. */
2188	if (!em_smart_pwr_down &&
2189	    (adapter->hw.mac_type == em_82571 || adapter->hw.mac_type == em_82572)) {
2190		uint16_t phy_tmp = 0;
2191
2192		/* Speed up time to link by disabling smart power down. */
2193		em_read_phy_reg(&adapter->hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
2194		phy_tmp &= ~IGP02E1000_PM_SPD;
2195		em_write_phy_reg(&adapter->hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp);
2196	}
2197
2198	/*
2199	 * These parameters control the automatic generation (Tx) and
2200	 * response (Rx) to Ethernet PAUSE frames.
2201	 * - High water mark should allow for at least two frames to be
2202	 *   received after sending an XOFF.
2203	 * - Low water mark works best when it is very near the high water mark.
2204	 *   This allows the receiver to restart by sending XON when it has
2205	 *   drained a bit. Here we use an arbitary value of 1500 which will
2206	 *   restart after one full frame is pulled from the buffer. There
2207	 *   could be several smaller frames in the buffer and if so they will
2208	 *   not trigger the XON until their total number reduces the buffer
2209	 *   by 1500.
2210	 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2211	 */
2212	rx_buffer_size = ((E1000_READ_REG(&adapter->hw, PBA) & 0xffff) << 10 );
2213
2214	adapter->hw.fc_high_water = rx_buffer_size -
2215	    roundup2(adapter->hw.max_frame_size, 1024);
2216	adapter->hw.fc_low_water = adapter->hw.fc_high_water - 1500;
2217	if (adapter->hw.mac_type == em_80003es2lan)
2218		adapter->hw.fc_pause_time = 0xFFFF;
2219	else
2220		adapter->hw.fc_pause_time = 0x1000;
2221	adapter->hw.fc_send_xon = TRUE;
2222	adapter->hw.fc = em_fc_full;
2223
2224	if (em_init_hw(&adapter->hw) < 0) {
2225		device_printf(dev, "Hardware Initialization Failed");
2226		return (EIO);
2227	}
2228
2229	em_check_for_link(&adapter->hw);
2230
2231	return (0);
2232}
2233
2234/*********************************************************************
2235 *
2236 *  Setup networking device structure and register an interface.
2237 *
2238 **********************************************************************/
2239static void
2240em_setup_interface(device_t dev, struct adapter *adapter)
2241{
2242	struct ifnet   *ifp;
2243	INIT_DEBUGOUT("em_setup_interface: begin");
2244
2245	ifp = adapter->ifp = if_alloc(IFT_ETHER);
2246	if (ifp == NULL)
2247		panic("%s: can not if_alloc()", device_get_nameunit(dev));
2248	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2249	ifp->if_mtu = ETHERMTU;
2250	ifp->if_init =  em_init;
2251	ifp->if_softc = adapter;
2252	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2253	ifp->if_ioctl = em_ioctl;
2254	ifp->if_start = em_start;
2255	ifp->if_watchdog = em_watchdog;
2256	IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2257	ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2258	IFQ_SET_READY(&ifp->if_snd);
2259
2260	ether_ifattach(ifp, adapter->hw.mac_addr);
2261
2262	ifp->if_capabilities = ifp->if_capenable = 0;
2263
2264	if (adapter->hw.mac_type >= em_82543) {
2265		ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2266		ifp->if_capenable |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM;
2267	}
2268
2269	/*
2270	 * Tell the upper layer(s) we support long frames.
2271	 */
2272	ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2273	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2274	ifp->if_capenable |= IFCAP_VLAN_MTU;
2275
2276#ifdef DEVICE_POLLING
2277	ifp->if_capabilities |= IFCAP_POLLING;
2278#endif
2279
2280	/*
2281	 * Specify the media types supported by this adapter and register
2282	 * callbacks to update media and link information
2283	 */
2284	ifmedia_init(&adapter->media, IFM_IMASK, em_media_change,
2285	    em_media_status);
2286	if ((adapter->hw.media_type == em_media_type_fiber) ||
2287	    (adapter->hw.media_type == em_media_type_internal_serdes)) {
2288		u_char fiber_type = IFM_1000_SX;	/* default type; */
2289
2290		if (adapter->hw.mac_type == em_82545)
2291			fiber_type = IFM_1000_LX;
2292		ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2293		    0, NULL);
2294		ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2295	} else {
2296		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2297		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2298			    0, NULL);
2299		ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2300			    0, NULL);
2301		ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2302			    0, NULL);
2303		if (adapter->hw.phy_type != em_phy_ife) {
2304			ifmedia_add(&adapter->media,
2305				IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2306			ifmedia_add(&adapter->media,
2307				IFM_ETHER | IFM_1000_T, 0, NULL);
2308		}
2309	}
2310	ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2311	ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2312}
2313
2314
2315/*********************************************************************
2316 *
2317 *  Workaround for SmartSpeed on 82541 and 82547 controllers
2318 *
2319 **********************************************************************/
2320static void
2321em_smartspeed(struct adapter *adapter)
2322{
2323	uint16_t phy_tmp;
2324
2325	if (adapter->link_active || (adapter->hw.phy_type != em_phy_igp) ||
2326	    adapter->hw.autoneg == 0 ||
2327	    (adapter->hw.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
2328		return;
2329
2330	if (adapter->smartspeed == 0) {
2331		/* If Master/Slave config fault is asserted twice,
2332		 * we assume back-to-back */
2333		em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2334		if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
2335			return;
2336		em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2337		if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
2338			em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2339			if(phy_tmp & CR_1000T_MS_ENABLE) {
2340				phy_tmp &= ~CR_1000T_MS_ENABLE;
2341				em_write_phy_reg(&adapter->hw, PHY_1000T_CTRL,
2342				    phy_tmp);
2343				adapter->smartspeed++;
2344				if(adapter->hw.autoneg &&
2345				   !em_phy_setup_autoneg(&adapter->hw) &&
2346				   !em_read_phy_reg(&adapter->hw, PHY_CTRL,
2347				    &phy_tmp)) {
2348					phy_tmp |= (MII_CR_AUTO_NEG_EN |
2349						    MII_CR_RESTART_AUTO_NEG);
2350					em_write_phy_reg(&adapter->hw, PHY_CTRL,
2351					    phy_tmp);
2352				}
2353			}
2354		}
2355		return;
2356	} else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
2357		/* If still no link, perhaps using 2/3 pair cable */
2358		em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2359		phy_tmp |= CR_1000T_MS_ENABLE;
2360		em_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
2361		if(adapter->hw.autoneg &&
2362		   !em_phy_setup_autoneg(&adapter->hw) &&
2363		   !em_read_phy_reg(&adapter->hw, PHY_CTRL, &phy_tmp)) {
2364			phy_tmp |= (MII_CR_AUTO_NEG_EN |
2365				    MII_CR_RESTART_AUTO_NEG);
2366			em_write_phy_reg(&adapter->hw, PHY_CTRL, phy_tmp);
2367		}
2368	}
2369	/* Restart process after EM_SMARTSPEED_MAX iterations */
2370	if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
2371		adapter->smartspeed = 0;
2372}
2373
2374
2375/*
2376 * Manage DMA'able memory.
2377 */
2378static void
2379em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2380{
2381	if (error)
2382		return;
2383	*(bus_addr_t *) arg = segs[0].ds_addr;
2384}
2385
2386static int
2387em_dma_malloc(struct adapter *adapter, bus_size_t size, struct em_dma_alloc *dma,
2388	int mapflags)
2389{
2390	int error;
2391
2392	error = bus_dma_tag_create(NULL,		/* parent */
2393				EM_DBA_ALIGN, 0,	/* alignment, bounds */
2394				BUS_SPACE_MAXADDR,	/* lowaddr */
2395				BUS_SPACE_MAXADDR,	/* highaddr */
2396				NULL, NULL,		/* filter, filterarg */
2397				size,			/* maxsize */
2398				1,			/* nsegments */
2399				size,			/* maxsegsize */
2400				0,			/* flags */
2401				NULL,			/* lockfunc */
2402				NULL,			/* lockarg */
2403				&dma->dma_tag);
2404	if (error) {
2405		device_printf(adapter->dev, "%s: bus_dma_tag_create failed: %d\n",
2406		    __func__, error);
2407		goto fail_0;
2408	}
2409
2410	error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
2411	    BUS_DMA_NOWAIT, &dma->dma_map);
2412	if (error) {
2413		device_printf(adapter->dev, "%s: bus_dmamem_alloc(%ju) failed: %d\n",
2414		    __func__, (uintmax_t)size, error);
2415		goto fail_2;
2416	}
2417
2418	dma->dma_paddr = 0;
2419	error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2420	    size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
2421	if (error || dma->dma_paddr == 0) {
2422		device_printf(adapter->dev, "%s: bus_dmamap_load failed: %d\n",
2423		    __func__, error);
2424		goto fail_3;
2425	}
2426
2427	return (0);
2428
2429fail_3:
2430	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2431fail_2:
2432	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2433	bus_dma_tag_destroy(dma->dma_tag);
2434fail_0:
2435	dma->dma_map = NULL;
2436	dma->dma_tag = NULL;
2437
2438	return (error);
2439}
2440
2441static void
2442em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
2443{
2444	if (dma->dma_tag == NULL)
2445		return;
2446	if (dma->dma_map != NULL) {
2447		bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2448		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2449		bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2450		bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2451		dma->dma_map = NULL;
2452	}
2453	bus_dma_tag_destroy(dma->dma_tag);
2454	dma->dma_tag = NULL;
2455}
2456
2457
2458/*********************************************************************
2459 *
2460 *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2461 *  the information needed to transmit a packet on the wire.
2462 *
2463 **********************************************************************/
2464static int
2465em_allocate_transmit_structures(struct adapter *adapter)
2466{
2467	adapter->tx_buffer_area =  malloc(sizeof(struct em_buffer) *
2468	    adapter->num_tx_desc, M_DEVBUF, M_NOWAIT);
2469	if (adapter->tx_buffer_area == NULL) {
2470		device_printf(adapter->dev, "Unable to allocate tx_buffer memory\n");
2471		return (ENOMEM);
2472	}
2473
2474	bzero(adapter->tx_buffer_area, sizeof(struct em_buffer) * adapter->num_tx_desc);
2475
2476	return (0);
2477}
2478
2479/*********************************************************************
2480 *
2481 *  Allocate and initialize transmit structures.
2482 *
2483 **********************************************************************/
2484static int
2485em_setup_transmit_structures(struct adapter *adapter)
2486{
2487	device_t dev = adapter->dev;
2488	struct em_buffer *tx_buffer;
2489	bus_size_t size;
2490	int error, i;
2491
2492	/*
2493	 * Setup DMA descriptor areas.
2494	 */
2495	size = roundup2(adapter->hw.max_frame_size, MCLBYTES);
2496	if ((error = bus_dma_tag_create(NULL,		/* parent */
2497				1, 0,			/* alignment, bounds */
2498				BUS_SPACE_MAXADDR,	/* lowaddr */
2499				BUS_SPACE_MAXADDR,	/* highaddr */
2500				NULL, NULL,		/* filter, filterarg */
2501				size,			/* maxsize */
2502				EM_MAX_SCATTER,		/* nsegments */
2503				size,			/* maxsegsize */
2504				0,			/* flags */
2505				NULL,		/* lockfunc */
2506				NULL,		/* lockarg */
2507				&adapter->txtag)) != 0) {
2508		device_printf(dev, "Unable to allocate TX DMA tag\n");
2509		goto fail;
2510	}
2511
2512	if ((error = em_allocate_transmit_structures(adapter)) != 0)
2513		goto fail;
2514
2515	bzero(adapter->tx_desc_base, (sizeof(struct em_tx_desc)) * adapter->num_tx_desc);
2516	tx_buffer = adapter->tx_buffer_area;
2517	for (i = 0; i < adapter->num_tx_desc; i++) {
2518		error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
2519		if (error != 0) {
2520			device_printf(dev, "Unable to create TX DMA map\n");
2521			goto fail;
2522		}
2523		tx_buffer++;
2524	}
2525
2526	adapter->next_avail_tx_desc = 0;
2527	adapter->oldest_used_tx_desc = 0;
2528
2529	/* Set number of descriptors available */
2530	adapter->num_tx_desc_avail = adapter->num_tx_desc;
2531
2532	/* Set checksum context */
2533	adapter->active_checksum_context = OFFLOAD_NONE;
2534	bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2535	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2536
2537	return (0);
2538
2539fail:
2540	em_free_transmit_structures(adapter);
2541	return (error);
2542}
2543
2544/*********************************************************************
2545 *
2546 *  Enable transmit unit.
2547 *
2548 **********************************************************************/
2549static void
2550em_initialize_transmit_unit(struct adapter *adapter)
2551{
2552	uint32_t	reg_tctl, reg_tarc;
2553	uint32_t	reg_tipg = 0;
2554	uint64_t	bus_addr;
2555
2556	 INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
2557	/* Setup the Base and Length of the Tx Descriptor Ring */
2558	bus_addr = adapter->txdma.dma_paddr;
2559	E1000_WRITE_REG(&adapter->hw, TDLEN,
2560	    adapter->num_tx_desc * sizeof(struct em_tx_desc));
2561	E1000_WRITE_REG(&adapter->hw, TDBAH, (uint32_t)(bus_addr >> 32));
2562	E1000_WRITE_REG(&adapter->hw, TDBAL, (uint32_t)bus_addr);
2563
2564	/* Setup the HW Tx Head and Tail descriptor pointers */
2565	E1000_WRITE_REG(&adapter->hw, TDT, 0);
2566	E1000_WRITE_REG(&adapter->hw, TDH, 0);
2567
2568
2569	HW_DEBUGOUT2("Base = %x, Length = %x\n", E1000_READ_REG(&adapter->hw, TDBAL),
2570	    E1000_READ_REG(&adapter->hw, TDLEN));
2571
2572	/* Set the default values for the Tx Inter Packet Gap timer */
2573	switch (adapter->hw.mac_type) {
2574	case em_82542_rev2_0:
2575	case em_82542_rev2_1:
2576		reg_tipg = DEFAULT_82542_TIPG_IPGT;
2577		reg_tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2578		reg_tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2579		break;
2580	case em_80003es2lan:
2581		reg_tipg = DEFAULT_82543_TIPG_IPGR1;
2582		reg_tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
2583		    E1000_TIPG_IPGR2_SHIFT;
2584		break;
2585	default:
2586		if ((adapter->hw.media_type == em_media_type_fiber) ||
2587		    (adapter->hw.media_type == em_media_type_internal_serdes))
2588			reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
2589		else
2590			reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
2591		reg_tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2592		reg_tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2593	}
2594
2595	E1000_WRITE_REG(&adapter->hw, TIPG, reg_tipg);
2596	E1000_WRITE_REG(&adapter->hw, TIDV, adapter->tx_int_delay.value);
2597	if(adapter->hw.mac_type >= em_82540)
2598		E1000_WRITE_REG(&adapter->hw, TADV, adapter->tx_abs_int_delay.value);
2599
2600	/* Do adapter specific tweaks before we enable the transmitter. */
2601	if (adapter->hw.mac_type == em_82571 || adapter->hw.mac_type == em_82572) {
2602		reg_tarc = E1000_READ_REG(&adapter->hw, TARC0);
2603		reg_tarc |= (1 << 25);
2604		E1000_WRITE_REG(&adapter->hw, TARC0, reg_tarc);
2605		reg_tarc = E1000_READ_REG(&adapter->hw, TARC1);
2606		reg_tarc |= (1 << 25);
2607		reg_tarc &= ~(1 << 28);
2608		E1000_WRITE_REG(&adapter->hw, TARC1, reg_tarc);
2609	} else if (adapter->hw.mac_type == em_80003es2lan) {
2610		reg_tarc = E1000_READ_REG(&adapter->hw, TARC0);
2611		reg_tarc |= 1;
2612		E1000_WRITE_REG(&adapter->hw, TARC0, reg_tarc);
2613		reg_tarc = E1000_READ_REG(&adapter->hw, TARC1);
2614		reg_tarc |= 1;
2615		E1000_WRITE_REG(&adapter->hw, TARC1, reg_tarc);
2616	}
2617
2618	/* Program the Transmit Control Register */
2619	reg_tctl = E1000_TCTL_PSP | E1000_TCTL_EN |
2620		   (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2621	if (adapter->hw.mac_type >= em_82571)
2622		reg_tctl |= E1000_TCTL_MULR;
2623	if (adapter->link_duplex == FULL_DUPLEX) {
2624		reg_tctl |= E1000_FDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2625	} else {
2626		reg_tctl |= E1000_HDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2627	}
2628	/* This write will effectively turn on the transmit unit. */
2629	E1000_WRITE_REG(&adapter->hw, TCTL, reg_tctl);
2630
2631	/* Setup Transmit Descriptor Settings for this adapter */
2632	adapter->txd_cmd = E1000_TXD_CMD_IFCS | E1000_TXD_CMD_RS;
2633
2634	if (adapter->tx_int_delay.value > 0)
2635		adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2636}
2637
2638/*********************************************************************
2639 *
2640 *  Free all transmit related data structures.
2641 *
2642 **********************************************************************/
2643static void
2644em_free_transmit_structures(struct adapter *adapter)
2645{
2646	struct em_buffer *tx_buffer;
2647	int i;
2648
2649	INIT_DEBUGOUT("free_transmit_structures: begin");
2650
2651	if (adapter->tx_buffer_area != NULL) {
2652		tx_buffer = adapter->tx_buffer_area;
2653		for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2654			if (tx_buffer->m_head != NULL) {
2655				bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2656				    BUS_DMASYNC_POSTWRITE);
2657				bus_dmamap_unload(adapter->txtag,
2658				    tx_buffer->map);
2659				m_freem(tx_buffer->m_head);
2660				tx_buffer->m_head = NULL;
2661			} else if (tx_buffer->map != NULL)
2662				bus_dmamap_unload(adapter->txtag,
2663				    tx_buffer->map);
2664			if (tx_buffer->map != NULL) {
2665				bus_dmamap_destroy(adapter->txtag,
2666				    tx_buffer->map);
2667				tx_buffer->map = NULL;
2668			}
2669		}
2670	}
2671	if (adapter->tx_buffer_area != NULL) {
2672		free(adapter->tx_buffer_area, M_DEVBUF);
2673		adapter->tx_buffer_area = NULL;
2674	}
2675	if (adapter->txtag != NULL) {
2676		bus_dma_tag_destroy(adapter->txtag);
2677		adapter->txtag = NULL;
2678	}
2679}
2680
2681/*********************************************************************
2682 *
2683 *  The offload context needs to be set when we transfer the first
2684 *  packet of a particular protocol (TCP/UDP). We change the
2685 *  context only if the protocol type changes.
2686 *
2687 **********************************************************************/
2688static void
2689em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
2690    uint32_t *txd_upper, uint32_t *txd_lower)
2691{
2692	struct em_context_desc *TXD;
2693	struct em_buffer *tx_buffer;
2694	int curr_txd;
2695
2696	if (mp->m_pkthdr.csum_flags) {
2697
2698		if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
2699			*txd_upper = E1000_TXD_POPTS_TXSM << 8;
2700			*txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2701			if (adapter->active_checksum_context == OFFLOAD_TCP_IP)
2702				return;
2703			else
2704				adapter->active_checksum_context = OFFLOAD_TCP_IP;
2705
2706		} else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
2707			*txd_upper = E1000_TXD_POPTS_TXSM << 8;
2708			*txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2709			if (adapter->active_checksum_context == OFFLOAD_UDP_IP)
2710				return;
2711			else
2712				adapter->active_checksum_context = OFFLOAD_UDP_IP;
2713		} else {
2714			*txd_upper = 0;
2715			*txd_lower = 0;
2716			return;
2717		}
2718	} else {
2719		*txd_upper = 0;
2720		*txd_lower = 0;
2721		return;
2722	}
2723
2724	/* If we reach this point, the checksum offload context
2725	 * needs to be reset.
2726	 */
2727	curr_txd = adapter->next_avail_tx_desc;
2728	tx_buffer = &adapter->tx_buffer_area[curr_txd];
2729	TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd];
2730
2731	TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2732	TXD->lower_setup.ip_fields.ipcso =
2733		ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2734	TXD->lower_setup.ip_fields.ipcse =
2735		htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1);
2736
2737	TXD->upper_setup.tcp_fields.tucss =
2738		ETHER_HDR_LEN + sizeof(struct ip);
2739	TXD->upper_setup.tcp_fields.tucse = htole16(0);
2740
2741	if (adapter->active_checksum_context == OFFLOAD_TCP_IP) {
2742		TXD->upper_setup.tcp_fields.tucso =
2743			ETHER_HDR_LEN + sizeof(struct ip) +
2744			offsetof(struct tcphdr, th_sum);
2745	} else if (adapter->active_checksum_context == OFFLOAD_UDP_IP) {
2746		TXD->upper_setup.tcp_fields.tucso =
2747			ETHER_HDR_LEN + sizeof(struct ip) +
2748			offsetof(struct udphdr, uh_sum);
2749	}
2750
2751	TXD->tcp_seg_setup.data = htole32(0);
2752	TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT);
2753
2754	tx_buffer->m_head = NULL;
2755
2756	if (++curr_txd == adapter->num_tx_desc)
2757		curr_txd = 0;
2758
2759	adapter->num_tx_desc_avail--;
2760	adapter->next_avail_tx_desc = curr_txd;
2761}
2762
2763/**********************************************************************
2764 *
2765 *  Examine each tx_buffer in the used queue. If the hardware is done
2766 *  processing the packet then free associated resources. The
2767 *  tx_buffer is put back on the free queue.
2768 *
2769 **********************************************************************/
2770static void
2771em_txeof(struct adapter *adapter)
2772{
2773	int i, num_avail;
2774	struct em_buffer *tx_buffer;
2775	struct em_tx_desc   *tx_desc;
2776	struct ifnet   *ifp = adapter->ifp;
2777
2778	EM_LOCK_ASSERT(adapter);
2779
2780	if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2781		return;
2782
2783	num_avail = adapter->num_tx_desc_avail;
2784	i = adapter->oldest_used_tx_desc;
2785
2786	tx_buffer = &adapter->tx_buffer_area[i];
2787	tx_desc = &adapter->tx_desc_base[i];
2788
2789	bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2790	    BUS_DMASYNC_POSTREAD);
2791	while (tx_desc->upper.fields.status & E1000_TXD_STAT_DD) {
2792
2793		tx_desc->upper.data = 0;
2794		num_avail++;
2795
2796		if (tx_buffer->m_head) {
2797			ifp->if_opackets++;
2798			bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2799			    BUS_DMASYNC_POSTWRITE);
2800			bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2801
2802			m_freem(tx_buffer->m_head);
2803			tx_buffer->m_head = NULL;
2804		}
2805
2806		if (++i == adapter->num_tx_desc)
2807			i = 0;
2808
2809		tx_buffer = &adapter->tx_buffer_area[i];
2810		tx_desc = &adapter->tx_desc_base[i];
2811	}
2812	bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2813	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2814
2815	adapter->oldest_used_tx_desc = i;
2816
2817	/*
2818	 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack
2819	 * that it is OK to send packets.
2820	 * If there are no pending descriptors, clear the timeout. Otherwise,
2821	 * if some descriptors have been freed, restart the timeout.
2822	 */
2823	if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
2824		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2825		if (num_avail == adapter->num_tx_desc)
2826			ifp->if_timer = 0;
2827		else if (num_avail != adapter->num_tx_desc_avail)
2828			ifp->if_timer = EM_TX_TIMEOUT;
2829	}
2830	adapter->num_tx_desc_avail = num_avail;
2831}
2832
2833/*********************************************************************
2834 *
2835 *  Get a buffer from system mbuf buffer pool.
2836 *
2837 **********************************************************************/
2838static int
2839em_get_buf(struct adapter *adapter, int i)
2840{
2841	struct mbuf		*m;
2842	bus_dma_segment_t	segs[1];
2843	bus_dmamap_t		map;
2844	struct em_buffer	*rx_buffer;
2845	int			error, nsegs;
2846
2847	m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
2848	if (m == NULL) {
2849		adapter->mbuf_cluster_failed++;
2850		return (ENOBUFS);
2851	}
2852	m->m_len = m->m_pkthdr.len = MCLBYTES;
2853	if (adapter->hw.max_frame_size <= (MCLBYTES - ETHER_ALIGN))
2854		m_adj(m, ETHER_ALIGN);
2855
2856	/*
2857	 * Using memory from the mbuf cluster pool, invoke the
2858	 * bus_dma machinery to arrange the memory mapping.
2859	 */
2860	error = bus_dmamap_load_mbuf_sg(adapter->rxtag, adapter->rx_sparemap,
2861	    m, segs, &nsegs, BUS_DMA_NOWAIT);
2862	if (error != 0) {
2863		m_free(m);
2864		return (error);
2865	}
2866	/* If nsegs is wrong then the stack is corrupt. */
2867	KASSERT(nsegs == 1, ("Too many segments returned!"));
2868
2869	rx_buffer = &adapter->rx_buffer_area[i];
2870	if (rx_buffer->m_head != NULL)
2871		bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
2872
2873	map = rx_buffer->map;
2874	rx_buffer->map = adapter->rx_sparemap;
2875	adapter->rx_sparemap = map;
2876	bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
2877	rx_buffer->m_head = m;
2878
2879	adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr);
2880
2881	return (0);
2882}
2883
2884/*********************************************************************
2885 *
2886 *  Allocate memory for rx_buffer structures. Since we use one
2887 *  rx_buffer per received packet, the maximum number of rx_buffer's
2888 *  that we'll need is equal to the number of receive descriptors
2889 *  that we've allocated.
2890 *
2891 **********************************************************************/
2892static int
2893em_allocate_receive_structures(struct adapter *adapter)
2894{
2895	device_t dev = adapter->dev;
2896	struct em_buffer *rx_buffer;
2897	int i, error;
2898
2899	adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) * adapter->num_rx_desc,
2900	    M_DEVBUF, M_NOWAIT);
2901	if (adapter->rx_buffer_area == NULL) {
2902		device_printf(dev, "Unable to allocate rx_buffer memory\n");
2903		return (ENOMEM);
2904	}
2905
2906	bzero(adapter->rx_buffer_area, sizeof(struct em_buffer) * adapter->num_rx_desc);
2907
2908	error = bus_dma_tag_create(NULL,		/* parent */
2909				1, 0,			/* alignment, bounds */
2910				BUS_SPACE_MAXADDR,	/* lowaddr */
2911				BUS_SPACE_MAXADDR,	/* highaddr */
2912				NULL, NULL,		/* filter, filterarg */
2913				MCLBYTES,		/* maxsize */
2914				1,			/* nsegments */
2915				MCLBYTES,		/* maxsegsize */
2916				0,			/* flags */
2917				NULL,			/* lockfunc */
2918				NULL,			/* lockarg */
2919				&adapter->rxtag);
2920	if (error) {
2921		device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
2922		    __func__, error);
2923		goto fail;
2924	}
2925
2926	error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
2927	    &adapter->rx_sparemap);
2928	if (error) {
2929		device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
2930		    __func__, error);
2931		goto fail;
2932	}
2933	rx_buffer = adapter->rx_buffer_area;
2934	for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
2935		error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
2936		    &rx_buffer->map);
2937		if (error) {
2938			device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
2939			    __func__, error);
2940			goto fail;
2941		}
2942	}
2943
2944	for (i = 0; i < adapter->num_rx_desc; i++) {
2945		error = em_get_buf(adapter, i);
2946		if (error)
2947			goto fail;
2948	}
2949	bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
2950	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2951
2952	return (0);
2953
2954fail:
2955	em_free_receive_structures(adapter);
2956	return (error);
2957}
2958
2959/*********************************************************************
2960 *
2961 *  Allocate and initialize receive structures.
2962 *
2963 **********************************************************************/
2964static int
2965em_setup_receive_structures(struct adapter *adapter)
2966{
2967	int error;
2968
2969	bzero(adapter->rx_desc_base, (sizeof(struct em_rx_desc)) * adapter->num_rx_desc);
2970
2971	if ((error = em_allocate_receive_structures(adapter)) != 0)
2972		return (error);
2973
2974	/* Setup our descriptor pointers */
2975	adapter->next_rx_desc_to_check = 0;
2976
2977	return (0);
2978}
2979
2980/*********************************************************************
2981 *
2982 *  Enable receive unit.
2983 *
2984 **********************************************************************/
2985static void
2986em_initialize_receive_unit(struct adapter *adapter)
2987{
2988	struct ifnet	*ifp = adapter->ifp;
2989	uint64_t	bus_addr;
2990	uint32_t	reg_rctl;
2991	uint32_t	reg_rxcsum;
2992
2993	INIT_DEBUGOUT("em_initialize_receive_unit: begin");
2994
2995	/*
2996	 * Make sure receives are disabled while setting
2997	 * up the descriptor ring
2998	 */
2999	E1000_WRITE_REG(&adapter->hw, RCTL, 0);
3000
3001	/* Set the Receive Delay Timer Register */
3002	E1000_WRITE_REG(&adapter->hw, RDTR, adapter->rx_int_delay.value | E1000_RDT_FPDB);
3003
3004	if(adapter->hw.mac_type >= em_82540) {
3005		E1000_WRITE_REG(&adapter->hw, RADV, adapter->rx_abs_int_delay.value);
3006
3007		/*
3008		 * Set the interrupt throttling rate. Value is calculated
3009		 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
3010		 */
3011#define MAX_INTS_PER_SEC	8000
3012#define DEFAULT_ITR	     1000000000/(MAX_INTS_PER_SEC * 256)
3013		E1000_WRITE_REG(&adapter->hw, ITR, DEFAULT_ITR);
3014	}
3015
3016	/* Setup the Base and Length of the Rx Descriptor Ring */
3017	bus_addr = adapter->rxdma.dma_paddr;
3018	E1000_WRITE_REG(&adapter->hw, RDLEN, adapter->num_rx_desc *
3019			sizeof(struct em_rx_desc));
3020	E1000_WRITE_REG(&adapter->hw, RDBAH, (uint32_t)(bus_addr >> 32));
3021	E1000_WRITE_REG(&adapter->hw, RDBAL, (uint32_t)bus_addr);
3022
3023	/* Setup the HW Rx Head and Tail Descriptor Pointers */
3024	E1000_WRITE_REG(&adapter->hw, RDT, adapter->num_rx_desc - 1);
3025	E1000_WRITE_REG(&adapter->hw, RDH, 0);
3026
3027	/* Setup the Receive Control Register */
3028	reg_rctl = E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
3029		   E1000_RCTL_RDMTS_HALF |
3030		   (adapter->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
3031
3032	if (adapter->hw.tbi_compatibility_on == TRUE)
3033		reg_rctl |= E1000_RCTL_SBP;
3034
3035
3036	switch (adapter->rx_buffer_len) {
3037	default:
3038	case EM_RXBUFFER_2048:
3039		reg_rctl |= E1000_RCTL_SZ_2048;
3040		break;
3041	case EM_RXBUFFER_4096:
3042		reg_rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
3043		break;
3044	case EM_RXBUFFER_8192:
3045		reg_rctl |= E1000_RCTL_SZ_8192 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
3046		break;
3047	case EM_RXBUFFER_16384:
3048		reg_rctl |= E1000_RCTL_SZ_16384 | E1000_RCTL_BSEX | E1000_RCTL_LPE;
3049		break;
3050	}
3051
3052	if (ifp->if_mtu > ETHERMTU)
3053		reg_rctl |= E1000_RCTL_LPE;
3054
3055	/* Enable 82543 Receive Checksum Offload for TCP and UDP */
3056	if ((adapter->hw.mac_type >= em_82543) &&
3057	    (ifp->if_capenable & IFCAP_RXCSUM)) {
3058		reg_rxcsum = E1000_READ_REG(&adapter->hw, RXCSUM);
3059		reg_rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
3060		E1000_WRITE_REG(&adapter->hw, RXCSUM, reg_rxcsum);
3061	}
3062
3063	/* Enable Receives */
3064	E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
3065}
3066
3067/*********************************************************************
3068 *
3069 *  Free receive related data structures.
3070 *
3071 **********************************************************************/
3072static void
3073em_free_receive_structures(struct adapter *adapter)
3074{
3075	struct em_buffer *rx_buffer;
3076	int i;
3077
3078	INIT_DEBUGOUT("free_receive_structures: begin");
3079
3080	if (adapter->rx_sparemap) {
3081		bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap);
3082		adapter->rx_sparemap = NULL;
3083	}
3084	if (adapter->rx_buffer_area != NULL) {
3085		rx_buffer = adapter->rx_buffer_area;
3086		for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
3087			if (rx_buffer->m_head != NULL) {
3088				bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
3089				    BUS_DMASYNC_POSTREAD);
3090				bus_dmamap_unload(adapter->rxtag,
3091				    rx_buffer->map);
3092				m_freem(rx_buffer->m_head);
3093				rx_buffer->m_head = NULL;
3094			} else if (rx_buffer->map != NULL)
3095				bus_dmamap_unload(adapter->rxtag,
3096				    rx_buffer->map);
3097			if (rx_buffer->map != NULL) {
3098				bus_dmamap_destroy(adapter->rxtag,
3099				    rx_buffer->map);
3100				rx_buffer->map = NULL;
3101			}
3102		}
3103	}
3104	if (adapter->rx_buffer_area != NULL) {
3105		free(adapter->rx_buffer_area, M_DEVBUF);
3106		adapter->rx_buffer_area = NULL;
3107	}
3108	if (adapter->rxtag != NULL) {
3109		bus_dma_tag_destroy(adapter->rxtag);
3110		adapter->rxtag = NULL;
3111	}
3112}
3113
3114/*********************************************************************
3115 *
3116 *  This routine executes in interrupt context. It replenishes
3117 *  the mbufs in the descriptor and sends data which has been
3118 *  dma'ed into host memory to upper layer.
3119 *
3120 *  We loop at most count times if count is > 0, or until done if
3121 *  count < 0.
3122 *
3123 *********************************************************************/
3124static int
3125em_rxeof(struct adapter *adapter, int count)
3126{
3127	struct ifnet	*ifp;
3128	struct mbuf	*mp;
3129	uint8_t		accept_frame = 0;
3130	uint8_t		eop = 0;
3131	uint16_t 	len, desc_len, prev_len_adj;
3132	int		i;
3133
3134	/* Pointer to the receive descriptor being examined. */
3135	struct em_rx_desc   *current_desc;
3136	uint8_t		status;
3137
3138	ifp = adapter->ifp;
3139	i = adapter->next_rx_desc_to_check;
3140	current_desc = &adapter->rx_desc_base[i];
3141	bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3142	    BUS_DMASYNC_POSTREAD);
3143
3144	if (!((current_desc->status) & E1000_RXD_STAT_DD))
3145		return (0);
3146
3147	while ((current_desc->status & E1000_RXD_STAT_DD) &&
3148	    (count != 0) &&
3149	    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3150		struct mbuf *m = NULL;
3151
3152		mp = adapter->rx_buffer_area[i].m_head;
3153		/*
3154		 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
3155		 * needs to access the last received byte in the mbuf.
3156		 */
3157		bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
3158		    BUS_DMASYNC_POSTREAD);
3159
3160		accept_frame = 1;
3161		prev_len_adj = 0;
3162		desc_len = le16toh(current_desc->length);
3163		status = current_desc->status;
3164		if (status & E1000_RXD_STAT_EOP) {
3165			count--;
3166			eop = 1;
3167			if (desc_len < ETHER_CRC_LEN) {
3168				len = 0;
3169				prev_len_adj = ETHER_CRC_LEN - desc_len;
3170			} else
3171				len = desc_len - ETHER_CRC_LEN;
3172		} else {
3173			eop = 0;
3174			len = desc_len;
3175		}
3176
3177		if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
3178			uint8_t		last_byte;
3179			uint32_t	pkt_len = desc_len;
3180
3181			if (adapter->fmp != NULL)
3182				pkt_len += adapter->fmp->m_pkthdr.len;
3183
3184			last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
3185			if (TBI_ACCEPT(&adapter->hw, status,
3186			    current_desc->errors, pkt_len, last_byte)) {
3187				em_tbi_adjust_stats(&adapter->hw,
3188				    &adapter->stats, pkt_len,
3189				    adapter->hw.mac_addr);
3190				if (len > 0)
3191					len--;
3192			} else
3193				accept_frame = 0;
3194		}
3195
3196		if (accept_frame) {
3197			if (em_get_buf(adapter, i) != 0) {
3198				ifp->if_iqdrops++;
3199				goto discard;
3200			}
3201
3202			/* Assign correct length to the current fragment */
3203			mp->m_len = len;
3204
3205			if (adapter->fmp == NULL) {
3206				mp->m_pkthdr.len = len;
3207				adapter->fmp = mp; /* Store the first mbuf */
3208				adapter->lmp = mp;
3209			} else {
3210				/* Chain mbuf's together */
3211				mp->m_flags &= ~M_PKTHDR;
3212				/*
3213				 * Adjust length of previous mbuf in chain if
3214				 * we received less than 4 bytes in the last
3215				 * descriptor.
3216				 */
3217				if (prev_len_adj > 0) {
3218					adapter->lmp->m_len -= prev_len_adj;
3219					adapter->fmp->m_pkthdr.len -=
3220					    prev_len_adj;
3221				}
3222				adapter->lmp->m_next = mp;
3223				adapter->lmp = adapter->lmp->m_next;
3224				adapter->fmp->m_pkthdr.len += len;
3225			}
3226
3227			if (eop) {
3228				adapter->fmp->m_pkthdr.rcvif = ifp;
3229				ifp->if_ipackets++;
3230				em_receive_checksum(adapter, current_desc,
3231				    adapter->fmp);
3232#ifndef __NO_STRICT_ALIGNMENT
3233				if (adapter->hw.max_frame_size >
3234				    (MCLBYTES - ETHER_ALIGN) &&
3235				    em_fixup_rx(adapter) != 0)
3236					goto skip;
3237#endif
3238				if (status & E1000_RXD_STAT_VP)
3239					VLAN_INPUT_TAG(ifp, adapter->fmp,
3240					    (le16toh(current_desc->special) &
3241					    E1000_RXD_SPC_VLAN_MASK));
3242#ifndef __NO_STRICT_ALIGNMENT
3243skip:
3244#endif
3245				m = adapter->fmp;
3246				adapter->fmp = NULL;
3247				adapter->lmp = NULL;
3248			}
3249		} else {
3250			ifp->if_ierrors++;
3251discard:
3252			/* Reuse loaded DMA map and just update mbuf chain */
3253			mp = adapter->rx_buffer_area[i].m_head;
3254			mp->m_len = mp->m_pkthdr.len = MCLBYTES;
3255			mp->m_data = mp->m_ext.ext_buf;
3256			mp->m_next = NULL;
3257			if (adapter->hw.max_frame_size <= (MCLBYTES - ETHER_ALIGN))
3258				m_adj(mp, ETHER_ALIGN);
3259			if (adapter->fmp != NULL) {
3260				m_freem(adapter->fmp);
3261				adapter->fmp = NULL;
3262				adapter->lmp = NULL;
3263			}
3264			m = NULL;
3265		}
3266
3267		/* Zero out the receive descriptors status. */
3268		current_desc->status = 0;
3269		bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3270		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3271
3272		/* Advance our pointers to the next descriptor. */
3273		if (++i == adapter->num_rx_desc)
3274			i = 0;
3275		if (m != NULL) {
3276			adapter->next_rx_desc_to_check = i;
3277#ifdef DEVICE_POLLING
3278			EM_UNLOCK(adapter);
3279			(*ifp->if_input)(ifp, m);
3280			EM_LOCK(adapter);
3281#else
3282			(*ifp->if_input)(ifp, m);
3283#endif
3284			i = adapter->next_rx_desc_to_check;
3285		}
3286		current_desc = &adapter->rx_desc_base[i];
3287	}
3288	adapter->next_rx_desc_to_check = i;
3289
3290	/* Advance the E1000's Receive Queue #0  "Tail Pointer". */
3291	if (--i < 0)
3292		i = adapter->num_rx_desc - 1;
3293	E1000_WRITE_REG(&adapter->hw, RDT, i);
3294	if (!((current_desc->status) & E1000_RXD_STAT_DD))
3295		return (0);
3296
3297	return (1);
3298}
3299
3300#ifndef __NO_STRICT_ALIGNMENT
3301/*
3302 * When jumbo frames are enabled we should realign entire payload on
3303 * architecures with strict alignment. This is serious design mistake of 8254x
3304 * as it nullifies DMA operations. 8254x just allows RX buffer size to be
3305 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
3306 * payload. On architecures without strict alignment restrictions 8254x still
3307 * performs unaligned memory access which would reduce the performance too.
3308 * To avoid copying over an entire frame to align, we allocate a new mbuf and
3309 * copy ethernet header to the new mbuf. The new mbuf is prepended into the
3310 * existing mbuf chain.
3311 *
3312 * Be aware, best performance of the 8254x is achived only when jumbo frame is
3313 * not used at all on architectures with strict alignment.
3314 */
3315static int
3316em_fixup_rx(struct adapter *adapter)
3317{
3318	struct mbuf *m, *n;
3319	int error;
3320
3321	error = 0;
3322	m = adapter->fmp;
3323	if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
3324		bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
3325		m->m_data += ETHER_HDR_LEN;
3326	} else {
3327		MGETHDR(n, M_DONTWAIT, MT_DATA);
3328		if (n != NULL) {
3329			bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
3330			m->m_data += ETHER_HDR_LEN;
3331			m->m_len -= ETHER_HDR_LEN;
3332			n->m_len = ETHER_HDR_LEN;
3333			M_MOVE_PKTHDR(n, m);
3334			n->m_next = m;
3335			adapter->fmp = n;
3336		} else {
3337			adapter->ifp->if_iqdrops++;
3338			adapter->mbuf_alloc_failed++;
3339			m_freem(adapter->fmp);
3340			adapter->fmp = NULL;
3341			adapter->lmp = NULL;
3342			error = ENOBUFS;
3343		}
3344	}
3345
3346	return (error);
3347}
3348#endif
3349
3350/*********************************************************************
3351 *
3352 *  Verify that the hardware indicated that the checksum is valid.
3353 *  Inform the stack about the status of checksum so that stack
3354 *  doesn't spend time verifying the checksum.
3355 *
3356 *********************************************************************/
3357static void
3358em_receive_checksum(struct adapter *adapter, struct em_rx_desc *rx_desc,
3359		    struct mbuf *mp)
3360{
3361	/* 82543 or newer only */
3362	if ((adapter->hw.mac_type < em_82543) ||
3363	    /* Ignore Checksum bit is set */
3364	    (rx_desc->status & E1000_RXD_STAT_IXSM)) {
3365		mp->m_pkthdr.csum_flags = 0;
3366		return;
3367	}
3368
3369	if (rx_desc->status & E1000_RXD_STAT_IPCS) {
3370		/* Did it pass? */
3371		if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
3372			/* IP Checksum Good */
3373			mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
3374			mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
3375
3376		} else {
3377			mp->m_pkthdr.csum_flags = 0;
3378		}
3379	}
3380
3381	if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
3382		/* Did it pass? */
3383		if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
3384			mp->m_pkthdr.csum_flags |=
3385			(CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
3386			mp->m_pkthdr.csum_data = htons(0xffff);
3387		}
3388	}
3389}
3390
3391
3392static void
3393em_enable_vlans(struct adapter *adapter)
3394{
3395	uint32_t ctrl;
3396
3397	E1000_WRITE_REG(&adapter->hw, VET, ETHERTYPE_VLAN);
3398
3399	ctrl = E1000_READ_REG(&adapter->hw, CTRL);
3400	ctrl |= E1000_CTRL_VME;
3401	E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
3402}
3403
3404static void
3405em_disable_vlans(struct adapter *adapter)
3406{
3407	uint32_t ctrl;
3408
3409	ctrl = E1000_READ_REG(&adapter->hw, CTRL);
3410	ctrl &= ~E1000_CTRL_VME;
3411	E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
3412}
3413
3414static void
3415em_enable_intr(struct adapter *adapter)
3416{
3417	E1000_WRITE_REG(&adapter->hw, IMS, (IMS_ENABLE_MASK));
3418}
3419
3420static void
3421em_disable_intr(struct adapter *adapter)
3422{
3423	/*
3424	 * The first version of 82542 had an errata where when link was forced
3425	 * it would stay up even up even if the cable was disconnected.
3426	 * Sequence errors were used to detect the disconnect and then the
3427	 * driver would unforce the link. This code in the in the ISR. For this
3428	 * to work correctly the Sequence error interrupt had to be enabled
3429	 * all the time.
3430	 */
3431
3432	if (adapter->hw.mac_type == em_82542_rev2_0)
3433	    E1000_WRITE_REG(&adapter->hw, IMC,
3434		(0xffffffff & ~E1000_IMC_RXSEQ));
3435	else
3436	    E1000_WRITE_REG(&adapter->hw, IMC,
3437		0xffffffff);
3438}
3439
3440static int
3441em_is_valid_ether_addr(uint8_t *addr)
3442{
3443	char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
3444
3445	if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
3446		return (FALSE);
3447	}
3448
3449	return (TRUE);
3450}
3451
3452void
3453em_write_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3454{
3455	pci_write_config(((struct em_osdep *)hw->back)->dev, reg, *value, 2);
3456}
3457
3458void
3459em_read_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3460{
3461	*value = pci_read_config(((struct em_osdep *)hw->back)->dev, reg, 2);
3462}
3463
3464void
3465em_pci_set_mwi(struct em_hw *hw)
3466{
3467	pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
3468	    (hw->pci_cmd_word | CMD_MEM_WRT_INVALIDATE), 2);
3469}
3470
3471void
3472em_pci_clear_mwi(struct em_hw *hw)
3473{
3474	pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
3475	    (hw->pci_cmd_word & ~CMD_MEM_WRT_INVALIDATE), 2);
3476}
3477
3478/*********************************************************************
3479* 82544 Coexistence issue workaround.
3480*    There are 2 issues.
3481*       1. Transmit Hang issue.
3482*    To detect this issue, following equation can be used...
3483*	  SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3484*	  If SUM[3:0] is in between 1 to 4, we will have this issue.
3485*
3486*       2. DAC issue.
3487*    To detect this issue, following equation can be used...
3488*	  SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3489*	  If SUM[3:0] is in between 9 to c, we will have this issue.
3490*
3491*
3492*    WORKAROUND:
3493*	  Make sure we do not have ending address as 1,2,3,4(Hang) or 9,a,b,c (DAC)
3494*
3495*** *********************************************************************/
3496static uint32_t
3497em_fill_descriptors (bus_addr_t address, uint32_t length,
3498		PDESC_ARRAY desc_array)
3499{
3500	/* Since issue is sensitive to length and address.*/
3501	/* Let us first check the address...*/
3502	uint32_t safe_terminator;
3503	if (length <= 4) {
3504		desc_array->descriptor[0].address = address;
3505		desc_array->descriptor[0].length = length;
3506		desc_array->elements = 1;
3507		return (desc_array->elements);
3508	}
3509	safe_terminator = (uint32_t)((((uint32_t)address & 0x7) + (length & 0xF)) & 0xF);
3510	/* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
3511	if (safe_terminator == 0   ||
3512	(safe_terminator > 4   &&
3513	safe_terminator < 9)   ||
3514	(safe_terminator > 0xC &&
3515	safe_terminator <= 0xF)) {
3516		desc_array->descriptor[0].address = address;
3517		desc_array->descriptor[0].length = length;
3518		desc_array->elements = 1;
3519		return (desc_array->elements);
3520	}
3521
3522	desc_array->descriptor[0].address = address;
3523	desc_array->descriptor[0].length = length - 4;
3524	desc_array->descriptor[1].address = address + (length - 4);
3525	desc_array->descriptor[1].length = 4;
3526	desc_array->elements = 2;
3527	return (desc_array->elements);
3528}
3529
3530/**********************************************************************
3531 *
3532 *  Update the board statistics counters.
3533 *
3534 **********************************************************************/
3535static void
3536em_update_stats_counters(struct adapter *adapter)
3537{
3538	struct ifnet   *ifp;
3539
3540	if(adapter->hw.media_type == em_media_type_copper ||
3541	   (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)) {
3542		adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, SYMERRS);
3543		adapter->stats.sec += E1000_READ_REG(&adapter->hw, SEC);
3544	}
3545	adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, CRCERRS);
3546	adapter->stats.mpc += E1000_READ_REG(&adapter->hw, MPC);
3547	adapter->stats.scc += E1000_READ_REG(&adapter->hw, SCC);
3548	adapter->stats.ecol += E1000_READ_REG(&adapter->hw, ECOL);
3549
3550	adapter->stats.mcc += E1000_READ_REG(&adapter->hw, MCC);
3551	adapter->stats.latecol += E1000_READ_REG(&adapter->hw, LATECOL);
3552	adapter->stats.colc += E1000_READ_REG(&adapter->hw, COLC);
3553	adapter->stats.dc += E1000_READ_REG(&adapter->hw, DC);
3554	adapter->stats.rlec += E1000_READ_REG(&adapter->hw, RLEC);
3555	adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, XONRXC);
3556	adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, XONTXC);
3557	adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, XOFFRXC);
3558	adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, XOFFTXC);
3559	adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, FCRUC);
3560	adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, PRC64);
3561	adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, PRC127);
3562	adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, PRC255);
3563	adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, PRC511);
3564	adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, PRC1023);
3565	adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, PRC1522);
3566	adapter->stats.gprc += E1000_READ_REG(&adapter->hw, GPRC);
3567	adapter->stats.bprc += E1000_READ_REG(&adapter->hw, BPRC);
3568	adapter->stats.mprc += E1000_READ_REG(&adapter->hw, MPRC);
3569	adapter->stats.gptc += E1000_READ_REG(&adapter->hw, GPTC);
3570
3571	/* For the 64-bit byte counters the low dword must be read first. */
3572	/* Both registers clear on the read of the high dword */
3573
3574	adapter->stats.gorcl += E1000_READ_REG(&adapter->hw, GORCL);
3575	adapter->stats.gorch += E1000_READ_REG(&adapter->hw, GORCH);
3576	adapter->stats.gotcl += E1000_READ_REG(&adapter->hw, GOTCL);
3577	adapter->stats.gotch += E1000_READ_REG(&adapter->hw, GOTCH);
3578
3579	adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, RNBC);
3580	adapter->stats.ruc += E1000_READ_REG(&adapter->hw, RUC);
3581	adapter->stats.rfc += E1000_READ_REG(&adapter->hw, RFC);
3582	adapter->stats.roc += E1000_READ_REG(&adapter->hw, ROC);
3583	adapter->stats.rjc += E1000_READ_REG(&adapter->hw, RJC);
3584
3585	adapter->stats.torl += E1000_READ_REG(&adapter->hw, TORL);
3586	adapter->stats.torh += E1000_READ_REG(&adapter->hw, TORH);
3587	adapter->stats.totl += E1000_READ_REG(&adapter->hw, TOTL);
3588	adapter->stats.toth += E1000_READ_REG(&adapter->hw, TOTH);
3589
3590	adapter->stats.tpr += E1000_READ_REG(&adapter->hw, TPR);
3591	adapter->stats.tpt += E1000_READ_REG(&adapter->hw, TPT);
3592	adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, PTC64);
3593	adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, PTC127);
3594	adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, PTC255);
3595	adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, PTC511);
3596	adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, PTC1023);
3597	adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, PTC1522);
3598	adapter->stats.mptc += E1000_READ_REG(&adapter->hw, MPTC);
3599	adapter->stats.bptc += E1000_READ_REG(&adapter->hw, BPTC);
3600
3601	if (adapter->hw.mac_type >= em_82543) {
3602		adapter->stats.algnerrc += E1000_READ_REG(&adapter->hw, ALGNERRC);
3603		adapter->stats.rxerrc += E1000_READ_REG(&adapter->hw, RXERRC);
3604		adapter->stats.tncrs += E1000_READ_REG(&adapter->hw, TNCRS);
3605		adapter->stats.cexterr += E1000_READ_REG(&adapter->hw, CEXTERR);
3606		adapter->stats.tsctc += E1000_READ_REG(&adapter->hw, TSCTC);
3607		adapter->stats.tsctfc += E1000_READ_REG(&adapter->hw, TSCTFC);
3608	}
3609	ifp = adapter->ifp;
3610
3611	ifp->if_collisions = adapter->stats.colc;
3612
3613	/* Rx Errors */
3614	ifp->if_ierrors = adapter->stats.rxerrc + adapter->stats.crcerrs +
3615	    adapter->stats.algnerrc + adapter->stats.ruc + adapter->stats.roc +
3616	    adapter->stats.mpc + adapter->stats.cexterr;
3617
3618	/* Tx Errors */
3619	ifp->if_oerrors = adapter->stats.ecol + adapter->stats.latecol +
3620	    adapter->watchdog_events;
3621}
3622
3623
3624/**********************************************************************
3625 *
3626 *  This routine is called only when em_display_debug_stats is enabled.
3627 *  This routine provides a way to take a look at important statistics
3628 *  maintained by the driver and hardware.
3629 *
3630 **********************************************************************/
3631static void
3632em_print_debug_info(struct adapter *adapter)
3633{
3634	device_t dev = adapter->dev;
3635	uint8_t *hw_addr = adapter->hw.hw_addr;
3636
3637	device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
3638	device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
3639	    E1000_READ_REG(&adapter->hw, CTRL),
3640	    E1000_READ_REG(&adapter->hw, RCTL));
3641	device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
3642	    ((E1000_READ_REG(&adapter->hw, PBA) & 0xffff0000) >> 16),\
3643	    (E1000_READ_REG(&adapter->hw, PBA) & 0xffff) );
3644	device_printf(dev, "Flow control watermarks high = %d low = %d\n",
3645	    adapter->hw.fc_high_water,
3646	    adapter->hw.fc_low_water);
3647	device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
3648	    E1000_READ_REG(&adapter->hw, TIDV),
3649	    E1000_READ_REG(&adapter->hw, TADV));
3650	device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
3651	    E1000_READ_REG(&adapter->hw, RDTR),
3652	    E1000_READ_REG(&adapter->hw, RADV));
3653	device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
3654	    (long long)adapter->tx_fifo_wrk_cnt,
3655	    (long long)adapter->tx_fifo_reset_cnt);
3656	device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
3657	    E1000_READ_REG(&adapter->hw, TDH),
3658	    E1000_READ_REG(&adapter->hw, TDT));
3659	device_printf(dev, "Num Tx descriptors avail = %d\n",
3660	    adapter->num_tx_desc_avail);
3661	device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
3662	    adapter->no_tx_desc_avail1);
3663	device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
3664	    adapter->no_tx_desc_avail2);
3665	device_printf(dev, "Std mbuf failed = %ld\n",
3666	    adapter->mbuf_alloc_failed);
3667	device_printf(dev, "Std mbuf cluster failed = %ld\n",
3668	    adapter->mbuf_cluster_failed);
3669}
3670
3671static void
3672em_print_hw_stats(struct adapter *adapter)
3673{
3674	device_t dev = adapter->dev;
3675
3676	device_printf(dev, "Excessive collisions = %lld\n",
3677	    (long long)adapter->stats.ecol);
3678	device_printf(dev, "Symbol errors = %lld\n",
3679	    (long long)adapter->stats.symerrs);
3680	device_printf(dev, "Sequence errors = %lld\n",
3681	    (long long)adapter->stats.sec);
3682	device_printf(dev, "Defer count = %lld\n", (long long)adapter->stats.dc);
3683
3684	device_printf(dev, "Missed Packets = %lld\n", (long long)adapter->stats.mpc);
3685	device_printf(dev, "Receive No Buffers = %lld\n",
3686	    (long long)adapter->stats.rnbc);
3687	/* RLEC is inaccurate on some hardware, calculate our own. */
3688	device_printf(dev, "Receive Length Errors = %lld\n",
3689	    ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
3690	device_printf(dev, "Receive errors = %lld\n",
3691	    (long long)adapter->stats.rxerrc);
3692	device_printf(dev, "Crc errors = %lld\n", (long long)adapter->stats.crcerrs);
3693	device_printf(dev, "Alignment errors = %lld\n",
3694	    (long long)adapter->stats.algnerrc);
3695	device_printf(dev, "Carrier extension errors = %lld\n",
3696	    (long long)adapter->stats.cexterr);
3697	device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
3698	device_printf(dev, "watchdog timeouts = %ld\n", adapter->watchdog_events);
3699
3700	device_printf(dev, "XON Rcvd = %lld\n", (long long)adapter->stats.xonrxc);
3701	device_printf(dev, "XON Xmtd = %lld\n", (long long)adapter->stats.xontxc);
3702	device_printf(dev, "XOFF Rcvd = %lld\n", (long long)adapter->stats.xoffrxc);
3703	device_printf(dev, "XOFF Xmtd = %lld\n", (long long)adapter->stats.xofftxc);
3704
3705	device_printf(dev, "Good Packets Rcvd = %lld\n",
3706	    (long long)adapter->stats.gprc);
3707	device_printf(dev, "Good Packets Xmtd = %lld\n",
3708	    (long long)adapter->stats.gptc);
3709}
3710
3711static int
3712em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
3713{
3714	struct adapter *adapter;
3715	int error;
3716	int result;
3717
3718	result = -1;
3719	error = sysctl_handle_int(oidp, &result, 0, req);
3720
3721	if (error || !req->newptr)
3722		return (error);
3723
3724	if (result == 1) {
3725		adapter = (struct adapter *)arg1;
3726		em_print_debug_info(adapter);
3727	}
3728
3729	return (error);
3730}
3731
3732
3733static int
3734em_sysctl_stats(SYSCTL_HANDLER_ARGS)
3735{
3736	struct adapter *adapter;
3737	int error;
3738	int result;
3739
3740	result = -1;
3741	error = sysctl_handle_int(oidp, &result, 0, req);
3742
3743	if (error || !req->newptr)
3744		return (error);
3745
3746	if (result == 1) {
3747		adapter = (struct adapter *)arg1;
3748		em_print_hw_stats(adapter);
3749	}
3750
3751	return (error);
3752}
3753
3754static int
3755em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
3756{
3757	struct em_int_delay_info *info;
3758	struct adapter *adapter;
3759	uint32_t regval;
3760	int error;
3761	int usecs;
3762	int ticks;
3763
3764	info = (struct em_int_delay_info *)arg1;
3765	usecs = info->value;
3766	error = sysctl_handle_int(oidp, &usecs, 0, req);
3767	if (error != 0 || req->newptr == NULL)
3768		return (error);
3769	if (usecs < 0 || usecs > E1000_TICKS_TO_USECS(65535))
3770		return (EINVAL);
3771	info->value = usecs;
3772	ticks = E1000_USECS_TO_TICKS(usecs);
3773
3774	adapter = info->adapter;
3775
3776	EM_LOCK(adapter);
3777	regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
3778	regval = (regval & ~0xffff) | (ticks & 0xffff);
3779	/* Handle a few special cases. */
3780	switch (info->offset) {
3781	case E1000_RDTR:
3782	case E1000_82542_RDTR:
3783		regval |= E1000_RDT_FPDB;
3784		break;
3785	case E1000_TIDV:
3786	case E1000_82542_TIDV:
3787		if (ticks == 0) {
3788			adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
3789			/* Don't write 0 into the TIDV register. */
3790			regval++;
3791		} else
3792			adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3793		break;
3794	}
3795	E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
3796	EM_UNLOCK(adapter);
3797	return (0);
3798}
3799
3800static void
3801em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
3802	const char *description, struct em_int_delay_info *info,
3803	int offset, int value)
3804{
3805	info->adapter = adapter;
3806	info->offset = offset;
3807	info->value = value;
3808	SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
3809	    SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
3810	    OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
3811	    info, 0, em_sysctl_int_delay, "I", description);
3812}
3813
3814#ifndef DEVICE_POLLING
3815static void
3816em_add_int_process_limit(struct adapter *adapter, const char *name,
3817	const char *description, int *limit, int value)
3818{
3819	*limit = value;
3820	SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev),
3821	    SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
3822	    OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description);
3823}
3824#endif
3825