if_em.c revision 163724 
1/**************************************************************************
2
3Copyright (c) 2001-2006, Intel Corporation
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without
7modification, are permitted provided that the following conditions are met:
8
9 1. Redistributions of source code must retain the above copyright notice,
10    this list of conditions and the following disclaimer.
11
12 2. Redistributions in binary form must reproduce the above copyright
13    notice, this list of conditions and the following disclaimer in the
14    documentation and/or other materials provided with the distribution.
15
16 3. Neither the name of the Intel Corporation nor the names of its
17    contributors may be used to endorse or promote products derived from
18    this software without specific prior written permission.
19
20THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30POSSIBILITY OF SUCH DAMAGE.
31
32***************************************************************************/
33
34/*$FreeBSD: head/sys/dev/em/if_em.c 163724 2006-10-28 00:47:55Z jfv $*/
35
36#ifdef HAVE_KERNEL_OPTION_HEADERS
37#include "opt_device_polling.h"
38#endif
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/bus.h>
43#include <sys/endian.h>
44#include <sys/kernel.h>
45#include <sys/kthread.h>
46#include <sys/malloc.h>
47#include <sys/mbuf.h>
48#include <sys/module.h>
49#include <sys/rman.h>
50#include <sys/socket.h>
51#include <sys/sockio.h>
52#include <sys/sysctl.h>
53#include <sys/taskqueue.h>
54
55#include <machine/bus.h>
56#include <machine/resource.h>
57
58#include <net/bpf.h>
59#include <net/ethernet.h>
60#include <net/if.h>
61#include <net/if_arp.h>
62#include <net/if_dl.h>
63#include <net/if_media.h>
64
65#include <net/if_types.h>
66#include <net/if_vlan_var.h>
67
68#include <netinet/in_systm.h>
69#include <netinet/in.h>
70#include <netinet/if_ether.h>
71#include <netinet/ip.h>
72#include <netinet/tcp.h>
73#include <netinet/udp.h>
74
75#include <machine/in_cksum.h>
76
77#include <dev/pci/pcivar.h>
78#include <dev/pci/pcireg.h>
79#include <dev/em/if_em_hw.h>
80#include <dev/em/if_em.h>
81
82/*********************************************************************
83 *  Set this to one to display debug statistics
84 *********************************************************************/
85int	em_display_debug_stats = 0;
86
87/*********************************************************************
88 *  Driver version
89 *********************************************************************/
90char em_driver_version[] = "Version - 6.2.9";
91
92
93/*********************************************************************
94 *  PCI Device ID Table
95 *
96 *  Used by probe to select devices to load on
97 *  Last field stores an index into em_strings
98 *  Last entry must be all 0s
99 *
100 *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
101 *********************************************************************/
102
103static em_vendor_info_t em_vendor_info_array[] =
104{
105	/* Intel(R) PRO/1000 Network Connection */
106	{ 0x8086, E1000_DEV_ID_82540EM,		PCI_ANY_ID, PCI_ANY_ID, 0},
107	{ 0x8086, E1000_DEV_ID_82540EM_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
108	{ 0x8086, E1000_DEV_ID_82540EP,		PCI_ANY_ID, PCI_ANY_ID, 0},
109	{ 0x8086, E1000_DEV_ID_82540EP_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
110	{ 0x8086, E1000_DEV_ID_82540EP_LP,	PCI_ANY_ID, PCI_ANY_ID, 0},
111
112	{ 0x8086, E1000_DEV_ID_82541EI,		PCI_ANY_ID, PCI_ANY_ID, 0},
113	{ 0x8086, E1000_DEV_ID_82541ER,		PCI_ANY_ID, PCI_ANY_ID, 0},
114	{ 0x8086, E1000_DEV_ID_82541ER_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
115	{ 0x8086, E1000_DEV_ID_82541EI_MOBILE,	PCI_ANY_ID, PCI_ANY_ID, 0},
116	{ 0x8086, E1000_DEV_ID_82541GI,		PCI_ANY_ID, PCI_ANY_ID, 0},
117	{ 0x8086, E1000_DEV_ID_82541GI_LF,	PCI_ANY_ID, PCI_ANY_ID, 0},
118	{ 0x8086, E1000_DEV_ID_82541GI_MOBILE,	PCI_ANY_ID, PCI_ANY_ID, 0},
119
120	{ 0x8086, E1000_DEV_ID_82543GC_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
121	{ 0x8086, E1000_DEV_ID_82543GC_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
122
123	{ 0x8086, E1000_DEV_ID_82544EI_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
124	{ 0x8086, E1000_DEV_ID_82544EI_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
125	{ 0x8086, E1000_DEV_ID_82544GC_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
126	{ 0x8086, E1000_DEV_ID_82544GC_LOM,	PCI_ANY_ID, PCI_ANY_ID, 0},
127
128	{ 0x8086, E1000_DEV_ID_82545EM_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
129	{ 0x8086, E1000_DEV_ID_82545EM_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
130	{ 0x8086, E1000_DEV_ID_82545GM_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
131	{ 0x8086, E1000_DEV_ID_82545GM_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
132	{ 0x8086, E1000_DEV_ID_82545GM_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
133
134	{ 0x8086, E1000_DEV_ID_82546EB_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
135	{ 0x8086, E1000_DEV_ID_82546EB_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
136	{ 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
137	{ 0x8086, E1000_DEV_ID_82546GB_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
138	{ 0x8086, E1000_DEV_ID_82546GB_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
139	{ 0x8086, E1000_DEV_ID_82546GB_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
140	{ 0x8086, E1000_DEV_ID_82546GB_PCIE,	PCI_ANY_ID, PCI_ANY_ID, 0},
141	{ 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0},
142	{ 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3,
143						PCI_ANY_ID, PCI_ANY_ID, 0},
144
145	{ 0x8086, E1000_DEV_ID_82547EI,		PCI_ANY_ID, PCI_ANY_ID, 0},
146	{ 0x8086, E1000_DEV_ID_82547EI_MOBILE,	PCI_ANY_ID, PCI_ANY_ID, 0},
147	{ 0x8086, E1000_DEV_ID_82547GI,		PCI_ANY_ID, PCI_ANY_ID, 0},
148
149	{ 0x8086, E1000_DEV_ID_82571EB_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
150	{ 0x8086, E1000_DEV_ID_82571EB_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
151	{ 0x8086, E1000_DEV_ID_82571EB_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
152	{ 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER,
153						PCI_ANY_ID, PCI_ANY_ID, 0},
154	{ 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LOWPROFILE,
155						PCI_ANY_ID, PCI_ANY_ID, 0},
156
157	{ 0x8086, E1000_DEV_ID_82572EI_COPPER,	PCI_ANY_ID, PCI_ANY_ID, 0},
158	{ 0x8086, E1000_DEV_ID_82572EI_FIBER,	PCI_ANY_ID, PCI_ANY_ID, 0},
159	{ 0x8086, E1000_DEV_ID_82572EI_SERDES,	PCI_ANY_ID, PCI_ANY_ID, 0},
160	{ 0x8086, E1000_DEV_ID_82572EI,		PCI_ANY_ID, PCI_ANY_ID, 0},
161
162	{ 0x8086, E1000_DEV_ID_82573E,		PCI_ANY_ID, PCI_ANY_ID, 0},
163	{ 0x8086, E1000_DEV_ID_82573E_IAMT,	PCI_ANY_ID, PCI_ANY_ID, 0},
164	{ 0x8086, E1000_DEV_ID_82573L,		PCI_ANY_ID, PCI_ANY_ID, 0},
165	{ 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT,
166						PCI_ANY_ID, PCI_ANY_ID, 0},
167	{ 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT,
168						PCI_ANY_ID, PCI_ANY_ID, 0},
169	{ 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT,
170						PCI_ANY_ID, PCI_ANY_ID, 0},
171	{ 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT,
172						PCI_ANY_ID, PCI_ANY_ID, 0},
173	{ 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT,	PCI_ANY_ID, PCI_ANY_ID, 0},
174	{ 0x8086, E1000_DEV_ID_ICH8_IGP_AMT,	PCI_ANY_ID, PCI_ANY_ID, 0},
175	{ 0x8086, E1000_DEV_ID_ICH8_IGP_C,	PCI_ANY_ID, PCI_ANY_ID, 0},
176	{ 0x8086, E1000_DEV_ID_ICH8_IFE,	PCI_ANY_ID, PCI_ANY_ID, 0},
177	{ 0x8086, E1000_DEV_ID_ICH8_IFE_GT,	PCI_ANY_ID, PCI_ANY_ID, 0},
178	{ 0x8086, E1000_DEV_ID_ICH8_IFE_G,	PCI_ANY_ID, PCI_ANY_ID, 0},
179	{ 0x8086, E1000_DEV_ID_ICH8_IGP_M,	PCI_ANY_ID, PCI_ANY_ID, 0},
180
181	/* required last entry */
182	{ 0, 0, 0, 0, 0}
183};
184
185/*********************************************************************
186 *  Table of branding strings for all supported NICs.
187 *********************************************************************/
188
189static char *em_strings[] = {
190	"Intel(R) PRO/1000 Network Connection"
191};
192
193/*********************************************************************
194 *  Function prototypes
195 *********************************************************************/
196static int	em_probe(device_t);
197static int	em_attach(device_t);
198static int	em_detach(device_t);
199static int	em_shutdown(device_t);
200static int	em_suspend(device_t);
201static int	em_resume(device_t);
202static void	em_start(struct ifnet *);
203static void	em_start_locked(struct ifnet *ifp);
204static int	em_ioctl(struct ifnet *, u_long, caddr_t);
205static void	em_watchdog(struct ifnet *);
206static void	em_init(void *);
207static void	em_init_locked(struct adapter *);
208static void	em_stop(void *);
209static void	em_media_status(struct ifnet *, struct ifmediareq *);
210static int	em_media_change(struct ifnet *);
211static void	em_identify_hardware(struct adapter *);
212static int	em_allocate_pci_resources(struct adapter *);
213static void	em_free_pci_resources(struct adapter *);
214static void	em_local_timer(void *);
215static int	em_hardware_init(struct adapter *);
216static void	em_setup_interface(device_t, struct adapter *);
217static void	em_setup_transmit_structures(struct adapter *);
218static void	em_initialize_transmit_unit(struct adapter *);
219static int	em_setup_receive_structures(struct adapter *);
220static void	em_initialize_receive_unit(struct adapter *);
221static void	em_enable_intr(struct adapter *);
222static void	em_disable_intr(struct adapter *);
223static void	em_free_transmit_structures(struct adapter *);
224static void	em_free_receive_structures(struct adapter *);
225static void	em_update_stats_counters(struct adapter *);
226static void	em_txeof(struct adapter *);
227static int	em_allocate_receive_structures(struct adapter *);
228static int	em_allocate_transmit_structures(struct adapter *);
229static int	em_rxeof(struct adapter *, int);
230#ifndef __NO_STRICT_ALIGNMENT
231static int	em_fixup_rx(struct adapter *);
232#endif
233static void	em_receive_checksum(struct adapter *, struct em_rx_desc *,
234		    struct mbuf *);
235static void	em_transmit_checksum_setup(struct adapter *, struct mbuf *,
236		    uint32_t *, uint32_t *);
237#ifdef EM_TSO
238static boolean_t em_tso_setup(struct adapter *, struct mbuf *, u_int32_t *,
239		    uint32_t *);
240#endif
241static void	em_set_promisc(struct adapter *);
242static void	em_disable_promisc(struct adapter *);
243static void	em_set_multi(struct adapter *);
244static void	em_print_hw_stats(struct adapter *);
245static void	em_update_link_status(struct adapter *);
246static int	em_get_buf(int i, struct adapter *, struct mbuf *);
247static void	em_enable_vlans(struct adapter *);
248static void	em_disable_vlans(struct adapter *);
249static int	em_encap(struct adapter *, struct mbuf **);
250static void	em_smartspeed(struct adapter *);
251static int	em_82547_fifo_workaround(struct adapter *, int);
252static void	em_82547_update_fifo_head(struct adapter *, int);
253static int	em_82547_tx_fifo_reset(struct adapter *);
254static void	em_82547_move_tail(void *arg);
255static void	em_82547_move_tail_locked(struct adapter *);
256static int	em_dma_malloc(struct adapter *, bus_size_t,
257		struct em_dma_alloc *, int);
258static void	em_dma_free(struct adapter *, struct em_dma_alloc *);
259static void	em_print_debug_info(struct adapter *);
260static int 	em_is_valid_ether_addr(uint8_t *);
261static int	em_sysctl_stats(SYSCTL_HANDLER_ARGS);
262static int	em_sysctl_debug_info(SYSCTL_HANDLER_ARGS);
263static uint32_t	em_fill_descriptors (bus_addr_t address, uint32_t length,
264		    PDESC_ARRAY desc_array);
265static int	em_sysctl_int_delay(SYSCTL_HANDLER_ARGS);
266static void	em_add_int_delay_sysctl(struct adapter *, const char *,
267		const char *, struct em_int_delay_info *, int, int);
268static void	em_intr(void *);
269
270#ifdef DEVICE_POLLING
271static poll_handler_t em_poll;
272#endif
273
274/*********************************************************************
275 *  FreeBSD Device Interface Entry Points
276 *********************************************************************/
277
278static device_method_t em_methods[] = {
279	/* Device interface */
280	DEVMETHOD(device_probe, em_probe),
281	DEVMETHOD(device_attach, em_attach),
282	DEVMETHOD(device_detach, em_detach),
283	DEVMETHOD(device_shutdown, em_shutdown),
284	DEVMETHOD(device_suspend, em_suspend),
285	DEVMETHOD(device_resume, em_resume),
286	{0, 0}
287};
288
289static driver_t em_driver = {
290	"em", em_methods, sizeof(struct adapter),
291};
292
293static devclass_t em_devclass;
294DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0);
295MODULE_DEPEND(em, pci, 1, 1, 1);
296MODULE_DEPEND(em, ether, 1, 1, 1);
297
298/*********************************************************************
299 *  Tunable default values.
300 *********************************************************************/
301
302#define E1000_TICKS_TO_USECS(ticks)	((1024 * (ticks) + 500) / 1000)
303#define E1000_USECS_TO_TICKS(usecs)	((1000 * (usecs) + 512) / 1024)
304#define M_TSO_LEN		66  /* mbuf with just hdr and TSO pkthdr */
305
306static int em_tx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TIDV);
307static int em_rx_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RDTR);
308static int em_tx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_TADV);
309static int em_rx_abs_int_delay_dflt = E1000_TICKS_TO_USECS(EM_RADV);
310static int em_rxd = EM_DEFAULT_RXD;
311static int em_txd = EM_DEFAULT_TXD;
312static int em_smart_pwr_down = FALSE;
313
314TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt);
315TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt);
316TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt);
317TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt);
318TUNABLE_INT("hw.em.rxd", &em_rxd);
319TUNABLE_INT("hw.em.txd", &em_txd);
320TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down);
321
322/*********************************************************************
323 *  Device identification routine
324 *
325 *  em_probe determines if the driver should be loaded on
326 *  adapter based on PCI vendor/device id of the adapter.
327 *
328 *  return BUS_PROBE_DEFAULT on success, positive on failure
329 *********************************************************************/
330
331static int
332em_probe(device_t dev)
333{
334	char		adapter_name[60];
335	uint16_t	pci_vendor_id = 0;
336	uint16_t	pci_device_id = 0;
337	uint16_t	pci_subvendor_id = 0;
338	uint16_t	pci_subdevice_id = 0;
339	em_vendor_info_t *ent;
340
341	INIT_DEBUGOUT("em_probe: begin");
342
343	pci_vendor_id = pci_get_vendor(dev);
344	if (pci_vendor_id != EM_VENDOR_ID)
345		return (ENXIO);
346
347	pci_device_id = pci_get_device(dev);
348	pci_subvendor_id = pci_get_subvendor(dev);
349	pci_subdevice_id = pci_get_subdevice(dev);
350
351	ent = em_vendor_info_array;
352	while (ent->vendor_id != 0) {
353		if ((pci_vendor_id == ent->vendor_id) &&
354		    (pci_device_id == ent->device_id) &&
355
356		    ((pci_subvendor_id == ent->subvendor_id) ||
357		    (ent->subvendor_id == PCI_ANY_ID)) &&
358
359		    ((pci_subdevice_id == ent->subdevice_id) ||
360		    (ent->subdevice_id == PCI_ANY_ID))) {
361			sprintf(adapter_name, "%s %s",
362				em_strings[ent->index],
363				em_driver_version);
364			device_set_desc_copy(dev, adapter_name);
365			return (BUS_PROBE_DEFAULT);
366		}
367		ent++;
368	}
369
370	return (ENXIO);
371}
372
373/*********************************************************************
374 *  Device initialization routine
375 *
376 *  The attach entry point is called when the driver is being loaded.
377 *  This routine identifies the type of hardware, allocates all resources
378 *  and initializes the hardware.
379 *
380 *  return 0 on success, positive on failure
381 *********************************************************************/
382
383static int
384em_attach(device_t dev)
385{
386	struct adapter	*adapter;
387	int		tsize, rsize;
388	int		error = 0;
389
390	INIT_DEBUGOUT("em_attach: begin");
391
392	adapter = device_get_softc(dev);
393	adapter->dev = adapter->osdep.dev = dev;
394	EM_LOCK_INIT(adapter, device_get_nameunit(dev));
395
396	/* SYSCTL stuff */
397	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
398	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
399	    OID_AUTO, "debug_info", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
400	    em_sysctl_debug_info, "I", "Debug Information");
401
402	SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
403	    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
404	    OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
405	    em_sysctl_stats, "I", "Statistics");
406
407	callout_init(&adapter->timer, CALLOUT_MPSAFE);
408	callout_init(&adapter->tx_fifo_timer, CALLOUT_MPSAFE);
409
410	/* Determine hardware revision */
411	em_identify_hardware(adapter);
412
413	/* Set up some sysctls for the tunable interrupt delays */
414	em_add_int_delay_sysctl(adapter, "rx_int_delay",
415	    "receive interrupt delay in usecs", &adapter->rx_int_delay,
416	    E1000_REG_OFFSET(&adapter->hw, RDTR), em_rx_int_delay_dflt);
417	em_add_int_delay_sysctl(adapter, "tx_int_delay",
418	    "transmit interrupt delay in usecs", &adapter->tx_int_delay,
419	    E1000_REG_OFFSET(&adapter->hw, TIDV), em_tx_int_delay_dflt);
420	if (adapter->hw.mac_type >= em_82540) {
421		em_add_int_delay_sysctl(adapter, "rx_abs_int_delay",
422		    "receive interrupt delay limit in usecs",
423		    &adapter->rx_abs_int_delay,
424		    E1000_REG_OFFSET(&adapter->hw, RADV),
425		    em_rx_abs_int_delay_dflt);
426		em_add_int_delay_sysctl(adapter, "tx_abs_int_delay",
427		    "transmit interrupt delay limit in usecs",
428		    &adapter->tx_abs_int_delay,
429		    E1000_REG_OFFSET(&adapter->hw, TADV),
430		    em_tx_abs_int_delay_dflt);
431	}
432
433	/*
434	 * Validate number of transmit and receive descriptors. It
435	 * must not exceed hardware maximum, and must be multiple
436	 * of EM_DBA_ALIGN.
437	 */
438	if (((em_txd * sizeof(struct em_tx_desc)) % EM_DBA_ALIGN) != 0 ||
439	    (adapter->hw.mac_type >= em_82544 && em_txd > EM_MAX_TXD) ||
440	    (adapter->hw.mac_type < em_82544 && em_txd > EM_MAX_TXD_82543) ||
441	    (em_txd < EM_MIN_TXD)) {
442		device_printf(dev, "Using %d TX descriptors instead of %d!\n",
443		    EM_DEFAULT_TXD, em_txd);
444		adapter->num_tx_desc = EM_DEFAULT_TXD;
445	} else
446		adapter->num_tx_desc = em_txd;
447	if (((em_rxd * sizeof(struct em_rx_desc)) % EM_DBA_ALIGN) != 0 ||
448	    (adapter->hw.mac_type >= em_82544 && em_rxd > EM_MAX_RXD) ||
449	    (adapter->hw.mac_type < em_82544 && em_rxd > EM_MAX_RXD_82543) ||
450	    (em_rxd < EM_MIN_RXD)) {
451		device_printf(dev, "Using %d RX descriptors instead of %d!\n",
452		    EM_DEFAULT_RXD, em_rxd);
453		adapter->num_rx_desc = EM_DEFAULT_RXD;
454	} else
455		adapter->num_rx_desc = em_rxd;
456
457	adapter->hw.autoneg = DO_AUTO_NEG;
458	adapter->hw.wait_autoneg_complete = WAIT_FOR_AUTO_NEG_DEFAULT;
459	adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
460	adapter->hw.tbi_compatibility_en = TRUE;
461	adapter->rx_buffer_len = EM_RXBUFFER_2048;
462
463	adapter->hw.phy_init_script = 1;
464	adapter->hw.phy_reset_disable = FALSE;
465
466#ifndef EM_MASTER_SLAVE
467	adapter->hw.master_slave = em_ms_hw_default;
468#else
469	adapter->hw.master_slave = EM_MASTER_SLAVE;
470#endif
471	/*
472	 * Set the max frame size assuming standard ethernet
473	 * sized frames.
474	 */
475	adapter->hw.max_frame_size =
476	    ETHERMTU + ETHER_HDR_LEN + ETHER_CRC_LEN;
477
478	adapter->hw.min_frame_size =
479	    MINIMUM_ETHERNET_PACKET_SIZE + ETHER_CRC_LEN;
480
481	/*
482	 * This controls when hardware reports transmit completion
483	 * status.
484	 */
485	adapter->hw.report_tx_early = 1;
486	if (em_allocate_pci_resources(adapter)) {
487		device_printf(dev, "Allocation of PCI resources failed\n");
488		error = ENXIO;
489		goto err_pci;
490	}
491
492	/* Initialize eeprom parameters */
493	em_init_eeprom_params(&adapter->hw);
494
495	tsize = roundup2(adapter->num_tx_desc * sizeof(struct em_tx_desc),
496	    EM_DBA_ALIGN);
497
498	/* Allocate Transmit Descriptor ring */
499	if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) {
500		device_printf(dev, "Unable to allocate tx_desc memory\n");
501		error = ENOMEM;
502		goto err_tx_desc;
503	}
504	adapter->tx_desc_base = (struct em_tx_desc *)adapter->txdma.dma_vaddr;
505
506	rsize = roundup2(adapter->num_rx_desc * sizeof(struct em_rx_desc),
507	    EM_DBA_ALIGN);
508
509	/* Allocate Receive Descriptor ring */
510	if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) {
511		device_printf(dev, "Unable to allocate rx_desc memory\n");
512		error = ENOMEM;
513		goto err_rx_desc;
514	}
515	adapter->rx_desc_base = (struct em_rx_desc *)adapter->rxdma.dma_vaddr;
516
517	/* Initialize the hardware */
518	if (em_hardware_init(adapter)) {
519		device_printf(dev, "Unable to initialize the hardware\n");
520		error = EIO;
521		goto err_hw_init;
522	}
523
524	/* Copy the permanent MAC address out of the EEPROM */
525	if (em_read_mac_addr(&adapter->hw) < 0) {
526		device_printf(dev, "EEPROM read error while reading MAC"
527		    " address\n");
528		error = EIO;
529		goto err_hw_init;
530	}
531
532	if (!em_is_valid_ether_addr(adapter->hw.mac_addr)) {
533		device_printf(dev, "Invalid MAC address\n");
534		error = EIO;
535		goto err_hw_init;
536	}
537
538	/* Allocate transmit descriptors and buffers */
539	if (em_allocate_transmit_structures(adapter)) {
540		device_printf(dev, "Could not setup transmit structures\n");
541		error = ENOMEM;
542		goto err_tx_struct;
543	}
544
545	/* Allocate receive descriptors and buffers */
546	if (em_allocate_receive_structures(adapter)) {
547		device_printf(dev, "Could not setup receive structures\n");
548		error = ENOMEM;
549		goto err_rx_struct;
550	}
551
552	/* Setup OS specific network interface */
553	em_setup_interface(dev, adapter);
554
555	/* Initialize statistics */
556	em_clear_hw_cntrs(&adapter->hw);
557	em_update_stats_counters(adapter);
558	adapter->hw.get_link_status = 1;
559	em_update_link_status(adapter);
560
561	/* Indicate SOL/IDER usage */
562	if (em_check_phy_reset_block(&adapter->hw))
563		device_printf(dev,
564		    "PHY reset is blocked due to SOL/IDER session.\n");
565
566	/* Identify 82544 on PCIX */
567	em_get_bus_info(&adapter->hw);
568	if (adapter->hw.bus_type == em_bus_type_pcix &&
569	    adapter->hw.mac_type == em_82544)
570		adapter->pcix_82544 = TRUE;
571	else
572		adapter->pcix_82544 = FALSE;
573
574	INIT_DEBUGOUT("em_attach: end");
575
576	return (0);
577
578err_rx_struct:
579	em_free_transmit_structures(adapter);
580err_hw_init:
581err_tx_struct:
582	em_dma_free(adapter, &adapter->rxdma);
583err_rx_desc:
584	em_dma_free(adapter, &adapter->txdma);
585err_tx_desc:
586err_pci:
587	em_free_pci_resources(adapter);
588	EM_LOCK_DESTROY(adapter);
589
590	return (error);
591}
592
593/*********************************************************************
594 *  Device removal routine
595 *
596 *  The detach entry point is called when the driver is being removed.
597 *  This routine stops the adapter and deallocates all the resources
598 *  that were allocated for driver operation.
599 *
600 *  return 0 on success, positive on failure
601 *********************************************************************/
602
603static int
604em_detach(device_t dev)
605{
606	struct adapter	*adapter = device_get_softc(dev);
607	struct ifnet	*ifp = adapter->ifp;
608
609	INIT_DEBUGOUT("em_detach: begin");
610
611#ifdef DEVICE_POLLING
612	if (ifp->if_capenable & IFCAP_POLLING)
613		ether_poll_deregister(ifp);
614#endif
615
616	EM_LOCK(adapter);
617	adapter->in_detach = 1;
618	em_stop(adapter);
619	em_phy_hw_reset(&adapter->hw);
620	EM_UNLOCK(adapter);
621	ether_ifdetach(adapter->ifp);
622
623	em_free_pci_resources(adapter);
624	bus_generic_detach(dev);
625	if_free(ifp);
626
627	em_free_transmit_structures(adapter);
628	em_free_receive_structures(adapter);
629
630	/* Free Transmit Descriptor ring */
631	if (adapter->tx_desc_base) {
632		em_dma_free(adapter, &adapter->txdma);
633		adapter->tx_desc_base = NULL;
634	}
635
636	/* Free Receive Descriptor ring */
637	if (adapter->rx_desc_base) {
638		em_dma_free(adapter, &adapter->rxdma);
639		adapter->rx_desc_base = NULL;
640	}
641
642	EM_LOCK_DESTROY(adapter);
643
644	return (0);
645}
646
647/*********************************************************************
648 *
649 *  Shutdown entry point
650 *
651 **********************************************************************/
652
653static int
654em_shutdown(device_t dev)
655{
656	struct adapter *adapter = device_get_softc(dev);
657	EM_LOCK(adapter);
658	em_stop(adapter);
659	EM_UNLOCK(adapter);
660	return (0);
661}
662
663/*
664 * Suspend/resume device methods.
665 */
666static int
667em_suspend(device_t dev)
668{
669	struct adapter *adapter = device_get_softc(dev);
670
671	EM_LOCK(adapter);
672	em_stop(adapter);
673	EM_UNLOCK(adapter);
674
675	return bus_generic_suspend(dev);
676}
677
678static int
679em_resume(device_t dev)
680{
681	struct adapter *adapter = device_get_softc(dev);
682	struct ifnet *ifp = adapter->ifp;
683
684	EM_LOCK(adapter);
685	em_init_locked(adapter);
686	if ((ifp->if_flags & IFF_UP) &&
687	    (ifp->if_drv_flags & IFF_DRV_RUNNING))
688		em_start_locked(ifp);
689	EM_UNLOCK(adapter);
690
691	return bus_generic_resume(dev);
692}
693
694
695/*********************************************************************
696 *  Transmit entry point
697 *
698 *  em_start is called by the stack to initiate a transmit.
699 *  The driver will remain in this routine as long as there are
700 *  packets to transmit and transmit resources are available.
701 *  In case resources are not available stack is notified and
702 *  the packet is requeued.
703 **********************************************************************/
704
705static void
706em_start_locked(struct ifnet *ifp)
707{
708	struct adapter	*adapter = ifp->if_softc;
709	struct mbuf	*m_head;
710
711	EM_LOCK_ASSERT(adapter);
712
713	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) !=
714	    IFF_DRV_RUNNING)
715		return;
716	if (!adapter->link_active)
717		return;
718
719	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
720
721		IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head);
722		if (m_head == NULL)
723			break;
724		/*
725		 * em_encap() can modify our pointer, and or make it NULL on
726		 * failure.  In that event, we can't requeue.
727		 */
728		if (em_encap(adapter, &m_head)) {
729			if (m_head == NULL)
730				break;
731			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
732			IFQ_DRV_PREPEND(&ifp->if_snd, m_head);
733			break;
734		}
735
736		/* Send a copy of the frame to the BPF listener */
737		BPF_MTAP(ifp, m_head);
738
739		/* Set timeout in case hardware has problems transmitting. */
740		ifp->if_timer = EM_TX_TIMEOUT;
741	}
742}
743
744static void
745em_start(struct ifnet *ifp)
746{
747	struct adapter *adapter = ifp->if_softc;
748
749	EM_LOCK(adapter);
750	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
751		em_start_locked(ifp);
752	EM_UNLOCK(adapter);
753}
754
755/*********************************************************************
756 *  Ioctl entry point
757 *
758 *  em_ioctl is called when the user wants to configure the
759 *  interface.
760 *
761 *  return 0 on success, positive on failure
762 **********************************************************************/
763
764static int
765em_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
766{
767	struct adapter	*adapter = ifp->if_softc;
768	struct ifreq *ifr = (struct ifreq *)data;
769	struct ifaddr *ifa = (struct ifaddr *)data;
770	int error = 0;
771
772	if (adapter->in_detach)
773		return (error);
774
775	switch (command) {
776	case SIOCSIFADDR:
777	case SIOCGIFADDR:
778		if (ifa->ifa_addr->sa_family == AF_INET) {
779			/*
780			 * XXX
781			 * Since resetting hardware takes a very long time
782			 * and results in link renegotiation we only
783			 * initialize the hardware only when it is absolutely
784			 * required.
785			 */
786			ifp->if_flags |= IFF_UP;
787			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) {
788				EM_LOCK(adapter);
789				em_init_locked(adapter);
790				EM_UNLOCK(adapter);
791			}
792			arp_ifinit(ifp, ifa);
793		} else
794			error = ether_ioctl(ifp, command, data);
795		break;
796	case SIOCSIFMTU:
797	    {
798		int max_frame_size;
799		uint16_t eeprom_data = 0;
800
801		IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)");
802
803		EM_LOCK(adapter);
804		switch (adapter->hw.mac_type) {
805		case em_82573:
806			/*
807			 * 82573 only supports jumbo frames
808			 * if ASPM is disabled.
809			 */
810			em_read_eeprom(&adapter->hw,
811			    EEPROM_INIT_3GIO_3, 1, &eeprom_data);
812			if (eeprom_data & EEPROM_WORD1A_ASPM_MASK) {
813				max_frame_size = ETHER_MAX_LEN;
814				break;
815			}
816			/* Allow Jumbo frames - fall thru */
817		case em_82571:
818		case em_82572:
819		case em_80003es2lan:	/* Limit Jumbo Frame size */
820			max_frame_size = 9234;
821			break;
822		case em_ich8lan:
823			/* ICH8 does not support jumbo frames */
824			max_frame_size = ETHER_MAX_LEN;
825			break;
826		default:
827			max_frame_size = MAX_JUMBO_FRAME_SIZE;
828		}
829		if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN -
830		    ETHER_CRC_LEN) {
831			EM_UNLOCK(adapter);
832			error = EINVAL;
833			break;
834		}
835
836		ifp->if_mtu = ifr->ifr_mtu;
837		adapter->hw.max_frame_size =
838		ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
839		em_init_locked(adapter);
840		EM_UNLOCK(adapter);
841		break;
842	    }
843	case SIOCSIFFLAGS:
844		IOCTL_DEBUGOUT("ioctl rcv'd:\
845		    SIOCSIFFLAGS (Set Interface Flags)");
846		EM_LOCK(adapter);
847		if (ifp->if_flags & IFF_UP) {
848			if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) {
849				if ((ifp->if_flags ^ adapter->if_flags) &
850				    IFF_PROMISC) {
851					em_disable_promisc(adapter);
852					em_set_promisc(adapter);
853				}
854			} else
855				em_init_locked(adapter);
856		} else {
857			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
858				em_stop(adapter);
859			}
860		}
861		adapter->if_flags = ifp->if_flags;
862		EM_UNLOCK(adapter);
863		break;
864	case SIOCADDMULTI:
865	case SIOCDELMULTI:
866		IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI");
867		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
868			EM_LOCK(adapter);
869			em_disable_intr(adapter);
870			em_set_multi(adapter);
871			if (adapter->hw.mac_type == em_82542_rev2_0) {
872				em_initialize_receive_unit(adapter);
873			}
874#ifdef DEVICE_POLLING
875			if (!(ifp->if_capenable & IFCAP_POLLING))
876#endif
877				em_enable_intr(adapter);
878			EM_UNLOCK(adapter);
879		}
880		break;
881	case SIOCSIFMEDIA:
882		/* Check SOL/IDER usage */
883		if (em_check_phy_reset_block(&adapter->hw)) {
884			device_printf(adapter->dev, "Media change is"
885			    "blocked due to SOL/IDER session.\n");
886			break;
887		}
888	case SIOCGIFMEDIA:
889		IOCTL_DEBUGOUT("ioctl rcv'd: \
890		    SIOCxIFMEDIA (Get/Set Interface Media)");
891		error = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
892		break;
893	case SIOCSIFCAP:
894	    {
895		int mask, reinit;
896
897		IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)");
898		reinit = 0;
899		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
900#ifdef DEVICE_POLLING
901		if (mask & IFCAP_POLLING) {
902			if (ifr->ifr_reqcap & IFCAP_POLLING) {
903				error = ether_poll_register(em_poll, ifp);
904				if (error)
905					return (error);
906				EM_LOCK(adapter);
907				em_disable_intr(adapter);
908				ifp->if_capenable |= IFCAP_POLLING;
909				EM_UNLOCK(adapter);
910			} else {
911				error = ether_poll_deregister(ifp);
912				/* Enable interrupt even in error case */
913				EM_LOCK(adapter);
914				em_enable_intr(adapter);
915				ifp->if_capenable &= ~IFCAP_POLLING;
916				EM_UNLOCK(adapter);
917			}
918		}
919#endif
920		if (mask & IFCAP_HWCSUM) {
921			ifp->if_capenable ^= IFCAP_HWCSUM;
922			reinit = 1;
923		}
924		if (mask & IFCAP_VLAN_HWTAGGING) {
925			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
926			reinit = 1;
927		}
928		if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING))
929			em_init(adapter);
930		break;
931	    }
932	default:
933		error = ether_ioctl(ifp, command, data);
934		break;
935	}
936
937	return (error);
938}
939
940/*********************************************************************
941 *  Watchdog entry point
942 *
943 *  This routine is called whenever hardware quits transmitting.
944 *
945 **********************************************************************/
946
947static void
948em_watchdog(struct ifnet *ifp)
949{
950	struct adapter *adapter = ifp->if_softc;
951
952	EM_LOCK(adapter);
953	/* If we are in this routine because of pause frames, then
954	 * don't reset the hardware.
955	 */
956	if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_TXOFF) {
957		ifp->if_timer = EM_TX_TIMEOUT;
958		EM_UNLOCK(adapter);
959		return;
960	}
961
962	if (em_check_for_link(&adapter->hw) == 0)
963		device_printf(adapter->dev, "watchdog timeout -- resetting\n");
964
965	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
966	adapter->watchdog_events++;
967
968	em_init_locked(adapter);
969	EM_UNLOCK(adapter);
970}
971
972/*********************************************************************
973 *  Init entry point
974 *
975 *  This routine is used in two ways. It is used by the stack as
976 *  init entry point in network interface structure. It is also used
977 *  by the driver as a hw/sw initialization routine to get to a
978 *  consistent state.
979 *
980 *  return 0 on success, positive on failure
981 **********************************************************************/
982
983static void
984em_init_locked(struct adapter *adapter)
985{
986	struct ifnet	*ifp = adapter->ifp;
987	device_t	dev = adapter->dev;
988	uint32_t	pba;
989
990	INIT_DEBUGOUT("em_init: begin");
991
992	EM_LOCK_ASSERT(adapter);
993
994	em_stop(adapter);
995
996	/*
997	 * Packet Buffer Allocation (PBA)
998	 * Writing PBA sets the receive portion of the buffer
999	 * the remainder is used for the transmit buffer.
1000	 *
1001	 * Devices before the 82547 had a Packet Buffer of 64K.
1002	 *   Default allocation: PBA=48K for Rx, leaving 16K for Tx.
1003	 * After the 82547 the buffer was reduced to 40K.
1004	 *   Default allocation: PBA=30K for Rx, leaving 10K for Tx.
1005	 *   Note: default does not leave enough room for Jumbo Frame >10k.
1006	 */
1007	switch (adapter->hw.mac_type) {
1008	case em_82547:
1009	case em_82547_rev_2: /* 82547: Total Packet Buffer is 40K */
1010		if (adapter->hw.max_frame_size > EM_RXBUFFER_8192)
1011			pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */
1012		else
1013			pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */
1014		adapter->tx_fifo_head = 0;
1015		adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT;
1016		adapter->tx_fifo_size =
1017		    (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT;
1018		break;
1019	/* Total Packet Buffer on these is 48K */
1020	case em_82571:
1021	case em_82572:
1022	case em_80003es2lan:
1023			pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */
1024		break;
1025	case em_82573: /* 82573: Total Packet Buffer is 32K */
1026			pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */
1027		break;
1028	case em_ich8lan:
1029		pba = E1000_PBA_8K;
1030		break;
1031	default:
1032		/* Devices before 82547 had a Packet Buffer of 64K.   */
1033		if(adapter->hw.max_frame_size > EM_RXBUFFER_8192)
1034			pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */
1035		else
1036			pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */
1037	}
1038
1039	INIT_DEBUGOUT1("em_init: pba=%dK",pba);
1040	E1000_WRITE_REG(&adapter->hw, PBA, pba);
1041
1042	/* Get the latest mac address, User can use a LAA */
1043        bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac_addr,
1044              ETHER_ADDR_LEN);
1045
1046	/* Initialize the hardware */
1047	if (em_hardware_init(adapter)) {
1048		device_printf(dev, "Unable to initialize the hardware\n");
1049		return;
1050	}
1051	em_update_link_status(adapter);
1052
1053	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1054		em_enable_vlans(adapter);
1055
1056	/* Set hardware offload abilities */
1057	ifp->if_hwassist = 0;
1058	if (adapter->hw.mac_type >= em_82543) {
1059		if (ifp->if_capenable & IFCAP_TXCSUM)
1060			ifp->if_hwassist |= EM_CHECKSUM_FEATURES;
1061#ifdef EM_TSO
1062		if (ifp->if_capenable & IFCAP_TSO)
1063			ifp->if_hwassist |= EM_TCPSEG_FEATURES;
1064#endif
1065	}
1066
1067	/* Prepare transmit descriptors and buffers */
1068	em_setup_transmit_structures(adapter);
1069	em_initialize_transmit_unit(adapter);
1070
1071	/* Setup Multicast table */
1072	em_set_multi(adapter);
1073
1074	/* Prepare receive descriptors and buffers */
1075	if (em_setup_receive_structures(adapter)) {
1076		device_printf(dev, "Could not setup receive structures\n");
1077		em_stop(adapter);
1078		return;
1079	}
1080	em_initialize_receive_unit(adapter);
1081
1082	/* Don't lose promiscuous settings */
1083	em_set_promisc(adapter);
1084
1085	ifp->if_drv_flags |= IFF_DRV_RUNNING;
1086	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1087
1088	callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1089	em_clear_hw_cntrs(&adapter->hw);
1090
1091#ifdef DEVICE_POLLING
1092	/*
1093	 * Only enable interrupts if we are not polling, make sure
1094	 * they are off otherwise.
1095	 */
1096	if (ifp->if_capenable & IFCAP_POLLING)
1097		em_disable_intr(adapter);
1098	else
1099#endif /* DEVICE_POLLING */
1100		em_enable_intr(adapter);
1101
1102	/* Don't reset the phy next time init gets called */
1103	adapter->hw.phy_reset_disable = TRUE;
1104}
1105
1106static void
1107em_init(void *arg)
1108{
1109	struct adapter *adapter = arg;
1110
1111	EM_LOCK(adapter);
1112	em_init_locked(adapter);
1113	EM_UNLOCK(adapter);
1114}
1115
1116
1117#ifdef DEVICE_POLLING
1118static void
1119em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
1120{
1121	struct adapter *adapter = ifp->if_softc;
1122	uint32_t reg_icr;
1123
1124	EM_LOCK(adapter);
1125	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
1126		EM_UNLOCK(adapter);
1127		return;
1128	}
1129
1130	if (cmd == POLL_AND_CHECK_STATUS) {
1131		reg_icr = E1000_READ_REG(&adapter->hw, ICR);
1132		if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1133			callout_stop(&adapter->timer);
1134			adapter->hw.get_link_status = 1;
1135			em_check_for_link(&adapter->hw);
1136			em_update_link_status(adapter);
1137			callout_reset(&adapter->timer, hz,
1138			    em_local_timer, adapter);
1139		}
1140	}
1141	em_rxeof(adapter, count);
1142	em_txeof(adapter);
1143
1144	if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1145		em_start_locked(ifp);
1146	EM_UNLOCK(adapter);
1147}
1148#endif /* DEVICE_POLLING */
1149
1150/*********************************************************************
1151 *
1152 *  Interrupt Service routine
1153 *
1154 *********************************************************************/
1155static void
1156em_intr(void *arg)
1157{
1158	struct adapter	*adapter = arg;
1159	struct ifnet	*ifp;
1160	uint32_t	reg_icr;
1161
1162	EM_LOCK(adapter);
1163
1164	ifp = adapter->ifp;
1165
1166#ifdef DEVICE_POLLING
1167	if (ifp->if_capenable & IFCAP_POLLING) {
1168		EM_UNLOCK(adapter);
1169		return;
1170	}
1171#endif /* DEVICE_POLLING */
1172
1173	for (;;) {
1174		reg_icr = E1000_READ_REG(&adapter->hw, ICR);
1175		if (adapter->hw.mac_type >= em_82571 &&
1176		    (reg_icr & E1000_ICR_INT_ASSERTED) == 0)
1177			break;
1178		else if (reg_icr == 0)
1179			break;
1180
1181		/*
1182		 * XXX: some laptops trigger several spurious interrupts
1183		 * on em(4) when in the resume cycle. The ICR register
1184		 * reports all-ones value in this case. Processing such
1185		 * interrupts would lead to a freeze. I don't know why.
1186		 */
1187		if (reg_icr == 0xffffffff)
1188			break;
1189
1190		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1191			em_rxeof(adapter, -1);
1192			em_txeof(adapter);
1193		}
1194
1195		/* Link status change */
1196		if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
1197			callout_stop(&adapter->timer);
1198			adapter->hw.get_link_status = 1;
1199			em_check_for_link(&adapter->hw);
1200			em_update_link_status(adapter);
1201			callout_reset(&adapter->timer, hz,
1202			    em_local_timer, adapter);
1203		}
1204
1205		if (reg_icr & E1000_ICR_RXO)
1206			adapter->rx_overruns++;
1207	}
1208
1209	if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
1210	    !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
1211		em_start_locked(ifp);
1212
1213	EM_UNLOCK(adapter);
1214}
1215
1216/*********************************************************************
1217 *
1218 *  Media Ioctl callback
1219 *
1220 *  This routine is called whenever the user queries the status of
1221 *  the interface using ifconfig.
1222 *
1223 **********************************************************************/
1224static void
1225em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
1226{
1227	struct adapter *adapter = ifp->if_softc;
1228	u_char fiber_type = IFM_1000_SX;
1229
1230	INIT_DEBUGOUT("em_media_status: begin");
1231
1232	EM_LOCK(adapter);
1233	em_check_for_link(&adapter->hw);
1234	em_update_link_status(adapter);
1235
1236	ifmr->ifm_status = IFM_AVALID;
1237	ifmr->ifm_active = IFM_ETHER;
1238
1239	if (!adapter->link_active) {
1240		EM_UNLOCK(adapter);
1241		return;
1242	}
1243
1244	ifmr->ifm_status |= IFM_ACTIVE;
1245
1246	if ((adapter->hw.media_type == em_media_type_fiber) ||
1247	    (adapter->hw.media_type == em_media_type_internal_serdes)) {
1248		if (adapter->hw.mac_type == em_82545)
1249			fiber_type = IFM_1000_LX;
1250		ifmr->ifm_active |= fiber_type | IFM_FDX;
1251	} else {
1252		switch (adapter->link_speed) {
1253		case 10:
1254			ifmr->ifm_active |= IFM_10_T;
1255			break;
1256		case 100:
1257			ifmr->ifm_active |= IFM_100_TX;
1258			break;
1259		case 1000:
1260			ifmr->ifm_active |= IFM_1000_T;
1261			break;
1262		}
1263		if (adapter->link_duplex == FULL_DUPLEX)
1264			ifmr->ifm_active |= IFM_FDX;
1265		else
1266			ifmr->ifm_active |= IFM_HDX;
1267	}
1268	EM_UNLOCK(adapter);
1269}
1270
1271/*********************************************************************
1272 *
1273 *  Media Ioctl callback
1274 *
1275 *  This routine is called when the user changes speed/duplex using
1276 *  media/mediopt option with ifconfig.
1277 *
1278 **********************************************************************/
1279static int
1280em_media_change(struct ifnet *ifp)
1281{
1282	struct adapter *adapter = ifp->if_softc;
1283	struct ifmedia  *ifm = &adapter->media;
1284
1285	INIT_DEBUGOUT("em_media_change: begin");
1286
1287	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
1288		return (EINVAL);
1289
1290	EM_LOCK(adapter);
1291	switch (IFM_SUBTYPE(ifm->ifm_media)) {
1292	case IFM_AUTO:
1293		adapter->hw.autoneg = DO_AUTO_NEG;
1294		adapter->hw.autoneg_advertised = AUTONEG_ADV_DEFAULT;
1295		break;
1296	case IFM_1000_LX:
1297	case IFM_1000_SX:
1298	case IFM_1000_T:
1299		adapter->hw.autoneg = DO_AUTO_NEG;
1300		adapter->hw.autoneg_advertised = ADVERTISE_1000_FULL;
1301		break;
1302	case IFM_100_TX:
1303		adapter->hw.autoneg = FALSE;
1304		adapter->hw.autoneg_advertised = 0;
1305		if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1306			adapter->hw.forced_speed_duplex = em_100_full;
1307		else
1308			adapter->hw.forced_speed_duplex = em_100_half;
1309		break;
1310	case IFM_10_T:
1311		adapter->hw.autoneg = FALSE;
1312		adapter->hw.autoneg_advertised = 0;
1313		if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX)
1314			adapter->hw.forced_speed_duplex = em_10_full;
1315		else
1316			adapter->hw.forced_speed_duplex = em_10_half;
1317		break;
1318	default:
1319		device_printf(adapter->dev, "Unsupported media type\n");
1320	}
1321
1322	/* As the speed/duplex settings my have changed we need to
1323	 * reset the PHY.
1324	 */
1325	adapter->hw.phy_reset_disable = FALSE;
1326
1327	em_init_locked(adapter);
1328	EM_UNLOCK(adapter);
1329
1330	return (0);
1331}
1332
1333/*********************************************************************
1334 *
1335 *  This routine maps the mbufs to tx descriptors.
1336 *
1337 *  return 0 on success, positive on failure
1338 **********************************************************************/
1339static int
1340em_encap(struct adapter *adapter, struct mbuf **m_headp)
1341{
1342	struct ifnet		*ifp = adapter->ifp;
1343	bus_dma_segment_t	segs[EM_MAX_SCATTER];
1344	bus_dmamap_t		map;
1345	struct em_buffer	*tx_buffer, *tx_buffer_mapped;
1346	struct em_tx_desc	*current_tx_desc;
1347	struct mbuf		*m_head;
1348	struct m_tag		*mtag;
1349	uint32_t		txd_upper, txd_lower, txd_used, txd_saved;
1350	int			nsegs, i, j, first, last = 0;
1351	int			error, do_tso, tso_desc = 0;
1352
1353	m_head = *m_headp;
1354	current_tx_desc = NULL;
1355	txd_upper = txd_lower = txd_used = txd_saved = 0;
1356
1357#ifdef EM_TSO
1358	do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0);
1359#else
1360	do_tso = 0;
1361#endif
1362
1363        /*
1364         * Force a cleanup if number of TX descriptors
1365         * available hits the threshold
1366         */
1367	if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1368		em_txeof(adapter);
1369		if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) {
1370			adapter->no_tx_desc_avail1++;
1371			return(ENOBUFS);
1372		}
1373        }
1374
1375	/*
1376	 * Capture the first descriptor index,
1377	 * this descriptor will have the index
1378	 * of the EOP which is the only one that
1379	 * now gets a DONE bit writeback.
1380	 */
1381	first = adapter->next_avail_tx_desc;
1382
1383        /* Find out if we are in vlan mode */
1384        mtag = VLAN_OUTPUT_TAG(ifp, m_head);
1385
1386	/*
1387	 * When operating in promiscuous mode, hardware encapsulation for
1388	 * packets is disabled.  This means we have to add the vlan
1389	 * encapsulation in the driver, since it will have come down from the
1390	 * VLAN layer with a tag instead of a VLAN header.
1391	 */
1392	if (mtag != NULL && adapter->em_insert_vlan_header) {
1393		struct ether_vlan_header *evl;
1394		struct ether_header eh;
1395
1396		m_head = m_pullup(m_head, sizeof(eh));
1397		if (m_head == NULL) {
1398			*m_headp = NULL;
1399			return (ENOBUFS);
1400		}
1401		eh = *mtod(m_head, struct ether_header *);
1402		M_PREPEND(m_head, sizeof(*evl), M_DONTWAIT);
1403		if (m_head == NULL) {
1404			*m_headp = NULL;
1405			return (ENOBUFS);
1406		}
1407		m_head = m_pullup(m_head, sizeof(*evl));
1408		if (m_head == NULL) {
1409			*m_headp = NULL;
1410			return (ENOBUFS);
1411		}
1412		evl = mtod(m_head, struct ether_vlan_header *);
1413		bcopy(&eh, evl, sizeof(*evl));
1414		evl->evl_proto = evl->evl_encap_proto;
1415		evl->evl_encap_proto = htons(ETHERTYPE_VLAN);
1416		evl->evl_tag = htons(VLAN_TAG_VALUE(mtag));
1417		m_tag_delete(m_head, mtag);
1418		mtag = NULL;
1419		*m_headp = m_head;
1420	}
1421
1422	/*
1423	 * TSO workaround:
1424	 *  If an mbuf is only header we need
1425	 *     to pull 4 bytes of data into it.
1426	 */
1427	if (do_tso && (m_head->m_len <= M_TSO_LEN)) {
1428		m_head = m_pullup(m_head, M_TSO_LEN + 4);
1429		*m_headp = m_head;
1430		if (m_head == NULL)
1431			return (ENOBUFS);
1432	}
1433
1434        /*
1435         * Map the packet for DMA.
1436         */
1437	tx_buffer = &adapter->tx_buffer_area[first];
1438	tx_buffer_mapped = tx_buffer;
1439	map = tx_buffer->map;
1440	error = bus_dmamap_load_mbuf_sg(adapter->txtag, tx_buffer->map,
1441	    *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1442	if (error == EFBIG) {
1443		struct mbuf *m;
1444
1445		m = m_defrag(*m_headp, M_DONTWAIT);
1446		if (m == NULL) {
1447			adapter->mbuf_alloc_failed++;
1448			m_freem(*m_headp);
1449			*m_headp = NULL;
1450			return (ENOBUFS);
1451		}
1452		*m_headp = m;
1453		/* Try it again */
1454		error = bus_dmamap_load_mbuf_sg(adapter->txtag, tx_buffer->map,
1455		    *m_headp, segs, &nsegs, BUS_DMA_NOWAIT);
1456
1457		if (error == ENOMEM) {
1458			adapter->no_tx_dma_setup++;
1459			return (error);
1460		} else if (error != 0) {
1461			adapter->no_tx_dma_setup++;
1462			m_freem(*m_headp);
1463			*m_headp = NULL;
1464			return (error);
1465		}
1466	} else if (error == ENOMEM) {
1467		adapter->no_tx_dma_setup++;
1468		return (error);
1469	} else if (error != 0) {
1470		adapter->no_tx_dma_setup++;
1471		m_freem(*m_headp);
1472		*m_headp = NULL;
1473		return (error);
1474	}
1475
1476	/*
1477	 * TSO Hardware workaround, if this packet is not
1478	 * TSO, and is only a single descriptor long, and
1479	 * it follows a TSO burst, then we need to add a
1480	 * sentinel descriptor to prevent premature writeback.
1481	 */
1482	if ((do_tso == 0) && (adapter->tx_tso == TRUE)) {
1483		if (nsegs == 1)
1484			tso_desc = TRUE;
1485		adapter->tx_tso = FALSE;
1486	}
1487
1488        if (nsegs > (adapter->num_tx_desc_avail - 2)) {
1489                adapter->no_tx_desc_avail2++;
1490                error = ENOBUFS;
1491		goto encap_fail;
1492        }
1493
1494	/* Do hardware assists */
1495	if (ifp->if_hwassist > 0) {
1496#ifdef EM_TSO
1497        	if (em_tso_setup(adapter, m_head, &txd_upper, &txd_lower)) {
1498			/* we need to make a final sentinel transmit desc */
1499			tso_desc = TRUE;
1500		} else
1501#endif
1502			em_transmit_checksum_setup(adapter,  m_head,
1503			    &txd_upper, &txd_lower);
1504	}
1505
1506	i = adapter->next_avail_tx_desc;
1507	if (adapter->pcix_82544)
1508		txd_saved = i;
1509
1510	/* Set up our transmit descriptors */
1511	for (j = 0; j < nsegs; j++) {
1512		bus_size_t seg_len;
1513		bus_addr_t seg_addr;
1514		/* If adapter is 82544 and on PCIX bus */
1515		if(adapter->pcix_82544) {
1516			DESC_ARRAY	desc_array;
1517			uint32_t	array_elements, counter;
1518			/*
1519			 * Check the Address and Length combination and
1520			 * split the data accordingly
1521			 */
1522			array_elements = em_fill_descriptors(segs[j].ds_addr,
1523			    segs[j].ds_len, &desc_array);
1524			for (counter = 0; counter < array_elements; counter++) {
1525				if (txd_used == adapter->num_tx_desc_avail) {
1526					adapter->next_avail_tx_desc = txd_saved;
1527					adapter->no_tx_desc_avail2++;
1528					error = ENOBUFS;
1529					goto encap_fail;
1530				}
1531				tx_buffer = &adapter->tx_buffer_area[i];
1532				current_tx_desc = &adapter->tx_desc_base[i];
1533				current_tx_desc->buffer_addr = htole64(
1534				    desc_array.descriptor[counter].address);
1535				current_tx_desc->lower.data = htole32(
1536				    (adapter->txd_cmd | txd_lower | (uint16_t)
1537				    desc_array.descriptor[counter].length));
1538				current_tx_desc->upper.data =
1539				    htole32((txd_upper));
1540				last = i;
1541				if (++i == adapter->num_tx_desc)
1542                                         i = 0;
1543				tx_buffer->m_head = NULL;
1544				tx_buffer->next_eop = -1;
1545				txd_used++;
1546                        }
1547		} else {
1548			tx_buffer = &adapter->tx_buffer_area[i];
1549			current_tx_desc = &adapter->tx_desc_base[i];
1550			seg_addr = htole64(segs[j].ds_addr);
1551			seg_len  = segs[j].ds_len;
1552			/*
1553			** TSO Workaround:
1554			** If this is the last descriptor, we want to
1555			** split it so we have a small final sentinel
1556			*/
1557			if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) {
1558				seg_len -= 4;
1559				current_tx_desc->buffer_addr = seg_addr;
1560				current_tx_desc->lower.data = htole32(
1561				adapter->txd_cmd | txd_lower | seg_len);
1562				current_tx_desc->upper.data =
1563				    htole32(txd_upper);
1564				if (++i == adapter->num_tx_desc)
1565					i = 0;
1566				/* Now make the sentinel */
1567				++txd_used; /* using an extra txd */
1568				current_tx_desc = &adapter->tx_desc_base[i];
1569				tx_buffer = &adapter->tx_buffer_area[i];
1570				current_tx_desc->buffer_addr =
1571				    seg_addr + seg_len;
1572				current_tx_desc->lower.data = htole32(
1573				adapter->txd_cmd | txd_lower | 4);
1574				current_tx_desc->upper.data =
1575				    htole32(txd_upper);
1576				last = i;
1577				if (++i == adapter->num_tx_desc)
1578					i = 0;
1579			} else {
1580				current_tx_desc->buffer_addr = seg_addr;
1581				current_tx_desc->lower.data = htole32(
1582				adapter->txd_cmd | txd_lower | seg_len);
1583				current_tx_desc->upper.data =
1584				    htole32(txd_upper);
1585				last = i;
1586				if (++i == adapter->num_tx_desc)
1587					i = 0;
1588			}
1589			tx_buffer->m_head = NULL;
1590			tx_buffer->next_eop = -1;
1591		}
1592	}
1593
1594	adapter->next_avail_tx_desc = i;
1595	if (adapter->pcix_82544)
1596		adapter->num_tx_desc_avail -= txd_used;
1597	else {
1598		adapter->num_tx_desc_avail -= nsegs;
1599		if (tso_desc) /* TSO used an extra for sentinel */
1600			adapter->num_tx_desc_avail -= txd_used;
1601	}
1602
1603	if (mtag != NULL) {
1604		/* Set the vlan id. */
1605		current_tx_desc->upper.fields.special =
1606		    htole16(VLAN_TAG_VALUE(mtag));
1607                /* Tell hardware to add tag */
1608                current_tx_desc->lower.data |=
1609		    htole32(E1000_TXD_CMD_VLE);
1610        }
1611
1612        tx_buffer->m_head = m_head;
1613	tx_buffer_mapped->map = tx_buffer->map;
1614	tx_buffer->map = map;
1615        bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE);
1616
1617        /*
1618         * Last Descriptor of Packet
1619	 * needs End Of Packet (EOP)
1620	 * and Report Status (RS)
1621         */
1622        current_tx_desc->lower.data |=
1623	    htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
1624	/*
1625	 * Keep track in the first buffer which
1626	 * descriptor will be written back
1627	 */
1628	tx_buffer = &adapter->tx_buffer_area[first];
1629	tx_buffer->next_eop = last;
1630
1631	/*
1632	 * Advance the Transmit Descriptor Tail (Tdt), this tells the E1000
1633	 * that this frame is available to transmit.
1634	 */
1635	bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
1636	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1637	if (adapter->hw.mac_type == em_82547 &&
1638	    adapter->link_duplex == HALF_DUPLEX)
1639		em_82547_move_tail_locked(adapter);
1640	else {
1641		E1000_WRITE_REG(&adapter->hw, TDT, i);
1642		if (adapter->hw.mac_type == em_82547)
1643			em_82547_update_fifo_head(adapter,
1644			    m_head->m_pkthdr.len);
1645	}
1646
1647	return (0);
1648
1649encap_fail:
1650	bus_dmamap_unload(adapter->txtag, tx_buffer->map);
1651	return (error);
1652}
1653
1654/*********************************************************************
1655 *
1656 * 82547 workaround to avoid controller hang in half-duplex environment.
1657 * The workaround is to avoid queuing a large packet that would span
1658 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers
1659 * in this case. We do that only when FIFO is quiescent.
1660 *
1661 **********************************************************************/
1662static void
1663em_82547_move_tail_locked(struct adapter *adapter)
1664{
1665	uint16_t hw_tdt;
1666	uint16_t sw_tdt;
1667	struct em_tx_desc *tx_desc;
1668	uint16_t length = 0;
1669	boolean_t eop = 0;
1670
1671	EM_LOCK_ASSERT(adapter);
1672
1673	hw_tdt = E1000_READ_REG(&adapter->hw, TDT);
1674	sw_tdt = adapter->next_avail_tx_desc;
1675
1676	while (hw_tdt != sw_tdt) {
1677		tx_desc = &adapter->tx_desc_base[hw_tdt];
1678		length += tx_desc->lower.flags.length;
1679		eop = tx_desc->lower.data & E1000_TXD_CMD_EOP;
1680		if (++hw_tdt == adapter->num_tx_desc)
1681			hw_tdt = 0;
1682
1683		if (eop) {
1684			if (em_82547_fifo_workaround(adapter, length)) {
1685				adapter->tx_fifo_wrk_cnt++;
1686				callout_reset(&adapter->tx_fifo_timer, 1,
1687					em_82547_move_tail, adapter);
1688				break;
1689			}
1690			E1000_WRITE_REG(&adapter->hw, TDT, hw_tdt);
1691			em_82547_update_fifo_head(adapter, length);
1692			length = 0;
1693		}
1694	}
1695}
1696
1697static void
1698em_82547_move_tail(void *arg)
1699{
1700	struct adapter *adapter = arg;
1701
1702	EM_LOCK(adapter);
1703	em_82547_move_tail_locked(adapter);
1704	EM_UNLOCK(adapter);
1705}
1706
1707static int
1708em_82547_fifo_workaround(struct adapter *adapter, int len)
1709{
1710	int fifo_space, fifo_pkt_len;
1711
1712	fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1713
1714	if (adapter->link_duplex == HALF_DUPLEX) {
1715		fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head;
1716
1717		if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) {
1718			if (em_82547_tx_fifo_reset(adapter))
1719				return (0);
1720			else
1721				return (1);
1722		}
1723	}
1724
1725	return (0);
1726}
1727
1728static void
1729em_82547_update_fifo_head(struct adapter *adapter, int len)
1730{
1731	int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR);
1732
1733	/* tx_fifo_head is always 16 byte aligned */
1734	adapter->tx_fifo_head += fifo_pkt_len;
1735	if (adapter->tx_fifo_head >= adapter->tx_fifo_size) {
1736		adapter->tx_fifo_head -= adapter->tx_fifo_size;
1737	}
1738}
1739
1740
1741static int
1742em_82547_tx_fifo_reset(struct adapter *adapter)
1743{
1744	uint32_t tctl;
1745
1746	if ((E1000_READ_REG(&adapter->hw, TDT) ==
1747	    E1000_READ_REG(&adapter->hw, TDH)) &&
1748	    (E1000_READ_REG(&adapter->hw, TDFT) ==
1749	    E1000_READ_REG(&adapter->hw, TDFH)) &&
1750	    (E1000_READ_REG(&adapter->hw, TDFTS) ==
1751	    E1000_READ_REG(&adapter->hw, TDFHS)) &&
1752	    (E1000_READ_REG(&adapter->hw, TDFPC) == 0)) {
1753		/* Disable TX unit */
1754		tctl = E1000_READ_REG(&adapter->hw, TCTL);
1755		E1000_WRITE_REG(&adapter->hw, TCTL, tctl & ~E1000_TCTL_EN);
1756
1757		/* Reset FIFO pointers */
1758		E1000_WRITE_REG(&adapter->hw, TDFT,  adapter->tx_head_addr);
1759		E1000_WRITE_REG(&adapter->hw, TDFH,  adapter->tx_head_addr);
1760		E1000_WRITE_REG(&adapter->hw, TDFTS, adapter->tx_head_addr);
1761		E1000_WRITE_REG(&adapter->hw, TDFHS, adapter->tx_head_addr);
1762
1763		/* Re-enable TX unit */
1764		E1000_WRITE_REG(&adapter->hw, TCTL, tctl);
1765		E1000_WRITE_FLUSH(&adapter->hw);
1766
1767		adapter->tx_fifo_head = 0;
1768		adapter->tx_fifo_reset_cnt++;
1769
1770		return (TRUE);
1771	}
1772	else {
1773		return (FALSE);
1774	}
1775}
1776
1777static void
1778em_set_promisc(struct adapter *adapter)
1779{
1780	struct ifnet	*ifp = adapter->ifp;
1781	uint32_t	reg_rctl;
1782
1783	reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1784
1785	if (ifp->if_flags & IFF_PROMISC) {
1786		reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
1787		E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1788		/* Disable VLAN stripping in promiscous mode
1789		 * This enables bridging of vlan tagged frames to occur
1790		 * and also allows vlan tags to be seen in tcpdump
1791		 */
1792		if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1793			em_disable_vlans(adapter);
1794		adapter->em_insert_vlan_header = 1;
1795	} else if (ifp->if_flags & IFF_ALLMULTI) {
1796		reg_rctl |= E1000_RCTL_MPE;
1797		reg_rctl &= ~E1000_RCTL_UPE;
1798		E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1799		adapter->em_insert_vlan_header = 0;
1800	} else
1801		adapter->em_insert_vlan_header = 0;
1802}
1803
1804static void
1805em_disable_promisc(struct adapter *adapter)
1806{
1807	struct ifnet	*ifp = adapter->ifp;
1808	uint32_t	reg_rctl;
1809
1810	reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1811
1812	reg_rctl &=  (~E1000_RCTL_UPE);
1813	reg_rctl &=  (~E1000_RCTL_MPE);
1814	E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1815
1816	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING)
1817		em_enable_vlans(adapter);
1818	adapter->em_insert_vlan_header = 0;
1819}
1820
1821
1822/*********************************************************************
1823 *  Multicast Update
1824 *
1825 *  This routine is called whenever multicast address list is updated.
1826 *
1827 **********************************************************************/
1828
1829static void
1830em_set_multi(struct adapter *adapter)
1831{
1832	struct ifnet	*ifp = adapter->ifp;
1833	struct ifmultiaddr *ifma;
1834	uint32_t reg_rctl = 0;
1835	uint8_t  mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_LENGTH_OF_ADDRESS];
1836	int mcnt = 0;
1837
1838	IOCTL_DEBUGOUT("em_set_multi: begin");
1839
1840	if (adapter->hw.mac_type == em_82542_rev2_0) {
1841		reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1842		if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1843			em_pci_clear_mwi(&adapter->hw);
1844		reg_rctl |= E1000_RCTL_RST;
1845		E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1846		msec_delay(5);
1847	}
1848
1849	IF_ADDR_LOCK(ifp);
1850	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
1851		if (ifma->ifma_addr->sa_family != AF_LINK)
1852			continue;
1853
1854		if (mcnt == MAX_NUM_MULTICAST_ADDRESSES)
1855			break;
1856
1857		bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr),
1858		    &mta[mcnt*ETH_LENGTH_OF_ADDRESS], ETH_LENGTH_OF_ADDRESS);
1859		mcnt++;
1860	}
1861	IF_ADDR_UNLOCK(ifp);
1862
1863	if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) {
1864		reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1865		reg_rctl |= E1000_RCTL_MPE;
1866		E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1867	} else
1868		em_mc_addr_list_update(&adapter->hw, mta, mcnt, 0, 1);
1869
1870	if (adapter->hw.mac_type == em_82542_rev2_0) {
1871		reg_rctl = E1000_READ_REG(&adapter->hw, RCTL);
1872		reg_rctl &= ~E1000_RCTL_RST;
1873		E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
1874		msec_delay(5);
1875		if (adapter->hw.pci_cmd_word & CMD_MEM_WRT_INVALIDATE)
1876			em_pci_set_mwi(&adapter->hw);
1877	}
1878}
1879
1880
1881/*********************************************************************
1882 *  Timer routine
1883 *
1884 *  This routine checks for link status and updates statistics.
1885 *
1886 **********************************************************************/
1887
1888static void
1889em_local_timer(void *arg)
1890{
1891	struct adapter	*adapter = arg;
1892	struct ifnet	*ifp = adapter->ifp;
1893
1894	EM_LOCK(adapter);
1895
1896	em_check_for_link(&adapter->hw);
1897	em_update_link_status(adapter);
1898	em_update_stats_counters(adapter);
1899	if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING)
1900		em_print_hw_stats(adapter);
1901	em_smartspeed(adapter);
1902
1903	callout_reset(&adapter->timer, hz, em_local_timer, adapter);
1904
1905	EM_UNLOCK(adapter);
1906}
1907
1908static void
1909em_update_link_status(struct adapter *adapter)
1910{
1911	struct ifnet *ifp = adapter->ifp;
1912	device_t dev = adapter->dev;
1913
1914	if (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU) {
1915		if (adapter->link_active == 0) {
1916			em_get_speed_and_duplex(&adapter->hw,
1917			    &adapter->link_speed, &adapter->link_duplex);
1918			/* Check if we may set SPEED_MODE bit on PCI-E */
1919			if ((adapter->link_speed == SPEED_1000) &&
1920			    ((adapter->hw.mac_type == em_82571) ||
1921			    (adapter->hw.mac_type == em_82572))) {
1922				int tarc0;
1923
1924				tarc0 = E1000_READ_REG(&adapter->hw, TARC0);
1925				tarc0 |= SPEED_MODE_BIT;
1926				E1000_WRITE_REG(&adapter->hw, TARC0, tarc0);
1927			}
1928			if (bootverbose)
1929				device_printf(dev, "Link is up %d Mbps %s\n",
1930				    adapter->link_speed,
1931				    ((adapter->link_duplex == FULL_DUPLEX) ?
1932				    "Full Duplex" : "Half Duplex"));
1933			adapter->link_active = 1;
1934			adapter->smartspeed = 0;
1935			ifp->if_baudrate = adapter->link_speed * 1000000;
1936			if_link_state_change(ifp, LINK_STATE_UP);
1937		}
1938	} else {
1939		if (adapter->link_active == 1) {
1940			ifp->if_baudrate = adapter->link_speed = 0;
1941			adapter->link_duplex = 0;
1942			if (bootverbose)
1943				device_printf(dev, "Link is Down\n");
1944			adapter->link_active = 0;
1945			if_link_state_change(ifp, LINK_STATE_DOWN);
1946		}
1947	}
1948}
1949
1950/*********************************************************************
1951 *
1952 *  This routine disables all traffic on the adapter by issuing a
1953 *  global reset on the MAC and deallocates TX/RX buffers.
1954 *
1955 **********************************************************************/
1956
1957static void
1958em_stop(void *arg)
1959{
1960	struct adapter	*adapter = arg;
1961	struct ifnet	*ifp = adapter->ifp;
1962
1963	EM_LOCK_ASSERT(adapter);
1964
1965	INIT_DEBUGOUT("em_stop: begin");
1966
1967	em_disable_intr(adapter);
1968	callout_stop(&adapter->timer);
1969	callout_stop(&adapter->tx_fifo_timer);
1970
1971	/* Tell the stack that the interface is no longer active */
1972	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1973	em_reset_hw(&adapter->hw);
1974}
1975
1976
1977/*********************************************************************
1978 *
1979 *  Determine hardware revision.
1980 *
1981 **********************************************************************/
1982static void
1983em_identify_hardware(struct adapter *adapter)
1984{
1985	device_t dev = adapter->dev;
1986
1987	/* Make sure our PCI config space has the necessary stuff set */
1988	adapter->hw.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
1989	if ((adapter->hw.pci_cmd_word & PCIM_CMD_BUSMASTEREN) == 0 &&
1990	    (adapter->hw.pci_cmd_word & PCIM_CMD_MEMEN)) {
1991		device_printf(dev, "Memory Access and/or Bus Master bits "
1992		    "were not set!\n");
1993		adapter->hw.pci_cmd_word |=
1994		(PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN);
1995		pci_write_config(dev, PCIR_COMMAND,
1996		    adapter->hw.pci_cmd_word, 2);
1997	}
1998
1999	/* Save off the information about this board */
2000	adapter->hw.vendor_id = pci_get_vendor(dev);
2001	adapter->hw.device_id = pci_get_device(dev);
2002	adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1);
2003	adapter->hw.subsystem_vendor_id =
2004	    pci_read_config(dev, PCIR_SUBVEND_0, 2);
2005	adapter->hw.subsystem_id = pci_read_config(dev, PCIR_SUBDEV_0, 2);
2006
2007	/* Identify the MAC */
2008	if (em_set_mac_type(&adapter->hw))
2009		device_printf(dev, "Unknown MAC Type\n");
2010
2011	if (adapter->hw.mac_type == em_82541 ||
2012	    adapter->hw.mac_type == em_82541_rev_2 ||
2013	    adapter->hw.mac_type == em_82547 ||
2014	    adapter->hw.mac_type == em_82547_rev_2)
2015		adapter->hw.phy_init_script = TRUE;
2016}
2017
2018static int
2019em_allocate_pci_resources(struct adapter *adapter)
2020{
2021	device_t	dev = adapter->dev;
2022	int		val, rid;
2023
2024	rid = PCIR_BAR(0);
2025	adapter->res_memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2026	    &rid, RF_ACTIVE);
2027	if (adapter->res_memory == NULL) {
2028		device_printf(dev, "Unable to allocate bus resource: memory\n");
2029		return (ENXIO);
2030	}
2031	adapter->osdep.mem_bus_space_tag =
2032	    rman_get_bustag(adapter->res_memory);
2033	adapter->osdep.mem_bus_space_handle =
2034	    rman_get_bushandle(adapter->res_memory);
2035	adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle;
2036
2037	if (adapter->hw.mac_type > em_82543) {
2038		/* Figure our where our IO BAR is ? */
2039		for (rid = PCIR_BAR(0); rid < PCIR_CIS;) {
2040			val = pci_read_config(dev, rid, 4);
2041			if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) {
2042				adapter->io_rid = rid;
2043				break;
2044			}
2045			rid += 4;
2046			/* check for 64bit BAR */
2047			if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT)
2048				rid += 4;
2049		}
2050		if (rid >= PCIR_CIS) {
2051			device_printf(dev, "Unable to locate IO BAR\n");
2052			return (ENXIO);
2053		}
2054		adapter->res_ioport = bus_alloc_resource_any(dev,
2055		    SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE);
2056		if (adapter->res_ioport == NULL) {
2057			device_printf(dev, "Unable to allocate bus resource: "
2058			    "ioport\n");
2059			return (ENXIO);
2060		}
2061		adapter->hw.io_base = 0;
2062		adapter->osdep.io_bus_space_tag =
2063		    rman_get_bustag(adapter->res_ioport);
2064		adapter->osdep.io_bus_space_handle =
2065		    rman_get_bushandle(adapter->res_ioport);
2066	}
2067
2068	/* For ICH8 we need to find the flash memory. */
2069	if (adapter->hw.mac_type == em_ich8lan) {
2070		rid = EM_FLASH;
2071		adapter->flash_mem = bus_alloc_resource_any(dev,
2072		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
2073		adapter->osdep.flash_bus_space_tag =
2074		    rman_get_bustag(adapter->flash_mem);
2075		adapter->osdep.flash_bus_space_handle =
2076		    rman_get_bushandle(adapter->flash_mem);
2077	}
2078
2079	rid = 0x0;
2080	adapter->res_interrupt = bus_alloc_resource_any(dev,
2081	    SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE);
2082	if (adapter->res_interrupt == NULL) {
2083		device_printf(dev, "Unable to allocate bus resource: "
2084		    "interrupt\n");
2085		return (ENXIO);
2086	}
2087	if (bus_setup_intr(dev, adapter->res_interrupt,
2088	    INTR_TYPE_NET | INTR_MPSAFE,
2089	    (void (*)(void *)) em_intr, adapter,
2090	    &adapter->int_handler_tag)) {
2091		device_printf(dev, "Error registering interrupt handler!\n");
2092		return(ENXIO);
2093	}
2094
2095	adapter->hw.back = &adapter->osdep;
2096
2097	return (0);
2098}
2099
2100static void
2101em_free_pci_resources(struct adapter *adapter)
2102{
2103	device_t dev = adapter->dev;
2104
2105	if (adapter->res_interrupt != NULL) {
2106		bus_teardown_intr(dev, adapter->res_interrupt,
2107			adapter->int_handler_tag);
2108		bus_release_resource(dev, SYS_RES_IRQ,
2109		    0, adapter->res_interrupt);
2110	}
2111
2112	if (adapter->res_memory != NULL)
2113		bus_release_resource(dev, SYS_RES_MEMORY,
2114		    PCIR_BAR(0), adapter->res_memory);
2115
2116	if (adapter->flash_mem != NULL)
2117		bus_release_resource(dev, SYS_RES_MEMORY,
2118		    EM_FLASH, adapter->flash_mem);
2119
2120	if (adapter->res_ioport != NULL)
2121		bus_release_resource(dev, SYS_RES_IOPORT,
2122		    adapter->io_rid, adapter->res_ioport);
2123}
2124
2125/*********************************************************************
2126 *
2127 *  Initialize the hardware to a configuration as specified by the
2128 *  adapter structure. The controller is reset, the EEPROM is
2129 *  verified, the MAC address is set, then the shared initialization
2130 *  routines are called.
2131 *
2132 **********************************************************************/
2133static int
2134em_hardware_init(struct adapter *adapter)
2135{
2136	device_t dev = adapter->dev;
2137	uint16_t rx_buffer_size;
2138
2139	INIT_DEBUGOUT("em_hardware_init: begin");
2140	/* Issue a global reset */
2141	em_reset_hw(&adapter->hw);
2142
2143	/* When hardware is reset, fifo_head is also reset */
2144	adapter->tx_fifo_head = 0;
2145
2146	/* Make sure we have a good EEPROM before we read from it */
2147	if (em_validate_eeprom_checksum(&adapter->hw) < 0) {
2148		device_printf(dev, "The EEPROM Checksum Is Not Valid\n");
2149		return (EIO);
2150	}
2151
2152	if (em_read_part_num(&adapter->hw, &(adapter->part_num)) < 0) {
2153		device_printf(dev, "EEPROM read error "
2154		    "reading part number\n");
2155		return (EIO);
2156	}
2157
2158	/* Set up smart power down as default off on newer adapters. */
2159	if (!em_smart_pwr_down && (adapter->hw.mac_type == em_82571 ||
2160	    adapter->hw.mac_type == em_82572)) {
2161		uint16_t phy_tmp = 0;
2162
2163		/* Speed up time to link by disabling smart power down. */
2164		em_read_phy_reg(&adapter->hw,
2165		    IGP02E1000_PHY_POWER_MGMT, &phy_tmp);
2166		phy_tmp &= ~IGP02E1000_PM_SPD;
2167		em_write_phy_reg(&adapter->hw,
2168		    IGP02E1000_PHY_POWER_MGMT, phy_tmp);
2169	}
2170
2171	/*
2172	 * These parameters control the automatic generation (Tx) and
2173	 * response (Rx) to Ethernet PAUSE frames.
2174	 * - High water mark should allow for at least two frames to be
2175	 *   received after sending an XOFF.
2176	 * - Low water mark works best when it is very near the high water mark.
2177	 *   This allows the receiver to restart by sending XON when it has
2178	 *   drained a bit. Here we use an arbitary value of 1500 which will
2179	 *   restart after one full frame is pulled from the buffer. There
2180	 *   could be several smaller frames in the buffer and if so they will
2181	 *   not trigger the XON until their total number reduces the buffer
2182	 *   by 1500.
2183	 * - The pause time is fairly large at 1000 x 512ns = 512 usec.
2184	 */
2185	rx_buffer_size = ((E1000_READ_REG(&adapter->hw, PBA) & 0xffff) << 10 );
2186
2187	adapter->hw.fc_high_water = rx_buffer_size -
2188	    roundup2(adapter->hw.max_frame_size, 1024);
2189	adapter->hw.fc_low_water = adapter->hw.fc_high_water - 1500;
2190	if (adapter->hw.mac_type == em_80003es2lan)
2191		adapter->hw.fc_pause_time = 0xFFFF;
2192	else
2193		adapter->hw.fc_pause_time = 0x1000;
2194	adapter->hw.fc_send_xon = TRUE;
2195	adapter->hw.fc = E1000_FC_FULL;
2196
2197	if (em_init_hw(&adapter->hw) < 0) {
2198		device_printf(dev, "Hardware Initialization Failed");
2199		return (EIO);
2200	}
2201
2202	em_check_for_link(&adapter->hw);
2203
2204	return (0);
2205}
2206
2207/*********************************************************************
2208 *
2209 *  Setup networking device structure and register an interface.
2210 *
2211 **********************************************************************/
2212static void
2213em_setup_interface(device_t dev, struct adapter *adapter)
2214{
2215	struct ifnet   *ifp;
2216	u_char fiber_type = IFM_1000_SX;	/* default type */
2217
2218	INIT_DEBUGOUT("em_setup_interface: begin");
2219
2220	ifp = adapter->ifp = if_alloc(IFT_ETHER);
2221	if (ifp == NULL)
2222		panic("%s: can not if_alloc()", device_get_nameunit(dev));
2223	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
2224	ifp->if_mtu = ETHERMTU;
2225	ifp->if_init =  em_init;
2226	ifp->if_softc = adapter;
2227	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
2228	ifp->if_ioctl = em_ioctl;
2229	ifp->if_start = em_start;
2230	ifp->if_watchdog = em_watchdog;
2231	IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1);
2232	ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1;
2233	IFQ_SET_READY(&ifp->if_snd);
2234
2235	ether_ifattach(ifp, adapter->hw.mac_addr);
2236
2237	ifp->if_capabilities = ifp->if_capenable = 0;
2238
2239	if (adapter->hw.mac_type >= em_82543) {
2240		ifp->if_capabilities |= IFCAP_HWCSUM;
2241		ifp->if_capenable |= IFCAP_HWCSUM;
2242	}
2243
2244#ifdef EM_TSO
2245	/* Enable TSO if available */
2246	if ((adapter->hw.mac_type > em_82544) &&
2247	    (adapter->hw.mac_type != em_82547)) {
2248		ifp->if_capabilities |= IFCAP_TSO;
2249		ifp->if_capenable |= IFCAP_TSO;
2250	}
2251#endif
2252
2253	/*
2254	 * Tell the upper layer(s) we support long frames.
2255	 */
2256	ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
2257	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU;
2258	ifp->if_capenable |= IFCAP_VLAN_MTU;
2259
2260#ifdef DEVICE_POLLING
2261	ifp->if_capabilities |= IFCAP_POLLING;
2262#endif
2263
2264	/*
2265	 * Specify the media types supported by this adapter and register
2266	 * callbacks to update media and link information
2267	 */
2268	ifmedia_init(&adapter->media, IFM_IMASK,
2269	    em_media_change, em_media_status);
2270	if ((adapter->hw.media_type == em_media_type_fiber) ||
2271	    (adapter->hw.media_type == em_media_type_internal_serdes)) {
2272		if (adapter->hw.mac_type == em_82545)
2273			fiber_type = IFM_1000_LX;
2274		ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX,
2275			    0, NULL);
2276		ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL);
2277	} else {
2278		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL);
2279		ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX,
2280			    0, NULL);
2281		ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX,
2282			    0, NULL);
2283		ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX,
2284			    0, NULL);
2285		if (adapter->hw.phy_type != em_phy_ife) {
2286			ifmedia_add(&adapter->media,
2287				IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL);
2288			ifmedia_add(&adapter->media,
2289				IFM_ETHER | IFM_1000_T, 0, NULL);
2290		}
2291	}
2292	ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2293	ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2294}
2295
2296
2297/*********************************************************************
2298 *
2299 *  Workaround for SmartSpeed on 82541 and 82547 controllers
2300 *
2301 **********************************************************************/
2302static void
2303em_smartspeed(struct adapter *adapter)
2304{
2305	uint16_t phy_tmp;
2306
2307	if (adapter->link_active || (adapter->hw.phy_type != em_phy_igp) ||
2308	    adapter->hw.autoneg == 0 ||
2309	    (adapter->hw.autoneg_advertised & ADVERTISE_1000_FULL) == 0)
2310		return;
2311
2312	if (adapter->smartspeed == 0) {
2313		/* If Master/Slave config fault is asserted twice,
2314		 * we assume back-to-back */
2315		em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2316		if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT))
2317			return;
2318		em_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp);
2319		if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) {
2320			em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2321			if(phy_tmp & CR_1000T_MS_ENABLE) {
2322				phy_tmp &= ~CR_1000T_MS_ENABLE;
2323				em_write_phy_reg(&adapter->hw, PHY_1000T_CTRL,
2324				    phy_tmp);
2325				adapter->smartspeed++;
2326				if(adapter->hw.autoneg &&
2327				   !em_phy_setup_autoneg(&adapter->hw) &&
2328				   !em_read_phy_reg(&adapter->hw, PHY_CTRL,
2329				    &phy_tmp)) {
2330					phy_tmp |= (MII_CR_AUTO_NEG_EN |
2331						    MII_CR_RESTART_AUTO_NEG);
2332					em_write_phy_reg(&adapter->hw, PHY_CTRL,
2333					    phy_tmp);
2334				}
2335			}
2336		}
2337		return;
2338	} else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) {
2339		/* If still no link, perhaps using 2/3 pair cable */
2340		em_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp);
2341		phy_tmp |= CR_1000T_MS_ENABLE;
2342		em_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp);
2343		if(adapter->hw.autoneg &&
2344		   !em_phy_setup_autoneg(&adapter->hw) &&
2345		   !em_read_phy_reg(&adapter->hw, PHY_CTRL, &phy_tmp)) {
2346			phy_tmp |= (MII_CR_AUTO_NEG_EN |
2347				    MII_CR_RESTART_AUTO_NEG);
2348			em_write_phy_reg(&adapter->hw, PHY_CTRL, phy_tmp);
2349		}
2350	}
2351	/* Restart process after EM_SMARTSPEED_MAX iterations */
2352	if(adapter->smartspeed++ == EM_SMARTSPEED_MAX)
2353		adapter->smartspeed = 0;
2354}
2355
2356
2357/*
2358 * Manage DMA'able memory.
2359 */
2360static void
2361em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
2362{
2363	if (error)
2364		return;
2365	*(bus_addr_t *) arg = segs[0].ds_addr;
2366}
2367
2368static int
2369em_dma_malloc(struct adapter *adapter, bus_size_t size,
2370        struct em_dma_alloc *dma, int mapflags)
2371{
2372	int error;
2373
2374	error = bus_dma_tag_create(NULL,		/* parent */
2375				EM_DBA_ALIGN, 0,	/* alignment, bounds */
2376				BUS_SPACE_MAXADDR,	/* lowaddr */
2377				BUS_SPACE_MAXADDR,	/* highaddr */
2378				NULL, NULL,		/* filter, filterarg */
2379				size,			/* maxsize */
2380				1,			/* nsegments */
2381				size,			/* maxsegsize */
2382				0,			/* flags */
2383				NULL,			/* lockfunc */
2384				NULL,			/* lockarg */
2385				&dma->dma_tag);
2386	if (error) {
2387		device_printf(adapter->dev,
2388		    "%s: bus_dma_tag_create failed: %d\n",
2389		    __func__, error);
2390		goto fail_0;
2391	}
2392
2393	error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
2394	    BUS_DMA_NOWAIT, &dma->dma_map);
2395	if (error) {
2396		device_printf(adapter->dev,
2397		    "%s: bus_dmamem_alloc(%ju) failed: %d\n",
2398		    __func__, (uintmax_t)size, error);
2399		goto fail_2;
2400	}
2401
2402	dma->dma_paddr = 0;
2403	error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
2404	    size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT);
2405	if (error || dma->dma_paddr == 0) {
2406		device_printf(adapter->dev,
2407		    "%s: bus_dmamap_load failed: %d\n",
2408		    __func__, error);
2409		goto fail_3;
2410	}
2411
2412	return (0);
2413
2414fail_3:
2415	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2416fail_2:
2417	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2418	bus_dma_tag_destroy(dma->dma_tag);
2419fail_0:
2420	dma->dma_map = NULL;
2421	dma->dma_tag = NULL;
2422
2423	return (error);
2424}
2425
2426static void
2427em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma)
2428{
2429	if (dma->dma_tag == NULL)
2430		return;
2431	if (dma->dma_map != NULL) {
2432		bus_dmamap_sync(dma->dma_tag, dma->dma_map,
2433		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2434		bus_dmamap_unload(dma->dma_tag, dma->dma_map);
2435		bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
2436		dma->dma_map = NULL;
2437	}
2438	bus_dma_tag_destroy(dma->dma_tag);
2439	dma->dma_tag = NULL;
2440}
2441
2442
2443/*********************************************************************
2444 *
2445 *  Allocate memory for tx_buffer structures. The tx_buffer stores all
2446 *  the information needed to transmit a packet on the wire.
2447 *
2448 **********************************************************************/
2449static int
2450em_allocate_transmit_structures(struct adapter *adapter)
2451{
2452	device_t dev = adapter->dev;
2453	struct em_buffer *tx_buffer;
2454	bus_size_t size, segsize;
2455	int error, i;
2456
2457#ifdef EM_TSO
2458	size = EM_TSO_SIZE;
2459	segsize = PAGE_SIZE;
2460#else
2461	segsize = size = roundup2(adapter->hw.max_frame_size, MCLBYTES);
2462#endif
2463	/*
2464	 * Setup DMA descriptor areas.
2465	 */
2466	if ((error = bus_dma_tag_create(NULL,		/* parent */
2467				1, 0,			/* alignment, bounds */
2468				BUS_SPACE_MAXADDR,	/* lowaddr */
2469				BUS_SPACE_MAXADDR,	/* highaddr */
2470				NULL, NULL,		/* filter, filterarg */
2471				size,			/* maxsize */
2472				EM_MAX_SCATTER,		/* nsegments */
2473				segsize,		/* maxsegsize */
2474				0,			/* flags */
2475				NULL,		/* lockfunc */
2476				NULL,		/* lockarg */
2477				&adapter->txtag)) != 0) {
2478		device_printf(dev, "Unable to allocate TX DMA tag\n");
2479		goto fail;
2480	}
2481
2482	adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) *
2483	    adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO);
2484	if (adapter->tx_buffer_area == NULL) {
2485		device_printf(dev, "Unable to allocate tx_buffer memory\n");
2486		error = ENOMEM;
2487		goto fail;
2488	}
2489
2490	tx_buffer = adapter->tx_buffer_area;
2491	for (i = 0; i < adapter->num_tx_desc; i++) {
2492		error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map);
2493		if (error != 0) {
2494			device_printf(dev, "Unable to create TX DMA map\n");
2495			goto fail;
2496		}
2497		tx_buffer++;
2498	}
2499
2500	return (0);
2501
2502fail:
2503	em_free_transmit_structures(adapter);
2504	return (error);
2505}
2506
2507/*********************************************************************
2508 *
2509 *  Initialize transmit structures.
2510 *
2511 **********************************************************************/
2512static void
2513em_setup_transmit_structures(struct adapter *adapter)
2514{
2515	struct em_buffer *tx_buffer;
2516	int i;
2517
2518	bzero(adapter->tx_desc_base,
2519	    (sizeof(struct em_tx_desc)) * adapter->num_tx_desc);
2520
2521	adapter->next_avail_tx_desc = 0;
2522	adapter->next_tx_to_clean = 0;
2523
2524	/* Free any existing tx buffers. */
2525	tx_buffer = adapter->tx_buffer_area;
2526	for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2527		if (tx_buffer->m_head != NULL) {
2528			bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2529			    BUS_DMASYNC_POSTWRITE);
2530			bus_dmamap_unload(adapter->txtag, tx_buffer->map);
2531			m_freem(tx_buffer->m_head);
2532			tx_buffer->m_head = NULL;
2533		}
2534	}
2535
2536	/* Set number of descriptors available */
2537	adapter->num_tx_desc_avail = adapter->num_tx_desc;
2538
2539	/* Set checksum context */
2540	adapter->active_checksum_context = OFFLOAD_NONE;
2541	bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2542	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2543}
2544
2545/*********************************************************************
2546 *
2547 *  Enable transmit unit.
2548 *
2549 **********************************************************************/
2550static void
2551em_initialize_transmit_unit(struct adapter *adapter)
2552{
2553	uint32_t	reg_tctl, reg_tipg = 0;
2554	uint64_t	bus_addr;
2555
2556	 INIT_DEBUGOUT("em_initialize_transmit_unit: begin");
2557	/* Setup the Base and Length of the Tx Descriptor Ring */
2558	bus_addr = adapter->txdma.dma_paddr;
2559	E1000_WRITE_REG(&adapter->hw, TDLEN,
2560	    adapter->num_tx_desc * sizeof(struct em_tx_desc));
2561	E1000_WRITE_REG(&adapter->hw, TDBAH, (uint32_t)(bus_addr >> 32));
2562	E1000_WRITE_REG(&adapter->hw, TDBAL, (uint32_t)bus_addr);
2563
2564	/* Setup the HW Tx Head and Tail descriptor pointers */
2565	E1000_WRITE_REG(&adapter->hw, TDT, 0);
2566	E1000_WRITE_REG(&adapter->hw, TDH, 0);
2567
2568	HW_DEBUGOUT2("Base = %x, Length = %x\n",
2569	    E1000_READ_REG(&adapter->hw, TDBAL),
2570	    E1000_READ_REG(&adapter->hw, TDLEN));
2571
2572	/* Set the default values for the Tx Inter Packet Gap timer */
2573	switch (adapter->hw.mac_type) {
2574	case em_82542_rev2_0:
2575	case em_82542_rev2_1:
2576		reg_tipg = DEFAULT_82542_TIPG_IPGT;
2577		reg_tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2578		reg_tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2579		break;
2580	case em_80003es2lan:
2581		reg_tipg = DEFAULT_82543_TIPG_IPGR1;
2582		reg_tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 <<
2583		    E1000_TIPG_IPGR2_SHIFT;
2584		break;
2585	default:
2586		if ((adapter->hw.media_type == em_media_type_fiber) ||
2587		    (adapter->hw.media_type == em_media_type_internal_serdes))
2588			reg_tipg = DEFAULT_82543_TIPG_IPGT_FIBER;
2589		else
2590			reg_tipg = DEFAULT_82543_TIPG_IPGT_COPPER;
2591		reg_tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT;
2592		reg_tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT;
2593	}
2594
2595	E1000_WRITE_REG(&adapter->hw, TIPG, reg_tipg);
2596	E1000_WRITE_REG(&adapter->hw, TIDV, adapter->tx_int_delay.value);
2597	if(adapter->hw.mac_type >= em_82540)
2598		E1000_WRITE_REG(&adapter->hw, TADV,
2599		    adapter->tx_abs_int_delay.value);
2600
2601	/* Program the Transmit Control Register */
2602	reg_tctl = E1000_TCTL_PSP | E1000_TCTL_EN |
2603		   (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2604	if (adapter->hw.mac_type >= em_82571)
2605		reg_tctl |= E1000_TCTL_MULR;
2606	if (adapter->link_duplex == FULL_DUPLEX) {
2607		reg_tctl |= E1000_FDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2608	} else {
2609		reg_tctl |= E1000_HDX_COLLISION_DISTANCE << E1000_COLD_SHIFT;
2610	}
2611	/* This write will effectively turn on the transmit unit. */
2612	E1000_WRITE_REG(&adapter->hw, TCTL, reg_tctl);
2613
2614	/* Setup Transmit Descriptor Base Settings */
2615	adapter->txd_cmd = E1000_TXD_CMD_IFCS;
2616
2617	if (adapter->tx_int_delay.value > 0)
2618		adapter->txd_cmd |= E1000_TXD_CMD_IDE;
2619}
2620
2621/*********************************************************************
2622 *
2623 *  Free all transmit related data structures.
2624 *
2625 **********************************************************************/
2626static void
2627em_free_transmit_structures(struct adapter *adapter)
2628{
2629	struct em_buffer *tx_buffer;
2630	int i;
2631
2632	INIT_DEBUGOUT("free_transmit_structures: begin");
2633
2634	if (adapter->tx_buffer_area != NULL) {
2635		tx_buffer = adapter->tx_buffer_area;
2636		for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) {
2637			if (tx_buffer->m_head != NULL) {
2638				bus_dmamap_sync(adapter->txtag, tx_buffer->map,
2639				    BUS_DMASYNC_POSTWRITE);
2640				bus_dmamap_unload(adapter->txtag,
2641				    tx_buffer->map);
2642				m_freem(tx_buffer->m_head);
2643				tx_buffer->m_head = NULL;
2644			} else if (tx_buffer->map != NULL)
2645				bus_dmamap_unload(adapter->txtag,
2646				    tx_buffer->map);
2647			if (tx_buffer->map != NULL) {
2648				bus_dmamap_destroy(adapter->txtag,
2649				    tx_buffer->map);
2650				tx_buffer->map = NULL;
2651			}
2652		}
2653	}
2654	if (adapter->tx_buffer_area != NULL) {
2655		free(adapter->tx_buffer_area, M_DEVBUF);
2656		adapter->tx_buffer_area = NULL;
2657	}
2658	if (adapter->txtag != NULL) {
2659		bus_dma_tag_destroy(adapter->txtag);
2660		adapter->txtag = NULL;
2661	}
2662}
2663
2664/*********************************************************************
2665 *
2666 *  The offload context needs to be set when we transfer the first
2667 *  packet of a particular protocol (TCP/UDP). We change the
2668 *  context only if the protocol type changes.
2669 *
2670 **********************************************************************/
2671static void
2672em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp,
2673    uint32_t *txd_upper, uint32_t *txd_lower)
2674{
2675	struct em_context_desc *TXD;
2676	struct em_buffer *tx_buffer;
2677	int curr_txd;
2678
2679	if (mp->m_pkthdr.csum_flags) {
2680
2681		if (mp->m_pkthdr.csum_flags & CSUM_TCP) {
2682			*txd_upper = E1000_TXD_POPTS_TXSM << 8;
2683			*txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2684			if (adapter->active_checksum_context == OFFLOAD_TCP_IP)
2685				return;
2686			else
2687				adapter->active_checksum_context =
2688				    OFFLOAD_TCP_IP;
2689		} else if (mp->m_pkthdr.csum_flags & CSUM_UDP) {
2690			*txd_upper = E1000_TXD_POPTS_TXSM << 8;
2691			*txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
2692			if (adapter->active_checksum_context == OFFLOAD_UDP_IP)
2693				return;
2694			else
2695				adapter->active_checksum_context =
2696				    OFFLOAD_UDP_IP;
2697		} else {
2698			*txd_upper = 0;
2699			*txd_lower = 0;
2700			return;
2701		}
2702	} else {
2703		*txd_upper = 0;
2704		*txd_lower = 0;
2705		return;
2706	}
2707
2708	/* If we reach this point, the checksum offload context
2709	 * needs to be reset.
2710	 */
2711	curr_txd = adapter->next_avail_tx_desc;
2712	tx_buffer = &adapter->tx_buffer_area[curr_txd];
2713	TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd];
2714
2715	TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2716	TXD->lower_setup.ip_fields.ipcso =
2717		ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2718	TXD->lower_setup.ip_fields.ipcse =
2719		htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1);
2720
2721	TXD->upper_setup.tcp_fields.tucss =
2722		ETHER_HDR_LEN + sizeof(struct ip);
2723	TXD->upper_setup.tcp_fields.tucse = htole16(0);
2724
2725	if (adapter->active_checksum_context == OFFLOAD_TCP_IP) {
2726		TXD->upper_setup.tcp_fields.tucso =
2727			ETHER_HDR_LEN + sizeof(struct ip) +
2728			offsetof(struct tcphdr, th_sum);
2729	} else if (adapter->active_checksum_context == OFFLOAD_UDP_IP) {
2730		TXD->upper_setup.tcp_fields.tucso =
2731			ETHER_HDR_LEN + sizeof(struct ip) +
2732			offsetof(struct udphdr, uh_sum);
2733	}
2734
2735	TXD->tcp_seg_setup.data = htole32(0);
2736	TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT);
2737
2738	tx_buffer->m_head = NULL;
2739	tx_buffer->next_eop = -1;
2740
2741	if (++curr_txd == adapter->num_tx_desc)
2742		curr_txd = 0;
2743
2744	adapter->num_tx_desc_avail--;
2745	adapter->next_avail_tx_desc = curr_txd;
2746}
2747
2748#ifdef EM_TSO
2749/**********************************************************************
2750 *
2751 *  Setup work for hardware segmentation offload (TSO)
2752 *
2753 **********************************************************************/
2754static boolean_t
2755em_tso_setup(struct adapter *adapter,
2756	     struct mbuf *mp,
2757	     u_int32_t *txd_upper,
2758	     u_int32_t *txd_lower)
2759{
2760	struct em_context_desc *TXD;
2761	struct em_buffer *tx_buffer;
2762	struct ip *ip;
2763	struct tcphdr *th;
2764	int curr_txd, hdr_len, ip_hlen, tcp_hlen;
2765
2766	if (((mp->m_pkthdr.csum_flags & CSUM_TSO) == 0) ||
2767	    (mp->m_pkthdr.len <= E1000_TX_BUFFER_SIZE)) {
2768		return FALSE;
2769	}
2770
2771	*txd_lower = (E1000_TXD_CMD_DEXT |
2772		      E1000_TXD_DTYP_D |
2773		      E1000_TXD_CMD_TSE);
2774
2775	*txd_upper = (E1000_TXD_POPTS_IXSM |
2776		      E1000_TXD_POPTS_TXSM) << 8;
2777
2778	curr_txd = adapter->next_avail_tx_desc;
2779	tx_buffer = &adapter->tx_buffer_area[curr_txd];
2780	TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd];
2781
2782	mp->m_data += sizeof(struct ether_header);
2783	ip = mtod(mp, struct ip *);
2784	ip->ip_len = 0;
2785	ip->ip_sum = 0;
2786	ip_hlen = ip->ip_hl << 2 ;
2787	th = (struct tcphdr *)((caddr_t)ip + ip_hlen);
2788	tcp_hlen = th->th_off << 2;
2789
2790	hdr_len = ETHER_HDR_LEN + ip_hlen + tcp_hlen;
2791	th->th_sum = in_pseudo(ip->ip_src.s_addr,
2792				ip->ip_dst.s_addr,
2793				htons(IPPROTO_TCP));
2794
2795	mp->m_data -= sizeof(struct ether_header);
2796	TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN;
2797	TXD->lower_setup.ip_fields.ipcso =
2798		ETHER_HDR_LEN + offsetof(struct ip, ip_sum);
2799	TXD->lower_setup.ip_fields.ipcse =
2800		htole16(ETHER_HDR_LEN + ip_hlen - 1);
2801
2802	TXD->upper_setup.tcp_fields.tucss =
2803		ETHER_HDR_LEN + ip_hlen;
2804	TXD->upper_setup.tcp_fields.tucse = 0;
2805	TXD->upper_setup.tcp_fields.tucso =
2806		ETHER_HDR_LEN + ip_hlen +
2807		offsetof(struct tcphdr, th_sum);
2808	TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz);
2809	TXD->tcp_seg_setup.fields.hdr_len = hdr_len;
2810	TXD->cmd_and_length = htole32(adapter->txd_cmd |
2811				E1000_TXD_CMD_DEXT |
2812				E1000_TXD_CMD_TSE |
2813				E1000_TXD_CMD_IP | E1000_TXD_CMD_TCP |
2814				(mp->m_pkthdr.len - (hdr_len)));
2815
2816	tx_buffer->m_head = NULL;
2817	tx_buffer->next_eop = -1;
2818
2819	if (++curr_txd == adapter->num_tx_desc)
2820		curr_txd = 0;
2821
2822	adapter->num_tx_desc_avail--;
2823	adapter->next_avail_tx_desc = curr_txd;
2824	adapter->tx_tso = TRUE;
2825
2826	return TRUE;
2827}
2828#endif /* EM_TSO */
2829
2830/**********************************************************************
2831 *
2832 *  Examine each tx_buffer in the used queue. If the hardware is done
2833 *  processing the packet then free associated resources. The
2834 *  tx_buffer is put back on the free queue.
2835 *
2836 **********************************************************************/
2837static void
2838em_txeof(struct adapter *adapter)
2839{
2840        int first, last, done, num_avail;
2841        struct em_buffer *tx_buffer;
2842        struct em_tx_desc   *tx_desc, *eop_desc;
2843	struct ifnet   *ifp = adapter->ifp;
2844
2845	EM_LOCK_ASSERT(adapter);
2846
2847        if (adapter->num_tx_desc_avail == adapter->num_tx_desc)
2848                return;
2849
2850        num_avail = adapter->num_tx_desc_avail;
2851        first = adapter->next_tx_to_clean;
2852        tx_desc = &adapter->tx_desc_base[first];
2853        tx_buffer = &adapter->tx_buffer_area[first];
2854	last = tx_buffer->next_eop;
2855        eop_desc = &adapter->tx_desc_base[last];
2856
2857	/*
2858	 * Now caculate the terminating index
2859	 * for the cleanup loop below
2860	 */
2861	if (++last == adapter->num_tx_desc) last = 0;
2862	done = last;
2863
2864        bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2865            BUS_DMASYNC_POSTREAD);
2866
2867        while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
2868		/* We clean the range of the packet */
2869		while (first != done) {
2870                	tx_desc->upper.data = 0;
2871                	tx_desc->lower.data = 0;
2872                	num_avail++;
2873
2874			if (tx_buffer->m_head) {
2875				ifp->if_opackets++;
2876				bus_dmamap_sync(adapter->txtag,
2877				    tx_buffer->map,
2878				    BUS_DMASYNC_POSTWRITE);
2879				bus_dmamap_unload(adapter->txtag,
2880				    tx_buffer->map);
2881
2882                        	m_freem(tx_buffer->m_head);
2883                        	tx_buffer->m_head = NULL;
2884                	}
2885			tx_buffer->next_eop = -1;
2886
2887	                if (++first == adapter->num_tx_desc)
2888				first = 0;
2889
2890	                tx_buffer = &adapter->tx_buffer_area[first];
2891			tx_desc = &adapter->tx_desc_base[first];
2892		}
2893		/* See if we can continue to the next packet */
2894		last = tx_buffer->next_eop;
2895		if (last != -1) {
2896        		eop_desc = &adapter->tx_desc_base[last];
2897			/* Get new done point */
2898			if (++last == adapter->num_tx_desc) last = 0;
2899			done = last;
2900		} else
2901			break;
2902        }
2903        bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map,
2904            BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
2905
2906        adapter->next_tx_to_clean = first;
2907
2908        /*
2909         * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack
2910         * that it is OK to send packets.
2911         * If there are no pending descriptors, clear the timeout. Otherwise,
2912         * if some descriptors have been freed, restart the timeout.
2913         */
2914        if (num_avail > EM_TX_CLEANUP_THRESHOLD) {
2915                ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
2916                if (num_avail == adapter->num_tx_desc)
2917                        ifp->if_timer = 0;
2918                else if (num_avail != adapter->num_tx_desc_avail)
2919                        ifp->if_timer = EM_TX_TIMEOUT;
2920        }
2921        adapter->num_tx_desc_avail = num_avail;
2922        return;
2923}
2924
2925/*********************************************************************
2926 *
2927 *  Get a buffer from system mbuf buffer pool.
2928 *
2929 **********************************************************************/
2930static int
2931em_get_buf(int i, struct adapter *adapter, struct mbuf *mp)
2932{
2933	struct ifnet		*ifp = adapter->ifp;
2934	bus_dma_segment_t	segs[1];
2935	struct em_buffer	*rx_buffer;
2936	int			error, nsegs;
2937
2938	if (mp == NULL) {
2939		mp = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR);
2940		if (mp == NULL) {
2941			adapter->mbuf_cluster_failed++;
2942			return (ENOBUFS);
2943		}
2944		mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2945	} else {
2946		mp->m_len = mp->m_pkthdr.len = MCLBYTES;
2947		mp->m_data = mp->m_ext.ext_buf;
2948		mp->m_next = NULL;
2949	}
2950
2951	if (ifp->if_mtu <= ETHERMTU)
2952		m_adj(mp, ETHER_ALIGN);
2953
2954	rx_buffer = &adapter->rx_buffer_area[i];
2955
2956	/*
2957	 * Using memory from the mbuf cluster pool, invoke the
2958	 * bus_dma machinery to arrange the memory mapping.
2959	 */
2960	error = bus_dmamap_load_mbuf_sg(adapter->rxtag, rx_buffer->map,
2961	    mp, segs, &nsegs, 0);
2962	if (error != 0) {
2963		m_free(mp);
2964		return (error);
2965	}
2966	/* If nsegs is wrong then the stack is corrupt. */
2967	KASSERT(nsegs == 1, ("Too many segments returned!"));
2968	rx_buffer->m_head = mp;
2969	adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr);
2970	bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD);
2971
2972	return (0);
2973}
2974
2975/*********************************************************************
2976 *
2977 *  Allocate memory for rx_buffer structures. Since we use one
2978 *  rx_buffer per received packet, the maximum number of rx_buffer's
2979 *  that we'll need is equal to the number of receive descriptors
2980 *  that we've allocated.
2981 *
2982 **********************************************************************/
2983static int
2984em_allocate_receive_structures(struct adapter *adapter)
2985{
2986	device_t dev = adapter->dev;
2987	struct em_buffer *rx_buffer;
2988	int i, error;
2989
2990	adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) *
2991	    adapter->num_rx_desc, M_DEVBUF, M_NOWAIT);
2992	if (adapter->rx_buffer_area == NULL) {
2993		device_printf(dev, "Unable to allocate rx_buffer memory\n");
2994		return (ENOMEM);
2995	}
2996
2997	bzero(adapter->rx_buffer_area,
2998	    sizeof(struct em_buffer) * adapter->num_rx_desc);
2999
3000	error = bus_dma_tag_create(NULL,		/* parent */
3001				1, 0,			/* alignment, bounds */
3002				BUS_SPACE_MAXADDR,	/* lowaddr */
3003				BUS_SPACE_MAXADDR,	/* highaddr */
3004				NULL, NULL,		/* filter, filterarg */
3005				MCLBYTES,		/* maxsize */
3006				1,			/* nsegments */
3007				MCLBYTES,		/* maxsegsize */
3008				0,			/* flags */
3009				NULL,			/* lockfunc */
3010				NULL,			/* lockarg */
3011				&adapter->rxtag);
3012	if (error) {
3013		device_printf(dev, "%s: bus_dma_tag_create failed %d\n",
3014		    __func__, error);
3015		goto fail;
3016	}
3017
3018	rx_buffer = adapter->rx_buffer_area;
3019	for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
3020		error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT,
3021		    &rx_buffer->map);
3022		if (error) {
3023			device_printf(dev, "%s: bus_dmamap_create failed: %d\n",
3024			    __func__, error);
3025			goto fail;
3026		}
3027	}
3028
3029	return (0);
3030
3031fail:
3032	em_free_receive_structures(adapter);
3033	return (error);
3034}
3035
3036/*********************************************************************
3037 *
3038 *  Allocate and initialize receive structures.
3039 *
3040 **********************************************************************/
3041static int
3042em_setup_receive_structures(struct adapter *adapter)
3043{
3044	struct em_buffer *rx_buffer;
3045	int i, error;
3046
3047	bzero(adapter->rx_desc_base,
3048	    (sizeof(struct em_rx_desc)) * adapter->num_rx_desc);
3049
3050	/* Free current RX buffers. */
3051	rx_buffer = adapter->rx_buffer_area;
3052	for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
3053		if (rx_buffer->m_head != NULL) {
3054			bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
3055			    BUS_DMASYNC_POSTREAD);
3056			bus_dmamap_unload(adapter->rxtag, rx_buffer->map);
3057			m_freem(rx_buffer->m_head);
3058			rx_buffer->m_head = NULL;
3059		}
3060	}
3061
3062	/* Allocate new ones. */
3063	for (i = 0; i < adapter->num_rx_desc; i++) {
3064		error = em_get_buf(i, adapter, NULL);
3065		if (error)
3066			return (error);
3067	}
3068
3069	/* Setup our descriptor pointers */
3070	adapter->next_rx_desc_to_check = 0;
3071	bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3072	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3073
3074	return (0);
3075}
3076
3077/*********************************************************************
3078 *
3079 *  Enable receive unit.
3080 *
3081 **********************************************************************/
3082static void
3083em_initialize_receive_unit(struct adapter *adapter)
3084{
3085	struct ifnet	*ifp = adapter->ifp;
3086	uint64_t	bus_addr;
3087	uint32_t	reg_rctl;
3088	uint32_t	reg_rxcsum;
3089
3090	INIT_DEBUGOUT("em_initialize_receive_unit: begin");
3091
3092	/*
3093	 * Make sure receives are disabled while setting
3094	 * up the descriptor ring
3095	 */
3096	E1000_WRITE_REG(&adapter->hw, RCTL, 0);
3097
3098	if(adapter->hw.mac_type >= em_82540) {
3099		E1000_WRITE_REG(&adapter->hw, RADV,
3100		    adapter->rx_abs_int_delay.value);
3101		/*
3102		 * Set the interrupt throttling rate. Value is calculated
3103		 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns)
3104		 */
3105#define MAX_INTS_PER_SEC	8000
3106#define DEFAULT_ITR	     1000000000/(MAX_INTS_PER_SEC * 256)
3107		E1000_WRITE_REG(&adapter->hw, ITR, DEFAULT_ITR);
3108	}
3109
3110	/* Setup the Base and Length of the Rx Descriptor Ring */
3111	bus_addr = adapter->rxdma.dma_paddr;
3112	E1000_WRITE_REG(&adapter->hw, RDLEN, adapter->num_rx_desc *
3113			sizeof(struct em_rx_desc));
3114	E1000_WRITE_REG(&adapter->hw, RDBAH, (uint32_t)(bus_addr >> 32));
3115	E1000_WRITE_REG(&adapter->hw, RDBAL, (uint32_t)bus_addr);
3116
3117	/* Setup the Receive Control Register */
3118	reg_rctl = E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO |
3119		   E1000_RCTL_RDMTS_HALF |
3120		   (adapter->hw.mc_filter_type << E1000_RCTL_MO_SHIFT);
3121
3122	if (adapter->hw.tbi_compatibility_on == TRUE)
3123		reg_rctl |= E1000_RCTL_SBP;
3124
3125
3126	switch (adapter->rx_buffer_len) {
3127	default:
3128	case EM_RXBUFFER_2048:
3129		reg_rctl |= E1000_RCTL_SZ_2048;
3130		break;
3131	case EM_RXBUFFER_4096:
3132		reg_rctl |= E1000_RCTL_SZ_4096 |
3133		    E1000_RCTL_BSEX | E1000_RCTL_LPE;
3134		break;
3135	case EM_RXBUFFER_8192:
3136		reg_rctl |= E1000_RCTL_SZ_8192 |
3137		    E1000_RCTL_BSEX | E1000_RCTL_LPE;
3138		break;
3139	case EM_RXBUFFER_16384:
3140		reg_rctl |= E1000_RCTL_SZ_16384 |
3141		    E1000_RCTL_BSEX | E1000_RCTL_LPE;
3142		break;
3143	}
3144
3145	if (ifp->if_mtu > ETHERMTU)
3146		reg_rctl |= E1000_RCTL_LPE;
3147
3148	/* Enable 82543 Receive Checksum Offload for TCP and UDP */
3149	if ((adapter->hw.mac_type >= em_82543) &&
3150	    (ifp->if_capenable & IFCAP_RXCSUM)) {
3151		reg_rxcsum = E1000_READ_REG(&adapter->hw, RXCSUM);
3152		reg_rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL);
3153		E1000_WRITE_REG(&adapter->hw, RXCSUM, reg_rxcsum);
3154	}
3155
3156	/* Enable Receives */
3157	E1000_WRITE_REG(&adapter->hw, RCTL, reg_rctl);
3158	/*
3159	 * Setup the HW Rx Head and
3160	 * Tail Descriptor Pointers
3161	 */
3162	E1000_WRITE_REG(&adapter->hw, RDH, 0);
3163	E1000_WRITE_REG(&adapter->hw, RDT, adapter->num_rx_desc - 1);
3164
3165	return;
3166}
3167
3168/*********************************************************************
3169 *
3170 *  Free receive related data structures.
3171 *
3172 **********************************************************************/
3173static void
3174em_free_receive_structures(struct adapter *adapter)
3175{
3176	struct em_buffer *rx_buffer;
3177	int i;
3178
3179	INIT_DEBUGOUT("free_receive_structures: begin");
3180
3181	if (adapter->rx_buffer_area != NULL) {
3182		rx_buffer = adapter->rx_buffer_area;
3183		for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) {
3184			if (rx_buffer->m_head != NULL) {
3185				bus_dmamap_sync(adapter->rxtag, rx_buffer->map,
3186				    BUS_DMASYNC_POSTREAD);
3187				bus_dmamap_unload(adapter->rxtag,
3188				    rx_buffer->map);
3189				m_freem(rx_buffer->m_head);
3190				rx_buffer->m_head = NULL;
3191			} else if (rx_buffer->map != NULL)
3192				bus_dmamap_unload(adapter->rxtag,
3193				    rx_buffer->map);
3194			if (rx_buffer->map != NULL) {
3195				bus_dmamap_destroy(adapter->rxtag,
3196				    rx_buffer->map);
3197				rx_buffer->map = NULL;
3198			}
3199		}
3200	}
3201	if (adapter->rx_buffer_area != NULL) {
3202		free(adapter->rx_buffer_area, M_DEVBUF);
3203		adapter->rx_buffer_area = NULL;
3204	}
3205	if (adapter->rxtag != NULL) {
3206		bus_dma_tag_destroy(adapter->rxtag);
3207		adapter->rxtag = NULL;
3208	}
3209}
3210
3211/*********************************************************************
3212 *
3213 *  This routine executes in interrupt context. It replenishes
3214 *  the mbufs in the descriptor and sends data which has been
3215 *  dma'ed into host memory to upper layer.
3216 *
3217 *  We loop at most count times if count is > 0, or until done if
3218 *  count < 0.
3219 *
3220 *********************************************************************/
3221static int
3222em_rxeof(struct adapter *adapter, int count)
3223{
3224	struct ifnet	*ifp;
3225	struct mbuf	*mp;
3226	uint8_t		accept_frame = 0;
3227	uint8_t		eop = 0;
3228	uint16_t 	len, desc_len, prev_len_adj;
3229	int		i;
3230
3231	/* Pointer to the receive descriptor being examined. */
3232	struct em_rx_desc   *current_desc;
3233	uint8_t		status;
3234
3235	ifp = adapter->ifp;
3236	i = adapter->next_rx_desc_to_check;
3237	current_desc = &adapter->rx_desc_base[i];
3238	bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3239	    BUS_DMASYNC_POSTREAD);
3240
3241	if (!((current_desc->status) & E1000_RXD_STAT_DD))
3242		return (0);
3243
3244	while ((current_desc->status & E1000_RXD_STAT_DD) &&
3245	    (count != 0) &&
3246	    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3247		struct mbuf *m = NULL;
3248
3249		mp = adapter->rx_buffer_area[i].m_head;
3250		/*
3251		 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT
3252		 * needs to access the last received byte in the mbuf.
3253		 */
3254		bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map,
3255		    BUS_DMASYNC_POSTREAD);
3256
3257		accept_frame = 1;
3258		prev_len_adj = 0;
3259		desc_len = le16toh(current_desc->length);
3260		status = current_desc->status;
3261		if (status & E1000_RXD_STAT_EOP) {
3262			count--;
3263			eop = 1;
3264			if (desc_len < ETHER_CRC_LEN) {
3265				len = 0;
3266				prev_len_adj = ETHER_CRC_LEN - desc_len;
3267			} else
3268				len = desc_len - ETHER_CRC_LEN;
3269		} else {
3270			eop = 0;
3271			len = desc_len;
3272		}
3273
3274		if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) {
3275			uint8_t		last_byte;
3276			uint32_t	pkt_len = desc_len;
3277
3278			if (adapter->fmp != NULL)
3279				pkt_len += adapter->fmp->m_pkthdr.len;
3280
3281			last_byte = *(mtod(mp, caddr_t) + desc_len - 1);
3282			if (TBI_ACCEPT(&adapter->hw, status,
3283			    current_desc->errors, pkt_len, last_byte)) {
3284				em_tbi_adjust_stats(&adapter->hw,
3285				    &adapter->stats, pkt_len,
3286				    adapter->hw.mac_addr);
3287				if (len > 0)
3288					len--;
3289			} else
3290				accept_frame = 0;
3291		}
3292
3293		if (accept_frame) {
3294			if (em_get_buf(i, adapter, NULL) == ENOBUFS) {
3295				adapter->dropped_pkts++;
3296				em_get_buf(i, adapter, mp);
3297				if (adapter->fmp != NULL)
3298					m_freem(adapter->fmp);
3299				adapter->fmp = NULL;
3300				adapter->lmp = NULL;
3301				break;
3302			}
3303
3304			/* Assign correct length to the current fragment */
3305			mp->m_len = len;
3306
3307			if (adapter->fmp == NULL) {
3308				mp->m_pkthdr.len = len;
3309				adapter->fmp = mp; /* Store the first mbuf */
3310				adapter->lmp = mp;
3311			} else {
3312				/* Chain mbuf's together */
3313				mp->m_flags &= ~M_PKTHDR;
3314				/*
3315				 * Adjust length of previous mbuf in chain if
3316				 * we received less than 4 bytes in the last
3317				 * descriptor.
3318				 */
3319				if (prev_len_adj > 0) {
3320					adapter->lmp->m_len -= prev_len_adj;
3321					adapter->fmp->m_pkthdr.len -=
3322					    prev_len_adj;
3323				}
3324				adapter->lmp->m_next = mp;
3325				adapter->lmp = adapter->lmp->m_next;
3326				adapter->fmp->m_pkthdr.len += len;
3327			}
3328
3329			if (eop) {
3330				adapter->fmp->m_pkthdr.rcvif = ifp;
3331				ifp->if_ipackets++;
3332				em_receive_checksum(adapter, current_desc,
3333				    adapter->fmp);
3334#ifndef __NO_STRICT_ALIGNMENT
3335				if (adapter->hw.max_frame_size >
3336				    (MCLBYTES - ETHER_ALIGN) &&
3337				    em_fixup_rx(adapter) != 0)
3338					goto skip;
3339#endif
3340				if (status & E1000_RXD_STAT_VP)
3341					VLAN_INPUT_TAG_NEW(ifp, adapter->fmp,
3342					    (le16toh(current_desc->special) &
3343					    E1000_RXD_SPC_VLAN_MASK));
3344#ifndef __NO_STRICT_ALIGNMENT
3345skip:
3346#endif
3347				m = adapter->fmp;
3348				adapter->fmp = NULL;
3349				adapter->lmp = NULL;
3350			}
3351		} else {
3352			adapter->dropped_pkts++;
3353			em_get_buf(i, adapter, mp);
3354			if (adapter->fmp != NULL)
3355				m_freem(adapter->fmp);
3356			adapter->fmp = NULL;
3357			adapter->lmp = NULL;
3358		}
3359
3360		/* Zero out the receive descriptors status. */
3361		current_desc->status = 0;
3362		bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map,
3363		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
3364
3365		/* Advance our pointers to the next descriptor. */
3366		if (++i == adapter->num_rx_desc)
3367			i = 0;
3368		if (m != NULL) {
3369			adapter->next_rx_desc_to_check = i;
3370			EM_UNLOCK(adapter);
3371			(*ifp->if_input)(ifp, m);
3372			EM_LOCK(adapter);
3373			i = adapter->next_rx_desc_to_check;
3374		}
3375		current_desc = &adapter->rx_desc_base[i];
3376	}
3377	adapter->next_rx_desc_to_check = i;
3378
3379	/* Advance the E1000's Receive Queue #0  "Tail Pointer". */
3380	if (--i < 0)
3381		i = adapter->num_rx_desc - 1;
3382	E1000_WRITE_REG(&adapter->hw, RDT, i);
3383	if (!((current_desc->status) & E1000_RXD_STAT_DD))
3384		return (0);
3385
3386	return (1);
3387}
3388
3389#ifndef __NO_STRICT_ALIGNMENT
3390/*
3391 * When jumbo frames are enabled we should realign entire payload on
3392 * architecures with strict alignment. This is serious design mistake of 8254x
3393 * as it nullifies DMA operations. 8254x just allows RX buffer size to be
3394 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its
3395 * payload. On architecures without strict alignment restrictions 8254x still
3396 * performs unaligned memory access which would reduce the performance too.
3397 * To avoid copying over an entire frame to align, we allocate a new mbuf and
3398 * copy ethernet header to the new mbuf. The new mbuf is prepended into the
3399 * existing mbuf chain.
3400 *
3401 * Be aware, best performance of the 8254x is achived only when jumbo frame is
3402 * not used at all on architectures with strict alignment.
3403 */
3404static int
3405em_fixup_rx(struct adapter *adapter)
3406{
3407	struct mbuf *m, *n;
3408	int error;
3409
3410	error = 0;
3411	m = adapter->fmp;
3412	if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) {
3413		bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len);
3414		m->m_data += ETHER_HDR_LEN;
3415	} else {
3416		MGETHDR(n, M_DONTWAIT, MT_DATA);
3417		if (n != NULL) {
3418			bcopy(m->m_data, n->m_data, ETHER_HDR_LEN);
3419			m->m_data += ETHER_HDR_LEN;
3420			m->m_len -= ETHER_HDR_LEN;
3421			n->m_len = ETHER_HDR_LEN;
3422			M_MOVE_PKTHDR(n, m);
3423			n->m_next = m;
3424			adapter->fmp = n;
3425		} else {
3426			adapter->dropped_pkts++;
3427			m_freem(adapter->fmp);
3428			adapter->fmp = NULL;
3429			error = ENOMEM;
3430		}
3431	}
3432
3433	return (error);
3434}
3435#endif
3436
3437/*********************************************************************
3438 *
3439 *  Verify that the hardware indicated that the checksum is valid.
3440 *  Inform the stack about the status of checksum so that stack
3441 *  doesn't spend time verifying the checksum.
3442 *
3443 *********************************************************************/
3444static void
3445em_receive_checksum(struct adapter *adapter,
3446	    struct em_rx_desc *rx_desc, struct mbuf *mp)
3447{
3448	/* 82543 or newer only */
3449	if ((adapter->hw.mac_type < em_82543) ||
3450	    /* Ignore Checksum bit is set */
3451	    (rx_desc->status & E1000_RXD_STAT_IXSM)) {
3452		mp->m_pkthdr.csum_flags = 0;
3453		return;
3454	}
3455
3456	if (rx_desc->status & E1000_RXD_STAT_IPCS) {
3457		/* Did it pass? */
3458		if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) {
3459			/* IP Checksum Good */
3460			mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
3461			mp->m_pkthdr.csum_flags |= CSUM_IP_VALID;
3462
3463		} else {
3464			mp->m_pkthdr.csum_flags = 0;
3465		}
3466	}
3467
3468	if (rx_desc->status & E1000_RXD_STAT_TCPCS) {
3469		/* Did it pass? */
3470		if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) {
3471			mp->m_pkthdr.csum_flags |=
3472			(CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
3473			mp->m_pkthdr.csum_data = htons(0xffff);
3474		}
3475	}
3476}
3477
3478
3479static void
3480em_enable_vlans(struct adapter *adapter)
3481{
3482	uint32_t ctrl;
3483
3484	E1000_WRITE_REG(&adapter->hw, VET, ETHERTYPE_VLAN);
3485
3486	ctrl = E1000_READ_REG(&adapter->hw, CTRL);
3487	ctrl |= E1000_CTRL_VME;
3488	E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
3489}
3490
3491static void
3492em_disable_vlans(struct adapter *adapter)
3493{
3494	uint32_t ctrl;
3495
3496	ctrl = E1000_READ_REG(&adapter->hw, CTRL);
3497	ctrl &= ~E1000_CTRL_VME;
3498	E1000_WRITE_REG(&adapter->hw, CTRL, ctrl);
3499}
3500
3501static void
3502em_enable_intr(struct adapter *adapter)
3503{
3504	E1000_WRITE_REG(&adapter->hw, IMS, (IMS_ENABLE_MASK));
3505}
3506
3507static void
3508em_disable_intr(struct adapter *adapter)
3509{
3510	/*
3511	 * The first version of 82542 had an errata where when link was forced
3512	 * it would stay up even up even if the cable was disconnected.
3513	 * Sequence errors were used to detect the disconnect and then the
3514	 * driver would unforce the link. This code in the in the ISR. For this
3515	 * to work correctly the Sequence error interrupt had to be enabled
3516	 * all the time.
3517	 */
3518
3519	if (adapter->hw.mac_type == em_82542_rev2_0)
3520	    E1000_WRITE_REG(&adapter->hw, IMC,
3521		(0xffffffff & ~E1000_IMC_RXSEQ));
3522	else
3523	    E1000_WRITE_REG(&adapter->hw, IMC, 0xffffffff);
3524}
3525
3526static int
3527em_is_valid_ether_addr(uint8_t *addr)
3528{
3529	char zero_addr[6] = { 0, 0, 0, 0, 0, 0 };
3530
3531	if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) {
3532		return (FALSE);
3533	}
3534
3535	return (TRUE);
3536}
3537
3538void
3539em_write_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3540{
3541	pci_write_config(((struct em_osdep *)hw->back)->dev, reg, *value, 2);
3542}
3543
3544void
3545em_read_pci_cfg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3546{
3547	*value = pci_read_config(((struct em_osdep *)hw->back)->dev, reg, 2);
3548}
3549
3550void
3551em_pci_set_mwi(struct em_hw *hw)
3552{
3553	pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
3554	    (hw->pci_cmd_word | CMD_MEM_WRT_INVALIDATE), 2);
3555}
3556
3557void
3558em_pci_clear_mwi(struct em_hw *hw)
3559{
3560	pci_write_config(((struct em_osdep *)hw->back)->dev, PCIR_COMMAND,
3561	    (hw->pci_cmd_word & ~CMD_MEM_WRT_INVALIDATE), 2);
3562}
3563
3564uint32_t
3565em_io_read(struct em_hw *hw, unsigned long port)
3566{
3567	struct em_osdep *io = hw->back;
3568	uint32_t ret;
3569
3570	ret = bus_space_read_4(io->io_bus_space_tag,
3571            io->io_bus_space_handle, port);
3572	return (ret);
3573}
3574
3575void
3576em_io_write(struct em_hw *hw, unsigned long port, uint32_t value)
3577{
3578	struct em_osdep *io = hw->back;
3579
3580	bus_space_write_4(io->io_bus_space_tag,
3581            io->io_bus_space_handle, port, value);
3582	return;
3583}
3584
3585/*
3586 * We may eventually really do this, but its unnecessary
3587 * for now so we just return unsupported.
3588 */
3589int32_t
3590em_read_pcie_cap_reg(struct em_hw *hw, uint32_t reg, uint16_t *value)
3591{
3592	return (0);
3593}
3594
3595/*********************************************************************
3596* 82544 Coexistence issue workaround.
3597*    There are 2 issues.
3598*       1. Transmit Hang issue.
3599*    To detect this issue, following equation can be used...
3600*	  SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3601*	  If SUM[3:0] is in between 1 to 4, we will have this issue.
3602*
3603*       2. DAC issue.
3604*    To detect this issue, following equation can be used...
3605*	  SIZE[3:0] + ADDR[2:0] = SUM[3:0].
3606*	  If SUM[3:0] is in between 9 to c, we will have this issue.
3607*
3608*
3609*    WORKAROUND:
3610*	  Make sure we do not have ending address
3611*	  as 1,2,3,4(Hang) or 9,a,b,c (DAC)
3612*
3613*************************************************************************/
3614static uint32_t
3615em_fill_descriptors (bus_addr_t address, uint32_t length,
3616		PDESC_ARRAY desc_array)
3617{
3618	/* Since issue is sensitive to length and address.*/
3619	/* Let us first check the address...*/
3620	uint32_t safe_terminator;
3621	if (length <= 4) {
3622		desc_array->descriptor[0].address = address;
3623		desc_array->descriptor[0].length = length;
3624		desc_array->elements = 1;
3625		return (desc_array->elements);
3626	}
3627	safe_terminator = (uint32_t)((((uint32_t)address & 0x7) +
3628	    (length & 0xF)) & 0xF);
3629	/* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */
3630	if (safe_terminator == 0   ||
3631	(safe_terminator > 4   &&
3632	safe_terminator < 9)   ||
3633	(safe_terminator > 0xC &&
3634	safe_terminator <= 0xF)) {
3635		desc_array->descriptor[0].address = address;
3636		desc_array->descriptor[0].length = length;
3637		desc_array->elements = 1;
3638		return (desc_array->elements);
3639	}
3640
3641	desc_array->descriptor[0].address = address;
3642	desc_array->descriptor[0].length = length - 4;
3643	desc_array->descriptor[1].address = address + (length - 4);
3644	desc_array->descriptor[1].length = 4;
3645	desc_array->elements = 2;
3646	return (desc_array->elements);
3647}
3648
3649/**********************************************************************
3650 *
3651 *  Update the board statistics counters.
3652 *
3653 **********************************************************************/
3654static void
3655em_update_stats_counters(struct adapter *adapter)
3656{
3657	struct ifnet   *ifp;
3658
3659	if(adapter->hw.media_type == em_media_type_copper ||
3660	   (E1000_READ_REG(&adapter->hw, STATUS) & E1000_STATUS_LU)) {
3661		adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, SYMERRS);
3662		adapter->stats.sec += E1000_READ_REG(&adapter->hw, SEC);
3663	}
3664	adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, CRCERRS);
3665	adapter->stats.mpc += E1000_READ_REG(&adapter->hw, MPC);
3666	adapter->stats.scc += E1000_READ_REG(&adapter->hw, SCC);
3667	adapter->stats.ecol += E1000_READ_REG(&adapter->hw, ECOL);
3668
3669	adapter->stats.mcc += E1000_READ_REG(&adapter->hw, MCC);
3670	adapter->stats.latecol += E1000_READ_REG(&adapter->hw, LATECOL);
3671	adapter->stats.colc += E1000_READ_REG(&adapter->hw, COLC);
3672	adapter->stats.dc += E1000_READ_REG(&adapter->hw, DC);
3673	adapter->stats.rlec += E1000_READ_REG(&adapter->hw, RLEC);
3674	adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, XONRXC);
3675	adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, XONTXC);
3676	adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, XOFFRXC);
3677	adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, XOFFTXC);
3678	adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, FCRUC);
3679	adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, PRC64);
3680	adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, PRC127);
3681	adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, PRC255);
3682	adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, PRC511);
3683	adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, PRC1023);
3684	adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, PRC1522);
3685	adapter->stats.gprc += E1000_READ_REG(&adapter->hw, GPRC);
3686	adapter->stats.bprc += E1000_READ_REG(&adapter->hw, BPRC);
3687	adapter->stats.mprc += E1000_READ_REG(&adapter->hw, MPRC);
3688	adapter->stats.gptc += E1000_READ_REG(&adapter->hw, GPTC);
3689
3690	/* For the 64-bit byte counters the low dword must be read first. */
3691	/* Both registers clear on the read of the high dword */
3692
3693	adapter->stats.gorcl += E1000_READ_REG(&adapter->hw, GORCL);
3694	adapter->stats.gorch += E1000_READ_REG(&adapter->hw, GORCH);
3695	adapter->stats.gotcl += E1000_READ_REG(&adapter->hw, GOTCL);
3696	adapter->stats.gotch += E1000_READ_REG(&adapter->hw, GOTCH);
3697
3698	adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, RNBC);
3699	adapter->stats.ruc += E1000_READ_REG(&adapter->hw, RUC);
3700	adapter->stats.rfc += E1000_READ_REG(&adapter->hw, RFC);
3701	adapter->stats.roc += E1000_READ_REG(&adapter->hw, ROC);
3702	adapter->stats.rjc += E1000_READ_REG(&adapter->hw, RJC);
3703
3704	adapter->stats.torl += E1000_READ_REG(&adapter->hw, TORL);
3705	adapter->stats.torh += E1000_READ_REG(&adapter->hw, TORH);
3706	adapter->stats.totl += E1000_READ_REG(&adapter->hw, TOTL);
3707	adapter->stats.toth += E1000_READ_REG(&adapter->hw, TOTH);
3708
3709	adapter->stats.tpr += E1000_READ_REG(&adapter->hw, TPR);
3710	adapter->stats.tpt += E1000_READ_REG(&adapter->hw, TPT);
3711	adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, PTC64);
3712	adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, PTC127);
3713	adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, PTC255);
3714	adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, PTC511);
3715	adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, PTC1023);
3716	adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, PTC1522);
3717	adapter->stats.mptc += E1000_READ_REG(&adapter->hw, MPTC);
3718	adapter->stats.bptc += E1000_READ_REG(&adapter->hw, BPTC);
3719
3720	if (adapter->hw.mac_type >= em_82543) {
3721		adapter->stats.algnerrc +=
3722		E1000_READ_REG(&adapter->hw, ALGNERRC);
3723		adapter->stats.rxerrc +=
3724		E1000_READ_REG(&adapter->hw, RXERRC);
3725		adapter->stats.tncrs +=
3726		E1000_READ_REG(&adapter->hw, TNCRS);
3727		adapter->stats.cexterr +=
3728		E1000_READ_REG(&adapter->hw, CEXTERR);
3729		adapter->stats.tsctc +=
3730		E1000_READ_REG(&adapter->hw, TSCTC);
3731		adapter->stats.tsctfc +=
3732		E1000_READ_REG(&adapter->hw, TSCTFC);
3733	}
3734	ifp = adapter->ifp;
3735
3736	ifp->if_collisions = adapter->stats.colc;
3737
3738	/* Rx Errors */
3739	ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc +
3740	    adapter->stats.crcerrs + adapter->stats.algnerrc +
3741	    adapter->stats.ruc + adapter->stats.roc +
3742	    adapter->stats.mpc + adapter->stats.cexterr;
3743
3744	/* Tx Errors */
3745	ifp->if_oerrors = adapter->stats.ecol +
3746	    adapter->stats.latecol + adapter->watchdog_events;
3747}
3748
3749
3750/**********************************************************************
3751 *
3752 *  This routine is called only when em_display_debug_stats is enabled.
3753 *  This routine provides a way to take a look at important statistics
3754 *  maintained by the driver and hardware.
3755 *
3756 **********************************************************************/
3757static void
3758em_print_debug_info(struct adapter *adapter)
3759{
3760	device_t dev = adapter->dev;
3761	uint8_t *hw_addr = adapter->hw.hw_addr;
3762
3763	device_printf(dev, "Adapter hardware address = %p \n", hw_addr);
3764	device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n",
3765	    E1000_READ_REG(&adapter->hw, CTRL),
3766	    E1000_READ_REG(&adapter->hw, RCTL));
3767	device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n",
3768	    ((E1000_READ_REG(&adapter->hw, PBA) & 0xffff0000) >> 16),\
3769	    (E1000_READ_REG(&adapter->hw, PBA) & 0xffff) );
3770	device_printf(dev, "Flow control watermarks high = %d low = %d\n",
3771	    adapter->hw.fc_high_water,
3772	    adapter->hw.fc_low_water);
3773	device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n",
3774	    E1000_READ_REG(&adapter->hw, TIDV),
3775	    E1000_READ_REG(&adapter->hw, TADV));
3776	device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n",
3777	    E1000_READ_REG(&adapter->hw, RDTR),
3778	    E1000_READ_REG(&adapter->hw, RADV));
3779	device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n",
3780	    (long long)adapter->tx_fifo_wrk_cnt,
3781	    (long long)adapter->tx_fifo_reset_cnt);
3782	device_printf(dev, "hw tdh = %d, hw tdt = %d\n",
3783	    E1000_READ_REG(&adapter->hw, TDH),
3784	    E1000_READ_REG(&adapter->hw, TDT));
3785	device_printf(dev, "Num Tx descriptors avail = %d\n",
3786	    adapter->num_tx_desc_avail);
3787	device_printf(dev, "Tx Descriptors not avail1 = %ld\n",
3788	    adapter->no_tx_desc_avail1);
3789	device_printf(dev, "Tx Descriptors not avail2 = %ld\n",
3790	    adapter->no_tx_desc_avail2);
3791	device_printf(dev, "Std mbuf failed = %ld\n",
3792	    adapter->mbuf_alloc_failed);
3793	device_printf(dev, "Std mbuf cluster failed = %ld\n",
3794	    adapter->mbuf_cluster_failed);
3795	device_printf(dev, "Driver dropped packets = %ld\n",
3796	    adapter->dropped_pkts);
3797}
3798
3799static void
3800em_print_hw_stats(struct adapter *adapter)
3801{
3802	device_t dev = adapter->dev;
3803
3804	device_printf(dev, "Excessive collisions = %lld\n",
3805	    (long long)adapter->stats.ecol);
3806#if	(DEBUG_HW > 0)  /* Dont output these errors normally */
3807	device_printf(dev, "Symbol errors = %lld\n",
3808	    (long long)adapter->stats.symerrs);
3809#endif
3810	device_printf(dev, "Sequence errors = %lld\n",
3811	    (long long)adapter->stats.sec);
3812	device_printf(dev, "Defer count = %lld\n",
3813	    (long long)adapter->stats.dc);
3814	device_printf(dev, "Missed Packets = %lld\n",
3815	    (long long)adapter->stats.mpc);
3816	device_printf(dev, "Receive No Buffers = %lld\n",
3817	    (long long)adapter->stats.rnbc);
3818	/* RLEC is inaccurate on some hardware, calculate our own. */
3819	device_printf(dev, "Receive Length Errors = %lld\n",
3820	    ((long long)adapter->stats.roc + (long long)adapter->stats.ruc));
3821	device_printf(dev, "Receive errors = %lld\n",
3822	    (long long)adapter->stats.rxerrc);
3823	device_printf(dev, "Crc errors = %lld\n",
3824	    (long long)adapter->stats.crcerrs);
3825	device_printf(dev, "Alignment errors = %lld\n",
3826	    (long long)adapter->stats.algnerrc);
3827	device_printf(dev, "Carrier extension errors = %lld\n",
3828	    (long long)adapter->stats.cexterr);
3829	device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns);
3830	device_printf(dev, "watchdog timeouts = %ld\n",
3831	    adapter->watchdog_events);
3832	device_printf(dev, "XON Rcvd = %lld\n",
3833	    (long long)adapter->stats.xonrxc);
3834	device_printf(dev, "XON Xmtd = %lld\n",
3835	    (long long)adapter->stats.xontxc);
3836	device_printf(dev, "XOFF Rcvd = %lld\n",
3837	    (long long)adapter->stats.xoffrxc);
3838	device_printf(dev, "XOFF Xmtd = %lld\n",
3839	    (long long)adapter->stats.xofftxc);
3840	device_printf(dev, "Good Packets Rcvd = %lld\n",
3841	    (long long)adapter->stats.gprc);
3842	device_printf(dev, "Good Packets Xmtd = %lld\n",
3843	    (long long)adapter->stats.gptc);
3844#ifdef EM_TSO
3845	device_printf(dev, "TSO Contexts Xmtd = %lld\n",
3846	    (long long)adapter->stats.tsctc);
3847	device_printf(dev, "TSO Contexts Failed = %lld\n",
3848	    (long long)adapter->stats.tsctfc);
3849#endif
3850}
3851
3852static int
3853em_sysctl_debug_info(SYSCTL_HANDLER_ARGS)
3854{
3855	struct adapter *adapter;
3856	int error;
3857	int result;
3858
3859	result = -1;
3860	error = sysctl_handle_int(oidp, &result, 0, req);
3861
3862	if (error || !req->newptr)
3863		return (error);
3864
3865	if (result == 1) {
3866		adapter = (struct adapter *)arg1;
3867		em_print_debug_info(adapter);
3868	}
3869
3870	return (error);
3871}
3872
3873
3874static int
3875em_sysctl_stats(SYSCTL_HANDLER_ARGS)
3876{
3877	struct adapter *adapter;
3878	int error;
3879	int result;
3880
3881	result = -1;
3882	error = sysctl_handle_int(oidp, &result, 0, req);
3883
3884	if (error || !req->newptr)
3885		return (error);
3886
3887	if (result == 1) {
3888		adapter = (struct adapter *)arg1;
3889		em_print_hw_stats(adapter);
3890	}
3891
3892	return (error);
3893}
3894
3895static int
3896em_sysctl_int_delay(SYSCTL_HANDLER_ARGS)
3897{
3898	struct em_int_delay_info *info;
3899	struct adapter *adapter;
3900	uint32_t regval;
3901	int error;
3902	int usecs;
3903	int ticks;
3904
3905	info = (struct em_int_delay_info *)arg1;
3906	usecs = info->value;
3907	error = sysctl_handle_int(oidp, &usecs, 0, req);
3908	if (error != 0 || req->newptr == NULL)
3909		return (error);
3910	if (usecs < 0 || usecs > E1000_TICKS_TO_USECS(65535))
3911		return (EINVAL);
3912	info->value = usecs;
3913	ticks = E1000_USECS_TO_TICKS(usecs);
3914
3915	adapter = info->adapter;
3916
3917	EM_LOCK(adapter);
3918	regval = E1000_READ_OFFSET(&adapter->hw, info->offset);
3919	regval = (regval & ~0xffff) | (ticks & 0xffff);
3920	/* Handle a few special cases. */
3921	switch (info->offset) {
3922	case E1000_RDTR:
3923	case E1000_82542_RDTR:
3924		regval |= E1000_RDT_FPDB;
3925		break;
3926	case E1000_TIDV:
3927	case E1000_82542_TIDV:
3928		if (ticks == 0) {
3929			adapter->txd_cmd &= ~E1000_TXD_CMD_IDE;
3930			/* Don't write 0 into the TIDV register. */
3931			regval++;
3932		} else
3933			adapter->txd_cmd |= E1000_TXD_CMD_IDE;
3934		break;
3935	}
3936	E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval);
3937	EM_UNLOCK(adapter);
3938	return (0);
3939}
3940
3941static void
3942em_add_int_delay_sysctl(struct adapter *adapter, const char *name,
3943	const char *description, struct em_int_delay_info *info,
3944	int offset, int value)
3945{
3946	info->adapter = adapter;
3947	info->offset = offset;
3948	info->value = value;
3949	SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev),
3950	    SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)),
3951	    OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW,
3952	    info, 0, em_sysctl_int_delay, "I", description);
3953}
3954