if_em.c revision 177867
1100894Srwatson/****************************************************************************** 2189503Srwatson 3100894Srwatson Copyright (c) 2001-2008, Intel Corporation 4126097Srwatson All rights reserved. 5145147Srwatson 6172930Srwatson Redistribution and use in source and binary forms, with or without 7182063Srwatson modification, are permitted provided that the following conditions are met: 8100894Srwatson 9100894Srwatson 1. Redistributions of source code must retain the above copyright notice, 10100894Srwatson this list of conditions and the following disclaimer. 11100894Srwatson 12100894Srwatson 2. Redistributions in binary form must reproduce the above copyright 13106392Srwatson notice, this list of conditions and the following disclaimer in the 14106392Srwatson documentation and/or other materials provided with the distribution. 15106392Srwatson 16106392Srwatson 3. Neither the name of the Intel Corporation nor the names of its 17100894Srwatson contributors may be used to endorse or promote products derived from 18172930Srwatson this software without specific prior written permission. 19172930Srwatson 20172930Srwatson THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21189503Srwatson AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22189503Srwatson IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23189503Srwatson ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24100894Srwatson LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25100894Srwatson CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26100894Srwatson SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27100894Srwatson INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28100894Srwatson CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29100894Srwatson ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30100894Srwatson POSSIBILITY OF SUCH DAMAGE. 31100894Srwatson 32100894Srwatson******************************************************************************/ 33100894Srwatson/*$FreeBSD: head/sys/dev/em/if_em.c 177867 2008-04-02 22:00:36Z jfv $*/ 34100894Srwatson 35100894Srwatson#ifdef HAVE_KERNEL_OPTION_HEADERS 36100894Srwatson#include "opt_device_polling.h" 37100894Srwatson#endif 38100894Srwatson 39100894Srwatson#include <sys/param.h> 40100894Srwatson#include <sys/systm.h> 41100894Srwatson#include <sys/bus.h> 42100894Srwatson#include <sys/endian.h> 43100894Srwatson#include <sys/kernel.h> 44100894Srwatson#include <sys/kthread.h> 45116182Sobrien#include <sys/malloc.h> 46116182Sobrien#include <sys/mbuf.h> 47116182Sobrien#include <sys/module.h> 48116182Sobrien#include <sys/rman.h> 49189503Srwatson#include <sys/socket.h> 50100894Srwatson#include <sys/sockio.h> 51101173Srwatson#include <sys/sysctl.h> 52100894Srwatson#include <sys/taskqueue.h> 53106856Srwatson#include <machine/bus.h> 54106468Srwatson#include <machine/resource.h> 55100979Srwatson 56100979Srwatson#include <net/bpf.h> 57102949Sbde#include <net/ethernet.h> 58100979Srwatson#include <net/if.h> 59100979Srwatson#include <net/if_arp.h> 60100979Srwatson#include <net/if_dl.h> 61116701Srwatson#include <net/if_media.h> 62189503Srwatson 63100979Srwatson#include <net/if_types.h> 64100979Srwatson#include <net/if_vlan_var.h> 65100979Srwatson 66100979Srwatson#include <netinet/in_systm.h> 67100979Srwatson#include <netinet/in.h> 68100979Srwatson#include <netinet/if_ether.h> 69100894Srwatson#include <netinet/ip.h> 70100979Srwatson#include <netinet/ip6.h> 71100979Srwatson#include <netinet/tcp.h> 72100979Srwatson#include <netinet/udp.h> 73100979Srwatson 74100979Srwatson#include <machine/in_cksum.h> 75163606Srwatson#include <dev/pci/pcivar.h> 76121361Srwatson#include <dev/pci/pcireg.h> 77165469Srwatson 78100979Srwatson#include "e1000_api.h" 79122524Srwatson#include "if_em.h" 80122524Srwatson 81104521Srwatson/********************************************************************* 82122524Srwatson * Set this to one to display debug statistics 83104521Srwatson *********************************************************************/ 84122524Srwatsonint em_display_debug_stats = 0; 85191731Srwatson 86122524Srwatson/********************************************************************* 87104521Srwatson * Driver version: 88104521Srwatson *********************************************************************/ 89104521Srwatsonchar em_driver_version[] = "6.8.8"; 90172930Srwatson 91105694Srwatson 92105694Srwatson/********************************************************************* 93182063Srwatson * PCI Device ID Table 94182063Srwatson * 95182063Srwatson * Used by probe to select devices to load on 96182063Srwatson * Last field stores an index into e1000_strings 97105694Srwatson * Last entry must be all 0s 98105694Srwatson * 99105694Srwatson * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 100122524Srwatson *********************************************************************/ 101104521Srwatson 102104521Srwatsonstatic em_vendor_info_t em_vendor_info_array[] = 103191731Srwatson{ 104122524Srwatson /* Intel(R) PRO/1000 Network Connection */ 105104521Srwatson { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0}, 106104521Srwatson { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 107104521Srwatson { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0}, 108172930Srwatson { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 109105694Srwatson { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0}, 110105694Srwatson 111182063Srwatson { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 112182063Srwatson { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0}, 113182063Srwatson { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 114182063Srwatson { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 115105694Srwatson { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0}, 116105694Srwatson { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 117184407Srwatson { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 118184407Srwatson 119184407Srwatson { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0}, 120184407Srwatson 121184407Srwatson { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 122184407Srwatson { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 123184407Srwatson 124184407Srwatson { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 125184407Srwatson { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 126184407Srwatson { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 127184407Srwatson { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 128105694Srwatson 129104522Srwatson { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 130191731Srwatson { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 131104522Srwatson { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 132104522Srwatson { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 133104522Srwatson { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 134165425Srwatson 135165425Srwatson { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 136104522Srwatson { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 137104521Srwatson { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 138184407Srwatson { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 139104522Srwatson { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 140104522Srwatson { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 141191731Srwatson { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0}, 142104522Srwatson { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 143104522Srwatson { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, 144104522Srwatson PCI_ANY_ID, PCI_ANY_ID, 0}, 145104522Srwatson 146104522Srwatson { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 147104522Srwatson { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 148104522Srwatson { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0}, 149184407Srwatson 150104522Srwatson { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 151104522Srwatson { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 152191731Srwatson { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 153104522Srwatson { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 154104522Srwatson PCI_ANY_ID, PCI_ANY_ID, 0}, 155184407Srwatson { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 156184407Srwatson PCI_ANY_ID, PCI_ANY_ID, 0}, 157184407Srwatson { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 158172957Srwatson PCI_ANY_ID, PCI_ANY_ID, 0}, 159184407Srwatson { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 160172957Srwatson PCI_ANY_ID, PCI_ANY_ID, 0}, 161191731Srwatson { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 162184407Srwatson { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 163184407Srwatson { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 164172957Srwatson { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 165172957Srwatson 166184407Srwatson { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, 167184407Srwatson { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 168184407Srwatson { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, 169184407Srwatson { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 170184407Srwatson PCI_ANY_ID, PCI_ANY_ID, 0}, 171191731Srwatson { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 172184407Srwatson PCI_ANY_ID, PCI_ANY_ID, 0}, 173184407Srwatson { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 174184407Srwatson PCI_ANY_ID, PCI_ANY_ID, 0}, 175184407Srwatson { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 176104522Srwatson PCI_ANY_ID, PCI_ANY_ID, 0}, 177104522Srwatson { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 178165425Srwatson { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 179165425Srwatson { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 180104522Srwatson { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 181104522Srwatson { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 182172930Srwatson { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 183104522Srwatson { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 184104522Srwatson 185191731Srwatson { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 186191731Srwatson { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 187104522Srwatson { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 188104522Srwatson { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 189100979Srwatson { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 190100979Srwatson /* required last entry */ 191100979Srwatson { 0, 0, 0, 0, 0} 192100979Srwatson}; 193100979Srwatson 194121361Srwatson/********************************************************************* 195172930Srwatson * Table of branding strings for all supported NICs. 196100979Srwatson *********************************************************************/ 197100979Srwatson 198191731Srwatsonstatic char *em_strings[] = { 199100979Srwatson "Intel(R) PRO/1000 Network Connection" 200100979Srwatson}; 201189503Srwatson 202189503Srwatson/********************************************************************* 203189503Srwatson * Function prototypes 204100979Srwatson *********************************************************************/ 205172930Srwatsonstatic int em_probe(device_t); 206100979Srwatsonstatic int em_attach(device_t); 207100979Srwatsonstatic int em_detach(device_t); 208100979Srwatsonstatic int em_shutdown(device_t); 209191731Srwatsonstatic int em_suspend(device_t); 210189503Srwatsonstatic int em_resume(device_t); 211100979Srwatsonstatic void em_start(struct ifnet *); 212100979Srwatsonstatic void em_start_locked(struct ifnet *ifp); 213100979Srwatsonstatic int em_ioctl(struct ifnet *, u_long, caddr_t); 214100979Srwatsonstatic void em_watchdog(struct adapter *); 215189529Srwatsonstatic void em_init(void *); 216189529Srwatsonstatic void em_init_locked(struct adapter *); 217189529Srwatsonstatic void em_stop(void *); 218189529Srwatsonstatic void em_media_status(struct ifnet *, struct ifmediareq *); 219189529Srwatsonstatic int em_media_change(struct ifnet *); 220189529Srwatsonstatic void em_identify_hardware(struct adapter *); 221189529Srwatsonstatic int em_allocate_pci_resources(struct adapter *); 222191731Srwatsonstatic int em_allocate_legacy(struct adapter *adapter); 223189529Srwatsonstatic int em_allocate_msix(struct adapter *adapter); 224189529Srwatsonstatic int em_setup_msix(struct adapter *); 225189529Srwatsonstatic void em_free_pci_resources(struct adapter *); 226189529Srwatsonstatic void em_local_timer(void *); 227189529Srwatsonstatic int em_hardware_init(struct adapter *); 228189529Srwatsonstatic void em_setup_interface(device_t, struct adapter *); 229189529Srwatsonstatic void em_setup_transmit_structures(struct adapter *); 230189529Srwatsonstatic void em_initialize_transmit_unit(struct adapter *); 231189529Srwatsonstatic int em_setup_receive_structures(struct adapter *); 232189529Srwatsonstatic void em_initialize_receive_unit(struct adapter *); 233189529Srwatsonstatic void em_enable_intr(struct adapter *); 234189529Srwatsonstatic void em_disable_intr(struct adapter *); 235191731Srwatsonstatic void em_free_transmit_structures(struct adapter *); 236189529Srwatsonstatic void em_free_receive_structures(struct adapter *); 237189529Srwatsonstatic void em_update_stats_counters(struct adapter *); 238189529Srwatsonstatic void em_txeof(struct adapter *); 239189529Srwatsonstatic void em_tx_purge(struct adapter *); 240189529Srwatsonstatic int em_allocate_receive_structures(struct adapter *); 241189529Srwatsonstatic int em_allocate_transmit_structures(struct adapter *); 242189529Srwatsonstatic int em_rxeof(struct adapter *, int); 243189529Srwatson#ifndef __NO_STRICT_ALIGNMENT 244189529Srwatsonstatic int em_fixup_rx(struct adapter *); 245189529Srwatson#endif 246189529Srwatsonstatic void em_receive_checksum(struct adapter *, struct e1000_rx_desc *, 247189529Srwatson struct mbuf *); 248191731Srwatsonstatic void em_transmit_checksum_setup(struct adapter *, struct mbuf *, 249189529Srwatson u32 *, u32 *); 250189529Srwatson#if __FreeBSD_version >= 700000 251189529Srwatsonstatic bool em_tso_setup(struct adapter *, struct mbuf *, 252189529Srwatson u32 *, u32 *); 253189529Srwatson#endif /* FreeBSD_version >= 700000 */ 254189529Srwatsonstatic void em_set_promisc(struct adapter *); 255189529Srwatsonstatic void em_disable_promisc(struct adapter *); 256189529Srwatsonstatic void em_set_multi(struct adapter *); 257189529Srwatsonstatic void em_print_hw_stats(struct adapter *); 258189529Srwatsonstatic void em_update_link_status(struct adapter *); 259189529Srwatsonstatic int em_get_buf(struct adapter *, int); 260189529Srwatsonstatic void em_enable_hw_vlans(struct adapter *); 261191731Srwatsonstatic int em_xmit(struct adapter *, struct mbuf **); 262189529Srwatsonstatic void em_smartspeed(struct adapter *); 263189529Srwatsonstatic int em_82547_fifo_workaround(struct adapter *, int); 264189529Srwatsonstatic void em_82547_update_fifo_head(struct adapter *, int); 265189529Srwatsonstatic int em_82547_tx_fifo_reset(struct adapter *); 266189529Srwatsonstatic void em_82547_move_tail(void *); 267189529Srwatsonstatic int em_dma_malloc(struct adapter *, bus_size_t, 268189529Srwatson struct em_dma_alloc *, int); 269189529Srwatsonstatic void em_dma_free(struct adapter *, struct em_dma_alloc *); 270189529Srwatsonstatic void em_print_debug_info(struct adapter *); 271189529Srwatsonstatic void em_print_nvm_info(struct adapter *); 272189529Srwatsonstatic int em_is_valid_ether_addr(u8 *); 273189529Srwatsonstatic int em_sysctl_stats(SYSCTL_HANDLER_ARGS); 274189529Srwatsonstatic int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 275191731Srwatsonstatic u32 em_fill_descriptors (bus_addr_t address, u32 length, 276189529Srwatson PDESC_ARRAY desc_array); 277189529Srwatsonstatic int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 278189529Srwatsonstatic void em_add_int_delay_sysctl(struct adapter *, const char *, 279189529Srwatson const char *, struct em_int_delay_info *, int, int); 280189529Srwatson/* Management and WOL Support */ 281189529Srwatsonstatic void em_init_manageability(struct adapter *); 282189529Srwatsonstatic void em_release_manageability(struct adapter *); 283189529Srwatsonstatic void em_get_hw_control(struct adapter *); 284189529Srwatsonstatic void em_release_hw_control(struct adapter *); 285189529Srwatsonstatic void em_enable_wakeup(device_t); 286189529Srwatson 287189529Srwatson 288189529Srwatson#ifdef EM_LEGACY_IRQ 289191731Srwatsonstatic void em_intr(void *); 290189529Srwatson#else /* FAST IRQ */ 291189529Srwatson#if __FreeBSD_version < 700000 292189529Srwatsonstatic void em_irq_fast(void *); 293189529Srwatson#else 294189529Srwatsonstatic int em_irq_fast(void *); 295189529Srwatson#endif 296189529Srwatsonstatic int em_tx_fast(void *); 297189529Srwatsonstatic int em_rx_fast(void *); 298189529Srwatsonstatic int em_link_fast(void *); 299189529Srwatsonstatic void em_add_rx_process_limit(struct adapter *, const char *, 300189529Srwatson const char *, int *, int); 301189529Srwatsonstatic void em_handle_rxtx(void *context, int pending); 302189529Srwatsonstatic void em_handle_rx(void *context, int pending); 303191731Srwatsonstatic void em_handle_tx(void *context, int pending); 304189529Srwatsonstatic void em_handle_link(void *context, int pending); 305189529Srwatson#endif /* EM_LEGACY_IRQ */ 306189529Srwatson 307189529Srwatson#ifdef DEVICE_POLLING 308189529Srwatsonstatic poll_handler_t em_poll; 309189529Srwatson#endif /* POLLING */ 310189529Srwatson 311189529Srwatson/********************************************************************* 312189529Srwatson * FreeBSD Device Interface Entry Points 313189529Srwatson *********************************************************************/ 314189529Srwatson 315189529Srwatsonstatic device_method_t em_methods[] = { 316189529Srwatson /* Device interface */ 317189529Srwatson DEVMETHOD(device_probe, em_probe), 318191731Srwatson DEVMETHOD(device_attach, em_attach), 319189529Srwatson DEVMETHOD(device_detach, em_detach), 320189529Srwatson DEVMETHOD(device_shutdown, em_shutdown), 321189529Srwatson DEVMETHOD(device_suspend, em_suspend), 322189529Srwatson DEVMETHOD(device_resume, em_resume), 323189529Srwatson {0, 0} 324189529Srwatson}; 325189529Srwatson 326189529Srwatsonstatic driver_t em_driver = { 327189529Srwatson "em", em_methods, sizeof(struct adapter), 328189529Srwatson}; 329189529Srwatson 330189529Srwatsonstatic devclass_t em_devclass; 331189529SrwatsonDRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); 332189529SrwatsonMODULE_DEPEND(em, pci, 1, 1, 1); 333189529SrwatsonMODULE_DEPEND(em, ether, 1, 1, 1); 334191731Srwatson 335189529Srwatson/********************************************************************* 336189529Srwatson * Tunable default values. 337189529Srwatson *********************************************************************/ 338189529Srwatson 339189529Srwatson#define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 340189529Srwatson#define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 341189503Srwatson#define M_TSO_LEN 66 342189503Srwatson 343189503Srwatson/* Allow common code without TSO */ 344100979Srwatson#ifndef CSUM_TSO 345172930Srwatson#define CSUM_TSO 0 346100979Srwatson#endif 347100979Srwatson 348100979Srwatsonstatic int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 349191731Srwatsonstatic int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 350189503Srwatsonstatic int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 351100979Srwatsonstatic int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 352100979Srwatsonstatic int em_rxd = EM_DEFAULT_RXD; 353100979Srwatsonstatic int em_txd = EM_DEFAULT_TXD; 354static int em_smart_pwr_down = FALSE; 355 356TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); 357TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); 358TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); 359TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); 360TUNABLE_INT("hw.em.rxd", &em_rxd); 361TUNABLE_INT("hw.em.txd", &em_txd); 362TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); 363 364#ifndef EM_LEGACY_IRQ 365/* How many packets rxeof tries to clean at a time */ 366static int em_rx_process_limit = 100; 367TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); 368#endif 369 370/* Global used in WOL setup with multiport cards */ 371static int global_quad_port_a = 0; 372 373/********************************************************************* 374 * Device identification routine 375 * 376 * em_probe determines if the driver should be loaded on 377 * adapter based on PCI vendor/device id of the adapter. 378 * 379 * return BUS_PROBE_DEFAULT on success, positive on failure 380 *********************************************************************/ 381 382static int 383em_probe(device_t dev) 384{ 385 char adapter_name[60]; 386 u16 pci_vendor_id = 0; 387 u16 pci_device_id = 0; 388 u16 pci_subvendor_id = 0; 389 u16 pci_subdevice_id = 0; 390 em_vendor_info_t *ent; 391 392 INIT_DEBUGOUT("em_probe: begin"); 393 394 pci_vendor_id = pci_get_vendor(dev); 395 if (pci_vendor_id != EM_VENDOR_ID) 396 return (ENXIO); 397 398 pci_device_id = pci_get_device(dev); 399 pci_subvendor_id = pci_get_subvendor(dev); 400 pci_subdevice_id = pci_get_subdevice(dev); 401 402 ent = em_vendor_info_array; 403 while (ent->vendor_id != 0) { 404 if ((pci_vendor_id == ent->vendor_id) && 405 (pci_device_id == ent->device_id) && 406 407 ((pci_subvendor_id == ent->subvendor_id) || 408 (ent->subvendor_id == PCI_ANY_ID)) && 409 410 ((pci_subdevice_id == ent->subdevice_id) || 411 (ent->subdevice_id == PCI_ANY_ID))) { 412 sprintf(adapter_name, "%s %s", 413 em_strings[ent->index], 414 em_driver_version); 415 device_set_desc_copy(dev, adapter_name); 416 return (BUS_PROBE_DEFAULT); 417 } 418 ent++; 419 } 420 421 return (ENXIO); 422} 423 424/********************************************************************* 425 * Device initialization routine 426 * 427 * The attach entry point is called when the driver is being loaded. 428 * This routine identifies the type of hardware, allocates all resources 429 * and initializes the hardware. 430 * 431 * return 0 on success, positive on failure 432 *********************************************************************/ 433 434static int 435em_attach(device_t dev) 436{ 437 struct adapter *adapter; 438 int tsize, rsize; 439 int error = 0; 440 u16 eeprom_data, device_id; 441 442 INIT_DEBUGOUT("em_attach: begin"); 443 444 adapter = device_get_softc(dev); 445 adapter->dev = adapter->osdep.dev = dev; 446 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 447 EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev)); 448 449 /* SYSCTL stuff */ 450 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 451 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 452 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 453 em_sysctl_debug_info, "I", "Debug Information"); 454 455 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 456 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 457 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 458 em_sysctl_stats, "I", "Statistics"); 459 460 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 461 callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0); 462 463 /* Determine hardware and mac info */ 464 em_identify_hardware(adapter); 465 466 /* Setup PCI resources */ 467 if (em_allocate_pci_resources(adapter)) { 468 device_printf(dev, "Allocation of PCI resources failed\n"); 469 error = ENXIO; 470 goto err_pci; 471 } 472 473 /* 474 ** For ICH8 and family we need to 475 ** map the flash memory, and this 476 ** must happen after the MAC is 477 ** identified 478 */ 479 if ((adapter->hw.mac.type == e1000_ich8lan) || 480 (adapter->hw.mac.type == e1000_ich9lan)) { 481 int rid = EM_BAR_TYPE_FLASH; 482 adapter->flash = bus_alloc_resource_any(dev, 483 SYS_RES_MEMORY, &rid, RF_ACTIVE); 484 if (adapter->flash == NULL) { 485 device_printf(dev, "Mapping of Flash failed\n"); 486 error = ENXIO; 487 goto err_pci; 488 } 489 /* This is used in the shared code */ 490 adapter->hw.flash_address = (u8 *)adapter->flash; 491 adapter->osdep.flash_bus_space_tag = 492 rman_get_bustag(adapter->flash); 493 adapter->osdep.flash_bus_space_handle = 494 rman_get_bushandle(adapter->flash); 495 } 496 497 /* Do Shared Code initialization */ 498 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 499 device_printf(dev, "Setup of Shared code failed\n"); 500 error = ENXIO; 501 goto err_pci; 502 } 503 504 e1000_get_bus_info(&adapter->hw); 505 506 /* Set up some sysctls for the tunable interrupt delays */ 507 em_add_int_delay_sysctl(adapter, "rx_int_delay", 508 "receive interrupt delay in usecs", &adapter->rx_int_delay, 509 E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt); 510 em_add_int_delay_sysctl(adapter, "tx_int_delay", 511 "transmit interrupt delay in usecs", &adapter->tx_int_delay, 512 E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt); 513 if (adapter->hw.mac.type >= e1000_82540) { 514 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay", 515 "receive interrupt delay limit in usecs", 516 &adapter->rx_abs_int_delay, 517 E1000_REGISTER(&adapter->hw, E1000_RADV), 518 em_rx_abs_int_delay_dflt); 519 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay", 520 "transmit interrupt delay limit in usecs", 521 &adapter->tx_abs_int_delay, 522 E1000_REGISTER(&adapter->hw, E1000_TADV), 523 em_tx_abs_int_delay_dflt); 524 } 525 526#ifndef EM_LEGACY_IRQ 527 /* Sysctls for limiting the amount of work done in the taskqueue */ 528 em_add_rx_process_limit(adapter, "rx_processing_limit", 529 "max number of rx packets to process", &adapter->rx_process_limit, 530 em_rx_process_limit); 531#endif 532 533 /* 534 * Validate number of transmit and receive descriptors. It 535 * must not exceed hardware maximum, and must be multiple 536 * of E1000_DBA_ALIGN. 537 */ 538 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || 539 (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) || 540 (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) || 541 (em_txd < EM_MIN_TXD)) { 542 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 543 EM_DEFAULT_TXD, em_txd); 544 adapter->num_tx_desc = EM_DEFAULT_TXD; 545 } else 546 adapter->num_tx_desc = em_txd; 547 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || 548 (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) || 549 (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) || 550 (em_rxd < EM_MIN_RXD)) { 551 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 552 EM_DEFAULT_RXD, em_rxd); 553 adapter->num_rx_desc = EM_DEFAULT_RXD; 554 } else 555 adapter->num_rx_desc = em_rxd; 556 557 adapter->hw.mac.autoneg = DO_AUTO_NEG; 558 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 559 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 560 adapter->rx_buffer_len = 2048; 561 562 e1000_init_script_state_82541(&adapter->hw, TRUE); 563 e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); 564 565 /* Copper options */ 566 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 567 adapter->hw.phy.mdix = AUTO_ALL_MODES; 568 adapter->hw.phy.disable_polarity_correction = FALSE; 569 adapter->hw.phy.ms_type = EM_MASTER_SLAVE; 570 } 571 572 /* 573 * Set the frame limits assuming 574 * standard ethernet sized frames. 575 */ 576 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 577 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 578 579 /* 580 * This controls when hardware reports transmit completion 581 * status. 582 */ 583 adapter->hw.mac.report_tx_early = 1; 584 585 tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc), 586 EM_DBA_ALIGN); 587 588 /* Allocate Transmit Descriptor ring */ 589 if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) { 590 device_printf(dev, "Unable to allocate tx_desc memory\n"); 591 error = ENOMEM; 592 goto err_tx_desc; 593 } 594 adapter->tx_desc_base = 595 (struct e1000_tx_desc *)adapter->txdma.dma_vaddr; 596 597 rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc), 598 EM_DBA_ALIGN); 599 600 /* Allocate Receive Descriptor ring */ 601 if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) { 602 device_printf(dev, "Unable to allocate rx_desc memory\n"); 603 error = ENOMEM; 604 goto err_rx_desc; 605 } 606 adapter->rx_desc_base = 607 (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr; 608 609 /* Make sure we have a good EEPROM before we read from it */ 610 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 611 /* 612 ** Some PCI-E parts fail the first check due to 613 ** the link being in sleep state, call it again, 614 ** if it fails a second time its a real issue. 615 */ 616 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 617 device_printf(dev, 618 "The EEPROM Checksum Is Not Valid\n"); 619 error = EIO; 620 goto err_hw_init; 621 } 622 } 623 624 /* Initialize the hardware */ 625 if (em_hardware_init(adapter)) { 626 device_printf(dev, "Unable to initialize the hardware\n"); 627 error = EIO; 628 goto err_hw_init; 629 } 630 631 /* Copy the permanent MAC address out of the EEPROM */ 632 if (e1000_read_mac_addr(&adapter->hw) < 0) { 633 device_printf(dev, "EEPROM read error while reading MAC" 634 " address\n"); 635 error = EIO; 636 goto err_hw_init; 637 } 638 639 if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) { 640 device_printf(dev, "Invalid MAC address\n"); 641 error = EIO; 642 goto err_hw_init; 643 } 644 645 /* Allocate transmit descriptors and buffers */ 646 if (em_allocate_transmit_structures(adapter)) { 647 device_printf(dev, "Could not setup transmit structures\n"); 648 error = ENOMEM; 649 goto err_tx_struct; 650 } 651 652 /* Allocate receive descriptors and buffers */ 653 if (em_allocate_receive_structures(adapter)) { 654 device_printf(dev, "Could not setup receive structures\n"); 655 error = ENOMEM; 656 goto err_rx_struct; 657 } 658 659 /* 660 ** Do interrupt configuration 661 */ 662 if (adapter->msi > 1) /* Do MSI/X */ 663 error = em_allocate_msix(adapter); 664 else /* MSI or Legacy */ 665 error = em_allocate_legacy(adapter); 666 if (error) 667 goto err_rx_struct; 668 669 /* Setup OS specific network interface */ 670 em_setup_interface(dev, adapter); 671 672 /* Initialize statistics */ 673 em_update_stats_counters(adapter); 674 675 adapter->hw.mac.get_link_status = 1; 676 em_update_link_status(adapter); 677 678 /* Indicate SOL/IDER usage */ 679 if (e1000_check_reset_block(&adapter->hw)) 680 device_printf(dev, 681 "PHY reset is blocked due to SOL/IDER session.\n"); 682 683 /* Determine if we have to control management hardware */ 684 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 685 686 /* 687 * Setup Wake-on-Lan 688 */ 689 switch (adapter->hw.mac.type) { 690 691 case e1000_82542: 692 case e1000_82543: 693 break; 694 case e1000_82546: 695 case e1000_82546_rev_3: 696 case e1000_82571: 697 case e1000_80003es2lan: 698 if (adapter->hw.bus.func == 1) 699 e1000_read_nvm(&adapter->hw, 700 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 701 else 702 e1000_read_nvm(&adapter->hw, 703 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 704 eeprom_data &= EM_EEPROM_APME; 705 break; 706 default: 707 /* APME bit in EEPROM is mapped to WUC.APME */ 708 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & 709 E1000_WUC_APME; 710 break; 711 } 712 if (eeprom_data) 713 adapter->wol = E1000_WUFC_MAG; 714 /* 715 * We have the eeprom settings, now apply the special cases 716 * where the eeprom may be wrong or the board won't support 717 * wake on lan on a particular port 718 */ 719 device_id = pci_get_device(dev); 720 switch (device_id) { 721 case E1000_DEV_ID_82546GB_PCIE: 722 adapter->wol = 0; 723 break; 724 case E1000_DEV_ID_82546EB_FIBER: 725 case E1000_DEV_ID_82546GB_FIBER: 726 case E1000_DEV_ID_82571EB_FIBER: 727 /* Wake events only supported on port A for dual fiber 728 * regardless of eeprom setting */ 729 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 730 E1000_STATUS_FUNC_1) 731 adapter->wol = 0; 732 break; 733 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: 734 case E1000_DEV_ID_82571EB_QUAD_COPPER: 735 case E1000_DEV_ID_82571EB_QUAD_FIBER: 736 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 737 /* if quad port adapter, disable WoL on all but port A */ 738 if (global_quad_port_a != 0) 739 adapter->wol = 0; 740 /* Reset for multiple quad port adapters */ 741 if (++global_quad_port_a == 4) 742 global_quad_port_a = 0; 743 break; 744 } 745 746 /* Do we need workaround for 82544 PCI-X adapter? */ 747 if (adapter->hw.bus.type == e1000_bus_type_pcix && 748 adapter->hw.mac.type == e1000_82544) 749 adapter->pcix_82544 = TRUE; 750 else 751 adapter->pcix_82544 = FALSE; 752 753 /* Tell the stack that the interface is not active */ 754 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 755 756 INIT_DEBUGOUT("em_attach: end"); 757 758 return (0); 759 760err_rx_struct: 761 em_free_transmit_structures(adapter); 762err_tx_struct: 763err_hw_init: 764 em_release_hw_control(adapter); 765 e1000_remove_device(&adapter->hw); 766 em_dma_free(adapter, &adapter->rxdma); 767err_rx_desc: 768 em_dma_free(adapter, &adapter->txdma); 769err_tx_desc: 770err_pci: 771 em_free_pci_resources(adapter); 772 EM_TX_LOCK_DESTROY(adapter); 773 EM_CORE_LOCK_DESTROY(adapter); 774 775 return (error); 776} 777 778/********************************************************************* 779 * Device removal routine 780 * 781 * The detach entry point is called when the driver is being removed. 782 * This routine stops the adapter and deallocates all the resources 783 * that were allocated for driver operation. 784 * 785 * return 0 on success, positive on failure 786 *********************************************************************/ 787 788static int 789em_detach(device_t dev) 790{ 791 struct adapter *adapter = device_get_softc(dev); 792 struct ifnet *ifp = adapter->ifp; 793 794 INIT_DEBUGOUT("em_detach: begin"); 795 796 /* Make sure VLANS are not using driver */ 797#if __FreeBSD_version >= 700000 798 if (adapter->ifp->if_vlantrunk != NULL) { 799#else 800 if (adapter->ifp->if_nvlans != 0) { 801#endif 802 device_printf(dev,"Vlan in use, detach first\n"); 803 return (EBUSY); 804 } 805 806#ifdef DEVICE_POLLING 807 if (ifp->if_capenable & IFCAP_POLLING) 808 ether_poll_deregister(ifp); 809#endif 810 811 EM_CORE_LOCK(adapter); 812 EM_TX_LOCK(adapter); 813 adapter->in_detach = 1; 814 em_stop(adapter); 815 e1000_phy_hw_reset(&adapter->hw); 816 817 em_release_manageability(adapter); 818 819 if (((adapter->hw.mac.type == e1000_82573) || 820 (adapter->hw.mac.type == e1000_ich8lan) || 821 (adapter->hw.mac.type == e1000_ich9lan)) && 822 e1000_check_mng_mode(&adapter->hw)) 823 em_release_hw_control(adapter); 824 825 if (adapter->wol) { 826 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 827 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 828 em_enable_wakeup(dev); 829 } 830 831 EM_TX_UNLOCK(adapter); 832 EM_CORE_UNLOCK(adapter); 833 834 ether_ifdetach(adapter->ifp); 835 callout_drain(&adapter->timer); 836 callout_drain(&adapter->tx_fifo_timer); 837 838 em_free_pci_resources(adapter); 839 bus_generic_detach(dev); 840 if_free(ifp); 841 842 e1000_remove_device(&adapter->hw); 843 em_free_transmit_structures(adapter); 844 em_free_receive_structures(adapter); 845 846 /* Free Transmit Descriptor ring */ 847 if (adapter->tx_desc_base) { 848 em_dma_free(adapter, &adapter->txdma); 849 adapter->tx_desc_base = NULL; 850 } 851 852 /* Free Receive Descriptor ring */ 853 if (adapter->rx_desc_base) { 854 em_dma_free(adapter, &adapter->rxdma); 855 adapter->rx_desc_base = NULL; 856 } 857 858 EM_TX_LOCK_DESTROY(adapter); 859 EM_CORE_LOCK_DESTROY(adapter); 860 861 return (0); 862} 863 864/********************************************************************* 865 * 866 * Shutdown entry point 867 * 868 **********************************************************************/ 869 870static int 871em_shutdown(device_t dev) 872{ 873 return em_suspend(dev); 874} 875 876/* 877 * Suspend/resume device methods. 878 */ 879static int 880em_suspend(device_t dev) 881{ 882 struct adapter *adapter = device_get_softc(dev); 883 884 EM_CORE_LOCK(adapter); 885 886 EM_TX_LOCK(adapter); 887 em_stop(adapter); 888 EM_TX_UNLOCK(adapter); 889 890 em_release_manageability(adapter); 891 892 if (((adapter->hw.mac.type == e1000_82573) || 893 (adapter->hw.mac.type == e1000_ich8lan) || 894 (adapter->hw.mac.type == e1000_ich9lan)) && 895 e1000_check_mng_mode(&adapter->hw)) 896 em_release_hw_control(adapter); 897 898 if (adapter->wol) { 899 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 900 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 901 em_enable_wakeup(dev); 902 } 903 904 EM_CORE_UNLOCK(adapter); 905 906 return bus_generic_suspend(dev); 907} 908 909static int 910em_resume(device_t dev) 911{ 912 struct adapter *adapter = device_get_softc(dev); 913 struct ifnet *ifp = adapter->ifp; 914 915 EM_CORE_LOCK(adapter); 916 em_init_locked(adapter); 917 em_init_manageability(adapter); 918 919 if ((ifp->if_flags & IFF_UP) && 920 (ifp->if_drv_flags & IFF_DRV_RUNNING)) 921 em_start_locked(ifp); 922 923 EM_CORE_UNLOCK(adapter); 924 925 return bus_generic_resume(dev); 926} 927 928 929/********************************************************************* 930 * Transmit entry point 931 * 932 * em_start is called by the stack to initiate a transmit. 933 * The driver will remain in this routine as long as there are 934 * packets to transmit and transmit resources are available. 935 * In case resources are not available stack is notified and 936 * the packet is requeued. 937 **********************************************************************/ 938 939static void 940em_start_locked(struct ifnet *ifp) 941{ 942 struct adapter *adapter = ifp->if_softc; 943 struct mbuf *m_head; 944 945 EM_TX_LOCK_ASSERT(adapter); 946 947 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 948 IFF_DRV_RUNNING) 949 return; 950 if (!adapter->link_active) 951 return; 952 953 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 954 955 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 956 if (m_head == NULL) 957 break; 958 /* 959 * Encapsulation can modify our pointer, and or make it 960 * NULL on failure. In that event, we can't requeue. 961 */ 962 if (em_xmit(adapter, &m_head)) { 963 if (m_head == NULL) 964 break; 965 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 966 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 967 break; 968 } 969 970 /* Send a copy of the frame to the BPF listener */ 971 ETHER_BPF_MTAP(ifp, m_head); 972 973 /* Set timeout in case hardware has problems transmitting. */ 974 adapter->watchdog_timer = EM_TX_TIMEOUT; 975 } 976} 977 978static void 979em_start(struct ifnet *ifp) 980{ 981 struct adapter *adapter = ifp->if_softc; 982 983 EM_TX_LOCK(adapter); 984 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 985 em_start_locked(ifp); 986 EM_TX_UNLOCK(adapter); 987} 988 989/********************************************************************* 990 * Ioctl entry point 991 * 992 * em_ioctl is called when the user wants to configure the 993 * interface. 994 * 995 * return 0 on success, positive on failure 996 **********************************************************************/ 997 998static int 999em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 1000{ 1001 struct adapter *adapter = ifp->if_softc; 1002 struct ifreq *ifr = (struct ifreq *)data; 1003 struct ifaddr *ifa = (struct ifaddr *)data; 1004 int error = 0; 1005 1006 if (adapter->in_detach) 1007 return (error); 1008 1009 switch (command) { 1010 case SIOCSIFADDR: 1011 if (ifa->ifa_addr->sa_family == AF_INET) { 1012 /* 1013 * XXX 1014 * Since resetting hardware takes a very long time 1015 * and results in link renegotiation we only 1016 * initialize the hardware only when it is absolutely 1017 * required. 1018 */ 1019 ifp->if_flags |= IFF_UP; 1020 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1021 EM_CORE_LOCK(adapter); 1022 em_init_locked(adapter); 1023 EM_CORE_UNLOCK(adapter); 1024 } 1025 arp_ifinit(ifp, ifa); 1026 } else 1027 error = ether_ioctl(ifp, command, data); 1028 break; 1029 case SIOCSIFMTU: 1030 { 1031 int max_frame_size; 1032 u16 eeprom_data = 0; 1033 1034 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1035 1036 EM_CORE_LOCK(adapter); 1037 switch (adapter->hw.mac.type) { 1038 case e1000_82573: 1039 /* 1040 * 82573 only supports jumbo frames 1041 * if ASPM is disabled. 1042 */ 1043 e1000_read_nvm(&adapter->hw, 1044 NVM_INIT_3GIO_3, 1, &eeprom_data); 1045 if (eeprom_data & NVM_WORD1A_ASPM_MASK) { 1046 max_frame_size = ETHER_MAX_LEN; 1047 break; 1048 } 1049 /* Allow Jumbo frames - fall thru */ 1050 case e1000_82571: 1051 case e1000_82572: 1052 case e1000_ich9lan: 1053 case e1000_80003es2lan: /* Limit Jumbo Frame size */ 1054 max_frame_size = 9234; 1055 break; 1056 /* Adapters that do not support jumbo frames */ 1057 case e1000_82542: 1058 case e1000_ich8lan: 1059 max_frame_size = ETHER_MAX_LEN; 1060 break; 1061 default: 1062 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1063 } 1064 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1065 ETHER_CRC_LEN) { 1066 EM_CORE_UNLOCK(adapter); 1067 error = EINVAL; 1068 break; 1069 } 1070 1071 ifp->if_mtu = ifr->ifr_mtu; 1072 adapter->max_frame_size = 1073 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1074 em_init_locked(adapter); 1075 EM_CORE_UNLOCK(adapter); 1076 break; 1077 } 1078 case SIOCSIFFLAGS: 1079 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1080 SIOCSIFFLAGS (Set Interface Flags)"); 1081 EM_CORE_LOCK(adapter); 1082 if (ifp->if_flags & IFF_UP) { 1083 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1084 if ((ifp->if_flags ^ adapter->if_flags) & 1085 IFF_PROMISC) { 1086 em_disable_promisc(adapter); 1087 em_set_promisc(adapter); 1088 } 1089 } else 1090 em_init_locked(adapter); 1091 } else 1092 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1093 EM_TX_LOCK(adapter); 1094 em_stop(adapter); 1095 EM_TX_UNLOCK(adapter); 1096 } 1097 adapter->if_flags = ifp->if_flags; 1098 EM_CORE_UNLOCK(adapter); 1099 break; 1100 case SIOCADDMULTI: 1101 case SIOCDELMULTI: 1102 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1103 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1104 EM_CORE_LOCK(adapter); 1105 em_disable_intr(adapter); 1106 em_set_multi(adapter); 1107 if (adapter->hw.mac.type == e1000_82542 && 1108 adapter->hw.revision_id == E1000_REVISION_2) { 1109 em_initialize_receive_unit(adapter); 1110 } 1111#ifdef DEVICE_POLLING 1112 if (!(ifp->if_capenable & IFCAP_POLLING)) 1113#endif 1114 em_enable_intr(adapter); 1115 EM_CORE_UNLOCK(adapter); 1116 } 1117 break; 1118 case SIOCSIFMEDIA: 1119 /* Check SOL/IDER usage */ 1120 EM_CORE_LOCK(adapter); 1121 if (e1000_check_reset_block(&adapter->hw)) { 1122 EM_CORE_UNLOCK(adapter); 1123 device_printf(adapter->dev, "Media change is" 1124 " blocked due to SOL/IDER session.\n"); 1125 break; 1126 } 1127 EM_CORE_UNLOCK(adapter); 1128 case SIOCGIFMEDIA: 1129 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1130 SIOCxIFMEDIA (Get/Set Interface Media)"); 1131 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1132 break; 1133 case SIOCSIFCAP: 1134 { 1135 int mask, reinit; 1136 1137 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1138 reinit = 0; 1139 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1140#ifdef DEVICE_POLLING 1141 if (mask & IFCAP_POLLING) { 1142 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1143 error = ether_poll_register(em_poll, ifp); 1144 if (error) 1145 return (error); 1146 EM_CORE_LOCK(adapter); 1147 em_disable_intr(adapter); 1148 ifp->if_capenable |= IFCAP_POLLING; 1149 EM_CORE_UNLOCK(adapter); 1150 } else { 1151 error = ether_poll_deregister(ifp); 1152 /* Enable interrupt even in error case */ 1153 EM_CORE_LOCK(adapter); 1154 em_enable_intr(adapter); 1155 ifp->if_capenable &= ~IFCAP_POLLING; 1156 EM_CORE_UNLOCK(adapter); 1157 } 1158 } 1159#endif 1160 if (mask & IFCAP_HWCSUM) { 1161 ifp->if_capenable ^= IFCAP_HWCSUM; 1162 reinit = 1; 1163 } 1164#if __FreeBSD_version >= 700000 1165 if (mask & IFCAP_TSO4) { 1166 ifp->if_capenable ^= IFCAP_TSO4; 1167 reinit = 1; 1168 } 1169#endif 1170 1171 if (mask & IFCAP_VLAN_HWTAGGING) { 1172 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1173 reinit = 1; 1174 } 1175 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1176 em_init(adapter); 1177#if __FreeBSD_version >= 700000 1178 VLAN_CAPABILITIES(ifp); 1179#endif 1180 break; 1181 } 1182 1183 1184 default: 1185 error = ether_ioctl(ifp, command, data); 1186 break; 1187 } 1188 1189 return (error); 1190} 1191 1192/********************************************************************* 1193 * Watchdog timer: 1194 * 1195 * This routine is called from the local timer every second. 1196 * As long as transmit descriptors are being cleaned the value 1197 * is non-zero and we do nothing. Reaching 0 indicates a tx hang 1198 * and we then reset the device. 1199 * 1200 **********************************************************************/ 1201 1202static void 1203em_watchdog(struct adapter *adapter) 1204{ 1205 1206 EM_CORE_LOCK_ASSERT(adapter); 1207 1208 /* 1209 ** The timer is set to 5 every time start queues a packet. 1210 ** Then txeof keeps resetting it as long as it cleans at 1211 ** least one descriptor. 1212 ** Finally, anytime all descriptors are clean the timer is 1213 ** set to 0. 1214 */ 1215 EM_TX_LOCK(adapter); 1216 if ((adapter->watchdog_timer == 0) || (--adapter->watchdog_timer)) { 1217 EM_TX_UNLOCK(adapter); 1218 return; 1219 } 1220 1221 /* If we are in this routine because of pause frames, then 1222 * don't reset the hardware. 1223 */ 1224 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 1225 E1000_STATUS_TXOFF) { 1226 adapter->watchdog_timer = EM_TX_TIMEOUT; 1227 EM_TX_UNLOCK(adapter); 1228 return; 1229 } 1230 1231 if (e1000_check_for_link(&adapter->hw) == 0) 1232 device_printf(adapter->dev, "watchdog timeout -- resetting\n"); 1233 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1234 adapter->watchdog_events++; 1235 1236 em_init_locked(adapter); 1237} 1238 1239/********************************************************************* 1240 * Init entry point 1241 * 1242 * This routine is used in two ways. It is used by the stack as 1243 * init entry point in network interface structure. It is also used 1244 * by the driver as a hw/sw initialization routine to get to a 1245 * consistent state. 1246 * 1247 * return 0 on success, positive on failure 1248 **********************************************************************/ 1249 1250static void 1251em_init_locked(struct adapter *adapter) 1252{ 1253 struct ifnet *ifp = adapter->ifp; 1254 device_t dev = adapter->dev; 1255 u32 pba; 1256 1257 INIT_DEBUGOUT("em_init: begin"); 1258 1259 EM_CORE_LOCK_ASSERT(adapter); 1260 1261 EM_TX_LOCK(adapter); 1262 em_stop(adapter); 1263 EM_TX_UNLOCK(adapter); 1264 1265 /* 1266 * Packet Buffer Allocation (PBA) 1267 * Writing PBA sets the receive portion of the buffer 1268 * the remainder is used for the transmit buffer. 1269 * 1270 * Devices before the 82547 had a Packet Buffer of 64K. 1271 * Default allocation: PBA=48K for Rx, leaving 16K for Tx. 1272 * After the 82547 the buffer was reduced to 40K. 1273 * Default allocation: PBA=30K for Rx, leaving 10K for Tx. 1274 * Note: default does not leave enough room for Jumbo Frame >10k. 1275 */ 1276 switch (adapter->hw.mac.type) { 1277 case e1000_82547: 1278 case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */ 1279 if (adapter->max_frame_size > 8192) 1280 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 1281 else 1282 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ 1283 adapter->tx_fifo_head = 0; 1284 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT; 1285 adapter->tx_fifo_size = 1286 (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT; 1287 break; 1288 /* Total Packet Buffer on these is 48K */ 1289 case e1000_82571: 1290 case e1000_82572: 1291 case e1000_80003es2lan: 1292 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1293 break; 1294 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1295 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1296 break; 1297 case e1000_ich9lan: 1298#define E1000_PBA_10K 0x000A 1299 pba = E1000_PBA_10K; 1300 break; 1301 case e1000_ich8lan: 1302 pba = E1000_PBA_8K; 1303 break; 1304 default: 1305 /* Devices before 82547 had a Packet Buffer of 64K. */ 1306 if (adapter->max_frame_size > 8192) 1307 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1308 else 1309 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1310 } 1311 1312 INIT_DEBUGOUT1("em_init: pba=%dK",pba); 1313 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1314 1315 /* Get the latest mac address, User can use a LAA */ 1316 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1317 ETHER_ADDR_LEN); 1318 1319 /* Put the address into the Receive Address Array */ 1320 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1321 1322 /* 1323 * With the 82571 adapter, RAR[0] may be overwritten 1324 * when the other port is reset, we make a duplicate 1325 * in RAR[14] for that eventuality, this assures 1326 * the interface continues to function. 1327 */ 1328 if (adapter->hw.mac.type == e1000_82571) { 1329 e1000_set_laa_state_82571(&adapter->hw, TRUE); 1330 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 1331 E1000_RAR_ENTRIES - 1); 1332 } 1333 1334 /* Initialize the hardware */ 1335 if (em_hardware_init(adapter)) { 1336 device_printf(dev, "Unable to initialize the hardware\n"); 1337 return; 1338 } 1339 em_update_link_status(adapter); 1340 1341 /* Setup VLAN support, basic and offload if available */ 1342 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1343 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) 1344 em_enable_hw_vlans(adapter); 1345 1346 /* Set hardware offload abilities */ 1347 ifp->if_hwassist = 0; 1348 if (adapter->hw.mac.type >= e1000_82543) { 1349 if (ifp->if_capenable & IFCAP_TXCSUM) 1350 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1351#if __FreeBSD_version >= 700000 1352 if (ifp->if_capenable & IFCAP_TSO4) 1353 ifp->if_hwassist |= CSUM_TSO; 1354#endif 1355 } 1356 1357 /* Configure for OS presence */ 1358 em_init_manageability(adapter); 1359 1360 /* Prepare transmit descriptors and buffers */ 1361 em_setup_transmit_structures(adapter); 1362 em_initialize_transmit_unit(adapter); 1363 1364 /* Setup Multicast table */ 1365 em_set_multi(adapter); 1366 1367 /* Prepare receive descriptors and buffers */ 1368 if (em_setup_receive_structures(adapter)) { 1369 device_printf(dev, "Could not setup receive structures\n"); 1370 EM_TX_LOCK(adapter); 1371 em_stop(adapter); 1372 EM_TX_UNLOCK(adapter); 1373 return; 1374 } 1375 em_initialize_receive_unit(adapter); 1376 1377 /* Don't lose promiscuous settings */ 1378 em_set_promisc(adapter); 1379 1380 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1381 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1382 1383 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1384 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1385 1386 1387#ifdef DEVICE_POLLING 1388 /* 1389 * Only enable interrupts if we are not polling, make sure 1390 * they are off otherwise. 1391 */ 1392 if (ifp->if_capenable & IFCAP_POLLING) 1393 em_disable_intr(adapter); 1394 else 1395#endif /* DEVICE_POLLING */ 1396 em_enable_intr(adapter); 1397 1398 1399 /* Don't reset the phy next time init gets called */ 1400 adapter->hw.phy.reset_disable = TRUE; 1401} 1402 1403static void 1404em_init(void *arg) 1405{ 1406 struct adapter *adapter = arg; 1407 1408 EM_CORE_LOCK(adapter); 1409 em_init_locked(adapter); 1410 EM_CORE_UNLOCK(adapter); 1411} 1412 1413 1414#ifdef DEVICE_POLLING 1415/********************************************************************* 1416 * 1417 * Legacy polling routine 1418 * 1419 *********************************************************************/ 1420static void 1421em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1422{ 1423 struct adapter *adapter = ifp->if_softc; 1424 u32 reg_icr; 1425 1426 EM_CORE_LOCK(adapter); 1427 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1428 EM_CORE_UNLOCK(adapter); 1429 return; 1430 } 1431 1432 if (cmd == POLL_AND_CHECK_STATUS) { 1433 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1434 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1435 callout_stop(&adapter->timer); 1436 adapter->hw.mac.get_link_status = 1; 1437 em_update_link_status(adapter); 1438 callout_reset(&adapter->timer, hz, 1439 em_local_timer, adapter); 1440 } 1441 } 1442 em_rxeof(adapter, count); 1443 EM_CORE_UNLOCK(adapter); 1444 1445 EM_TX_LOCK(adapter); 1446 em_txeof(adapter); 1447 1448 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1449 em_start_locked(ifp); 1450 EM_TX_UNLOCK(adapter); 1451} 1452#endif /* DEVICE_POLLING */ 1453 1454#ifdef EM_LEGACY_IRQ 1455/********************************************************************* 1456 * 1457 * Legacy Interrupt Service routine 1458 * 1459 *********************************************************************/ 1460 1461static void 1462em_intr(void *arg) 1463{ 1464 struct adapter *adapter = arg; 1465 struct ifnet *ifp; 1466 u32 reg_icr; 1467 1468 EM_CORE_LOCK(adapter); 1469 ifp = adapter->ifp; 1470 1471 if (ifp->if_capenable & IFCAP_POLLING) { 1472 EM_CORE_UNLOCK(adapter); 1473 return; 1474 } 1475 1476 for (;;) { 1477 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1478 1479 if (adapter->hw.mac.type >= e1000_82571 && 1480 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1481 break; 1482 else if (reg_icr == 0) 1483 break; 1484 1485 /* 1486 * XXX: some laptops trigger several spurious interrupts 1487 * on em(4) when in the resume cycle. The ICR register 1488 * reports all-ones value in this case. Processing such 1489 * interrupts would lead to a freeze. I don't know why. 1490 */ 1491 if (reg_icr == 0xffffffff) 1492 break; 1493 1494 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1495 em_rxeof(adapter, -1); 1496 EM_TX_LOCK(adapter); 1497 em_txeof(adapter); 1498 EM_TX_UNLOCK(adapter); 1499 } 1500 1501 /* Link status change */ 1502 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1503 callout_stop(&adapter->timer); 1504 adapter->hw.mac.get_link_status = 1; 1505 em_update_link_status(adapter); 1506 /* Deal with TX cruft when link lost */ 1507 em_tx_purge(adapter); 1508 callout_reset(&adapter->timer, hz, 1509 em_local_timer, adapter); 1510 } 1511 1512 if (reg_icr & E1000_ICR_RXO) 1513 adapter->rx_overruns++; 1514 } 1515 EM_CORE_UNLOCK(adapter); 1516 1517 if (ifp->if_drv_flags & IFF_DRV_RUNNING && 1518 !IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1519 em_start(ifp); 1520} 1521 1522#else /* EM_FAST_IRQ, then fast interrupt routines only */ 1523 1524static void 1525em_handle_link(void *context, int pending) 1526{ 1527 struct adapter *adapter = context; 1528 struct ifnet *ifp; 1529 1530 ifp = adapter->ifp; 1531 1532 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1533 return; 1534 1535 EM_CORE_LOCK(adapter); 1536 callout_stop(&adapter->timer); 1537 em_update_link_status(adapter); 1538 /* Deal with TX cruft when link lost */ 1539 em_tx_purge(adapter); 1540 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1541 EM_CORE_UNLOCK(adapter); 1542} 1543 1544 1545/* Combined RX/TX handler, used by Legacy and MSI */ 1546static void 1547em_handle_rxtx(void *context, int pending) 1548{ 1549 struct adapter *adapter = context; 1550 struct ifnet *ifp = adapter->ifp; 1551 1552 1553 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1554 if (em_rxeof(adapter, adapter->rx_process_limit) != 0) 1555 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1556 EM_TX_LOCK(adapter); 1557 em_txeof(adapter); 1558 1559 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1560 em_start_locked(ifp); 1561 EM_TX_UNLOCK(adapter); 1562 } 1563 1564 em_enable_intr(adapter); 1565} 1566 1567/* RX deferred handler: used with MSIX */ 1568static void 1569em_handle_rx(void *context, int pending) 1570{ 1571 struct adapter *adapter = context; 1572 struct ifnet *ifp = adapter->ifp; 1573 1574 ++adapter->rx_irq; 1575 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && 1576 (em_rxeof(adapter, adapter->rx_process_limit) != 0)) 1577 taskqueue_enqueue(adapter->tq, &adapter->rx_task); 1578 1579 em_enable_intr(adapter); 1580} 1581 1582static void 1583em_handle_tx(void *context, int pending) 1584{ 1585 struct adapter *adapter = context; 1586 struct ifnet *ifp = adapter->ifp; 1587 1588 ++adapter->tx_irq; 1589 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1590 EM_TX_LOCK(adapter); 1591 em_txeof(adapter); 1592 1593 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1594 em_start_locked(ifp); 1595 EM_TX_UNLOCK(adapter); 1596 } 1597 1598 em_enable_intr(adapter); 1599} 1600 1601/********************************************************************* 1602 * 1603 * Fast Legacy/MSI Combined Interrupt Service routine 1604 * 1605 *********************************************************************/ 1606#if __FreeBSD_version < 700000 1607#define FILTER_STRAY 1608#define FILTER_HANDLED 1609static void 1610#else 1611static int 1612#endif 1613em_irq_fast(void *arg) 1614{ 1615 struct adapter *adapter = arg; 1616 struct ifnet *ifp; 1617 u32 reg_icr; 1618 1619 ifp = adapter->ifp; 1620 1621 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1622 1623 /* Hot eject? */ 1624 if (reg_icr == 0xffffffff) 1625 return FILTER_STRAY; 1626 1627 /* Definitely not our interrupt. */ 1628 if (reg_icr == 0x0) 1629 return FILTER_STRAY; 1630 1631 /* 1632 * Starting with the 82571 chip, bit 31 should be used to 1633 * determine whether the interrupt belongs to us. 1634 */ 1635 if (adapter->hw.mac.type >= e1000_82571 && 1636 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1637 return FILTER_STRAY; 1638 1639 /* 1640 * Mask interrupts until the taskqueue is finished running. This is 1641 * cheap, just assume that it is needed. This also works around the 1642 * MSI message reordering errata on certain systems. 1643 */ 1644 em_disable_intr(adapter); 1645 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1646 1647 /* Link status change */ 1648 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1649 adapter->hw.mac.get_link_status = 1; 1650 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1651 } 1652 1653 if (reg_icr & E1000_ICR_RXO) 1654 adapter->rx_overruns++; 1655 return FILTER_HANDLED; 1656} 1657 1658/********************************************************************* 1659 * 1660 * MSIX TX Fast Interrupt Service routine 1661 * 1662 **********************************************************************/ 1663 1664static int 1665em_tx_fast(void *arg) 1666{ 1667 struct adapter *adapter = arg; 1668 u32 reg_icr; 1669 1670 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1671 /* 1672 * Mask interrupts until the taskqueue is finished running. This is 1673 * cheap, just assume that it is needed. This also works around the 1674 * MSI message reordering errata on certain systems. 1675 */ 1676 em_disable_intr(adapter); 1677 taskqueue_enqueue(adapter->tq, &adapter->tx_task); 1678 1679 return FILTER_HANDLED; 1680} 1681 1682/********************************************************************* 1683 * 1684 * MSIX RX Fast Interrupt Service routine 1685 * 1686 **********************************************************************/ 1687 1688static int 1689em_rx_fast(void *arg) 1690{ 1691 struct adapter *adapter = arg; 1692 u32 reg_icr; 1693 1694 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1695 /* 1696 * Mask interrupts until the taskqueue is finished running. This is 1697 * cheap, just assume that it is needed. This also works around the 1698 * MSI message reordering errata on certain systems. 1699 */ 1700 em_disable_intr(adapter); 1701 taskqueue_enqueue(adapter->tq, &adapter->rx_task); 1702 if (reg_icr & E1000_ICR_RXO) 1703 adapter->rx_overruns++; 1704 1705 return FILTER_HANDLED; 1706} 1707 1708/********************************************************************* 1709 * 1710 * MSIX Link Fast Interrupt Service routine 1711 * 1712 **********************************************************************/ 1713 1714static int 1715em_link_fast(void *arg) 1716{ 1717 struct adapter *adapter = arg; 1718 u32 reg_eicr; 1719 1720 reg_eicr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1721 1722 if (reg_eicr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 1723 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1724 1725 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); 1726 return FILTER_HANDLED; 1727} 1728#endif /* EM_FAST_IRQ */ 1729 1730/********************************************************************* 1731 * 1732 * Media Ioctl callback 1733 * 1734 * This routine is called whenever the user queries the status of 1735 * the interface using ifconfig. 1736 * 1737 **********************************************************************/ 1738static void 1739em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1740{ 1741 struct adapter *adapter = ifp->if_softc; 1742 u_char fiber_type = IFM_1000_SX; 1743 1744 INIT_DEBUGOUT("em_media_status: begin"); 1745 1746 EM_CORE_LOCK(adapter); 1747 em_update_link_status(adapter); 1748 1749 ifmr->ifm_status = IFM_AVALID; 1750 ifmr->ifm_active = IFM_ETHER; 1751 1752 if (!adapter->link_active) { 1753 EM_CORE_UNLOCK(adapter); 1754 return; 1755 } 1756 1757 ifmr->ifm_status |= IFM_ACTIVE; 1758 1759 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1760 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 1761 if (adapter->hw.mac.type == e1000_82545) 1762 fiber_type = IFM_1000_LX; 1763 ifmr->ifm_active |= fiber_type | IFM_FDX; 1764 } else { 1765 switch (adapter->link_speed) { 1766 case 10: 1767 ifmr->ifm_active |= IFM_10_T; 1768 break; 1769 case 100: 1770 ifmr->ifm_active |= IFM_100_TX; 1771 break; 1772 case 1000: 1773 ifmr->ifm_active |= IFM_1000_T; 1774 break; 1775 } 1776 if (adapter->link_duplex == FULL_DUPLEX) 1777 ifmr->ifm_active |= IFM_FDX; 1778 else 1779 ifmr->ifm_active |= IFM_HDX; 1780 } 1781 EM_CORE_UNLOCK(adapter); 1782} 1783 1784/********************************************************************* 1785 * 1786 * Media Ioctl callback 1787 * 1788 * This routine is called when the user changes speed/duplex using 1789 * media/mediopt option with ifconfig. 1790 * 1791 **********************************************************************/ 1792static int 1793em_media_change(struct ifnet *ifp) 1794{ 1795 struct adapter *adapter = ifp->if_softc; 1796 struct ifmedia *ifm = &adapter->media; 1797 1798 INIT_DEBUGOUT("em_media_change: begin"); 1799 1800 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1801 return (EINVAL); 1802 1803 EM_CORE_LOCK(adapter); 1804 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1805 case IFM_AUTO: 1806 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1807 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1808 break; 1809 case IFM_1000_LX: 1810 case IFM_1000_SX: 1811 case IFM_1000_T: 1812 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1813 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1814 break; 1815 case IFM_100_TX: 1816 adapter->hw.mac.autoneg = FALSE; 1817 adapter->hw.phy.autoneg_advertised = 0; 1818 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1819 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1820 else 1821 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1822 break; 1823 case IFM_10_T: 1824 adapter->hw.mac.autoneg = FALSE; 1825 adapter->hw.phy.autoneg_advertised = 0; 1826 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1827 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1828 else 1829 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1830 break; 1831 default: 1832 device_printf(adapter->dev, "Unsupported media type\n"); 1833 } 1834 1835 /* As the speed/duplex settings my have changed we need to 1836 * reset the PHY. 1837 */ 1838 adapter->hw.phy.reset_disable = FALSE; 1839 1840 em_init_locked(adapter); 1841 EM_CORE_UNLOCK(adapter); 1842 1843 return (0); 1844} 1845 1846/********************************************************************* 1847 * 1848 * This routine maps the mbufs to tx descriptors. 1849 * 1850 * return 0 on success, positive on failure 1851 **********************************************************************/ 1852 1853static int 1854em_xmit(struct adapter *adapter, struct mbuf **m_headp) 1855{ 1856 bus_dma_segment_t segs[EM_MAX_SCATTER]; 1857 bus_dmamap_t map; 1858 struct em_buffer *tx_buffer, *tx_buffer_mapped; 1859 struct e1000_tx_desc *ctxd = NULL; 1860 struct mbuf *m_head; 1861 u32 txd_upper, txd_lower, txd_used, txd_saved; 1862 int nsegs, i, j, first, last = 0; 1863 int error, do_tso, tso_desc = 0; 1864#if __FreeBSD_version < 700000 1865 struct m_tag *mtag; 1866#endif 1867 m_head = *m_headp; 1868 txd_upper = txd_lower = txd_used = txd_saved = 0; 1869 1870#if __FreeBSD_version >= 700000 1871 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); 1872#else 1873 do_tso = 0; 1874#endif 1875 1876 /* 1877 * Force a cleanup if number of TX descriptors 1878 * available hits the threshold 1879 */ 1880 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { 1881 em_txeof(adapter); 1882 /* Now do we at least have a minimal? */ 1883 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) { 1884 adapter->no_tx_desc_avail1++; 1885 return (ENOBUFS); 1886 } 1887 } 1888 1889 1890 /* 1891 * TSO workaround: 1892 * If an mbuf is only header we need 1893 * to pull 4 bytes of data into it. 1894 */ 1895 if (do_tso && (m_head->m_len <= M_TSO_LEN)) { 1896 m_head = m_pullup(m_head, M_TSO_LEN + 4); 1897 *m_headp = m_head; 1898 if (m_head == NULL) 1899 return (ENOBUFS); 1900 } 1901 1902 /* 1903 * Map the packet for DMA 1904 * 1905 * Capture the first descriptor index, 1906 * this descriptor will have the index 1907 * of the EOP which is the only one that 1908 * now gets a DONE bit writeback. 1909 */ 1910 first = adapter->next_avail_tx_desc; 1911 tx_buffer = &adapter->tx_buffer_area[first]; 1912 tx_buffer_mapped = tx_buffer; 1913 map = tx_buffer->map; 1914 1915 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, 1916 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1917 1918 /* 1919 * There are two types of errors we can (try) to handle: 1920 * - EFBIG means the mbuf chain was too long and bus_dma ran 1921 * out of segments. Defragment the mbuf chain and try again. 1922 * - ENOMEM means bus_dma could not obtain enough bounce buffers 1923 * at this point in time. Defer sending and try again later. 1924 * All other errors, in particular EINVAL, are fatal and prevent the 1925 * mbuf chain from ever going through. Drop it and report error. 1926 */ 1927 if (error == EFBIG) { 1928 struct mbuf *m; 1929 1930 m = m_defrag(*m_headp, M_DONTWAIT); 1931 if (m == NULL) { 1932 adapter->mbuf_alloc_failed++; 1933 m_freem(*m_headp); 1934 *m_headp = NULL; 1935 return (ENOBUFS); 1936 } 1937 *m_headp = m; 1938 1939 /* Try it again */ 1940 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, 1941 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1942 1943 if (error == ENOMEM) { 1944 adapter->no_tx_dma_setup++; 1945 return (error); 1946 } else if (error != 0) { 1947 adapter->no_tx_dma_setup++; 1948 m_freem(*m_headp); 1949 *m_headp = NULL; 1950 return (error); 1951 } 1952 } else if (error == ENOMEM) { 1953 adapter->no_tx_dma_setup++; 1954 return (error); 1955 } else if (error != 0) { 1956 adapter->no_tx_dma_setup++; 1957 m_freem(*m_headp); 1958 *m_headp = NULL; 1959 return (error); 1960 } 1961 1962 /* 1963 * TSO Hardware workaround, if this packet is not 1964 * TSO, and is only a single descriptor long, and 1965 * it follows a TSO burst, then we need to add a 1966 * sentinel descriptor to prevent premature writeback. 1967 */ 1968 if ((do_tso == 0) && (adapter->tx_tso == TRUE)) { 1969 if (nsegs == 1) 1970 tso_desc = TRUE; 1971 adapter->tx_tso = FALSE; 1972 } 1973 1974 if (nsegs > (adapter->num_tx_desc_avail - 2)) { 1975 adapter->no_tx_desc_avail2++; 1976 bus_dmamap_unload(adapter->txtag, map); 1977 return (ENOBUFS); 1978 } 1979 m_head = *m_headp; 1980 1981 /* Do hardware assists */ 1982#if __FreeBSD_version >= 700000 1983 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1984 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower); 1985 if (error != TRUE) 1986 return (ENXIO); /* something foobar */ 1987 /* we need to make a final sentinel transmit desc */ 1988 tso_desc = TRUE; 1989 } else 1990#endif 1991 /* 1992 ** Timesync needs to check the packet header 1993 ** so call checksum code to do so, but don't 1994 ** penalize the code if not defined. 1995 */ 1996 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) 1997 em_transmit_checksum_setup(adapter, m_head, 1998 &txd_upper, &txd_lower); 1999 2000 i = adapter->next_avail_tx_desc; 2001 if (adapter->pcix_82544) 2002 txd_saved = i; 2003 2004 /* Set up our transmit descriptors */ 2005 for (j = 0; j < nsegs; j++) { 2006 bus_size_t seg_len; 2007 bus_addr_t seg_addr; 2008 /* If adapter is 82544 and on PCIX bus */ 2009 if(adapter->pcix_82544) { 2010 DESC_ARRAY desc_array; 2011 u32 array_elements, counter; 2012 /* 2013 * Check the Address and Length combination and 2014 * split the data accordingly 2015 */ 2016 array_elements = em_fill_descriptors(segs[j].ds_addr, 2017 segs[j].ds_len, &desc_array); 2018 for (counter = 0; counter < array_elements; counter++) { 2019 if (txd_used == adapter->num_tx_desc_avail) { 2020 adapter->next_avail_tx_desc = txd_saved; 2021 adapter->no_tx_desc_avail2++; 2022 bus_dmamap_unload(adapter->txtag, map); 2023 return (ENOBUFS); 2024 } 2025 tx_buffer = &adapter->tx_buffer_area[i]; 2026 ctxd = &adapter->tx_desc_base[i]; 2027 ctxd->buffer_addr = htole64( 2028 desc_array.descriptor[counter].address); 2029 ctxd->lower.data = htole32( 2030 (adapter->txd_cmd | txd_lower | (u16) 2031 desc_array.descriptor[counter].length)); 2032 ctxd->upper.data = 2033 htole32((txd_upper)); 2034 last = i; 2035 if (++i == adapter->num_tx_desc) 2036 i = 0; 2037 tx_buffer->m_head = NULL; 2038 tx_buffer->next_eop = -1; 2039 txd_used++; 2040 } 2041 } else { 2042 tx_buffer = &adapter->tx_buffer_area[i]; 2043 ctxd = &adapter->tx_desc_base[i]; 2044 seg_addr = segs[j].ds_addr; 2045 seg_len = segs[j].ds_len; 2046 /* 2047 ** TSO Workaround: 2048 ** If this is the last descriptor, we want to 2049 ** split it so we have a small final sentinel 2050 */ 2051 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { 2052 seg_len -= 4; 2053 ctxd->buffer_addr = htole64(seg_addr); 2054 ctxd->lower.data = htole32( 2055 adapter->txd_cmd | txd_lower | seg_len); 2056 ctxd->upper.data = 2057 htole32(txd_upper); 2058 if (++i == adapter->num_tx_desc) 2059 i = 0; 2060 /* Now make the sentinel */ 2061 ++txd_used; /* using an extra txd */ 2062 ctxd = &adapter->tx_desc_base[i]; 2063 tx_buffer = &adapter->tx_buffer_area[i]; 2064 ctxd->buffer_addr = 2065 htole64(seg_addr + seg_len); 2066 ctxd->lower.data = htole32( 2067 adapter->txd_cmd | txd_lower | 4); 2068 ctxd->upper.data = 2069 htole32(txd_upper); 2070 last = i; 2071 if (++i == adapter->num_tx_desc) 2072 i = 0; 2073 } else { 2074 ctxd->buffer_addr = htole64(seg_addr); 2075 ctxd->lower.data = htole32( 2076 adapter->txd_cmd | txd_lower | seg_len); 2077 ctxd->upper.data = 2078 htole32(txd_upper); 2079 last = i; 2080 if (++i == adapter->num_tx_desc) 2081 i = 0; 2082 } 2083 tx_buffer->m_head = NULL; 2084 tx_buffer->next_eop = -1; 2085 } 2086 } 2087 2088 adapter->next_avail_tx_desc = i; 2089 if (adapter->pcix_82544) 2090 adapter->num_tx_desc_avail -= txd_used; 2091 else { 2092 adapter->num_tx_desc_avail -= nsegs; 2093 if (tso_desc) /* TSO used an extra for sentinel */ 2094 adapter->num_tx_desc_avail -= txd_used; 2095 } 2096 2097 /* 2098 ** Handle VLAN tag, this is the 2099 ** biggest difference between 2100 ** 6.x and 7 2101 */ 2102#if __FreeBSD_version < 700000 2103 /* Find out if we are in vlan mode. */ 2104 mtag = VLAN_OUTPUT_TAG(ifp, m_head); 2105 if (mtag != NULL) { 2106 ctxd->upper.fields.special = 2107 htole16(VLAN_TAG_VALUE(mtag)); 2108#else /* FreeBSD 7 */ 2109 if (m_head->m_flags & M_VLANTAG) { 2110 /* Set the vlan id. */ 2111 ctxd->upper.fields.special = 2112 htole16(m_head->m_pkthdr.ether_vtag); 2113#endif 2114 /* Tell hardware to add tag */ 2115 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); 2116 } 2117 2118 tx_buffer->m_head = m_head; 2119 tx_buffer_mapped->map = tx_buffer->map; 2120 tx_buffer->map = map; 2121 bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE); 2122 2123 /* 2124 * Last Descriptor of Packet 2125 * needs End Of Packet (EOP) 2126 * and Report Status (RS) 2127 */ 2128 ctxd->lower.data |= 2129 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 2130 /* 2131 * Keep track in the first buffer which 2132 * descriptor will be written back 2133 */ 2134 tx_buffer = &adapter->tx_buffer_area[first]; 2135 tx_buffer->next_eop = last; 2136 2137 /* 2138 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 2139 * that this frame is available to transmit. 2140 */ 2141 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 2142 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2143 if (adapter->hw.mac.type == e1000_82547 && 2144 adapter->link_duplex == HALF_DUPLEX) 2145 em_82547_move_tail(adapter); 2146 else { 2147 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i); 2148 if (adapter->hw.mac.type == e1000_82547) 2149 em_82547_update_fifo_head(adapter, 2150 m_head->m_pkthdr.len); 2151 } 2152 2153 return (0); 2154} 2155 2156/********************************************************************* 2157 * 2158 * 82547 workaround to avoid controller hang in half-duplex environment. 2159 * The workaround is to avoid queuing a large packet that would span 2160 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers 2161 * in this case. We do that only when FIFO is quiescent. 2162 * 2163 **********************************************************************/ 2164static void 2165em_82547_move_tail(void *arg) 2166{ 2167 struct adapter *adapter = arg; 2168 struct e1000_tx_desc *tx_desc; 2169 u16 hw_tdt, sw_tdt, length = 0; 2170 bool eop = 0; 2171 2172 EM_TX_LOCK_ASSERT(adapter); 2173 2174 hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0)); 2175 sw_tdt = adapter->next_avail_tx_desc; 2176 2177 while (hw_tdt != sw_tdt) { 2178 tx_desc = &adapter->tx_desc_base[hw_tdt]; 2179 length += tx_desc->lower.flags.length; 2180 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP; 2181 if (++hw_tdt == adapter->num_tx_desc) 2182 hw_tdt = 0; 2183 2184 if (eop) { 2185 if (em_82547_fifo_workaround(adapter, length)) { 2186 adapter->tx_fifo_wrk_cnt++; 2187 callout_reset(&adapter->tx_fifo_timer, 1, 2188 em_82547_move_tail, adapter); 2189 break; 2190 } 2191 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt); 2192 em_82547_update_fifo_head(adapter, length); 2193 length = 0; 2194 } 2195 } 2196} 2197 2198static int 2199em_82547_fifo_workaround(struct adapter *adapter, int len) 2200{ 2201 int fifo_space, fifo_pkt_len; 2202 2203 fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2204 2205 if (adapter->link_duplex == HALF_DUPLEX) { 2206 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head; 2207 2208 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) { 2209 if (em_82547_tx_fifo_reset(adapter)) 2210 return (0); 2211 else 2212 return (1); 2213 } 2214 } 2215 2216 return (0); 2217} 2218 2219static void 2220em_82547_update_fifo_head(struct adapter *adapter, int len) 2221{ 2222 int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2223 2224 /* tx_fifo_head is always 16 byte aligned */ 2225 adapter->tx_fifo_head += fifo_pkt_len; 2226 if (adapter->tx_fifo_head >= adapter->tx_fifo_size) { 2227 adapter->tx_fifo_head -= adapter->tx_fifo_size; 2228 } 2229} 2230 2231 2232static int 2233em_82547_tx_fifo_reset(struct adapter *adapter) 2234{ 2235 u32 tctl; 2236 2237 if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) == 2238 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) && 2239 (E1000_READ_REG(&adapter->hw, E1000_TDFT) == 2240 E1000_READ_REG(&adapter->hw, E1000_TDFH)) && 2241 (E1000_READ_REG(&adapter->hw, E1000_TDFTS) == 2242 E1000_READ_REG(&adapter->hw, E1000_TDFHS)) && 2243 (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) { 2244 /* Disable TX unit */ 2245 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 2246 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, 2247 tctl & ~E1000_TCTL_EN); 2248 2249 /* Reset FIFO pointers */ 2250 E1000_WRITE_REG(&adapter->hw, E1000_TDFT, 2251 adapter->tx_head_addr); 2252 E1000_WRITE_REG(&adapter->hw, E1000_TDFH, 2253 adapter->tx_head_addr); 2254 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS, 2255 adapter->tx_head_addr); 2256 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS, 2257 adapter->tx_head_addr); 2258 2259 /* Re-enable TX unit */ 2260 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 2261 E1000_WRITE_FLUSH(&adapter->hw); 2262 2263 adapter->tx_fifo_head = 0; 2264 adapter->tx_fifo_reset_cnt++; 2265 2266 return (TRUE); 2267 } 2268 else { 2269 return (FALSE); 2270 } 2271} 2272 2273static void 2274em_set_promisc(struct adapter *adapter) 2275{ 2276 struct ifnet *ifp = adapter->ifp; 2277 u32 reg_rctl; 2278 2279 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2280 2281 if (ifp->if_flags & IFF_PROMISC) { 2282 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2283 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2284 } else if (ifp->if_flags & IFF_ALLMULTI) { 2285 reg_rctl |= E1000_RCTL_MPE; 2286 reg_rctl &= ~E1000_RCTL_UPE; 2287 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2288 } 2289} 2290 2291static void 2292em_disable_promisc(struct adapter *adapter) 2293{ 2294 u32 reg_rctl; 2295 2296 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2297 2298 reg_rctl &= (~E1000_RCTL_UPE); 2299 reg_rctl &= (~E1000_RCTL_MPE); 2300 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2301} 2302 2303 2304/********************************************************************* 2305 * Multicast Update 2306 * 2307 * This routine is called whenever multicast address list is updated. 2308 * 2309 **********************************************************************/ 2310 2311static void 2312em_set_multi(struct adapter *adapter) 2313{ 2314 struct ifnet *ifp = adapter->ifp; 2315 struct ifmultiaddr *ifma; 2316 u32 reg_rctl = 0; 2317 u8 mta[512]; /* Largest MTS is 4096 bits */ 2318 int mcnt = 0; 2319 2320 IOCTL_DEBUGOUT("em_set_multi: begin"); 2321 2322 if (adapter->hw.mac.type == e1000_82542 && 2323 adapter->hw.revision_id == E1000_REVISION_2) { 2324 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2325 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2326 e1000_pci_clear_mwi(&adapter->hw); 2327 reg_rctl |= E1000_RCTL_RST; 2328 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2329 msec_delay(5); 2330 } 2331 2332 IF_ADDR_LOCK(ifp); 2333 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2334 if (ifma->ifma_addr->sa_family != AF_LINK) 2335 continue; 2336 2337 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 2338 break; 2339 2340 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2341 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 2342 mcnt++; 2343 } 2344 IF_ADDR_UNLOCK(ifp); 2345 2346 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 2347 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2348 reg_rctl |= E1000_RCTL_MPE; 2349 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2350 } else 2351 e1000_update_mc_addr_list(&adapter->hw, mta, 2352 mcnt, 1, adapter->hw.mac.rar_entry_count); 2353 2354 if (adapter->hw.mac.type == e1000_82542 && 2355 adapter->hw.revision_id == E1000_REVISION_2) { 2356 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2357 reg_rctl &= ~E1000_RCTL_RST; 2358 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2359 msec_delay(5); 2360 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2361 e1000_pci_set_mwi(&adapter->hw); 2362 } 2363} 2364 2365 2366/********************************************************************* 2367 * Timer routine 2368 * 2369 * This routine checks for link status and updates statistics. 2370 * 2371 **********************************************************************/ 2372 2373static void 2374em_local_timer(void *arg) 2375{ 2376 struct adapter *adapter = arg; 2377 struct ifnet *ifp = adapter->ifp; 2378 2379 EM_CORE_LOCK_ASSERT(adapter); 2380 2381 em_update_link_status(adapter); 2382 em_update_stats_counters(adapter); 2383 2384 /* Reset LAA into RAR[0] on 82571 */ 2385 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE) 2386 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 2387 2388 if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 2389 em_print_hw_stats(adapter); 2390 2391 em_smartspeed(adapter); 2392 2393 /* 2394 * Each second we check the watchdog to 2395 * protect against hardware hangs. 2396 */ 2397 em_watchdog(adapter); 2398 2399 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 2400 2401} 2402 2403static void 2404em_update_link_status(struct adapter *adapter) 2405{ 2406 struct e1000_hw *hw = &adapter->hw; 2407 struct ifnet *ifp = adapter->ifp; 2408 device_t dev = adapter->dev; 2409 u32 link_check = 0; 2410 2411 /* Get the cached link value or read phy for real */ 2412 switch (hw->phy.media_type) { 2413 case e1000_media_type_copper: 2414 if (hw->mac.get_link_status) { 2415 /* Do the work to read phy */ 2416 e1000_check_for_link(hw); 2417 link_check = !hw->mac.get_link_status; 2418 } else 2419 link_check = TRUE; 2420 break; 2421 case e1000_media_type_fiber: 2422 e1000_check_for_link(hw); 2423 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2424 E1000_STATUS_LU); 2425 break; 2426 case e1000_media_type_internal_serdes: 2427 e1000_check_for_link(hw); 2428 link_check = adapter->hw.mac.serdes_has_link; 2429 break; 2430 default: 2431 case e1000_media_type_unknown: 2432 break; 2433 } 2434 2435 /* Now check for a transition */ 2436 if (link_check && (adapter->link_active == 0)) { 2437 e1000_get_speed_and_duplex(hw, &adapter->link_speed, 2438 &adapter->link_duplex); 2439 /* Check if we must disable SPEED_MODE bit on PCI-E */ 2440 if ((adapter->link_speed != SPEED_1000) && 2441 ((hw->mac.type == e1000_82571) || 2442 (hw->mac.type == e1000_82572))) { 2443 int tarc0; 2444 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2445 tarc0 &= ~SPEED_MODE_BIT; 2446 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2447 } 2448 if (bootverbose) 2449 device_printf(dev, "Link is up %d Mbps %s\n", 2450 adapter->link_speed, 2451 ((adapter->link_duplex == FULL_DUPLEX) ? 2452 "Full Duplex" : "Half Duplex")); 2453 adapter->link_active = 1; 2454 adapter->smartspeed = 0; 2455 ifp->if_baudrate = adapter->link_speed * 1000000; 2456 if_link_state_change(ifp, LINK_STATE_UP); 2457 } else if (!link_check && (adapter->link_active == 1)) { 2458 ifp->if_baudrate = adapter->link_speed = 0; 2459 adapter->link_duplex = 0; 2460 if (bootverbose) 2461 device_printf(dev, "Link is Down\n"); 2462 adapter->link_active = 0; 2463 /* Link down, disable watchdog */ 2464 adapter->watchdog_timer = FALSE; 2465 if_link_state_change(ifp, LINK_STATE_DOWN); 2466 } 2467} 2468 2469/********************************************************************* 2470 * 2471 * This routine disables all traffic on the adapter by issuing a 2472 * global reset on the MAC and deallocates TX/RX buffers. 2473 * 2474 * This routine should always be called with BOTH the CORE 2475 * and TX locks. 2476 **********************************************************************/ 2477 2478static void 2479em_stop(void *arg) 2480{ 2481 struct adapter *adapter = arg; 2482 struct ifnet *ifp = adapter->ifp; 2483 2484 EM_CORE_LOCK_ASSERT(adapter); 2485 EM_TX_LOCK_ASSERT(adapter); 2486 2487 INIT_DEBUGOUT("em_stop: begin"); 2488 2489 em_disable_intr(adapter); 2490 callout_stop(&adapter->timer); 2491 callout_stop(&adapter->tx_fifo_timer); 2492 2493 /* Tell the stack that the interface is no longer active */ 2494 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2495 2496 2497 e1000_reset_hw(&adapter->hw); 2498 if (adapter->hw.mac.type >= e1000_82544) 2499 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2500} 2501 2502 2503/********************************************************************* 2504 * 2505 * Determine hardware revision. 2506 * 2507 **********************************************************************/ 2508static void 2509em_identify_hardware(struct adapter *adapter) 2510{ 2511 device_t dev = adapter->dev; 2512 2513 /* Make sure our PCI config space has the necessary stuff set */ 2514 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2515 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2516 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2517 device_printf(dev, "Memory Access and/or Bus Master bits " 2518 "were not set!\n"); 2519 adapter->hw.bus.pci_cmd_word |= 2520 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2521 pci_write_config(dev, PCIR_COMMAND, 2522 adapter->hw.bus.pci_cmd_word, 2); 2523 } 2524 2525 /* Save off the information about this board */ 2526 adapter->hw.vendor_id = pci_get_vendor(dev); 2527 adapter->hw.device_id = pci_get_device(dev); 2528 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2529 adapter->hw.subsystem_vendor_id = 2530 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2531 adapter->hw.subsystem_device_id = 2532 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2533 2534 /* Do Shared Code Init and Setup */ 2535 if (e1000_set_mac_type(&adapter->hw)) { 2536 device_printf(dev, "Setup init failure\n"); 2537 return; 2538 } 2539} 2540 2541static int 2542em_allocate_pci_resources(struct adapter *adapter) 2543{ 2544 device_t dev = adapter->dev; 2545 int val, rid, error = E1000_SUCCESS; 2546 2547 rid = PCIR_BAR(0); 2548 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2549 &rid, RF_ACTIVE); 2550 if (adapter->memory == NULL) { 2551 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2552 return (ENXIO); 2553 } 2554 adapter->osdep.mem_bus_space_tag = 2555 rman_get_bustag(adapter->memory); 2556 adapter->osdep.mem_bus_space_handle = 2557 rman_get_bushandle(adapter->memory); 2558 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2559 2560 /* Only older adapters use IO mapping */ 2561 if ((adapter->hw.mac.type > e1000_82543) && 2562 (adapter->hw.mac.type < e1000_82571)) { 2563 /* Figure our where our IO BAR is ? */ 2564 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { 2565 val = pci_read_config(dev, rid, 4); 2566 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { 2567 adapter->io_rid = rid; 2568 break; 2569 } 2570 rid += 4; 2571 /* check for 64bit BAR */ 2572 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) 2573 rid += 4; 2574 } 2575 if (rid >= PCIR_CIS) { 2576 device_printf(dev, "Unable to locate IO BAR\n"); 2577 return (ENXIO); 2578 } 2579 adapter->ioport = bus_alloc_resource_any(dev, 2580 SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); 2581 if (adapter->ioport == NULL) { 2582 device_printf(dev, "Unable to allocate bus resource: " 2583 "ioport\n"); 2584 return (ENXIO); 2585 } 2586 adapter->hw.io_base = 0; 2587 adapter->osdep.io_bus_space_tag = 2588 rman_get_bustag(adapter->ioport); 2589 adapter->osdep.io_bus_space_handle = 2590 rman_get_bushandle(adapter->ioport); 2591 } 2592 2593 /* 2594 ** Init the resource arrays 2595 ** used by MSIX setup 2596 */ 2597 for (int i = 0; i < 3; i++) { 2598 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 2599 adapter->tag[i] = NULL; 2600 adapter->res[i] = NULL; 2601 } 2602 2603 /* 2604 * Setup MSI/X or MSI if PCI Express 2605 */ 2606 adapter->msi = em_setup_msix(adapter); 2607 2608 adapter->hw.back = &adapter->osdep; 2609 2610 return (error); 2611} 2612 2613/********************************************************************* 2614 * 2615 * Setup the Legacy or MSI Interrupt handler 2616 * 2617 **********************************************************************/ 2618int 2619em_allocate_legacy(struct adapter *adapter) 2620{ 2621 device_t dev = adapter->dev; 2622 int error; 2623 2624 /* Manually turn off all interrupts */ 2625 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2626 2627 /* Legacy RID is 0 */ 2628 if (adapter->msi == 0) 2629 adapter->rid[0] = 0; 2630 2631 /* We allocate a single interrupt resource */ 2632 adapter->res[0] = bus_alloc_resource_any(dev, 2633 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); 2634 if (adapter->res[0] == NULL) { 2635 device_printf(dev, "Unable to allocate bus resource: " 2636 "interrupt\n"); 2637 return (ENXIO); 2638 } 2639 2640#ifdef EM_LEGACY_IRQ 2641 /* We do Legacy setup */ 2642 if ((error = bus_setup_intr(dev, adapter->res[0], 2643#if __FreeBSD_version > 700000 2644 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter, 2645#else /* 6.X */ 2646 INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter, 2647#endif 2648 &adapter->tag[0])) != 0) { 2649 device_printf(dev, "Failed to register interrupt handler"); 2650 return (error); 2651 } 2652 2653#else /* FAST_IRQ */ 2654 /* 2655 * Try allocating a fast interrupt and the associated deferred 2656 * processing contexts. 2657 */ 2658 TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter); 2659 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2660 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2661 taskqueue_thread_enqueue, &adapter->tq); 2662 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2663 device_get_nameunit(adapter->dev)); 2664#if __FreeBSD_version < 700000 2665 if ((error = bus_setup_intr(dev, adapter->res[0], 2666 INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter, 2667#else 2668 if ((error = bus_setup_intr(dev, adapter->res[0], 2669 INTR_TYPE_NET, em_irq_fast, NULL, adapter, 2670#endif 2671 &adapter->tag[0])) != 0) { 2672 device_printf(dev, "Failed to register fast interrupt " 2673 "handler: %d\n", error); 2674 taskqueue_free(adapter->tq); 2675 adapter->tq = NULL; 2676 return (error); 2677 } 2678#endif /* EM_LEGACY_IRQ */ 2679 2680 return (0); 2681} 2682 2683/********************************************************************* 2684 * 2685 * Setup the MSIX Interrupt handlers 2686 * This is not really Multiqueue, rather 2687 * its just multiple interrupt vectors. 2688 * 2689 **********************************************************************/ 2690int 2691em_allocate_msix(struct adapter *adapter) 2692{ 2693 device_t dev = adapter->dev; 2694 int error; 2695 2696 /* Make sure all interrupts are disabled */ 2697 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2698 2699 /* First get the resources */ 2700 for (int i = 0; i < adapter->msi; i++) { 2701 adapter->res[i] = bus_alloc_resource_any(dev, 2702 SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE); 2703 if (adapter->res[i] == NULL) { 2704 device_printf(dev, 2705 "Unable to allocate bus resource: " 2706 "MSIX Interrupt\n"); 2707 return (ENXIO); 2708 } 2709 } 2710 2711 /* 2712 * Now allocate deferred processing contexts. 2713 */ 2714 TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter); 2715 TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter); 2716 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2717 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2718 taskqueue_thread_enqueue, &adapter->tq); 2719 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2720 device_get_nameunit(adapter->dev)); 2721 2722 /* 2723 * And setup the interrupt handlers 2724 */ 2725 2726 /* First slot to RX */ 2727 if ((error = bus_setup_intr(dev, adapter->res[0], 2728 INTR_TYPE_NET, em_rx_fast, NULL, adapter, 2729 &adapter->tag[0])) != 0) { 2730 device_printf(dev, "Failed to register RX handler"); 2731 return (error); 2732 } 2733 2734 /* Next TX */ 2735 if ((error = bus_setup_intr(dev, adapter->res[1], 2736 INTR_TYPE_NET, em_tx_fast, NULL, adapter, 2737 &adapter->tag[1])) != 0) { 2738 device_printf(dev, "Failed to register TX handler"); 2739 return (error); 2740 } 2741 2742 /* And Link */ 2743 if ((error = bus_setup_intr(dev, adapter->res[2], 2744 INTR_TYPE_NET, em_link_fast, NULL, adapter, 2745 &adapter->tag[2])) != 0) { 2746 device_printf(dev, "Failed to register TX handler"); 2747 return (error); 2748 } 2749 2750 return (0); 2751} 2752 2753static void 2754em_free_pci_resources(struct adapter *adapter) 2755{ 2756 device_t dev = adapter->dev; 2757 2758 /* Make sure the for loop below runs once */ 2759 if (adapter->msi == 0) 2760 adapter->msi = 1; 2761 2762 /* 2763 * First release all the interrupt resources: 2764 * notice that since these are just kept 2765 * in an array we can do the same logic 2766 * whether its MSIX or just legacy. 2767 */ 2768 for (int i = 0; i < adapter->msi; i++) { 2769 if (adapter->tag[i] != NULL) { 2770 bus_teardown_intr(dev, adapter->res[i], 2771 adapter->tag[i]); 2772 adapter->tag[i] = NULL; 2773 } 2774 if (adapter->res[i] != NULL) { 2775 bus_release_resource(dev, SYS_RES_IRQ, 2776 adapter->rid[i], adapter->res[i]); 2777 } 2778 } 2779 2780 if (adapter->msi) 2781 pci_release_msi(dev); 2782 2783 if (adapter->msix != NULL) 2784 bus_release_resource(dev, SYS_RES_MEMORY, 2785 PCIR_BAR(EM_MSIX_BAR), adapter->msix); 2786 2787 if (adapter->memory != NULL) 2788 bus_release_resource(dev, SYS_RES_MEMORY, 2789 PCIR_BAR(0), adapter->memory); 2790 2791 if (adapter->flash != NULL) 2792 bus_release_resource(dev, SYS_RES_MEMORY, 2793 EM_FLASH, adapter->flash); 2794 2795 if (adapter->ioport != NULL) 2796 bus_release_resource(dev, SYS_RES_IOPORT, 2797 adapter->io_rid, adapter->ioport); 2798} 2799 2800/* 2801 * Setup MSI/X 2802 */ 2803static int 2804em_setup_msix(struct adapter *adapter) 2805{ 2806 device_t dev = adapter->dev; 2807 int val = 0; 2808 2809 if (adapter->hw.mac.type < e1000_82571) 2810 return (0); 2811 2812 val = pci_msi_count(dev); 2813 if (val == 1 && pci_alloc_msi(dev, &val) == 0) { 2814 adapter->msi = 1; 2815 device_printf(adapter->dev,"Using MSI interrupt\n"); 2816 return (val); 2817 } 2818 return (0); 2819} 2820 2821/********************************************************************* 2822 * 2823 * Initialize the hardware to a configuration 2824 * as specified by the adapter structure. 2825 * 2826 **********************************************************************/ 2827static int 2828em_hardware_init(struct adapter *adapter) 2829{ 2830 device_t dev = adapter->dev; 2831 u16 rx_buffer_size; 2832 2833 INIT_DEBUGOUT("em_hardware_init: begin"); 2834 2835 /* Issue a global reset */ 2836 e1000_reset_hw(&adapter->hw); 2837 2838 /* Get control from any management/hw control */ 2839 if (((adapter->hw.mac.type == e1000_82573) || 2840 (adapter->hw.mac.type == e1000_ich8lan) || 2841 (adapter->hw.mac.type == e1000_ich9lan)) && 2842 e1000_check_mng_mode(&adapter->hw)) 2843 em_get_hw_control(adapter); 2844 2845 /* When hardware is reset, fifo_head is also reset */ 2846 adapter->tx_fifo_head = 0; 2847 2848 /* Set up smart power down as default off on newer adapters. */ 2849 if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 || 2850 adapter->hw.mac.type == e1000_82572)) { 2851 u16 phy_tmp = 0; 2852 2853 /* Speed up time to link by disabling smart power down. */ 2854 e1000_read_phy_reg(&adapter->hw, 2855 IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 2856 phy_tmp &= ~IGP02E1000_PM_SPD; 2857 e1000_write_phy_reg(&adapter->hw, 2858 IGP02E1000_PHY_POWER_MGMT, phy_tmp); 2859 } 2860 2861 /* 2862 * These parameters control the automatic generation (Tx) and 2863 * response (Rx) to Ethernet PAUSE frames. 2864 * - High water mark should allow for at least two frames to be 2865 * received after sending an XOFF. 2866 * - Low water mark works best when it is very near the high water mark. 2867 * This allows the receiver to restart by sending XON when it has 2868 * drained a bit. Here we use an arbitary value of 1500 which will 2869 * restart after one full frame is pulled from the buffer. There 2870 * could be several smaller frames in the buffer and if so they will 2871 * not trigger the XON until their total number reduces the buffer 2872 * by 1500. 2873 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 2874 */ 2875 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 2876 0xffff) << 10 ); 2877 2878 adapter->hw.fc.high_water = rx_buffer_size - 2879 roundup2(adapter->max_frame_size, 1024); 2880 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; 2881 2882 if (adapter->hw.mac.type == e1000_80003es2lan) 2883 adapter->hw.fc.pause_time = 0xFFFF; 2884 else 2885 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME; 2886 adapter->hw.fc.send_xon = TRUE; 2887 adapter->hw.fc.type = e1000_fc_full; 2888 2889 if (e1000_init_hw(&adapter->hw) < 0) { 2890 device_printf(dev, "Hardware Initialization Failed\n"); 2891 return (EIO); 2892 } 2893 2894 e1000_check_for_link(&adapter->hw); 2895 2896 return (0); 2897} 2898 2899/********************************************************************* 2900 * 2901 * Setup networking device structure and register an interface. 2902 * 2903 **********************************************************************/ 2904static void 2905em_setup_interface(device_t dev, struct adapter *adapter) 2906{ 2907 struct ifnet *ifp; 2908 2909 INIT_DEBUGOUT("em_setup_interface: begin"); 2910 2911 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2912 if (ifp == NULL) 2913 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 2914 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2915 ifp->if_mtu = ETHERMTU; 2916 ifp->if_init = em_init; 2917 ifp->if_softc = adapter; 2918 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2919 ifp->if_ioctl = em_ioctl; 2920 ifp->if_start = em_start; 2921 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 2922 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 2923 IFQ_SET_READY(&ifp->if_snd); 2924 2925 ether_ifattach(ifp, adapter->hw.mac.addr); 2926 2927 ifp->if_capabilities = ifp->if_capenable = 0; 2928 2929 if (adapter->hw.mac.type >= e1000_82543) { 2930 int version_cap; 2931#if __FreeBSD_version < 700000 2932 version_cap = IFCAP_HWCSUM; 2933#else 2934 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2935#endif 2936 ifp->if_capabilities |= version_cap; 2937 ifp->if_capenable |= version_cap; 2938 } 2939 2940#if __FreeBSD_version >= 700000 2941 /* Identify TSO capable adapters */ 2942 if ((adapter->hw.mac.type > e1000_82544) && 2943 (adapter->hw.mac.type != e1000_82547)) 2944 ifp->if_capabilities |= IFCAP_TSO4; 2945 /* 2946 * By default only enable on PCI-E, this 2947 * can be overriden by ifconfig. 2948 */ 2949 if (adapter->hw.mac.type >= e1000_82571) 2950 ifp->if_capenable |= IFCAP_TSO4; 2951#endif 2952 2953 /* 2954 * Tell the upper layer(s) we support long frames. 2955 */ 2956 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2957 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2958 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2959 2960#ifdef DEVICE_POLLING 2961 ifp->if_capabilities |= IFCAP_POLLING; 2962#endif 2963 2964 /* 2965 * Specify the media types supported by this adapter and register 2966 * callbacks to update media and link information 2967 */ 2968 ifmedia_init(&adapter->media, IFM_IMASK, 2969 em_media_change, em_media_status); 2970 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2971 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2972 u_char fiber_type = IFM_1000_SX; /* default type */ 2973 2974 if (adapter->hw.mac.type == e1000_82545) 2975 fiber_type = IFM_1000_LX; 2976 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 2977 0, NULL); 2978 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); 2979 } else { 2980 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2981 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2982 0, NULL); 2983 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2984 0, NULL); 2985 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2986 0, NULL); 2987 if (adapter->hw.phy.type != e1000_phy_ife) { 2988 ifmedia_add(&adapter->media, 2989 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2990 ifmedia_add(&adapter->media, 2991 IFM_ETHER | IFM_1000_T, 0, NULL); 2992 } 2993 } 2994 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2995 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2996} 2997 2998 2999/********************************************************************* 3000 * 3001 * Workaround for SmartSpeed on 82541 and 82547 controllers 3002 * 3003 **********************************************************************/ 3004static void 3005em_smartspeed(struct adapter *adapter) 3006{ 3007 u16 phy_tmp; 3008 3009 if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || 3010 adapter->hw.mac.autoneg == 0 || 3011 (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 3012 return; 3013 3014 if (adapter->smartspeed == 0) { 3015 /* If Master/Slave config fault is asserted twice, 3016 * we assume back-to-back */ 3017 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 3018 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 3019 return; 3020 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 3021 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 3022 e1000_read_phy_reg(&adapter->hw, 3023 PHY_1000T_CTRL, &phy_tmp); 3024 if(phy_tmp & CR_1000T_MS_ENABLE) { 3025 phy_tmp &= ~CR_1000T_MS_ENABLE; 3026 e1000_write_phy_reg(&adapter->hw, 3027 PHY_1000T_CTRL, phy_tmp); 3028 adapter->smartspeed++; 3029 if(adapter->hw.mac.autoneg && 3030 !e1000_phy_setup_autoneg(&adapter->hw) && 3031 !e1000_read_phy_reg(&adapter->hw, 3032 PHY_CONTROL, &phy_tmp)) { 3033 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3034 MII_CR_RESTART_AUTO_NEG); 3035 e1000_write_phy_reg(&adapter->hw, 3036 PHY_CONTROL, phy_tmp); 3037 } 3038 } 3039 } 3040 return; 3041 } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { 3042 /* If still no link, perhaps using 2/3 pair cable */ 3043 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); 3044 phy_tmp |= CR_1000T_MS_ENABLE; 3045 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); 3046 if(adapter->hw.mac.autoneg && 3047 !e1000_phy_setup_autoneg(&adapter->hw) && 3048 !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { 3049 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3050 MII_CR_RESTART_AUTO_NEG); 3051 e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); 3052 } 3053 } 3054 /* Restart process after EM_SMARTSPEED_MAX iterations */ 3055 if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) 3056 adapter->smartspeed = 0; 3057} 3058 3059 3060/* 3061 * Manage DMA'able memory. 3062 */ 3063static void 3064em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3065{ 3066 if (error) 3067 return; 3068 *(bus_addr_t *) arg = segs[0].ds_addr; 3069} 3070 3071static int 3072em_dma_malloc(struct adapter *adapter, bus_size_t size, 3073 struct em_dma_alloc *dma, int mapflags) 3074{ 3075 int error; 3076 3077#if __FreeBSD_version >= 700000 3078 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 3079#else 3080 error = bus_dma_tag_create(NULL, /* parent */ 3081#endif 3082 EM_DBA_ALIGN, 0, /* alignment, bounds */ 3083 BUS_SPACE_MAXADDR, /* lowaddr */ 3084 BUS_SPACE_MAXADDR, /* highaddr */ 3085 NULL, NULL, /* filter, filterarg */ 3086 size, /* maxsize */ 3087 1, /* nsegments */ 3088 size, /* maxsegsize */ 3089 0, /* flags */ 3090 NULL, /* lockfunc */ 3091 NULL, /* lockarg */ 3092 &dma->dma_tag); 3093 if (error) { 3094 device_printf(adapter->dev, 3095 "%s: bus_dma_tag_create failed: %d\n", 3096 __func__, error); 3097 goto fail_0; 3098 } 3099 3100 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 3101 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); 3102 if (error) { 3103 device_printf(adapter->dev, 3104 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 3105 __func__, (uintmax_t)size, error); 3106 goto fail_2; 3107 } 3108 3109 dma->dma_paddr = 0; 3110 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3111 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 3112 if (error || dma->dma_paddr == 0) { 3113 device_printf(adapter->dev, 3114 "%s: bus_dmamap_load failed: %d\n", 3115 __func__, error); 3116 goto fail_3; 3117 } 3118 3119 return (0); 3120 3121fail_3: 3122 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3123fail_2: 3124 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3125 bus_dma_tag_destroy(dma->dma_tag); 3126fail_0: 3127 dma->dma_map = NULL; 3128 dma->dma_tag = NULL; 3129 3130 return (error); 3131} 3132 3133static void 3134em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) 3135{ 3136 if (dma->dma_tag == NULL) 3137 return; 3138 if (dma->dma_map != NULL) { 3139 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3140 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3141 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3142 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3143 dma->dma_map = NULL; 3144 } 3145 bus_dma_tag_destroy(dma->dma_tag); 3146 dma->dma_tag = NULL; 3147} 3148 3149 3150/********************************************************************* 3151 * 3152 * Allocate memory for tx_buffer structures. The tx_buffer stores all 3153 * the information needed to transmit a packet on the wire. 3154 * 3155 **********************************************************************/ 3156static int 3157em_allocate_transmit_structures(struct adapter *adapter) 3158{ 3159 device_t dev = adapter->dev; 3160 struct em_buffer *tx_buffer; 3161 int error; 3162 3163 /* 3164 * Create DMA tags for tx descriptors 3165 */ 3166#if __FreeBSD_version >= 700000 3167 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 3168#else 3169 if ((error = bus_dma_tag_create(NULL, /* parent */ 3170#endif 3171 1, 0, /* alignment, bounds */ 3172 BUS_SPACE_MAXADDR, /* lowaddr */ 3173 BUS_SPACE_MAXADDR, /* highaddr */ 3174 NULL, NULL, /* filter, filterarg */ 3175 EM_TSO_SIZE, /* maxsize */ 3176 EM_MAX_SCATTER, /* nsegments */ 3177 EM_TSO_SEG_SIZE, /* maxsegsize */ 3178 0, /* flags */ 3179 NULL, /* lockfunc */ 3180 NULL, /* lockarg */ 3181 &adapter->txtag)) != 0) { 3182 device_printf(dev, "Unable to allocate TX DMA tag\n"); 3183 goto fail; 3184 } 3185 3186 adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) * 3187 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 3188 if (adapter->tx_buffer_area == NULL) { 3189 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 3190 error = ENOMEM; 3191 goto fail; 3192 } 3193 3194 /* Create the descriptor buffer dma maps */ 3195 for (int i = 0; i < adapter->num_tx_desc; i++) { 3196 tx_buffer = &adapter->tx_buffer_area[i]; 3197 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map); 3198 if (error != 0) { 3199 device_printf(dev, "Unable to create TX DMA map\n"); 3200 goto fail; 3201 } 3202 tx_buffer->next_eop = -1; 3203 } 3204 3205 return (0); 3206fail: 3207 em_free_transmit_structures(adapter); 3208 return (error); 3209} 3210 3211/********************************************************************* 3212 * 3213 * (Re)Initialize transmit structures. 3214 * 3215 **********************************************************************/ 3216static void 3217em_setup_transmit_structures(struct adapter *adapter) 3218{ 3219 struct em_buffer *tx_buffer; 3220 3221 /* Clear the old ring contents */ 3222 bzero(adapter->tx_desc_base, 3223 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); 3224 3225 /* Free any existing TX buffers */ 3226 for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3227 tx_buffer = &adapter->tx_buffer_area[i]; 3228 bus_dmamap_sync(adapter->txtag, tx_buffer->map, 3229 BUS_DMASYNC_POSTWRITE); 3230 bus_dmamap_unload(adapter->txtag, tx_buffer->map); 3231 m_freem(tx_buffer->m_head); 3232 tx_buffer->m_head = NULL; 3233 tx_buffer->next_eop = -1; 3234 } 3235 3236 /* Reset state */ 3237 adapter->next_avail_tx_desc = 0; 3238 adapter->next_tx_to_clean = 0; 3239 adapter->num_tx_desc_avail = adapter->num_tx_desc; 3240 3241 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 3242 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3243 3244 return; 3245} 3246 3247/********************************************************************* 3248 * 3249 * Enable transmit unit. 3250 * 3251 **********************************************************************/ 3252static void 3253em_initialize_transmit_unit(struct adapter *adapter) 3254{ 3255 u32 tctl, tarc, tipg = 0; 3256 u64 bus_addr; 3257 3258 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 3259 /* Setup the Base and Length of the Tx Descriptor Ring */ 3260 bus_addr = adapter->txdma.dma_paddr; 3261 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0), 3262 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3263 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0), 3264 (u32)(bus_addr >> 32)); 3265 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0), 3266 (u32)bus_addr); 3267 /* Setup the HW Tx Head and Tail descriptor pointers */ 3268 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0); 3269 E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0); 3270 3271 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3272 E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)), 3273 E1000_READ_REG(&adapter->hw, E1000_TDLEN(0))); 3274 3275 /* Set the default values for the Tx Inter Packet Gap timer */ 3276 switch (adapter->hw.mac.type) { 3277 case e1000_82542: 3278 tipg = DEFAULT_82542_TIPG_IPGT; 3279 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3280 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3281 break; 3282 case e1000_80003es2lan: 3283 tipg = DEFAULT_82543_TIPG_IPGR1; 3284 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 3285 E1000_TIPG_IPGR2_SHIFT; 3286 break; 3287 default: 3288 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3289 (adapter->hw.phy.media_type == 3290 e1000_media_type_internal_serdes)) 3291 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 3292 else 3293 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 3294 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3295 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3296 } 3297 3298 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); 3299 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); 3300 if(adapter->hw.mac.type >= e1000_82540) 3301 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 3302 adapter->tx_abs_int_delay.value); 3303 3304 if ((adapter->hw.mac.type == e1000_82571) || 3305 (adapter->hw.mac.type == e1000_82572)) { 3306 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3307 tarc |= SPEED_MODE_BIT; 3308 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3309 } else if (adapter->hw.mac.type == e1000_80003es2lan) { 3310 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3311 tarc |= 1; 3312 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3313 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 3314 tarc |= 1; 3315 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 3316 } 3317 3318 /* Program the Transmit Control Register */ 3319 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 3320 tctl &= ~E1000_TCTL_CT; 3321 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3322 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3323 3324 if (adapter->hw.mac.type >= e1000_82571) 3325 tctl |= E1000_TCTL_MULR; 3326 3327 /* This write will effectively turn on the transmit unit. */ 3328 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 3329 3330 /* Setup Transmit Descriptor Base Settings */ 3331 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 3332 3333 if (adapter->tx_int_delay.value > 0) 3334 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3335} 3336 3337/********************************************************************* 3338 * 3339 * Free all transmit related data structures. 3340 * 3341 **********************************************************************/ 3342static void 3343em_free_transmit_structures(struct adapter *adapter) 3344{ 3345 struct em_buffer *tx_buffer; 3346 3347 INIT_DEBUGOUT("free_transmit_structures: begin"); 3348 3349 if (adapter->tx_buffer_area != NULL) { 3350 for (int i = 0; i < adapter->num_tx_desc; i++) { 3351 tx_buffer = &adapter->tx_buffer_area[i]; 3352 if (tx_buffer->m_head != NULL) { 3353 bus_dmamap_sync(adapter->txtag, tx_buffer->map, 3354 BUS_DMASYNC_POSTWRITE); 3355 bus_dmamap_unload(adapter->txtag, 3356 tx_buffer->map); 3357 m_freem(tx_buffer->m_head); 3358 tx_buffer->m_head = NULL; 3359 } else if (tx_buffer->map != NULL) 3360 bus_dmamap_unload(adapter->txtag, 3361 tx_buffer->map); 3362 if (tx_buffer->map != NULL) { 3363 bus_dmamap_destroy(adapter->txtag, 3364 tx_buffer->map); 3365 tx_buffer->map = NULL; 3366 } 3367 } 3368 } 3369 if (adapter->tx_buffer_area != NULL) { 3370 free(adapter->tx_buffer_area, M_DEVBUF); 3371 adapter->tx_buffer_area = NULL; 3372 } 3373 if (adapter->txtag != NULL) { 3374 bus_dma_tag_destroy(adapter->txtag); 3375 adapter->txtag = NULL; 3376 } 3377} 3378 3379/********************************************************************* 3380 * 3381 * The offload context needs to be set when we transfer the first 3382 * packet of a particular protocol (TCP/UDP). This routine has been 3383 * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) 3384 * 3385 **********************************************************************/ 3386static void 3387em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp, 3388 u32 *txd_upper, u32 *txd_lower) 3389{ 3390 struct e1000_context_desc *TXD; 3391 struct em_buffer *tx_buffer; 3392 struct ether_vlan_header *eh; 3393 struct ip *ip = NULL; 3394 struct ip6_hdr *ip6; 3395 struct tcp_hdr *th; 3396 int curr_txd, ehdrlen; 3397 u32 cmd, hdr_len, ip_hlen; 3398 u16 etype; 3399 u8 ipproto; 3400 3401 cmd = hdr_len = ipproto = 0; 3402 /* Setup checksum offload context. */ 3403 curr_txd = adapter->next_avail_tx_desc; 3404 tx_buffer = &adapter->tx_buffer_area[curr_txd]; 3405 TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; 3406 3407 /* 3408 * Determine where frame payload starts. 3409 * Jump over vlan headers if already present, 3410 * helpful for QinQ too. 3411 */ 3412 eh = mtod(mp, struct ether_vlan_header *); 3413 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3414 etype = ntohs(eh->evl_proto); 3415 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3416 } else { 3417 etype = ntohs(eh->evl_encap_proto); 3418 ehdrlen = ETHER_HDR_LEN; 3419 } 3420 3421 /* 3422 * We only support TCP/UDP for IPv4 and IPv6 for the moment. 3423 * TODO: Support SCTP too when it hits the tree. 3424 */ 3425 switch (etype) { 3426 case ETHERTYPE_IP: 3427 ip = (struct ip *)(mp->m_data + ehdrlen); 3428 ip_hlen = ip->ip_hl << 2; 3429 3430 /* Setup of IP header checksum. */ 3431 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 3432 /* 3433 * Start offset for header checksum calculation. 3434 * End offset for header checksum calculation. 3435 * Offset of place to put the checksum. 3436 */ 3437 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3438 TXD->lower_setup.ip_fields.ipcse = 3439 htole16(ehdrlen + ip_hlen); 3440 TXD->lower_setup.ip_fields.ipcso = 3441 ehdrlen + offsetof(struct ip, ip_sum); 3442 cmd |= E1000_TXD_CMD_IP; 3443 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 3444 } 3445 3446 if (mp->m_len < ehdrlen + ip_hlen) 3447 return; /* failure */ 3448 3449 hdr_len = ehdrlen + ip_hlen; 3450 ipproto = ip->ip_p; 3451 3452 break; 3453 case ETHERTYPE_IPV6: 3454 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3455 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ 3456 3457 if (mp->m_len < ehdrlen + ip_hlen) 3458 return; /* failure */ 3459 3460 /* IPv6 doesn't have a header checksum. */ 3461 3462 hdr_len = ehdrlen + ip_hlen; 3463 ipproto = ip6->ip6_nxt; 3464 3465 break; 3466 default: 3467 *txd_upper = 0; 3468 *txd_lower = 0; 3469 return; 3470 } 3471 3472 switch (ipproto) { 3473 case IPPROTO_TCP: 3474 if (mp->m_pkthdr.csum_flags & CSUM_TCP) { 3475 /* 3476 * Start offset for payload checksum calculation. 3477 * End offset for payload checksum calculation. 3478 * Offset of place to put the checksum. 3479 */ 3480 th = (struct tcp_hdr *)(mp->m_data + hdr_len); 3481 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3482 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3483 TXD->upper_setup.tcp_fields.tucso = 3484 hdr_len + offsetof(struct tcphdr, th_sum); 3485 cmd |= E1000_TXD_CMD_TCP; 3486 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3487 } 3488 break; 3489 case IPPROTO_UDP: 3490 { 3491 if (mp->m_pkthdr.csum_flags & CSUM_UDP) { 3492 /* 3493 * Start offset for header checksum calculation. 3494 * End offset for header checksum calculation. 3495 * Offset of place to put the checksum. 3496 */ 3497 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3498 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3499 TXD->upper_setup.tcp_fields.tucso = 3500 hdr_len + offsetof(struct udphdr, uh_sum); 3501 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3502 } 3503 /* Fall Thru */ 3504 } 3505 default: 3506 break; 3507 } 3508 3509 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 3510 E1000_TXD_DTYP_D; /* Data descr */ 3511 TXD->tcp_seg_setup.data = htole32(0); 3512 TXD->cmd_and_length = 3513 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); 3514 tx_buffer->m_head = NULL; 3515 tx_buffer->next_eop = -1; 3516 3517 if (++curr_txd == adapter->num_tx_desc) 3518 curr_txd = 0; 3519 3520 adapter->num_tx_desc_avail--; 3521 adapter->next_avail_tx_desc = curr_txd; 3522} 3523 3524 3525#if __FreeBSD_version >= 700000 3526/********************************************************************** 3527 * 3528 * Setup work for hardware segmentation offload (TSO) 3529 * 3530 **********************************************************************/ 3531static bool 3532em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper, 3533 u32 *txd_lower) 3534{ 3535 struct e1000_context_desc *TXD; 3536 struct em_buffer *tx_buffer; 3537 struct ether_vlan_header *eh; 3538 struct ip *ip; 3539 struct ip6_hdr *ip6; 3540 struct tcphdr *th; 3541 int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6; 3542 u16 etype; 3543 3544 /* 3545 * This function could/should be extended to support IP/IPv6 3546 * fragmentation as well. But as they say, one step at a time. 3547 */ 3548 3549 /* 3550 * Determine where frame payload starts. 3551 * Jump over vlan headers if already present, 3552 * helpful for QinQ too. 3553 */ 3554 eh = mtod(mp, struct ether_vlan_header *); 3555 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3556 etype = ntohs(eh->evl_proto); 3557 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3558 } else { 3559 etype = ntohs(eh->evl_encap_proto); 3560 ehdrlen = ETHER_HDR_LEN; 3561 } 3562 3563 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3564 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3565 return FALSE; /* -1 */ 3566 3567 /* 3568 * We only support TCP for IPv4 and IPv6 (notyet) for the moment. 3569 * TODO: Support SCTP too when it hits the tree. 3570 */ 3571 switch (etype) { 3572 case ETHERTYPE_IP: 3573 isip6 = 0; 3574 ip = (struct ip *)(mp->m_data + ehdrlen); 3575 if (ip->ip_p != IPPROTO_TCP) 3576 return FALSE; /* 0 */ 3577 ip->ip_len = 0; 3578 ip->ip_sum = 0; 3579 ip_hlen = ip->ip_hl << 2; 3580 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3581 return FALSE; /* -1 */ 3582 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3583#if 1 3584 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3585 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3586#else 3587 th->th_sum = mp->m_pkthdr.csum_data; 3588#endif 3589 break; 3590 case ETHERTYPE_IPV6: 3591 isip6 = 1; 3592 return FALSE; /* Not supported yet. */ 3593 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3594 if (ip6->ip6_nxt != IPPROTO_TCP) 3595 return FALSE; /* 0 */ 3596 ip6->ip6_plen = 0; 3597 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */ 3598 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3599 return FALSE; /* -1 */ 3600 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 3601#if 0 3602 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst, 3603 htons(IPPROTO_TCP)); /* XXX: function notyet. */ 3604#else 3605 th->th_sum = mp->m_pkthdr.csum_data; 3606#endif 3607 break; 3608 default: 3609 return FALSE; 3610 } 3611 hdr_len = ehdrlen + ip_hlen + (th->th_off << 2); 3612 3613 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 3614 E1000_TXD_DTYP_D | /* Data descr type */ 3615 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 3616 3617 /* IP and/or TCP header checksum calculation and insertion. */ 3618 *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) | 3619 E1000_TXD_POPTS_TXSM) << 8; 3620 3621 curr_txd = adapter->next_avail_tx_desc; 3622 tx_buffer = &adapter->tx_buffer_area[curr_txd]; 3623 TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; 3624 3625 /* IPv6 doesn't have a header checksum. */ 3626 if (!isip6) { 3627 /* 3628 * Start offset for header checksum calculation. 3629 * End offset for header checksum calculation. 3630 * Offset of place put the checksum. 3631 */ 3632 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3633 TXD->lower_setup.ip_fields.ipcse = 3634 htole16(ehdrlen + ip_hlen - 1); 3635 TXD->lower_setup.ip_fields.ipcso = 3636 ehdrlen + offsetof(struct ip, ip_sum); 3637 } 3638 /* 3639 * Start offset for payload checksum calculation. 3640 * End offset for payload checksum calculation. 3641 * Offset of place to put the checksum. 3642 */ 3643 TXD->upper_setup.tcp_fields.tucss = 3644 ehdrlen + ip_hlen; 3645 TXD->upper_setup.tcp_fields.tucse = 0; 3646 TXD->upper_setup.tcp_fields.tucso = 3647 ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum); 3648 /* 3649 * Payload size per packet w/o any headers. 3650 * Length of all headers up to payload. 3651 */ 3652 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); 3653 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 3654 3655 TXD->cmd_and_length = htole32(adapter->txd_cmd | 3656 E1000_TXD_CMD_DEXT | /* Extended descr */ 3657 E1000_TXD_CMD_TSE | /* TSE context */ 3658 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */ 3659 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 3660 (mp->m_pkthdr.len - (hdr_len))); /* Total len */ 3661 3662 tx_buffer->m_head = NULL; 3663 tx_buffer->next_eop = -1; 3664 3665 if (++curr_txd == adapter->num_tx_desc) 3666 curr_txd = 0; 3667 3668 adapter->num_tx_desc_avail--; 3669 adapter->next_avail_tx_desc = curr_txd; 3670 adapter->tx_tso = TRUE; 3671 3672 return TRUE; 3673} 3674 3675#endif /* __FreeBSD_version >= 700000 */ 3676 3677/********************************************************************** 3678 * 3679 * Examine each tx_buffer in the used queue. If the hardware is done 3680 * processing the packet then free associated resources. The 3681 * tx_buffer is put back on the free queue. 3682 * 3683 **********************************************************************/ 3684static void 3685em_txeof(struct adapter *adapter) 3686{ 3687 int first, last, done, num_avail; 3688 struct em_buffer *tx_buffer; 3689 struct e1000_tx_desc *tx_desc, *eop_desc; 3690 struct ifnet *ifp = adapter->ifp; 3691 3692 EM_TX_LOCK_ASSERT(adapter); 3693 3694 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) 3695 return; 3696 3697 num_avail = adapter->num_tx_desc_avail; 3698 first = adapter->next_tx_to_clean; 3699 tx_desc = &adapter->tx_desc_base[first]; 3700 tx_buffer = &adapter->tx_buffer_area[first]; 3701 last = tx_buffer->next_eop; 3702 eop_desc = &adapter->tx_desc_base[last]; 3703 3704 /* 3705 * What this does is get the index of the 3706 * first descriptor AFTER the EOP of the 3707 * first packet, that way we can do the 3708 * simple comparison on the inner while loop. 3709 */ 3710 if (++last == adapter->num_tx_desc) 3711 last = 0; 3712 done = last; 3713 3714 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 3715 BUS_DMASYNC_POSTREAD); 3716 3717 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3718 /* We clean the range of the packet */ 3719 while (first != done) { 3720 tx_desc->upper.data = 0; 3721 tx_desc->lower.data = 0; 3722 tx_desc->buffer_addr = 0; 3723 num_avail++; 3724 3725 if (tx_buffer->m_head) { 3726 ifp->if_opackets++; 3727 bus_dmamap_sync(adapter->txtag, 3728 tx_buffer->map, 3729 BUS_DMASYNC_POSTWRITE); 3730 bus_dmamap_unload(adapter->txtag, 3731 tx_buffer->map); 3732 3733 m_freem(tx_buffer->m_head); 3734 tx_buffer->m_head = NULL; 3735 } 3736 tx_buffer->next_eop = -1; 3737 3738 if (++first == adapter->num_tx_desc) 3739 first = 0; 3740 3741 tx_buffer = &adapter->tx_buffer_area[first]; 3742 tx_desc = &adapter->tx_desc_base[first]; 3743 } 3744 /* See if we can continue to the next packet */ 3745 last = tx_buffer->next_eop; 3746 if (last != -1) { 3747 eop_desc = &adapter->tx_desc_base[last]; 3748 /* Get new done point */ 3749 if (++last == adapter->num_tx_desc) last = 0; 3750 done = last; 3751 } else 3752 break; 3753 } 3754 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 3755 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3756 3757 adapter->next_tx_to_clean = first; 3758 3759 /* 3760 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack 3761 * that it is OK to send packets. 3762 * If there are no pending descriptors, clear the timeout. Otherwise, 3763 * if some descriptors have been freed, restart the timeout. 3764 */ 3765 if (num_avail > EM_TX_CLEANUP_THRESHOLD) { 3766 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3767 /* All clean, turn off the timer */ 3768 if (num_avail == adapter->num_tx_desc) { 3769 adapter->watchdog_timer = 0; 3770 } else 3771 /* Some cleaned, reset the timer */ 3772 if (num_avail != adapter->num_tx_desc_avail) 3773 adapter->watchdog_timer = EM_TX_TIMEOUT; 3774 } 3775 adapter->num_tx_desc_avail = num_avail; 3776 return; 3777} 3778 3779/********************************************************************* 3780 * 3781 * When Link is lost sometimes there is work still in the TX ring 3782 * which will result in a watchdog, rather than allow that do an 3783 * attempted cleanup and then reinit here. Note that this has been 3784 * seens mostly with fiber adapters. 3785 * 3786 **********************************************************************/ 3787static void 3788em_tx_purge(struct adapter *adapter) 3789{ 3790 if ((!adapter->link_active) && (adapter->watchdog_timer)) { 3791 EM_TX_LOCK(adapter); 3792 em_txeof(adapter); 3793 EM_TX_UNLOCK(adapter); 3794 if (adapter->watchdog_timer) { /* Still not clean? */ 3795 adapter->watchdog_timer = 0; 3796 em_init_locked(adapter); 3797 } 3798 } 3799} 3800 3801/********************************************************************* 3802 * 3803 * Get a buffer from system mbuf buffer pool. 3804 * 3805 **********************************************************************/ 3806static int 3807em_get_buf(struct adapter *adapter, int i) 3808{ 3809 struct mbuf *m; 3810 bus_dma_segment_t segs[1]; 3811 bus_dmamap_t map; 3812 struct em_buffer *rx_buffer; 3813 int error, nsegs; 3814 3815 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 3816 if (m == NULL) { 3817 adapter->mbuf_cluster_failed++; 3818 return (ENOBUFS); 3819 } 3820 m->m_len = m->m_pkthdr.len = MCLBYTES; 3821 3822 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 3823 m_adj(m, ETHER_ALIGN); 3824 3825 /* 3826 * Using memory from the mbuf cluster pool, invoke the 3827 * bus_dma machinery to arrange the memory mapping. 3828 */ 3829 error = bus_dmamap_load_mbuf_sg(adapter->rxtag, 3830 adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT); 3831 if (error != 0) { 3832 m_free(m); 3833 return (error); 3834 } 3835 3836 /* If nsegs is wrong then the stack is corrupt. */ 3837 KASSERT(nsegs == 1, ("Too many segments returned!")); 3838 3839 rx_buffer = &adapter->rx_buffer_area[i]; 3840 if (rx_buffer->m_head != NULL) 3841 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 3842 3843 map = rx_buffer->map; 3844 rx_buffer->map = adapter->rx_sparemap; 3845 adapter->rx_sparemap = map; 3846 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD); 3847 rx_buffer->m_head = m; 3848 3849 adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr); 3850 return (0); 3851} 3852 3853/********************************************************************* 3854 * 3855 * Allocate memory for rx_buffer structures. Since we use one 3856 * rx_buffer per received packet, the maximum number of rx_buffer's 3857 * that we'll need is equal to the number of receive descriptors 3858 * that we've allocated. 3859 * 3860 **********************************************************************/ 3861static int 3862em_allocate_receive_structures(struct adapter *adapter) 3863{ 3864 device_t dev = adapter->dev; 3865 struct em_buffer *rx_buffer; 3866 int i, error; 3867 3868 adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) * 3869 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 3870 if (adapter->rx_buffer_area == NULL) { 3871 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3872 return (ENOMEM); 3873 } 3874 3875#if __FreeBSD_version >= 700000 3876 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 3877#else 3878 error = bus_dma_tag_create(NULL, /* parent */ 3879#endif 3880 1, 0, /* alignment, bounds */ 3881 BUS_SPACE_MAXADDR, /* lowaddr */ 3882 BUS_SPACE_MAXADDR, /* highaddr */ 3883 NULL, NULL, /* filter, filterarg */ 3884 MCLBYTES, /* maxsize */ 3885 1, /* nsegments */ 3886 MCLBYTES, /* maxsegsize */ 3887 0, /* flags */ 3888 NULL, /* lockfunc */ 3889 NULL, /* lockarg */ 3890 &adapter->rxtag); 3891 if (error) { 3892 device_printf(dev, "%s: bus_dma_tag_create failed %d\n", 3893 __func__, error); 3894 goto fail; 3895 } 3896 3897 /* Create the spare map (used by getbuf) */ 3898 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, 3899 &adapter->rx_sparemap); 3900 if (error) { 3901 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3902 __func__, error); 3903 goto fail; 3904 } 3905 3906 rx_buffer = adapter->rx_buffer_area; 3907 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 3908 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, 3909 &rx_buffer->map); 3910 if (error) { 3911 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3912 __func__, error); 3913 goto fail; 3914 } 3915 } 3916 3917 return (0); 3918 3919fail: 3920 em_free_receive_structures(adapter); 3921 return (error); 3922} 3923 3924/********************************************************************* 3925 * 3926 * (Re)initialize receive structures. 3927 * 3928 **********************************************************************/ 3929static int 3930em_setup_receive_structures(struct adapter *adapter) 3931{ 3932 struct em_buffer *rx_buffer; 3933 int i, error; 3934 3935 /* Reset descriptor ring */ 3936 bzero(adapter->rx_desc_base, 3937 (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc); 3938 3939 /* Free current RX buffers. */ 3940 rx_buffer = adapter->rx_buffer_area; 3941 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 3942 if (rx_buffer->m_head != NULL) { 3943 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, 3944 BUS_DMASYNC_POSTREAD); 3945 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 3946 m_freem(rx_buffer->m_head); 3947 rx_buffer->m_head = NULL; 3948 } 3949 } 3950 3951 /* Allocate new ones. */ 3952 for (i = 0; i < adapter->num_rx_desc; i++) { 3953 error = em_get_buf(adapter, i); 3954 if (error) 3955 return (error); 3956 } 3957 3958 /* Setup our descriptor pointers */ 3959 adapter->next_rx_desc_to_check = 0; 3960 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 3961 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3962 3963 return (0); 3964} 3965 3966/********************************************************************* 3967 * 3968 * Enable receive unit. 3969 * 3970 **********************************************************************/ 3971static void 3972em_initialize_receive_unit(struct adapter *adapter) 3973{ 3974 struct ifnet *ifp = adapter->ifp; 3975 u64 bus_addr; 3976 u32 rctl, rxcsum; 3977 3978 INIT_DEBUGOUT("em_initialize_receive_unit: begin"); 3979 3980 /* 3981 * Make sure receives are disabled while setting 3982 * up the descriptor ring 3983 */ 3984 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 3985 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3986 3987 if(adapter->hw.mac.type >= e1000_82540) { 3988 E1000_WRITE_REG(&adapter->hw, E1000_RADV, 3989 adapter->rx_abs_int_delay.value); 3990 /* 3991 * Set the interrupt throttling rate. Value is calculated 3992 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3993 */ 3994#define MAX_INTS_PER_SEC 8000 3995#define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256) 3996 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR); 3997 } 3998 3999 /* Setup the Base and Length of the Rx Descriptor Ring */ 4000 bus_addr = adapter->rxdma.dma_paddr; 4001 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0), 4002 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 4003 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0), 4004 (u32)(bus_addr >> 32)); 4005 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0), 4006 (u32)bus_addr); 4007 4008 /* Setup the Receive Control Register */ 4009 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4010 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 4011 E1000_RCTL_RDMTS_HALF | 4012 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4013 4014 /* Make sure VLAN Filters are off */ 4015 rctl &= ~E1000_RCTL_VFE; 4016 4017 if (e1000_tbi_sbp_enabled_82543(&adapter->hw)) 4018 rctl |= E1000_RCTL_SBP; 4019 else 4020 rctl &= ~E1000_RCTL_SBP; 4021 4022 switch (adapter->rx_buffer_len) { 4023 default: 4024 case 2048: 4025 rctl |= E1000_RCTL_SZ_2048; 4026 break; 4027 case 4096: 4028 rctl |= E1000_RCTL_SZ_4096 | 4029 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4030 break; 4031 case 8192: 4032 rctl |= E1000_RCTL_SZ_8192 | 4033 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4034 break; 4035 case 16384: 4036 rctl |= E1000_RCTL_SZ_16384 | 4037 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4038 break; 4039 } 4040 4041 if (ifp->if_mtu > ETHERMTU) 4042 rctl |= E1000_RCTL_LPE; 4043 else 4044 rctl &= ~E1000_RCTL_LPE; 4045 4046 /* Enable 82543 Receive Checksum Offload for TCP and UDP */ 4047 if ((adapter->hw.mac.type >= e1000_82543) && 4048 (ifp->if_capenable & IFCAP_RXCSUM)) { 4049 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); 4050 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 4051 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); 4052 } 4053 4054 /* 4055 ** XXX TEMPORARY WORKAROUND: on some systems with 82573 4056 ** long latencies are observed, like Lenovo X60. This 4057 ** change eliminates the problem, but since having positive 4058 ** values in RDTR is a known source of problems on other 4059 ** platforms another solution is being sought. 4060 */ 4061 if (adapter->hw.mac.type == e1000_82573) 4062 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20); 4063 4064 /* Enable Receives */ 4065 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4066 4067 /* 4068 * Setup the HW Rx Head and 4069 * Tail Descriptor Pointers 4070 */ 4071 E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0); 4072 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1); 4073 4074 return; 4075} 4076 4077/********************************************************************* 4078 * 4079 * Free receive related data structures. 4080 * 4081 **********************************************************************/ 4082static void 4083em_free_receive_structures(struct adapter *adapter) 4084{ 4085 struct em_buffer *rx_buffer; 4086 int i; 4087 4088 INIT_DEBUGOUT("free_receive_structures: begin"); 4089 4090 if (adapter->rx_sparemap) { 4091 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap); 4092 adapter->rx_sparemap = NULL; 4093 } 4094 4095 /* Cleanup any existing buffers */ 4096 if (adapter->rx_buffer_area != NULL) { 4097 rx_buffer = adapter->rx_buffer_area; 4098 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4099 if (rx_buffer->m_head != NULL) { 4100 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, 4101 BUS_DMASYNC_POSTREAD); 4102 bus_dmamap_unload(adapter->rxtag, 4103 rx_buffer->map); 4104 m_freem(rx_buffer->m_head); 4105 rx_buffer->m_head = NULL; 4106 } else if (rx_buffer->map != NULL) 4107 bus_dmamap_unload(adapter->rxtag, 4108 rx_buffer->map); 4109 if (rx_buffer->map != NULL) { 4110 bus_dmamap_destroy(adapter->rxtag, 4111 rx_buffer->map); 4112 rx_buffer->map = NULL; 4113 } 4114 } 4115 } 4116 4117 if (adapter->rx_buffer_area != NULL) { 4118 free(adapter->rx_buffer_area, M_DEVBUF); 4119 adapter->rx_buffer_area = NULL; 4120 } 4121 4122 if (adapter->rxtag != NULL) { 4123 bus_dma_tag_destroy(adapter->rxtag); 4124 adapter->rxtag = NULL; 4125 } 4126} 4127 4128/********************************************************************* 4129 * 4130 * This routine executes in interrupt context. It replenishes 4131 * the mbufs in the descriptor and sends data which has been 4132 * dma'ed into host memory to upper layer. 4133 * 4134 * We loop at most count times if count is > 0, or until done if 4135 * count < 0. 4136 * 4137 *********************************************************************/ 4138static int 4139em_rxeof(struct adapter *adapter, int count) 4140{ 4141 struct ifnet *ifp; 4142 struct mbuf *mp; 4143 u8 status, accept_frame = 0, eop = 0; 4144 u16 len, desc_len, prev_len_adj; 4145 int i; 4146 struct e1000_rx_desc *current_desc; 4147 4148 ifp = adapter->ifp; 4149 i = adapter->next_rx_desc_to_check; 4150 current_desc = &adapter->rx_desc_base[i]; 4151 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4152 BUS_DMASYNC_POSTREAD); 4153 4154 if (!((current_desc->status) & E1000_RXD_STAT_DD)) 4155 return (0); 4156 4157 while ((current_desc->status & E1000_RXD_STAT_DD) && 4158 (count != 0) && 4159 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4160 struct mbuf *m = NULL; 4161 4162 mp = adapter->rx_buffer_area[i].m_head; 4163 /* 4164 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT 4165 * needs to access the last received byte in the mbuf. 4166 */ 4167 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map, 4168 BUS_DMASYNC_POSTREAD); 4169 4170 accept_frame = 1; 4171 prev_len_adj = 0; 4172 desc_len = le16toh(current_desc->length); 4173 status = current_desc->status; 4174 if (status & E1000_RXD_STAT_EOP) { 4175 count--; 4176 eop = 1; 4177 if (desc_len < ETHER_CRC_LEN) { 4178 len = 0; 4179 prev_len_adj = ETHER_CRC_LEN - desc_len; 4180 } else 4181 len = desc_len - ETHER_CRC_LEN; 4182 } else { 4183 eop = 0; 4184 len = desc_len; 4185 } 4186 4187 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 4188 u8 last_byte; 4189 u32 pkt_len = desc_len; 4190 4191 if (adapter->fmp != NULL) 4192 pkt_len += adapter->fmp->m_pkthdr.len; 4193 4194 last_byte = *(mtod(mp, caddr_t) + desc_len - 1); 4195 if (TBI_ACCEPT(&adapter->hw, status, 4196 current_desc->errors, pkt_len, last_byte, 4197 adapter->min_frame_size, adapter->max_frame_size)) { 4198 e1000_tbi_adjust_stats_82543(&adapter->hw, 4199 &adapter->stats, pkt_len, 4200 adapter->hw.mac.addr, 4201 adapter->max_frame_size); 4202 if (len > 0) 4203 len--; 4204 } else 4205 accept_frame = 0; 4206 } 4207 4208 if (accept_frame) { 4209 if (em_get_buf(adapter, i) != 0) { 4210 ifp->if_iqdrops++; 4211 goto discard; 4212 } 4213 4214 /* Assign correct length to the current fragment */ 4215 mp->m_len = len; 4216 4217 if (adapter->fmp == NULL) { 4218 mp->m_pkthdr.len = len; 4219 adapter->fmp = mp; /* Store the first mbuf */ 4220 adapter->lmp = mp; 4221 } else { 4222 /* Chain mbuf's together */ 4223 mp->m_flags &= ~M_PKTHDR; 4224 /* 4225 * Adjust length of previous mbuf in chain if 4226 * we received less than 4 bytes in the last 4227 * descriptor. 4228 */ 4229 if (prev_len_adj > 0) { 4230 adapter->lmp->m_len -= prev_len_adj; 4231 adapter->fmp->m_pkthdr.len -= 4232 prev_len_adj; 4233 } 4234 adapter->lmp->m_next = mp; 4235 adapter->lmp = adapter->lmp->m_next; 4236 adapter->fmp->m_pkthdr.len += len; 4237 } 4238 4239 if (eop) { 4240 adapter->fmp->m_pkthdr.rcvif = ifp; 4241 ifp->if_ipackets++; 4242 em_receive_checksum(adapter, current_desc, 4243 adapter->fmp); 4244#ifndef __NO_STRICT_ALIGNMENT 4245 if (adapter->max_frame_size > 4246 (MCLBYTES - ETHER_ALIGN) && 4247 em_fixup_rx(adapter) != 0) 4248 goto skip; 4249#endif 4250 if (status & E1000_RXD_STAT_VP) { 4251#if __FreeBSD_version < 700000 4252 VLAN_INPUT_TAG_NEW(ifp, adapter->fmp, 4253 (le16toh(current_desc->special) & 4254 E1000_RXD_SPC_VLAN_MASK)); 4255#else 4256 adapter->fmp->m_pkthdr.ether_vtag = 4257 (le16toh(current_desc->special) & 4258 E1000_RXD_SPC_VLAN_MASK); 4259 adapter->fmp->m_flags |= M_VLANTAG; 4260#endif 4261 } 4262#ifndef __NO_STRICT_ALIGNMENT 4263skip: 4264#endif 4265 m = adapter->fmp; 4266 adapter->fmp = NULL; 4267 adapter->lmp = NULL; 4268 } 4269 } else { 4270 ifp->if_ierrors++; 4271discard: 4272 /* Reuse loaded DMA map and just update mbuf chain */ 4273 mp = adapter->rx_buffer_area[i].m_head; 4274 mp->m_len = mp->m_pkthdr.len = MCLBYTES; 4275 mp->m_data = mp->m_ext.ext_buf; 4276 mp->m_next = NULL; 4277 if (adapter->max_frame_size <= 4278 (MCLBYTES - ETHER_ALIGN)) 4279 m_adj(mp, ETHER_ALIGN); 4280 if (adapter->fmp != NULL) { 4281 m_freem(adapter->fmp); 4282 adapter->fmp = NULL; 4283 adapter->lmp = NULL; 4284 } 4285 m = NULL; 4286 } 4287 4288 /* Zero out the receive descriptors status. */ 4289 current_desc->status = 0; 4290 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4291 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4292 4293 /* Advance our pointers to the next descriptor. */ 4294 if (++i == adapter->num_rx_desc) 4295 i = 0; 4296 if (m != NULL) { 4297 adapter->next_rx_desc_to_check = i; 4298#ifdef EM_LEGACY_IRQ 4299 EM_CORE_UNLOCK(adapter); 4300 (*ifp->if_input)(ifp, m); 4301 EM_CORE_LOCK(adapter); 4302#else 4303 /* Already running unlocked */ 4304 (*ifp->if_input)(ifp, m); 4305#endif 4306 i = adapter->next_rx_desc_to_check; 4307 } 4308 current_desc = &adapter->rx_desc_base[i]; 4309 } 4310 adapter->next_rx_desc_to_check = i; 4311 4312 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ 4313 if (--i < 0) 4314 i = adapter->num_rx_desc - 1; 4315 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); 4316 if (!((current_desc->status) & E1000_RXD_STAT_DD)) 4317 return (0); 4318 4319 return (1); 4320} 4321 4322#ifndef __NO_STRICT_ALIGNMENT 4323/* 4324 * When jumbo frames are enabled we should realign entire payload on 4325 * architecures with strict alignment. This is serious design mistake of 8254x 4326 * as it nullifies DMA operations. 8254x just allows RX buffer size to be 4327 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its 4328 * payload. On architecures without strict alignment restrictions 8254x still 4329 * performs unaligned memory access which would reduce the performance too. 4330 * To avoid copying over an entire frame to align, we allocate a new mbuf and 4331 * copy ethernet header to the new mbuf. The new mbuf is prepended into the 4332 * existing mbuf chain. 4333 * 4334 * Be aware, best performance of the 8254x is achived only when jumbo frame is 4335 * not used at all on architectures with strict alignment. 4336 */ 4337static int 4338em_fixup_rx(struct adapter *adapter) 4339{ 4340 struct mbuf *m, *n; 4341 int error; 4342 4343 error = 0; 4344 m = adapter->fmp; 4345 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { 4346 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); 4347 m->m_data += ETHER_HDR_LEN; 4348 } else { 4349 MGETHDR(n, M_DONTWAIT, MT_DATA); 4350 if (n != NULL) { 4351 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); 4352 m->m_data += ETHER_HDR_LEN; 4353 m->m_len -= ETHER_HDR_LEN; 4354 n->m_len = ETHER_HDR_LEN; 4355 M_MOVE_PKTHDR(n, m); 4356 n->m_next = m; 4357 adapter->fmp = n; 4358 } else { 4359 adapter->dropped_pkts++; 4360 m_freem(adapter->fmp); 4361 adapter->fmp = NULL; 4362 error = ENOMEM; 4363 } 4364 } 4365 4366 return (error); 4367} 4368#endif 4369 4370/********************************************************************* 4371 * 4372 * Verify that the hardware indicated that the checksum is valid. 4373 * Inform the stack about the status of checksum so that stack 4374 * doesn't spend time verifying the checksum. 4375 * 4376 *********************************************************************/ 4377static void 4378em_receive_checksum(struct adapter *adapter, 4379 struct e1000_rx_desc *rx_desc, struct mbuf *mp) 4380{ 4381 /* 82543 or newer only */ 4382 if ((adapter->hw.mac.type < e1000_82543) || 4383 /* Ignore Checksum bit is set */ 4384 (rx_desc->status & E1000_RXD_STAT_IXSM)) { 4385 mp->m_pkthdr.csum_flags = 0; 4386 return; 4387 } 4388 4389 if (rx_desc->status & E1000_RXD_STAT_IPCS) { 4390 /* Did it pass? */ 4391 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { 4392 /* IP Checksum Good */ 4393 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4394 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4395 4396 } else { 4397 mp->m_pkthdr.csum_flags = 0; 4398 } 4399 } 4400 4401 if (rx_desc->status & E1000_RXD_STAT_TCPCS) { 4402 /* Did it pass? */ 4403 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { 4404 mp->m_pkthdr.csum_flags |= 4405 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4406 mp->m_pkthdr.csum_data = htons(0xffff); 4407 } 4408 } 4409} 4410 4411/* 4412 * This turns on the hardware offload of the VLAN 4413 * tag insertion and strip 4414 */ 4415static void 4416em_enable_hw_vlans(struct adapter *adapter) 4417{ 4418 u32 ctrl; 4419 4420 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4421 ctrl |= E1000_CTRL_VME; 4422 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4423} 4424 4425#define QUEUE_MASK 0x01500000 4426 4427static void 4428em_enable_intr(struct adapter *adapter) 4429{ 4430 u32 ims_mask = IMS_ENABLE_MASK; 4431 4432 if (adapter->msix) { 4433 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, QUEUE_MASK); 4434 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, QUEUE_MASK); 4435 ims_mask |= QUEUE_MASK; 4436 } 4437 E1000_WRITE_REG(&adapter->hw, E1000_IMS, ims_mask); 4438} 4439 4440static void 4441em_disable_intr(struct adapter *adapter) 4442{ 4443 if (adapter->msix) { 4444 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); 4445 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); 4446 } 4447 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 4448} 4449 4450/* 4451 * Bit of a misnomer, what this really means is 4452 * to enable OS management of the system... aka 4453 * to disable special hardware management features 4454 */ 4455static void 4456em_init_manageability(struct adapter *adapter) 4457{ 4458 /* A shared code workaround */ 4459#define E1000_82542_MANC2H E1000_MANC2H 4460 if (adapter->has_manage) { 4461 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4462 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4463 4464 /* disable hardware interception of ARP */ 4465 manc &= ~(E1000_MANC_ARP_EN); 4466 4467 /* enable receiving management packets to the host */ 4468 if (adapter->hw.mac.type >= e1000_82571) { 4469 manc |= E1000_MANC_EN_MNG2HOST; 4470#define E1000_MNG2HOST_PORT_623 (1 << 5) 4471#define E1000_MNG2HOST_PORT_664 (1 << 6) 4472 manc2h |= E1000_MNG2HOST_PORT_623; 4473 manc2h |= E1000_MNG2HOST_PORT_664; 4474 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4475 } 4476 4477 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4478 } 4479} 4480 4481/* 4482 * Give control back to hardware management 4483 * controller if there is one. 4484 */ 4485static void 4486em_release_manageability(struct adapter *adapter) 4487{ 4488 if (adapter->has_manage) { 4489 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4490 4491 /* re-enable hardware interception of ARP */ 4492 manc |= E1000_MANC_ARP_EN; 4493 4494 if (adapter->hw.mac.type >= e1000_82571) 4495 manc &= ~E1000_MANC_EN_MNG2HOST; 4496 4497 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4498 } 4499} 4500 4501/* 4502 * em_get_hw_control sets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4503 * For ASF and Pass Through versions of f/w this means that 4504 * the driver is loaded. For AMT version (only with 82573) 4505 * of the f/w this means that the network i/f is open. 4506 * 4507 */ 4508static void 4509em_get_hw_control(struct adapter *adapter) 4510{ 4511 u32 ctrl_ext, swsm; 4512 4513 /* Let firmware know the driver has taken over */ 4514 switch (adapter->hw.mac.type) { 4515 case e1000_82573: 4516 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4517 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4518 swsm | E1000_SWSM_DRV_LOAD); 4519 break; 4520 case e1000_82571: 4521 case e1000_82572: 4522 case e1000_80003es2lan: 4523 case e1000_ich8lan: 4524 case e1000_ich9lan: 4525 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4526 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4527 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4528 break; 4529 default: 4530 break; 4531 } 4532} 4533 4534/* 4535 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4536 * For ASF and Pass Through versions of f/w this means that the 4537 * driver is no longer loaded. For AMT version (only with 82573) i 4538 * of the f/w this means that the network i/f is closed. 4539 * 4540 */ 4541static void 4542em_release_hw_control(struct adapter *adapter) 4543{ 4544 u32 ctrl_ext, swsm; 4545 4546 /* Let firmware taken over control of h/w */ 4547 switch (adapter->hw.mac.type) { 4548 case e1000_82573: 4549 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4550 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4551 swsm & ~E1000_SWSM_DRV_LOAD); 4552 break; 4553 case e1000_82571: 4554 case e1000_82572: 4555 case e1000_80003es2lan: 4556 case e1000_ich8lan: 4557 case e1000_ich9lan: 4558 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4559 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4560 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4561 break; 4562 default: 4563 break; 4564 4565 } 4566} 4567 4568static int 4569em_is_valid_ether_addr(u8 *addr) 4570{ 4571 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4572 4573 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4574 return (FALSE); 4575 } 4576 4577 return (TRUE); 4578} 4579 4580/* 4581 * Enable PCI Wake On Lan capability 4582 */ 4583void 4584em_enable_wakeup(device_t dev) 4585{ 4586 u16 cap, status; 4587 u8 id; 4588 4589 /* First find the capabilities pointer*/ 4590 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 4591 /* Read the PM Capabilities */ 4592 id = pci_read_config(dev, cap, 1); 4593 if (id != PCIY_PMG) /* Something wrong */ 4594 return; 4595 /* OK, we have the power capabilities, so 4596 now get the status register */ 4597 cap += PCIR_POWER_STATUS; 4598 status = pci_read_config(dev, cap, 2); 4599 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4600 pci_write_config(dev, cap, status, 2); 4601 return; 4602} 4603 4604 4605/********************************************************************* 4606* 82544 Coexistence issue workaround. 4607* There are 2 issues. 4608* 1. Transmit Hang issue. 4609* To detect this issue, following equation can be used... 4610* SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 4611* If SUM[3:0] is in between 1 to 4, we will have this issue. 4612* 4613* 2. DAC issue. 4614* To detect this issue, following equation can be used... 4615* SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 4616* If SUM[3:0] is in between 9 to c, we will have this issue. 4617* 4618* 4619* WORKAROUND: 4620* Make sure we do not have ending address 4621* as 1,2,3,4(Hang) or 9,a,b,c (DAC) 4622* 4623*************************************************************************/ 4624static u32 4625em_fill_descriptors (bus_addr_t address, u32 length, 4626 PDESC_ARRAY desc_array) 4627{ 4628 u32 safe_terminator; 4629 4630 /* Since issue is sensitive to length and address.*/ 4631 /* Let us first check the address...*/ 4632 if (length <= 4) { 4633 desc_array->descriptor[0].address = address; 4634 desc_array->descriptor[0].length = length; 4635 desc_array->elements = 1; 4636 return (desc_array->elements); 4637 } 4638 safe_terminator = (u32)((((u32)address & 0x7) + 4639 (length & 0xF)) & 0xF); 4640 /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */ 4641 if (safe_terminator == 0 || 4642 (safe_terminator > 4 && 4643 safe_terminator < 9) || 4644 (safe_terminator > 0xC && 4645 safe_terminator <= 0xF)) { 4646 desc_array->descriptor[0].address = address; 4647 desc_array->descriptor[0].length = length; 4648 desc_array->elements = 1; 4649 return (desc_array->elements); 4650 } 4651 4652 desc_array->descriptor[0].address = address; 4653 desc_array->descriptor[0].length = length - 4; 4654 desc_array->descriptor[1].address = address + (length - 4); 4655 desc_array->descriptor[1].length = 4; 4656 desc_array->elements = 2; 4657 return (desc_array->elements); 4658} 4659 4660/********************************************************************** 4661 * 4662 * Update the board statistics counters. 4663 * 4664 **********************************************************************/ 4665static void 4666em_update_stats_counters(struct adapter *adapter) 4667{ 4668 struct ifnet *ifp; 4669 4670 if(adapter->hw.phy.media_type == e1000_media_type_copper || 4671 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 4672 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 4673 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 4674 } 4675 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 4676 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 4677 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 4678 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 4679 4680 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 4681 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 4682 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 4683 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 4684 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 4685 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 4686 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 4687 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 4688 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 4689 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 4690 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 4691 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 4692 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 4693 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 4694 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 4695 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 4696 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 4697 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 4698 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 4699 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 4700 4701 /* For the 64-bit byte counters the low dword must be read first. */ 4702 /* Both registers clear on the read of the high dword */ 4703 4704 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 4705 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 4706 4707 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 4708 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 4709 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 4710 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 4711 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 4712 4713 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 4714 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 4715 4716 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 4717 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 4718 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 4719 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 4720 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 4721 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 4722 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 4723 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 4724 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 4725 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 4726 4727 if (adapter->hw.mac.type >= e1000_82543) { 4728 adapter->stats.algnerrc += 4729 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 4730 adapter->stats.rxerrc += 4731 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 4732 adapter->stats.tncrs += 4733 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 4734 adapter->stats.cexterr += 4735 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 4736 adapter->stats.tsctc += 4737 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 4738 adapter->stats.tsctfc += 4739 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 4740 } 4741 ifp = adapter->ifp; 4742 4743 ifp->if_collisions = adapter->stats.colc; 4744 4745 /* Rx Errors */ 4746 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 4747 adapter->stats.crcerrs + adapter->stats.algnerrc + 4748 adapter->stats.ruc + adapter->stats.roc + 4749 adapter->stats.mpc + adapter->stats.cexterr; 4750 4751 /* Tx Errors */ 4752 ifp->if_oerrors = adapter->stats.ecol + 4753 adapter->stats.latecol + adapter->watchdog_events; 4754} 4755 4756 4757/********************************************************************** 4758 * 4759 * This routine is called only when em_display_debug_stats is enabled. 4760 * This routine provides a way to take a look at important statistics 4761 * maintained by the driver and hardware. 4762 * 4763 **********************************************************************/ 4764static void 4765em_print_debug_info(struct adapter *adapter) 4766{ 4767 device_t dev = adapter->dev; 4768 u8 *hw_addr = adapter->hw.hw_addr; 4769 4770 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 4771 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 4772 E1000_READ_REG(&adapter->hw, E1000_CTRL), 4773 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 4774 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 4775 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 4776 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 4777 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 4778 adapter->hw.fc.high_water, 4779 adapter->hw.fc.low_water); 4780 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 4781 E1000_READ_REG(&adapter->hw, E1000_TIDV), 4782 E1000_READ_REG(&adapter->hw, E1000_TADV)); 4783 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 4784 E1000_READ_REG(&adapter->hw, E1000_RDTR), 4785 E1000_READ_REG(&adapter->hw, E1000_RADV)); 4786 device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n", 4787 (long long)adapter->tx_fifo_wrk_cnt, 4788 (long long)adapter->tx_fifo_reset_cnt); 4789 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 4790 E1000_READ_REG(&adapter->hw, E1000_TDH(0)), 4791 E1000_READ_REG(&adapter->hw, E1000_TDT(0))); 4792 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 4793 E1000_READ_REG(&adapter->hw, E1000_RDH(0)), 4794 E1000_READ_REG(&adapter->hw, E1000_RDT(0))); 4795 device_printf(dev, "Num Tx descriptors avail = %d\n", 4796 adapter->num_tx_desc_avail); 4797 device_printf(dev, "Tx Descriptors not avail1 = %ld\n", 4798 adapter->no_tx_desc_avail1); 4799 device_printf(dev, "Tx Descriptors not avail2 = %ld\n", 4800 adapter->no_tx_desc_avail2); 4801 device_printf(dev, "Std mbuf failed = %ld\n", 4802 adapter->mbuf_alloc_failed); 4803 device_printf(dev, "Std mbuf cluster failed = %ld\n", 4804 adapter->mbuf_cluster_failed); 4805 device_printf(dev, "Driver dropped packets = %ld\n", 4806 adapter->dropped_pkts); 4807 device_printf(dev, "Driver tx dma failure in encap = %ld\n", 4808 adapter->no_tx_dma_setup); 4809} 4810 4811static void 4812em_print_hw_stats(struct adapter *adapter) 4813{ 4814 device_t dev = adapter->dev; 4815 4816 device_printf(dev, "Excessive collisions = %lld\n", 4817 (long long)adapter->stats.ecol); 4818#if (DEBUG_HW > 0) /* Dont output these errors normally */ 4819 device_printf(dev, "Symbol errors = %lld\n", 4820 (long long)adapter->stats.symerrs); 4821#endif 4822 device_printf(dev, "Sequence errors = %lld\n", 4823 (long long)adapter->stats.sec); 4824 device_printf(dev, "Defer count = %lld\n", 4825 (long long)adapter->stats.dc); 4826 device_printf(dev, "Missed Packets = %lld\n", 4827 (long long)adapter->stats.mpc); 4828 device_printf(dev, "Receive No Buffers = %lld\n", 4829 (long long)adapter->stats.rnbc); 4830 /* RLEC is inaccurate on some hardware, calculate our own. */ 4831 device_printf(dev, "Receive Length Errors = %lld\n", 4832 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 4833 device_printf(dev, "Receive errors = %lld\n", 4834 (long long)adapter->stats.rxerrc); 4835 device_printf(dev, "Crc errors = %lld\n", 4836 (long long)adapter->stats.crcerrs); 4837 device_printf(dev, "Alignment errors = %lld\n", 4838 (long long)adapter->stats.algnerrc); 4839 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 4840 (long long)adapter->stats.cexterr); 4841 device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); 4842 device_printf(dev, "watchdog timeouts = %ld\n", 4843 adapter->watchdog_events); 4844 device_printf(dev, "TX Interrupts = %ld ", adapter->tx_irq); 4845 device_printf(dev, "RX Interrupts = %ld\n", adapter->rx_irq); 4846 device_printf(dev, "XON Rcvd = %lld\n", 4847 (long long)adapter->stats.xonrxc); 4848 device_printf(dev, "XON Xmtd = %lld\n", 4849 (long long)adapter->stats.xontxc); 4850 device_printf(dev, "XOFF Rcvd = %lld\n", 4851 (long long)adapter->stats.xoffrxc); 4852 device_printf(dev, "XOFF Xmtd = %lld\n", 4853 (long long)adapter->stats.xofftxc); 4854 device_printf(dev, "Good Packets Rcvd = %lld\n", 4855 (long long)adapter->stats.gprc); 4856 device_printf(dev, "Good Packets Xmtd = %lld\n", 4857 (long long)adapter->stats.gptc); 4858 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 4859 (long long)adapter->stats.tsctc); 4860 device_printf(dev, "TSO Contexts Failed = %lld\n", 4861 (long long)adapter->stats.tsctfc); 4862} 4863 4864/********************************************************************** 4865 * 4866 * This routine provides a way to dump out the adapter eeprom, 4867 * often a useful debug/service tool. This only dumps the first 4868 * 32 words, stuff that matters is in that extent. 4869 * 4870 **********************************************************************/ 4871static void 4872em_print_nvm_info(struct adapter *adapter) 4873{ 4874 u16 eeprom_data; 4875 int i, j, row = 0; 4876 4877 /* Its a bit crude, but it gets the job done */ 4878 printf("\nInterface EEPROM Dump:\n"); 4879 printf("Offset\n0x0000 "); 4880 for (i = 0, j = 0; i < 32; i++, j++) { 4881 if (j == 8) { /* Make the offset block */ 4882 j = 0; ++row; 4883 printf("\n0x00%x0 ",row); 4884 } 4885 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 4886 printf("%04x ", eeprom_data); 4887 } 4888 printf("\n"); 4889} 4890 4891static int 4892em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 4893{ 4894 struct adapter *adapter; 4895 int error; 4896 int result; 4897 4898 result = -1; 4899 error = sysctl_handle_int(oidp, &result, 0, req); 4900 4901 if (error || !req->newptr) 4902 return (error); 4903 4904 if (result == 1) { 4905 adapter = (struct adapter *)arg1; 4906 em_print_debug_info(adapter); 4907 } 4908 /* 4909 * This value will cause a hex dump of the 4910 * first 32 16-bit words of the EEPROM to 4911 * the screen. 4912 */ 4913 if (result == 2) { 4914 adapter = (struct adapter *)arg1; 4915 em_print_nvm_info(adapter); 4916 } 4917 4918 return (error); 4919} 4920 4921 4922static int 4923em_sysctl_stats(SYSCTL_HANDLER_ARGS) 4924{ 4925 struct adapter *adapter; 4926 int error; 4927 int result; 4928 4929 result = -1; 4930 error = sysctl_handle_int(oidp, &result, 0, req); 4931 4932 if (error || !req->newptr) 4933 return (error); 4934 4935 if (result == 1) { 4936 adapter = (struct adapter *)arg1; 4937 em_print_hw_stats(adapter); 4938 } 4939 4940 return (error); 4941} 4942 4943static int 4944em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 4945{ 4946 struct em_int_delay_info *info; 4947 struct adapter *adapter; 4948 u32 regval; 4949 int error; 4950 int usecs; 4951 int ticks; 4952 4953 info = (struct em_int_delay_info *)arg1; 4954 usecs = info->value; 4955 error = sysctl_handle_int(oidp, &usecs, 0, req); 4956 if (error != 0 || req->newptr == NULL) 4957 return (error); 4958 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 4959 return (EINVAL); 4960 info->value = usecs; 4961 ticks = EM_USECS_TO_TICKS(usecs); 4962 4963 adapter = info->adapter; 4964 4965 EM_CORE_LOCK(adapter); 4966 regval = E1000_READ_OFFSET(&adapter->hw, info->offset); 4967 regval = (regval & ~0xffff) | (ticks & 0xffff); 4968 /* Handle a few special cases. */ 4969 switch (info->offset) { 4970 case E1000_RDTR: 4971 break; 4972 case E1000_TIDV: 4973 if (ticks == 0) { 4974 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; 4975 /* Don't write 0 into the TIDV register. */ 4976 regval++; 4977 } else 4978 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 4979 break; 4980 } 4981 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); 4982 EM_CORE_UNLOCK(adapter); 4983 return (0); 4984} 4985 4986static void 4987em_add_int_delay_sysctl(struct adapter *adapter, const char *name, 4988 const char *description, struct em_int_delay_info *info, 4989 int offset, int value) 4990{ 4991 info->adapter = adapter; 4992 info->offset = offset; 4993 info->value = value; 4994 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), 4995 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 4996 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, 4997 info, 0, em_sysctl_int_delay, "I", description); 4998} 4999 5000#ifndef EM_LEGACY_IRQ 5001static void 5002em_add_rx_process_limit(struct adapter *adapter, const char *name, 5003 const char *description, int *limit, int value) 5004{ 5005 *limit = value; 5006 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5007 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5008 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5009} 5010#endif 5011 5012