if_em.c revision 191038
1/****************************************************************************** 2 3 Copyright (c) 2001-2009, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ 33/*$FreeBSD: head/sys/dev/e1000/if_em.c 191038 2009-04-14 03:36:59Z kmacy $*/ 34 35#ifdef HAVE_KERNEL_OPTION_HEADERS 36#include "opt_device_polling.h" 37#include "opt_inet.h" 38#endif 39 40#include <sys/param.h> 41#include <sys/systm.h> 42#include <sys/buf_ring.h> 43#include <sys/bus.h> 44#include <sys/endian.h> 45#include <sys/kernel.h> 46#include <sys/kthread.h> 47#include <sys/malloc.h> 48#include <sys/mbuf.h> 49#include <sys/module.h> 50#include <sys/rman.h> 51#include <sys/socket.h> 52#include <sys/sockio.h> 53#include <sys/sysctl.h> 54#include <sys/taskqueue.h> 55#include <sys/eventhandler.h> 56#ifdef EM_TIMESYNC 57#include <sys/ioccom.h> 58#include <sys/time.h> 59#endif 60#include <machine/bus.h> 61#include <machine/resource.h> 62 63#include <net/bpf.h> 64#include <net/ethernet.h> 65#include <net/if.h> 66#include <net/if_arp.h> 67#include <net/if_dl.h> 68#include <net/if_media.h> 69 70#include <net/if_types.h> 71#include <net/if_vlan_var.h> 72 73#include <netinet/in_systm.h> 74#include <netinet/in.h> 75#include <netinet/if_ether.h> 76#include <netinet/ip.h> 77#include <netinet/ip6.h> 78#include <netinet/tcp.h> 79#include <netinet/udp.h> 80 81#include <machine/in_cksum.h> 82#include <dev/pci/pcivar.h> 83#include <dev/pci/pcireg.h> 84 85#include "e1000_api.h" 86#include "e1000_82571.h" 87#include "if_em.h" 88 89/********************************************************************* 90 * Set this to one to display debug statistics 91 *********************************************************************/ 92int em_display_debug_stats = 0; 93 94/********************************************************************* 95 * Driver version: 96 *********************************************************************/ 97char em_driver_version[] = "6.9.9"; 98 99 100/********************************************************************* 101 * PCI Device ID Table 102 * 103 * Used by probe to select devices to load on 104 * Last field stores an index into e1000_strings 105 * Last entry must be all 0s 106 * 107 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 108 *********************************************************************/ 109 110static em_vendor_info_t em_vendor_info_array[] = 111{ 112 /* Intel(R) PRO/1000 Network Connection */ 113 { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0}, 114 { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 115 { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0}, 116 { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 117 { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0}, 118 119 { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 120 { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0}, 121 { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 122 { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 123 { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0}, 124 { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 125 { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 126 127 { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0}, 128 129 { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 130 { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 131 132 { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 133 { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 136 137 { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 138 { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 139 { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 140 { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 141 { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 142 143 { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 144 { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 145 { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 146 { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 147 { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 148 { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 149 { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0}, 150 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 151 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, 152 PCI_ANY_ID, PCI_ANY_ID, 0}, 153 154 { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 155 { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 156 { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0}, 157 158 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 159 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 160 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 161 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 162 PCI_ANY_ID, PCI_ANY_ID, 0}, 163 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 164 PCI_ANY_ID, PCI_ANY_ID, 0}, 165 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 166 PCI_ANY_ID, PCI_ANY_ID, 0}, 167 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 168 PCI_ANY_ID, PCI_ANY_ID, 0}, 169 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 170 PCI_ANY_ID, PCI_ANY_ID, 0}, 171 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 172 PCI_ANY_ID, PCI_ANY_ID, 0}, 173 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 174 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 175 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 176 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 177 178 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, 179 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 180 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, 181 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 182 PCI_ANY_ID, PCI_ANY_ID, 0}, 183 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 184 PCI_ANY_ID, PCI_ANY_ID, 0}, 185 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 186 PCI_ANY_ID, PCI_ANY_ID, 0}, 187 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 188 PCI_ANY_ID, PCI_ANY_ID, 0}, 189 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 190 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 191 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 192 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 193 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 194 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 195 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 196 197 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 198 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 199 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 200 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 201 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 202 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 203 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 204 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 205 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, 206 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, 207 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 208 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 209 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 210 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 211 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 212 /* required last entry */ 213 { 0, 0, 0, 0, 0} 214}; 215 216/********************************************************************* 217 * Table of branding strings for all supported NICs. 218 *********************************************************************/ 219 220static char *em_strings[] = { 221 "Intel(R) PRO/1000 Network Connection" 222}; 223 224/********************************************************************* 225 * Function prototypes 226 *********************************************************************/ 227static int em_probe(device_t); 228static int em_attach(device_t); 229static int em_detach(device_t); 230static int em_shutdown(device_t); 231static int em_suspend(device_t); 232static int em_resume(device_t); 233static void em_start(struct ifnet *); 234static void em_start_locked(struct ifnet *ifp); 235static int em_ioctl(struct ifnet *, u_long, caddr_t); 236static void em_watchdog(struct adapter *); 237static void em_init(void *); 238static void em_init_locked(struct adapter *); 239static void em_stop(void *); 240static void em_media_status(struct ifnet *, struct ifmediareq *); 241static int em_media_change(struct ifnet *); 242static void em_identify_hardware(struct adapter *); 243static int em_allocate_pci_resources(struct adapter *); 244static int em_allocate_legacy(struct adapter *adapter); 245static int em_allocate_msix(struct adapter *adapter); 246static int em_setup_msix(struct adapter *); 247static void em_free_pci_resources(struct adapter *); 248static void em_local_timer(void *); 249static int em_hardware_init(struct adapter *); 250static void em_setup_interface(device_t, struct adapter *); 251static void em_setup_transmit_structures(struct adapter *); 252static void em_initialize_transmit_unit(struct adapter *); 253static int em_setup_receive_structures(struct adapter *); 254static void em_initialize_receive_unit(struct adapter *); 255static void em_enable_intr(struct adapter *); 256static void em_disable_intr(struct adapter *); 257static void em_free_transmit_structures(struct adapter *); 258static void em_free_receive_structures(struct adapter *); 259static void em_update_stats_counters(struct adapter *); 260static void em_txeof(struct adapter *); 261static void em_tx_purge(struct adapter *); 262static int em_allocate_receive_structures(struct adapter *); 263static int em_allocate_transmit_structures(struct adapter *); 264static int em_rxeof(struct adapter *, int); 265#ifndef __NO_STRICT_ALIGNMENT 266static int em_fixup_rx(struct adapter *); 267#endif 268static void em_receive_checksum(struct adapter *, struct e1000_rx_desc *, 269 struct mbuf *); 270static void em_transmit_checksum_setup(struct adapter *, struct mbuf *, 271 u32 *, u32 *); 272#if __FreeBSD_version >= 700000 273static bool em_tso_setup(struct adapter *, struct mbuf *, 274 u32 *, u32 *); 275#endif /* FreeBSD_version >= 700000 */ 276static void em_set_promisc(struct adapter *); 277static void em_disable_promisc(struct adapter *); 278static void em_set_multi(struct adapter *); 279static void em_print_hw_stats(struct adapter *); 280static void em_update_link_status(struct adapter *); 281static int em_get_buf(struct adapter *, int); 282static void em_register_vlan(void *, struct ifnet *, u16); 283static void em_unregister_vlan(void *, struct ifnet *, u16); 284static int em_xmit(struct adapter *, struct mbuf **); 285static void em_smartspeed(struct adapter *); 286static int em_82547_fifo_workaround(struct adapter *, int); 287static void em_82547_update_fifo_head(struct adapter *, int); 288static int em_82547_tx_fifo_reset(struct adapter *); 289static void em_82547_move_tail(void *); 290static int em_dma_malloc(struct adapter *, bus_size_t, 291 struct em_dma_alloc *, int); 292static void em_dma_free(struct adapter *, struct em_dma_alloc *); 293static void em_print_debug_info(struct adapter *); 294static void em_print_nvm_info(struct adapter *); 295static int em_is_valid_ether_addr(u8 *); 296static int em_sysctl_stats(SYSCTL_HANDLER_ARGS); 297static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 298static u32 em_fill_descriptors (bus_addr_t address, u32 length, 299 PDESC_ARRAY desc_array); 300static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 301static void em_add_int_delay_sysctl(struct adapter *, const char *, 302 const char *, struct em_int_delay_info *, int, int); 303/* Management and WOL Support */ 304static void em_init_manageability(struct adapter *); 305static void em_release_manageability(struct adapter *); 306static void em_get_hw_control(struct adapter *); 307static void em_release_hw_control(struct adapter *); 308static void em_enable_wakeup(device_t); 309 310#ifdef EM_TIMESYNC 311/* Precision Time sync support */ 312static int em_tsync_init(struct adapter *); 313static void em_tsync_disable(struct adapter *); 314#endif 315 316#ifdef EM_LEGACY_IRQ 317static void em_intr(void *); 318#else /* FAST IRQ */ 319#if __FreeBSD_version < 700000 320static void em_irq_fast(void *); 321#else 322static int em_irq_fast(void *); 323#endif 324 325/* MSIX handlers */ 326static void em_msix_tx(void *); 327static void em_msix_rx(void *); 328static void em_msix_link(void *); 329static void em_handle_rx(void *context, int pending); 330static void em_handle_tx(void *context, int pending); 331 332static void em_handle_rxtx(void *context, int pending); 333static void em_handle_link(void *context, int pending); 334static void em_add_rx_process_limit(struct adapter *, const char *, 335 const char *, int *, int); 336#endif /* ~EM_LEGACY_IRQ */ 337 338#ifdef DEVICE_POLLING 339static poll_handler_t em_poll; 340#endif /* POLLING */ 341 342/********************************************************************* 343 * FreeBSD Device Interface Entry Points 344 *********************************************************************/ 345 346static device_method_t em_methods[] = { 347 /* Device interface */ 348 DEVMETHOD(device_probe, em_probe), 349 DEVMETHOD(device_attach, em_attach), 350 DEVMETHOD(device_detach, em_detach), 351 DEVMETHOD(device_shutdown, em_shutdown), 352 DEVMETHOD(device_suspend, em_suspend), 353 DEVMETHOD(device_resume, em_resume), 354 {0, 0} 355}; 356 357static driver_t em_driver = { 358 "em", em_methods, sizeof(struct adapter), 359}; 360 361static devclass_t em_devclass; 362DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); 363MODULE_DEPEND(em, pci, 1, 1, 1); 364MODULE_DEPEND(em, ether, 1, 1, 1); 365 366/********************************************************************* 367 * Tunable default values. 368 *********************************************************************/ 369 370#define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 371#define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 372#define M_TSO_LEN 66 373 374/* Allow common code without TSO */ 375#ifndef CSUM_TSO 376#define CSUM_TSO 0 377#endif 378 379static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 380static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 381static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 382static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 383static int em_rxd = EM_DEFAULT_RXD; 384static int em_txd = EM_DEFAULT_TXD; 385static int em_smart_pwr_down = FALSE; 386/* Controls whether promiscuous also shows bad packets */ 387static int em_debug_sbp = FALSE; 388/* Local switch for MSI/MSIX */ 389static int em_enable_msi = TRUE; 390 391TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); 392TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); 393TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); 394TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); 395TUNABLE_INT("hw.em.rxd", &em_rxd); 396TUNABLE_INT("hw.em.txd", &em_txd); 397TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); 398TUNABLE_INT("hw.em.sbp", &em_debug_sbp); 399TUNABLE_INT("hw.em.enable_msi", &em_enable_msi); 400 401#ifndef EM_LEGACY_IRQ 402/* How many packets rxeof tries to clean at a time */ 403static int em_rx_process_limit = 100; 404TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); 405#endif 406 407/* Global used in WOL setup with multiport cards */ 408static int global_quad_port_a = 0; 409 410/********************************************************************* 411 * Device identification routine 412 * 413 * em_probe determines if the driver should be loaded on 414 * adapter based on PCI vendor/device id of the adapter. 415 * 416 * return BUS_PROBE_DEFAULT on success, positive on failure 417 *********************************************************************/ 418 419static int 420em_probe(device_t dev) 421{ 422 char adapter_name[60]; 423 u16 pci_vendor_id = 0; 424 u16 pci_device_id = 0; 425 u16 pci_subvendor_id = 0; 426 u16 pci_subdevice_id = 0; 427 em_vendor_info_t *ent; 428 429 INIT_DEBUGOUT("em_probe: begin"); 430 431 pci_vendor_id = pci_get_vendor(dev); 432 if (pci_vendor_id != EM_VENDOR_ID) 433 return (ENXIO); 434 435 pci_device_id = pci_get_device(dev); 436 pci_subvendor_id = pci_get_subvendor(dev); 437 pci_subdevice_id = pci_get_subdevice(dev); 438 439 ent = em_vendor_info_array; 440 while (ent->vendor_id != 0) { 441 if ((pci_vendor_id == ent->vendor_id) && 442 (pci_device_id == ent->device_id) && 443 444 ((pci_subvendor_id == ent->subvendor_id) || 445 (ent->subvendor_id == PCI_ANY_ID)) && 446 447 ((pci_subdevice_id == ent->subdevice_id) || 448 (ent->subdevice_id == PCI_ANY_ID))) { 449 sprintf(adapter_name, "%s %s", 450 em_strings[ent->index], 451 em_driver_version); 452 device_set_desc_copy(dev, adapter_name); 453 return (BUS_PROBE_DEFAULT); 454 } 455 ent++; 456 } 457 458 return (ENXIO); 459} 460 461/********************************************************************* 462 * Device initialization routine 463 * 464 * The attach entry point is called when the driver is being loaded. 465 * This routine identifies the type of hardware, allocates all resources 466 * and initializes the hardware. 467 * 468 * return 0 on success, positive on failure 469 *********************************************************************/ 470 471static int 472em_attach(device_t dev) 473{ 474 struct adapter *adapter; 475 int tsize, rsize; 476 int error = 0; 477 u16 eeprom_data, device_id; 478 479 INIT_DEBUGOUT("em_attach: begin"); 480 481 adapter = device_get_softc(dev); 482 adapter->dev = adapter->osdep.dev = dev; 483 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 484 EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev)); 485 EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev)); 486 487 /* SYSCTL stuff */ 488 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 489 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 490 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 491 em_sysctl_debug_info, "I", "Debug Information"); 492 493 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 494 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 495 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 496 em_sysctl_stats, "I", "Statistics"); 497 498 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 499 callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0); 500 501 /* Determine hardware and mac info */ 502 em_identify_hardware(adapter); 503 504 /* Setup PCI resources */ 505 if (em_allocate_pci_resources(adapter)) { 506 device_printf(dev, "Allocation of PCI resources failed\n"); 507 error = ENXIO; 508 goto err_pci; 509 } 510 511 /* 512 ** For ICH8 and family we need to 513 ** map the flash memory, and this 514 ** must happen after the MAC is 515 ** identified 516 */ 517 if ((adapter->hw.mac.type == e1000_ich8lan) || 518 (adapter->hw.mac.type == e1000_ich9lan) || 519 (adapter->hw.mac.type == e1000_ich10lan)) { 520 int rid = EM_BAR_TYPE_FLASH; 521 adapter->flash = bus_alloc_resource_any(dev, 522 SYS_RES_MEMORY, &rid, RF_ACTIVE); 523 if (adapter->flash == NULL) { 524 device_printf(dev, "Mapping of Flash failed\n"); 525 error = ENXIO; 526 goto err_pci; 527 } 528 /* This is used in the shared code */ 529 adapter->hw.flash_address = (u8 *)adapter->flash; 530 adapter->osdep.flash_bus_space_tag = 531 rman_get_bustag(adapter->flash); 532 adapter->osdep.flash_bus_space_handle = 533 rman_get_bushandle(adapter->flash); 534 } 535 536 /* Do Shared Code initialization */ 537 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 538 device_printf(dev, "Setup of Shared code failed\n"); 539 error = ENXIO; 540 goto err_pci; 541 } 542 543 e1000_get_bus_info(&adapter->hw); 544 545 /* Set up some sysctls for the tunable interrupt delays */ 546 em_add_int_delay_sysctl(adapter, "rx_int_delay", 547 "receive interrupt delay in usecs", &adapter->rx_int_delay, 548 E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt); 549 em_add_int_delay_sysctl(adapter, "tx_int_delay", 550 "transmit interrupt delay in usecs", &adapter->tx_int_delay, 551 E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt); 552 if (adapter->hw.mac.type >= e1000_82540) { 553 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay", 554 "receive interrupt delay limit in usecs", 555 &adapter->rx_abs_int_delay, 556 E1000_REGISTER(&adapter->hw, E1000_RADV), 557 em_rx_abs_int_delay_dflt); 558 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay", 559 "transmit interrupt delay limit in usecs", 560 &adapter->tx_abs_int_delay, 561 E1000_REGISTER(&adapter->hw, E1000_TADV), 562 em_tx_abs_int_delay_dflt); 563 } 564 565#ifndef EM_LEGACY_IRQ 566 /* Sysctls for limiting the amount of work done in the taskqueue */ 567 em_add_rx_process_limit(adapter, "rx_processing_limit", 568 "max number of rx packets to process", &adapter->rx_process_limit, 569 em_rx_process_limit); 570#endif 571 572 /* 573 * Validate number of transmit and receive descriptors. It 574 * must not exceed hardware maximum, and must be multiple 575 * of E1000_DBA_ALIGN. 576 */ 577 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || 578 (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) || 579 (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) || 580 (em_txd < EM_MIN_TXD)) { 581 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 582 EM_DEFAULT_TXD, em_txd); 583 adapter->num_tx_desc = EM_DEFAULT_TXD; 584 } else 585 adapter->num_tx_desc = em_txd; 586 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || 587 (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) || 588 (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) || 589 (em_rxd < EM_MIN_RXD)) { 590 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 591 EM_DEFAULT_RXD, em_rxd); 592 adapter->num_rx_desc = EM_DEFAULT_RXD; 593 } else 594 adapter->num_rx_desc = em_rxd; 595 596 adapter->hw.mac.autoneg = DO_AUTO_NEG; 597 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 598 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 599 adapter->rx_buffer_len = 2048; 600 601 e1000_init_script_state_82541(&adapter->hw, TRUE); 602 e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); 603 604 /* Copper options */ 605 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 606 adapter->hw.phy.mdix = AUTO_ALL_MODES; 607 adapter->hw.phy.disable_polarity_correction = FALSE; 608 adapter->hw.phy.ms_type = EM_MASTER_SLAVE; 609 } 610 611 /* 612 * Set the frame limits assuming 613 * standard ethernet sized frames. 614 */ 615 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 616 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 617 618 /* 619 * This controls when hardware reports transmit completion 620 * status. 621 */ 622 adapter->hw.mac.report_tx_early = 1; 623 624 tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc), 625 EM_DBA_ALIGN); 626 627 /* Allocate Transmit Descriptor ring */ 628 if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) { 629 device_printf(dev, "Unable to allocate tx_desc memory\n"); 630 error = ENOMEM; 631 goto err_tx_desc; 632 } 633 adapter->tx_desc_base = 634 (struct e1000_tx_desc *)adapter->txdma.dma_vaddr; 635 636 rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc), 637 EM_DBA_ALIGN); 638 639 /* Allocate Receive Descriptor ring */ 640 if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) { 641 device_printf(dev, "Unable to allocate rx_desc memory\n"); 642 error = ENOMEM; 643 goto err_rx_desc; 644 } 645 adapter->rx_desc_base = 646 (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr; 647 648 /* 649 ** Start from a known state, this is 650 ** important in reading the nvm and 651 ** mac from that. 652 */ 653 e1000_reset_hw(&adapter->hw); 654 655 /* Make sure we have a good EEPROM before we read from it */ 656 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 657 /* 658 ** Some PCI-E parts fail the first check due to 659 ** the link being in sleep state, call it again, 660 ** if it fails a second time its a real issue. 661 */ 662 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 663 device_printf(dev, 664 "The EEPROM Checksum Is Not Valid\n"); 665 error = EIO; 666 goto err_hw_init; 667 } 668 } 669 670 /* Copy the permanent MAC address out of the EEPROM */ 671 if (e1000_read_mac_addr(&adapter->hw) < 0) { 672 device_printf(dev, "EEPROM read error while reading MAC" 673 " address\n"); 674 error = EIO; 675 goto err_hw_init; 676 } 677 678 if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) { 679 device_printf(dev, "Invalid MAC address\n"); 680 error = EIO; 681 goto err_hw_init; 682 } 683 684 /* Initialize the hardware */ 685 if (em_hardware_init(adapter)) { 686 device_printf(dev, "Unable to initialize the hardware\n"); 687 error = EIO; 688 goto err_hw_init; 689 } 690 691 /* Allocate transmit descriptors and buffers */ 692 if (em_allocate_transmit_structures(adapter)) { 693 device_printf(dev, "Could not setup transmit structures\n"); 694 error = ENOMEM; 695 goto err_tx_struct; 696 } 697 698 /* Allocate receive descriptors and buffers */ 699 if (em_allocate_receive_structures(adapter)) { 700 device_printf(dev, "Could not setup receive structures\n"); 701 error = ENOMEM; 702 goto err_rx_struct; 703 } 704 705 /* 706 ** Do interrupt configuration 707 */ 708 if (adapter->msi > 1) /* Do MSI/X */ 709 error = em_allocate_msix(adapter); 710 else /* MSI or Legacy */ 711 error = em_allocate_legacy(adapter); 712 if (error) 713 goto err_rx_struct; 714 715 /* Setup OS specific network interface */ 716 em_setup_interface(dev, adapter); 717 718 /* Initialize statistics */ 719 em_update_stats_counters(adapter); 720 721 adapter->hw.mac.get_link_status = 1; 722 em_update_link_status(adapter); 723 724 /* Indicate SOL/IDER usage */ 725 if (e1000_check_reset_block(&adapter->hw)) 726 device_printf(dev, 727 "PHY reset is blocked due to SOL/IDER session.\n"); 728 729 /* Determine if we have to control management hardware */ 730 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 731 732 /* 733 * Setup Wake-on-Lan 734 */ 735 switch (adapter->hw.mac.type) { 736 737 case e1000_82542: 738 case e1000_82543: 739 break; 740 case e1000_82546: 741 case e1000_82546_rev_3: 742 case e1000_82571: 743 case e1000_80003es2lan: 744 if (adapter->hw.bus.func == 1) 745 e1000_read_nvm(&adapter->hw, 746 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 747 else 748 e1000_read_nvm(&adapter->hw, 749 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 750 eeprom_data &= EM_EEPROM_APME; 751 break; 752 default: 753 /* APME bit in EEPROM is mapped to WUC.APME */ 754 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & 755 E1000_WUC_APME; 756 break; 757 } 758 if (eeprom_data) 759 adapter->wol = E1000_WUFC_MAG; 760 /* 761 * We have the eeprom settings, now apply the special cases 762 * where the eeprom may be wrong or the board won't support 763 * wake on lan on a particular port 764 */ 765 device_id = pci_get_device(dev); 766 switch (device_id) { 767 case E1000_DEV_ID_82546GB_PCIE: 768 adapter->wol = 0; 769 break; 770 case E1000_DEV_ID_82546EB_FIBER: 771 case E1000_DEV_ID_82546GB_FIBER: 772 case E1000_DEV_ID_82571EB_FIBER: 773 /* Wake events only supported on port A for dual fiber 774 * regardless of eeprom setting */ 775 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 776 E1000_STATUS_FUNC_1) 777 adapter->wol = 0; 778 break; 779 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: 780 case E1000_DEV_ID_82571EB_QUAD_COPPER: 781 case E1000_DEV_ID_82571EB_QUAD_FIBER: 782 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 783 /* if quad port adapter, disable WoL on all but port A */ 784 if (global_quad_port_a != 0) 785 adapter->wol = 0; 786 /* Reset for multiple quad port adapters */ 787 if (++global_quad_port_a == 4) 788 global_quad_port_a = 0; 789 break; 790 } 791 792 /* Do we need workaround for 82544 PCI-X adapter? */ 793 if (adapter->hw.bus.type == e1000_bus_type_pcix && 794 adapter->hw.mac.type == e1000_82544) 795 adapter->pcix_82544 = TRUE; 796 else 797 adapter->pcix_82544 = FALSE; 798 799 /* Register for VLAN events */ 800 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 801 em_register_vlan, 0, EVENTHANDLER_PRI_FIRST); 802 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 803 em_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST); 804 805 /* Tell the stack that the interface is not active */ 806 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 807 808 INIT_DEBUGOUT("em_attach: end"); 809 810 return (0); 811 812err_rx_struct: 813 em_free_transmit_structures(adapter); 814err_tx_struct: 815err_hw_init: 816 em_release_hw_control(adapter); 817 em_dma_free(adapter, &adapter->rxdma); 818err_rx_desc: 819 em_dma_free(adapter, &adapter->txdma); 820err_tx_desc: 821err_pci: 822 em_free_pci_resources(adapter); 823 EM_TX_LOCK_DESTROY(adapter); 824 EM_RX_LOCK_DESTROY(adapter); 825 EM_CORE_LOCK_DESTROY(adapter); 826 827 return (error); 828} 829 830/********************************************************************* 831 * Device removal routine 832 * 833 * The detach entry point is called when the driver is being removed. 834 * This routine stops the adapter and deallocates all the resources 835 * that were allocated for driver operation. 836 * 837 * return 0 on success, positive on failure 838 *********************************************************************/ 839 840static int 841em_detach(device_t dev) 842{ 843 struct adapter *adapter = device_get_softc(dev); 844 struct ifnet *ifp = adapter->ifp; 845 846 INIT_DEBUGOUT("em_detach: begin"); 847 848 /* Make sure VLANS are not using driver */ 849#if __FreeBSD_version >= 700000 850 if (adapter->ifp->if_vlantrunk != NULL) { 851#else 852 if (adapter->ifp->if_nvlans != 0) { 853#endif 854 device_printf(dev,"Vlan in use, detach first\n"); 855 return (EBUSY); 856 } 857 858#ifdef DEVICE_POLLING 859 if (ifp->if_capenable & IFCAP_POLLING) 860 ether_poll_deregister(ifp); 861#endif 862 863 EM_CORE_LOCK(adapter); 864 EM_TX_LOCK(adapter); 865 adapter->in_detach = 1; 866 em_stop(adapter); 867 e1000_phy_hw_reset(&adapter->hw); 868 869 em_release_manageability(adapter); 870 871 if (((adapter->hw.mac.type == e1000_82573) || 872 (adapter->hw.mac.type == e1000_ich8lan) || 873 (adapter->hw.mac.type == e1000_ich10lan) || 874 (adapter->hw.mac.type == e1000_ich9lan)) && 875 e1000_check_mng_mode(&adapter->hw)) 876 em_release_hw_control(adapter); 877 878 if (adapter->wol) { 879 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 880 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 881 em_enable_wakeup(dev); 882 } 883 884 EM_TX_UNLOCK(adapter); 885 EM_CORE_UNLOCK(adapter); 886 887 /* Unregister VLAN events */ 888 if (adapter->vlan_attach != NULL) 889 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 890 if (adapter->vlan_detach != NULL) 891 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 892 893 ether_ifdetach(adapter->ifp); 894 callout_drain(&adapter->timer); 895 callout_drain(&adapter->tx_fifo_timer); 896 897 em_free_pci_resources(adapter); 898 bus_generic_detach(dev); 899 if_free(ifp); 900 901#ifdef IFNET_BUF_RING 902 drbr_free(adapter->br, M_DEVBUF); 903#endif 904 em_free_transmit_structures(adapter); 905 em_free_receive_structures(adapter); 906 907 /* Free Transmit Descriptor ring */ 908 if (adapter->tx_desc_base) { 909 em_dma_free(adapter, &adapter->txdma); 910 adapter->tx_desc_base = NULL; 911 } 912 913 /* Free Receive Descriptor ring */ 914 if (adapter->rx_desc_base) { 915 em_dma_free(adapter, &adapter->rxdma); 916 adapter->rx_desc_base = NULL; 917 } 918 919 EM_TX_LOCK_DESTROY(adapter); 920 EM_RX_LOCK_DESTROY(adapter); 921 EM_CORE_LOCK_DESTROY(adapter); 922 923 return (0); 924} 925 926/********************************************************************* 927 * 928 * Shutdown entry point 929 * 930 **********************************************************************/ 931 932static int 933em_shutdown(device_t dev) 934{ 935 return em_suspend(dev); 936} 937 938/* 939 * Suspend/resume device methods. 940 */ 941static int 942em_suspend(device_t dev) 943{ 944 struct adapter *adapter = device_get_softc(dev); 945 946 EM_CORE_LOCK(adapter); 947 948 EM_TX_LOCK(adapter); 949 em_stop(adapter); 950 EM_TX_UNLOCK(adapter); 951 952 em_release_manageability(adapter); 953 954 if (((adapter->hw.mac.type == e1000_82573) || 955 (adapter->hw.mac.type == e1000_ich8lan) || 956 (adapter->hw.mac.type == e1000_ich10lan) || 957 (adapter->hw.mac.type == e1000_ich9lan)) && 958 e1000_check_mng_mode(&adapter->hw)) 959 em_release_hw_control(adapter); 960 961 if (adapter->wol) { 962 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 963 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 964 em_enable_wakeup(dev); 965 } 966 967 EM_CORE_UNLOCK(adapter); 968 969 return bus_generic_suspend(dev); 970} 971 972static int 973em_resume(device_t dev) 974{ 975 struct adapter *adapter = device_get_softc(dev); 976 struct ifnet *ifp = adapter->ifp; 977 978 EM_CORE_LOCK(adapter); 979 em_init_locked(adapter); 980 em_init_manageability(adapter); 981 EM_CORE_UNLOCK(adapter); 982 em_start(ifp); 983 984 return bus_generic_resume(dev); 985} 986 987 988/********************************************************************* 989 * Transmit entry point 990 * 991 * em_start is called by the stack to initiate a transmit. 992 * The driver will remain in this routine as long as there are 993 * packets to transmit and transmit resources are available. 994 * In case resources are not available stack is notified and 995 * the packet is requeued. 996 **********************************************************************/ 997 998#ifdef IFNET_BUF_RING 999static int 1000em_transmit_locked(struct ifnet *ifp, struct mbuf *m) 1001{ 1002 struct adapter *adapter = ifp->if_softc; 1003 int error; 1004 1005 EM_TX_LOCK_ASSERT(adapter); 1006 if (((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 1007 IFF_DRV_RUNNING) 1008 || (!adapter->link_active)) { 1009 error = drbr_enqueue(ifp, adapter->br, m); 1010 return (error); 1011 } 1012 1013 if (ADAPTER_RING_EMPTY(adapter) && 1014 (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) { 1015 if (em_xmit(adapter, &m)) { 1016 if (m && (error = drbr_enqueue(ifp, adapter->br, m)) != 0) { 1017 return (error); 1018 } 1019 } else{ 1020 /* Send a copy of the frame to the BPF listener */ 1021 ETHER_BPF_MTAP(ifp, m); 1022 } 1023 } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0) 1024 return (error); 1025 1026 if (!ADAPTER_RING_EMPTY(adapter)) 1027 em_start_locked(ifp); 1028 1029 return (0); 1030} 1031 1032static int 1033em_transmit(struct ifnet *ifp, struct mbuf *m) 1034{ 1035 1036 struct adapter *adapter = ifp->if_softc; 1037 int error = 0; 1038 1039 if(EM_TX_TRYLOCK(adapter)) { 1040 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1041 error = em_transmit_locked(ifp, m); 1042 EM_TX_UNLOCK(adapter); 1043 } else 1044 error = drbr_enqueue(ifp, adapter->br, m); 1045 1046 return (error); 1047} 1048 1049static void 1050em_qflush(struct ifnet *ifp) 1051{ 1052 struct mbuf *m; 1053 struct adapter *adapter = (struct adapter *)ifp->if_softc; 1054 1055 EM_TX_LOCK(adapter); 1056 while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL) 1057 m_freem(m); 1058 EM_TX_UNLOCK(adapter); 1059} 1060 1061static void 1062em_start_locked(struct ifnet *ifp) 1063{ 1064 struct adapter *adapter = ifp->if_softc; 1065 struct mbuf *m_head; 1066 1067 EM_TX_LOCK_ASSERT(adapter); 1068 1069 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 1070 IFF_DRV_RUNNING) 1071 return; 1072 if (!adapter->link_active) 1073 return; 1074 1075 while ((adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD) 1076 && (!ADAPTER_RING_EMPTY(adapter))) { 1077 1078 m_head = buf_ring_dequeue_sc(adapter->br); 1079 if (m_head == NULL) 1080 break; 1081 /* 1082 * Encapsulation can modify our pointer, and or make it 1083 * NULL on failure. In that event, we can't requeue. 1084 */ 1085 if (em_xmit(adapter, &m_head)) { 1086 if (m_head == NULL) 1087 break; 1088 break; 1089 } 1090 1091 /* Send a copy of the frame to the BPF listener */ 1092 ETHER_BPF_MTAP(ifp, m_head); 1093 1094 /* Set timeout in case hardware has problems transmitting. */ 1095 adapter->watchdog_timer = EM_TX_TIMEOUT; 1096 } 1097 if ((adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)) 1098 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1099 1100} 1101#else 1102static void 1103em_start_locked(struct ifnet *ifp) 1104{ 1105 struct adapter *adapter = ifp->if_softc; 1106 struct mbuf *m_head; 1107 1108 EM_TX_LOCK_ASSERT(adapter); 1109 1110 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 1111 IFF_DRV_RUNNING) 1112 return; 1113 if (!adapter->link_active) 1114 return; 1115 1116 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 1117 1118 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 1119 if (m_head == NULL) 1120 break; 1121 /* 1122 * Encapsulation can modify our pointer, and or make it 1123 * NULL on failure. In that event, we can't requeue. 1124 */ 1125 if (em_xmit(adapter, &m_head)) { 1126 if (m_head == NULL) 1127 break; 1128 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1129 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 1130 break; 1131 } 1132 1133 /* Send a copy of the frame to the BPF listener */ 1134 ETHER_BPF_MTAP(ifp, m_head); 1135 1136 /* Set timeout in case hardware has problems transmitting. */ 1137 adapter->watchdog_timer = EM_TX_TIMEOUT; 1138 } 1139} 1140 1141#endif 1142 1143static void 1144em_start(struct ifnet *ifp) 1145{ 1146 struct adapter *adapter = ifp->if_softc; 1147 1148 EM_TX_LOCK(adapter); 1149 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1150 em_start_locked(ifp); 1151 EM_TX_UNLOCK(adapter); 1152} 1153 1154/********************************************************************* 1155 * Ioctl entry point 1156 * 1157 * em_ioctl is called when the user wants to configure the 1158 * interface. 1159 * 1160 * return 0 on success, positive on failure 1161 **********************************************************************/ 1162 1163static int 1164em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 1165{ 1166 struct adapter *adapter = ifp->if_softc; 1167 struct ifreq *ifr = (struct ifreq *)data; 1168#ifdef INET 1169 struct ifaddr *ifa = (struct ifaddr *)data; 1170#endif 1171 int error = 0; 1172 1173 if (adapter->in_detach) 1174 return (error); 1175 1176 switch (command) { 1177 case SIOCSIFADDR: 1178#ifdef INET 1179 if (ifa->ifa_addr->sa_family == AF_INET) { 1180 /* 1181 * XXX 1182 * Since resetting hardware takes a very long time 1183 * and results in link renegotiation we only 1184 * initialize the hardware only when it is absolutely 1185 * required. 1186 */ 1187 ifp->if_flags |= IFF_UP; 1188 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1189 EM_CORE_LOCK(adapter); 1190 em_init_locked(adapter); 1191 EM_CORE_UNLOCK(adapter); 1192 } 1193 arp_ifinit(ifp, ifa); 1194 } else 1195#endif 1196 error = ether_ioctl(ifp, command, data); 1197 break; 1198 case SIOCSIFMTU: 1199 { 1200 int max_frame_size; 1201 u16 eeprom_data = 0; 1202 1203 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1204 1205 EM_CORE_LOCK(adapter); 1206 switch (adapter->hw.mac.type) { 1207 case e1000_82573: 1208 /* 1209 * 82573 only supports jumbo frames 1210 * if ASPM is disabled. 1211 */ 1212 e1000_read_nvm(&adapter->hw, 1213 NVM_INIT_3GIO_3, 1, &eeprom_data); 1214 if (eeprom_data & NVM_WORD1A_ASPM_MASK) { 1215 max_frame_size = ETHER_MAX_LEN; 1216 break; 1217 } 1218 /* Allow Jumbo frames - fall thru */ 1219 case e1000_82571: 1220 case e1000_82572: 1221 case e1000_ich9lan: 1222 case e1000_ich10lan: 1223 case e1000_82574: 1224 case e1000_80003es2lan: /* Limit Jumbo Frame size */ 1225 max_frame_size = 9234; 1226 break; 1227 /* Adapters that do not support jumbo frames */ 1228 case e1000_82542: 1229 case e1000_ich8lan: 1230 max_frame_size = ETHER_MAX_LEN; 1231 break; 1232 default: 1233 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1234 } 1235 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1236 ETHER_CRC_LEN) { 1237 EM_CORE_UNLOCK(adapter); 1238 error = EINVAL; 1239 break; 1240 } 1241 1242 ifp->if_mtu = ifr->ifr_mtu; 1243 adapter->max_frame_size = 1244 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1245 em_init_locked(adapter); 1246 EM_CORE_UNLOCK(adapter); 1247 break; 1248 } 1249 case SIOCSIFFLAGS: 1250 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1251 SIOCSIFFLAGS (Set Interface Flags)"); 1252 EM_CORE_LOCK(adapter); 1253 if (ifp->if_flags & IFF_UP) { 1254 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1255 if ((ifp->if_flags ^ adapter->if_flags) & 1256 (IFF_PROMISC | IFF_ALLMULTI)) { 1257 em_disable_promisc(adapter); 1258 em_set_promisc(adapter); 1259 } 1260 } else 1261 em_init_locked(adapter); 1262 } else 1263 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1264 EM_TX_LOCK(adapter); 1265 em_stop(adapter); 1266 EM_TX_UNLOCK(adapter); 1267 } 1268 adapter->if_flags = ifp->if_flags; 1269 EM_CORE_UNLOCK(adapter); 1270 break; 1271 case SIOCADDMULTI: 1272 case SIOCDELMULTI: 1273 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1274 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1275 EM_CORE_LOCK(adapter); 1276 em_disable_intr(adapter); 1277 em_set_multi(adapter); 1278 if (adapter->hw.mac.type == e1000_82542 && 1279 adapter->hw.revision_id == E1000_REVISION_2) { 1280 em_initialize_receive_unit(adapter); 1281 } 1282#ifdef DEVICE_POLLING 1283 if (!(ifp->if_capenable & IFCAP_POLLING)) 1284#endif 1285 em_enable_intr(adapter); 1286 EM_CORE_UNLOCK(adapter); 1287 } 1288 break; 1289 case SIOCSIFMEDIA: 1290 /* Check SOL/IDER usage */ 1291 EM_CORE_LOCK(adapter); 1292 if (e1000_check_reset_block(&adapter->hw)) { 1293 EM_CORE_UNLOCK(adapter); 1294 device_printf(adapter->dev, "Media change is" 1295 " blocked due to SOL/IDER session.\n"); 1296 break; 1297 } 1298 EM_CORE_UNLOCK(adapter); 1299 case SIOCGIFMEDIA: 1300 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1301 SIOCxIFMEDIA (Get/Set Interface Media)"); 1302 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1303 break; 1304 case SIOCSIFCAP: 1305 { 1306 int mask, reinit; 1307 1308 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1309 reinit = 0; 1310 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1311#ifdef DEVICE_POLLING 1312 if (mask & IFCAP_POLLING) { 1313 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1314 error = ether_poll_register(em_poll, ifp); 1315 if (error) 1316 return (error); 1317 EM_CORE_LOCK(adapter); 1318 em_disable_intr(adapter); 1319 ifp->if_capenable |= IFCAP_POLLING; 1320 EM_CORE_UNLOCK(adapter); 1321 } else { 1322 error = ether_poll_deregister(ifp); 1323 /* Enable interrupt even in error case */ 1324 EM_CORE_LOCK(adapter); 1325 em_enable_intr(adapter); 1326 ifp->if_capenable &= ~IFCAP_POLLING; 1327 EM_CORE_UNLOCK(adapter); 1328 } 1329 } 1330#endif 1331 if (mask & IFCAP_HWCSUM) { 1332 ifp->if_capenable ^= IFCAP_HWCSUM; 1333 reinit = 1; 1334 } 1335#if __FreeBSD_version >= 700000 1336 if (mask & IFCAP_TSO4) { 1337 ifp->if_capenable ^= IFCAP_TSO4; 1338 reinit = 1; 1339 } 1340#endif 1341 1342 if (mask & IFCAP_VLAN_HWTAGGING) { 1343 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1344 reinit = 1; 1345 } 1346 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1347 em_init(adapter); 1348#if __FreeBSD_version >= 700000 1349 VLAN_CAPABILITIES(ifp); 1350#endif 1351 break; 1352 } 1353 1354#ifdef EM_TIMESYNC 1355 /* 1356 ** IOCTL support for Precision Time (IEEE 1588) Support 1357 */ 1358 case EM_TIMESYNC_READTS: 1359 { 1360 u32 rx_ctl, tx_ctl; 1361 struct em_tsync_read *tdata; 1362 1363 tdata = (struct em_tsync_read *) ifr->ifr_data; 1364 1365 IOCTL_DEBUGOUT("Reading Timestamp\n"); 1366 1367 if (tdata->read_current_time) { 1368 getnanotime(&tdata->system_time); 1369 tdata->network_time = E1000_READ_REG(&adapter->hw, E1000_SYSTIML); 1370 tdata->network_time |= 1371 (u64)E1000_READ_REG(&adapter->hw, E1000_SYSTIMH ) << 32; 1372 } 1373 1374 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 1375 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 1376 1377 IOCTL_DEBUGOUT1("RX_CTL value = %u\n", rx_ctl); 1378 IOCTL_DEBUGOUT1("TX_CTL value = %u\n", tx_ctl); 1379 1380 if (rx_ctl & 0x1) { 1381 IOCTL_DEBUGOUT("RX timestamp is valid\n"); 1382 u32 tmp; 1383 unsigned char *tmp_cp; 1384 1385 tdata->rx_valid = 1; 1386 tdata->rx_stamp = E1000_READ_REG(&adapter->hw, E1000_RXSTMPL); 1387 tdata->rx_stamp |= (u64)E1000_READ_REG(&adapter->hw, 1388 E1000_RXSTMPH) << 32; 1389 1390 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRL); 1391 tmp_cp = (unsigned char *) &tmp; 1392 tdata->srcid[0] = tmp_cp[0]; 1393 tdata->srcid[1] = tmp_cp[1]; 1394 tdata->srcid[2] = tmp_cp[2]; 1395 tdata->srcid[3] = tmp_cp[3]; 1396 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRH); 1397 tmp_cp = (unsigned char *) &tmp; 1398 tdata->srcid[4] = tmp_cp[0]; 1399 tdata->srcid[5] = tmp_cp[1]; 1400 tdata->seqid = tmp >> 16; 1401 tdata->seqid = htons(tdata->seqid); 1402 } else 1403 tdata->rx_valid = 0; 1404 1405 if (tx_ctl & 0x1) { 1406 IOCTL_DEBUGOUT("TX timestamp is valid\n"); 1407 tdata->tx_valid = 1; 1408 tdata->tx_stamp = E1000_READ_REG(&adapter->hw, E1000_TXSTMPL); 1409 tdata->tx_stamp |= (u64) E1000_READ_REG(&adapter->hw, 1410 E1000_TXSTMPH) << 32; 1411 } else 1412 tdata->tx_valid = 0; 1413 1414 return (0); 1415 } 1416#endif /* EM_TIMESYNC */ 1417 1418 default: 1419 error = ether_ioctl(ifp, command, data); 1420 break; 1421 } 1422 1423 return (error); 1424} 1425 1426/********************************************************************* 1427 * Watchdog timer: 1428 * 1429 * This routine is called from the local timer every second. 1430 * As long as transmit descriptors are being cleaned the value 1431 * is non-zero and we do nothing. Reaching 0 indicates a tx hang 1432 * and we then reset the device. 1433 * 1434 **********************************************************************/ 1435 1436static void 1437em_watchdog(struct adapter *adapter) 1438{ 1439 1440 EM_CORE_LOCK_ASSERT(adapter); 1441 1442 /* 1443 ** The timer is set to 5 every time start queues a packet. 1444 ** Then txeof keeps resetting it as long as it cleans at 1445 ** least one descriptor. 1446 ** Finally, anytime all descriptors are clean the timer is 1447 ** set to 0. 1448 */ 1449 EM_TX_LOCK(adapter); 1450 if ((adapter->watchdog_timer == 0) || (--adapter->watchdog_timer)) { 1451 EM_TX_UNLOCK(adapter); 1452 return; 1453 } 1454 1455 /* If we are in this routine because of pause frames, then 1456 * don't reset the hardware. 1457 */ 1458 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 1459 E1000_STATUS_TXOFF) { 1460 adapter->watchdog_timer = EM_TX_TIMEOUT; 1461 EM_TX_UNLOCK(adapter); 1462 return; 1463 } 1464 1465 if (e1000_check_for_link(&adapter->hw) == 0) 1466 device_printf(adapter->dev, "watchdog timeout -- resetting\n"); 1467 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1468 adapter->watchdog_events++; 1469 EM_TX_UNLOCK(adapter); 1470 1471 em_init_locked(adapter); 1472} 1473 1474/********************************************************************* 1475 * Init entry point 1476 * 1477 * This routine is used in two ways. It is used by the stack as 1478 * init entry point in network interface structure. It is also used 1479 * by the driver as a hw/sw initialization routine to get to a 1480 * consistent state. 1481 * 1482 * return 0 on success, positive on failure 1483 **********************************************************************/ 1484 1485static void 1486em_init_locked(struct adapter *adapter) 1487{ 1488 struct ifnet *ifp = adapter->ifp; 1489 device_t dev = adapter->dev; 1490 u32 pba; 1491 1492 INIT_DEBUGOUT("em_init: begin"); 1493 1494 EM_CORE_LOCK_ASSERT(adapter); 1495 1496 EM_TX_LOCK(adapter); 1497 em_stop(adapter); 1498 EM_TX_UNLOCK(adapter); 1499 1500 /* 1501 * Packet Buffer Allocation (PBA) 1502 * Writing PBA sets the receive portion of the buffer 1503 * the remainder is used for the transmit buffer. 1504 * 1505 * Devices before the 82547 had a Packet Buffer of 64K. 1506 * Default allocation: PBA=48K for Rx, leaving 16K for Tx. 1507 * After the 82547 the buffer was reduced to 40K. 1508 * Default allocation: PBA=30K for Rx, leaving 10K for Tx. 1509 * Note: default does not leave enough room for Jumbo Frame >10k. 1510 */ 1511 switch (adapter->hw.mac.type) { 1512 case e1000_82547: 1513 case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */ 1514 if (adapter->max_frame_size > 8192) 1515 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 1516 else 1517 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ 1518 adapter->tx_fifo_head = 0; 1519 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT; 1520 adapter->tx_fifo_size = 1521 (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT; 1522 break; 1523 /* Total Packet Buffer on these is 48K */ 1524 case e1000_82571: 1525 case e1000_82572: 1526 case e1000_80003es2lan: 1527 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1528 break; 1529 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1530 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1531 break; 1532 case e1000_82574: 1533 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1534 break; 1535 case e1000_ich9lan: 1536 case e1000_ich10lan: 1537#define E1000_PBA_10K 0x000A 1538 pba = E1000_PBA_10K; 1539 break; 1540 case e1000_ich8lan: 1541 pba = E1000_PBA_8K; 1542 break; 1543 default: 1544 /* Devices before 82547 had a Packet Buffer of 64K. */ 1545 if (adapter->max_frame_size > 8192) 1546 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1547 else 1548 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1549 } 1550 1551 INIT_DEBUGOUT1("em_init: pba=%dK",pba); 1552 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1553 1554 /* Get the latest mac address, User can use a LAA */ 1555 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1556 ETHER_ADDR_LEN); 1557 1558 /* Put the address into the Receive Address Array */ 1559 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1560 1561 /* 1562 * With the 82571 adapter, RAR[0] may be overwritten 1563 * when the other port is reset, we make a duplicate 1564 * in RAR[14] for that eventuality, this assures 1565 * the interface continues to function. 1566 */ 1567 if (adapter->hw.mac.type == e1000_82571) { 1568 e1000_set_laa_state_82571(&adapter->hw, TRUE); 1569 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 1570 E1000_RAR_ENTRIES - 1); 1571 } 1572 1573 /* Initialize the hardware */ 1574 if (em_hardware_init(adapter)) { 1575 device_printf(dev, "Unable to initialize the hardware\n"); 1576 return; 1577 } 1578 em_update_link_status(adapter); 1579 1580 /* Setup VLAN support, basic and offload if available */ 1581 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1582 1583 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) && 1584 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) { 1585 u32 ctrl; 1586 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1587 ctrl |= E1000_CTRL_VME; 1588 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1589 } 1590 1591 1592 /* Set hardware offload abilities */ 1593 ifp->if_hwassist = 0; 1594 if (adapter->hw.mac.type >= e1000_82543) { 1595 if (ifp->if_capenable & IFCAP_TXCSUM) 1596 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1597#if __FreeBSD_version >= 700000 1598 if (ifp->if_capenable & IFCAP_TSO4) 1599 ifp->if_hwassist |= CSUM_TSO; 1600#endif 1601 } 1602 1603 /* Configure for OS presence */ 1604 em_init_manageability(adapter); 1605 1606 /* Prepare transmit descriptors and buffers */ 1607 em_setup_transmit_structures(adapter); 1608 em_initialize_transmit_unit(adapter); 1609 1610 /* Setup Multicast table */ 1611 em_set_multi(adapter); 1612 1613 /* Prepare receive descriptors and buffers */ 1614 if (em_setup_receive_structures(adapter)) { 1615 device_printf(dev, "Could not setup receive structures\n"); 1616 EM_TX_LOCK(adapter); 1617 em_stop(adapter); 1618 EM_TX_UNLOCK(adapter); 1619 return; 1620 } 1621 em_initialize_receive_unit(adapter); 1622 1623 /* Don't lose promiscuous settings */ 1624 em_set_promisc(adapter); 1625 1626 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1627 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1628 1629 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1630 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1631 1632 /* MSI/X configuration for 82574 */ 1633 if (adapter->hw.mac.type == e1000_82574) { 1634 int tmp; 1635 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 1636 tmp |= E1000_CTRL_EXT_PBA_CLR; 1637 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); 1638 /* 1639 ** Set the IVAR - interrupt vector routing. 1640 ** Each nibble represents a vector, high bit 1641 ** is enable, other 3 bits are the MSIX table 1642 ** entry, we map RXQ0 to 0, TXQ0 to 1, and 1643 ** Link (other) to 2, hence the magic number. 1644 */ 1645 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908); 1646 } 1647 1648#ifdef DEVICE_POLLING 1649 /* 1650 * Only enable interrupts if we are not polling, make sure 1651 * they are off otherwise. 1652 */ 1653 if (ifp->if_capenable & IFCAP_POLLING) 1654 em_disable_intr(adapter); 1655 else 1656#endif /* DEVICE_POLLING */ 1657 em_enable_intr(adapter); 1658 1659#ifdef EM_TIMESYNC 1660 /* Initializae IEEE 1588 Precision Time hardware */ 1661 if ((adapter->hw.mac.type == e1000_82574) || 1662 (adapter->hw.mac.type == e1000_ich10lan)) 1663 em_tsync_init(adapter); 1664#endif 1665 1666 /* Don't reset the phy next time init gets called */ 1667 adapter->hw.phy.reset_disable = TRUE; 1668} 1669 1670static void 1671em_init(void *arg) 1672{ 1673 struct adapter *adapter = arg; 1674 1675 EM_CORE_LOCK(adapter); 1676 em_init_locked(adapter); 1677 EM_CORE_UNLOCK(adapter); 1678} 1679 1680 1681#ifdef DEVICE_POLLING 1682/********************************************************************* 1683 * 1684 * Legacy polling routine 1685 * 1686 *********************************************************************/ 1687static void 1688em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1689{ 1690 struct adapter *adapter = ifp->if_softc; 1691 u32 reg_icr; 1692 1693 EM_CORE_LOCK(adapter); 1694 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1695 EM_CORE_UNLOCK(adapter); 1696 return; 1697 } 1698 1699 if (cmd == POLL_AND_CHECK_STATUS) { 1700 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1701 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1702 callout_stop(&adapter->timer); 1703 adapter->hw.mac.get_link_status = 1; 1704 em_update_link_status(adapter); 1705 callout_reset(&adapter->timer, hz, 1706 em_local_timer, adapter); 1707 } 1708 } 1709 EM_CORE_UNLOCK(adapter); 1710 1711 em_rxeof(adapter, count); 1712 1713 EM_TX_LOCK(adapter); 1714 em_txeof(adapter); 1715 1716 if (!ADAPTER_RING_EMPTY(adapter)) 1717 em_start_locked(ifp); 1718 EM_TX_UNLOCK(adapter); 1719} 1720#endif /* DEVICE_POLLING */ 1721 1722#ifdef EM_LEGACY_IRQ 1723/********************************************************************* 1724 * 1725 * Legacy Interrupt Service routine 1726 * 1727 *********************************************************************/ 1728 1729static void 1730em_intr(void *arg) 1731{ 1732 struct adapter *adapter = arg; 1733 struct ifnet *ifp = adapter->ifp; 1734 u32 reg_icr; 1735 1736 1737 if (ifp->if_capenable & IFCAP_POLLING) 1738 return; 1739 1740 EM_CORE_LOCK(adapter); 1741 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1742 if ((reg_icr == 0xffffffff) || (reg_icr == 0)|| 1743 (adapter->hw.mac.type >= e1000_82571 && 1744 (reg_icr & E1000_ICR_INT_ASSERTED) == 0)) 1745 goto out; 1746 1747 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1748 goto out; 1749 1750 EM_TX_LOCK(adapter); 1751 em_txeof(adapter); 1752 em_rxeof(adapter, -1); 1753 em_txeof(adapter); 1754 EM_TX_UNLOCK(adapter); 1755 1756 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1757 callout_stop(&adapter->timer); 1758 adapter->hw.mac.get_link_status = 1; 1759 em_update_link_status(adapter); 1760 /* Deal with TX cruft when link lost */ 1761 em_tx_purge(adapter); 1762 callout_reset(&adapter->timer, hz, 1763 em_local_timer, adapter); 1764 } 1765 1766 if (reg_icr & E1000_ICR_RXO) 1767 adapter->rx_overruns++; 1768out: 1769 EM_CORE_UNLOCK(adapter); 1770 if (ifp->if_drv_flags & IFF_DRV_RUNNING && 1771 !ADAPTER_RING_EMPTY(adapter)) 1772 em_start(ifp); 1773} 1774 1775#else /* EM_FAST_IRQ, then fast interrupt routines only */ 1776 1777static void 1778em_handle_link(void *context, int pending) 1779{ 1780 struct adapter *adapter = context; 1781 struct ifnet *ifp = adapter->ifp; 1782 1783 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1784 return; 1785 1786 EM_CORE_LOCK(adapter); 1787 callout_stop(&adapter->timer); 1788 em_update_link_status(adapter); 1789 /* Deal with TX cruft when link lost */ 1790 em_tx_purge(adapter); 1791 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1792 EM_CORE_UNLOCK(adapter); 1793} 1794 1795 1796/* Combined RX/TX handler, used by Legacy and MSI */ 1797static void 1798em_handle_rxtx(void *context, int pending) 1799{ 1800 struct adapter *adapter = context; 1801 struct ifnet *ifp = adapter->ifp; 1802 1803 1804 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1805 if (em_rxeof(adapter, adapter->rx_process_limit) != 0) 1806 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1807 EM_TX_LOCK(adapter); 1808 em_txeof(adapter); 1809 1810 if (!ADAPTER_RING_EMPTY(adapter)) 1811 em_start_locked(ifp); 1812 EM_TX_UNLOCK(adapter); 1813 } 1814 1815 em_enable_intr(adapter); 1816} 1817 1818/********************************************************************* 1819 * 1820 * Fast Legacy/MSI Combined Interrupt Service routine 1821 * 1822 *********************************************************************/ 1823#if __FreeBSD_version < 700000 1824#define FILTER_STRAY 1825#define FILTER_HANDLED 1826static void 1827#else 1828static int 1829#endif 1830em_irq_fast(void *arg) 1831{ 1832 struct adapter *adapter = arg; 1833 struct ifnet *ifp; 1834 u32 reg_icr; 1835 1836 ifp = adapter->ifp; 1837 1838 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1839 1840 /* Hot eject? */ 1841 if (reg_icr == 0xffffffff) 1842 return FILTER_STRAY; 1843 1844 /* Definitely not our interrupt. */ 1845 if (reg_icr == 0x0) 1846 return FILTER_STRAY; 1847 1848 /* 1849 * Starting with the 82571 chip, bit 31 should be used to 1850 * determine whether the interrupt belongs to us. 1851 */ 1852 if (adapter->hw.mac.type >= e1000_82571 && 1853 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1854 return FILTER_STRAY; 1855 1856 /* 1857 * Mask interrupts until the taskqueue is finished running. This is 1858 * cheap, just assume that it is needed. This also works around the 1859 * MSI message reordering errata on certain systems. 1860 */ 1861 em_disable_intr(adapter); 1862 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1863 1864 /* Link status change */ 1865 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1866 adapter->hw.mac.get_link_status = 1; 1867 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1868 } 1869 1870 if (reg_icr & E1000_ICR_RXO) 1871 adapter->rx_overruns++; 1872 return FILTER_HANDLED; 1873} 1874 1875/********************************************************************* 1876 * 1877 * MSIX Interrupt Service Routines 1878 * 1879 **********************************************************************/ 1880#define EM_MSIX_TX 0x00040000 1881#define EM_MSIX_RX 0x00010000 1882#define EM_MSIX_LINK 0x00100000 1883 1884static void 1885em_msix_tx(void *arg) 1886{ 1887 struct adapter *adapter = arg; 1888 struct ifnet *ifp = adapter->ifp; 1889 1890 ++adapter->tx_irq; 1891 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1892 EM_TX_LOCK(adapter); 1893 em_txeof(adapter); 1894 EM_TX_UNLOCK(adapter); 1895 taskqueue_enqueue(adapter->tq, &adapter->tx_task); 1896 } 1897 /* Reenable this interrupt */ 1898 E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX); 1899 return; 1900} 1901 1902/********************************************************************* 1903 * 1904 * MSIX RX Interrupt Service routine 1905 * 1906 **********************************************************************/ 1907 1908static void 1909em_msix_rx(void *arg) 1910{ 1911 struct adapter *adapter = arg; 1912 struct ifnet *ifp = adapter->ifp; 1913 1914 ++adapter->rx_irq; 1915 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && 1916 (em_rxeof(adapter, adapter->rx_process_limit) != 0)) 1917 taskqueue_enqueue(adapter->tq, &adapter->rx_task); 1918 /* Reenable this interrupt */ 1919 E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX); 1920 return; 1921} 1922 1923/********************************************************************* 1924 * 1925 * MSIX Link Fast Interrupt Service routine 1926 * 1927 **********************************************************************/ 1928 1929static void 1930em_msix_link(void *arg) 1931{ 1932 struct adapter *adapter = arg; 1933 u32 reg_icr; 1934 1935 ++adapter->link_irq; 1936 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1937 1938 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1939 adapter->hw.mac.get_link_status = 1; 1940 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1941 } 1942 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1943 EM_MSIX_LINK | E1000_IMS_LSC); 1944 return; 1945} 1946 1947static void 1948em_handle_rx(void *context, int pending) 1949{ 1950 struct adapter *adapter = context; 1951 struct ifnet *ifp = adapter->ifp; 1952 1953 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && 1954 (em_rxeof(adapter, adapter->rx_process_limit) != 0)) 1955 taskqueue_enqueue(adapter->tq, &adapter->rx_task); 1956 1957} 1958 1959static void 1960em_handle_tx(void *context, int pending) 1961{ 1962 struct adapter *adapter = context; 1963 struct ifnet *ifp = adapter->ifp; 1964 1965 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1966#ifdef IFNET_BUF_RING 1967 if (!EM_TX_TRYLOCK(adapter)) 1968 return; 1969#else 1970 EM_TX_LOCK(adapter); 1971#endif 1972 1973 em_txeof(adapter); 1974 if (!ADAPTER_RING_EMPTY(adapter)) 1975 em_start_locked(ifp); 1976 EM_TX_UNLOCK(adapter); 1977 } 1978} 1979#endif /* EM_FAST_IRQ */ 1980 1981/********************************************************************* 1982 * 1983 * Media Ioctl callback 1984 * 1985 * This routine is called whenever the user queries the status of 1986 * the interface using ifconfig. 1987 * 1988 **********************************************************************/ 1989static void 1990em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1991{ 1992 struct adapter *adapter = ifp->if_softc; 1993 u_char fiber_type = IFM_1000_SX; 1994 1995 INIT_DEBUGOUT("em_media_status: begin"); 1996 1997 EM_CORE_LOCK(adapter); 1998 em_update_link_status(adapter); 1999 2000 ifmr->ifm_status = IFM_AVALID; 2001 ifmr->ifm_active = IFM_ETHER; 2002 2003 if (!adapter->link_active) { 2004 EM_CORE_UNLOCK(adapter); 2005 return; 2006 } 2007 2008 ifmr->ifm_status |= IFM_ACTIVE; 2009 2010 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2011 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2012 if (adapter->hw.mac.type == e1000_82545) 2013 fiber_type = IFM_1000_LX; 2014 ifmr->ifm_active |= fiber_type | IFM_FDX; 2015 } else { 2016 switch (adapter->link_speed) { 2017 case 10: 2018 ifmr->ifm_active |= IFM_10_T; 2019 break; 2020 case 100: 2021 ifmr->ifm_active |= IFM_100_TX; 2022 break; 2023 case 1000: 2024 ifmr->ifm_active |= IFM_1000_T; 2025 break; 2026 } 2027 if (adapter->link_duplex == FULL_DUPLEX) 2028 ifmr->ifm_active |= IFM_FDX; 2029 else 2030 ifmr->ifm_active |= IFM_HDX; 2031 } 2032 EM_CORE_UNLOCK(adapter); 2033} 2034 2035/********************************************************************* 2036 * 2037 * Media Ioctl callback 2038 * 2039 * This routine is called when the user changes speed/duplex using 2040 * media/mediopt option with ifconfig. 2041 * 2042 **********************************************************************/ 2043static int 2044em_media_change(struct ifnet *ifp) 2045{ 2046 struct adapter *adapter = ifp->if_softc; 2047 struct ifmedia *ifm = &adapter->media; 2048 2049 INIT_DEBUGOUT("em_media_change: begin"); 2050 2051 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2052 return (EINVAL); 2053 2054 EM_CORE_LOCK(adapter); 2055 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2056 case IFM_AUTO: 2057 adapter->hw.mac.autoneg = DO_AUTO_NEG; 2058 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2059 break; 2060 case IFM_1000_LX: 2061 case IFM_1000_SX: 2062 case IFM_1000_T: 2063 adapter->hw.mac.autoneg = DO_AUTO_NEG; 2064 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2065 break; 2066 case IFM_100_TX: 2067 adapter->hw.mac.autoneg = FALSE; 2068 adapter->hw.phy.autoneg_advertised = 0; 2069 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2070 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 2071 else 2072 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 2073 break; 2074 case IFM_10_T: 2075 adapter->hw.mac.autoneg = FALSE; 2076 adapter->hw.phy.autoneg_advertised = 0; 2077 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2078 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 2079 else 2080 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 2081 break; 2082 default: 2083 device_printf(adapter->dev, "Unsupported media type\n"); 2084 } 2085 2086 /* As the speed/duplex settings my have changed we need to 2087 * reset the PHY. 2088 */ 2089 adapter->hw.phy.reset_disable = FALSE; 2090 2091 em_init_locked(adapter); 2092 EM_CORE_UNLOCK(adapter); 2093 2094 return (0); 2095} 2096 2097/********************************************************************* 2098 * 2099 * This routine maps the mbufs to tx descriptors. 2100 * 2101 * return 0 on success, positive on failure 2102 **********************************************************************/ 2103 2104static int 2105em_xmit(struct adapter *adapter, struct mbuf **m_headp) 2106{ 2107 bus_dma_segment_t segs[EM_MAX_SCATTER]; 2108 bus_dmamap_t map; 2109 struct em_buffer *tx_buffer, *tx_buffer_mapped; 2110 struct e1000_tx_desc *ctxd = NULL; 2111 struct mbuf *m_head; 2112 u32 txd_upper, txd_lower, txd_used, txd_saved; 2113 int nsegs, i, j, first, last = 0; 2114 int error, do_tso, tso_desc = 0; 2115#if __FreeBSD_version < 700000 2116 struct m_tag *mtag; 2117#endif 2118 m_head = *m_headp; 2119 txd_upper = txd_lower = txd_used = txd_saved = 0; 2120 2121#if __FreeBSD_version >= 700000 2122 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); 2123#else 2124 do_tso = 0; 2125#endif 2126 2127 /* 2128 * Force a cleanup if number of TX descriptors 2129 * available hits the threshold 2130 */ 2131 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { 2132 em_txeof(adapter); 2133 /* Now do we at least have a minimal? */ 2134 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) { 2135 adapter->no_tx_desc_avail1++; 2136 return (ENOBUFS); 2137 } 2138 } 2139 2140 2141 /* 2142 * TSO workaround: 2143 * If an mbuf is only header we need 2144 * to pull 4 bytes of data into it. 2145 */ 2146 if (do_tso && (m_head->m_len <= M_TSO_LEN)) { 2147 m_head = m_pullup(m_head, M_TSO_LEN + 4); 2148 *m_headp = m_head; 2149 if (m_head == NULL) 2150 return (ENOBUFS); 2151 } 2152 2153 /* 2154 * Map the packet for DMA 2155 * 2156 * Capture the first descriptor index, 2157 * this descriptor will have the index 2158 * of the EOP which is the only one that 2159 * now gets a DONE bit writeback. 2160 */ 2161 first = adapter->next_avail_tx_desc; 2162 tx_buffer = &adapter->tx_buffer_area[first]; 2163 tx_buffer_mapped = tx_buffer; 2164 map = tx_buffer->map; 2165 2166 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, 2167 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 2168 2169 /* 2170 * There are two types of errors we can (try) to handle: 2171 * - EFBIG means the mbuf chain was too long and bus_dma ran 2172 * out of segments. Defragment the mbuf chain and try again. 2173 * - ENOMEM means bus_dma could not obtain enough bounce buffers 2174 * at this point in time. Defer sending and try again later. 2175 * All other errors, in particular EINVAL, are fatal and prevent the 2176 * mbuf chain from ever going through. Drop it and report error. 2177 */ 2178 if (error == EFBIG) { 2179 struct mbuf *m; 2180 2181 m = m_defrag(*m_headp, M_DONTWAIT); 2182 if (m == NULL) { 2183 adapter->mbuf_alloc_failed++; 2184 m_freem(*m_headp); 2185 *m_headp = NULL; 2186 return (ENOBUFS); 2187 } 2188 *m_headp = m; 2189 2190 /* Try it again */ 2191 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, 2192 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 2193 2194 if (error) { 2195 adapter->no_tx_dma_setup++; 2196 m_freem(*m_headp); 2197 *m_headp = NULL; 2198 return (error); 2199 } 2200 } else if (error != 0) { 2201 adapter->no_tx_dma_setup++; 2202 return (error); 2203 } 2204 2205 /* 2206 * TSO Hardware workaround, if this packet is not 2207 * TSO, and is only a single descriptor long, and 2208 * it follows a TSO burst, then we need to add a 2209 * sentinel descriptor to prevent premature writeback. 2210 */ 2211 if ((do_tso == 0) && (adapter->tx_tso == TRUE)) { 2212 if (nsegs == 1) 2213 tso_desc = TRUE; 2214 adapter->tx_tso = FALSE; 2215 } 2216 2217 if (nsegs > (adapter->num_tx_desc_avail - 2)) { 2218 adapter->no_tx_desc_avail2++; 2219 bus_dmamap_unload(adapter->txtag, map); 2220 return (ENOBUFS); 2221 } 2222 m_head = *m_headp; 2223 2224 /* Do hardware assists */ 2225#if __FreeBSD_version >= 700000 2226 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 2227 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower); 2228 if (error != TRUE) 2229 return (ENXIO); /* something foobar */ 2230 /* we need to make a final sentinel transmit desc */ 2231 tso_desc = TRUE; 2232 } else 2233#endif 2234#ifndef EM_TIMESYNC 2235 /* 2236 ** Timesync needs to check the packet header 2237 ** so call checksum code to do so, but don't 2238 ** penalize the code if not defined. 2239 */ 2240 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) 2241#endif 2242 em_transmit_checksum_setup(adapter, m_head, 2243 &txd_upper, &txd_lower); 2244 2245 i = adapter->next_avail_tx_desc; 2246 if (adapter->pcix_82544) 2247 txd_saved = i; 2248 2249 /* Set up our transmit descriptors */ 2250 for (j = 0; j < nsegs; j++) { 2251 bus_size_t seg_len; 2252 bus_addr_t seg_addr; 2253 /* If adapter is 82544 and on PCIX bus */ 2254 if(adapter->pcix_82544) { 2255 DESC_ARRAY desc_array; 2256 u32 array_elements, counter; 2257 /* 2258 * Check the Address and Length combination and 2259 * split the data accordingly 2260 */ 2261 array_elements = em_fill_descriptors(segs[j].ds_addr, 2262 segs[j].ds_len, &desc_array); 2263 for (counter = 0; counter < array_elements; counter++) { 2264 if (txd_used == adapter->num_tx_desc_avail) { 2265 adapter->next_avail_tx_desc = txd_saved; 2266 adapter->no_tx_desc_avail2++; 2267 bus_dmamap_unload(adapter->txtag, map); 2268 return (ENOBUFS); 2269 } 2270 tx_buffer = &adapter->tx_buffer_area[i]; 2271 ctxd = &adapter->tx_desc_base[i]; 2272 ctxd->buffer_addr = htole64( 2273 desc_array.descriptor[counter].address); 2274 ctxd->lower.data = htole32( 2275 (adapter->txd_cmd | txd_lower | (u16) 2276 desc_array.descriptor[counter].length)); 2277 ctxd->upper.data = 2278 htole32((txd_upper)); 2279 last = i; 2280 if (++i == adapter->num_tx_desc) 2281 i = 0; 2282 tx_buffer->m_head = NULL; 2283 tx_buffer->next_eop = -1; 2284 txd_used++; 2285 } 2286 } else { 2287 tx_buffer = &adapter->tx_buffer_area[i]; 2288 ctxd = &adapter->tx_desc_base[i]; 2289 seg_addr = segs[j].ds_addr; 2290 seg_len = segs[j].ds_len; 2291 /* 2292 ** TSO Workaround: 2293 ** If this is the last descriptor, we want to 2294 ** split it so we have a small final sentinel 2295 */ 2296 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { 2297 seg_len -= 4; 2298 ctxd->buffer_addr = htole64(seg_addr); 2299 ctxd->lower.data = htole32( 2300 adapter->txd_cmd | txd_lower | seg_len); 2301 ctxd->upper.data = 2302 htole32(txd_upper); 2303 if (++i == adapter->num_tx_desc) 2304 i = 0; 2305 /* Now make the sentinel */ 2306 ++txd_used; /* using an extra txd */ 2307 ctxd = &adapter->tx_desc_base[i]; 2308 tx_buffer = &adapter->tx_buffer_area[i]; 2309 ctxd->buffer_addr = 2310 htole64(seg_addr + seg_len); 2311 ctxd->lower.data = htole32( 2312 adapter->txd_cmd | txd_lower | 4); 2313 ctxd->upper.data = 2314 htole32(txd_upper); 2315 last = i; 2316 if (++i == adapter->num_tx_desc) 2317 i = 0; 2318 } else { 2319 ctxd->buffer_addr = htole64(seg_addr); 2320 ctxd->lower.data = htole32( 2321 adapter->txd_cmd | txd_lower | seg_len); 2322 ctxd->upper.data = 2323 htole32(txd_upper); 2324 last = i; 2325 if (++i == adapter->num_tx_desc) 2326 i = 0; 2327 } 2328 tx_buffer->m_head = NULL; 2329 tx_buffer->next_eop = -1; 2330 } 2331 } 2332 2333 adapter->next_avail_tx_desc = i; 2334 if (adapter->pcix_82544) 2335 adapter->num_tx_desc_avail -= txd_used; 2336 else { 2337 adapter->num_tx_desc_avail -= nsegs; 2338 if (tso_desc) /* TSO used an extra for sentinel */ 2339 adapter->num_tx_desc_avail -= txd_used; 2340 } 2341 2342 /* 2343 ** Handle VLAN tag, this is the 2344 ** biggest difference between 2345 ** 6.x and 7 2346 */ 2347#if __FreeBSD_version < 700000 2348 /* Find out if we are in vlan mode. */ 2349 mtag = VLAN_OUTPUT_TAG(ifp, m_head); 2350 if (mtag != NULL) { 2351 ctxd->upper.fields.special = 2352 htole16(VLAN_TAG_VALUE(mtag)); 2353#else /* FreeBSD 7 */ 2354 if (m_head->m_flags & M_VLANTAG) { 2355 /* Set the vlan id. */ 2356 ctxd->upper.fields.special = 2357 htole16(m_head->m_pkthdr.ether_vtag); 2358#endif 2359 /* Tell hardware to add tag */ 2360 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); 2361 } 2362 2363 tx_buffer->m_head = m_head; 2364 tx_buffer_mapped->map = tx_buffer->map; 2365 tx_buffer->map = map; 2366 bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE); 2367 2368 /* 2369 * Last Descriptor of Packet 2370 * needs End Of Packet (EOP) 2371 * and Report Status (RS) 2372 */ 2373 ctxd->lower.data |= 2374 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 2375 /* 2376 * Keep track in the first buffer which 2377 * descriptor will be written back 2378 */ 2379 tx_buffer = &adapter->tx_buffer_area[first]; 2380 tx_buffer->next_eop = last; 2381 2382 /* 2383 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 2384 * that this frame is available to transmit. 2385 */ 2386 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 2387 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2388 if (adapter->hw.mac.type == e1000_82547 && 2389 adapter->link_duplex == HALF_DUPLEX) 2390 em_82547_move_tail(adapter); 2391 else { 2392 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i); 2393 if (adapter->hw.mac.type == e1000_82547) 2394 em_82547_update_fifo_head(adapter, 2395 m_head->m_pkthdr.len); 2396 } 2397 2398#ifdef EM_TIMESYNC 2399 if (ctxd->upper.data & E1000_TXD_EXTCMD_TSTAMP) { 2400 HW_DEBUGOUT( "@@@ Timestamp bit is set in transmit descriptor\n" ); 2401 } 2402#endif 2403 return (0); 2404} 2405 2406/********************************************************************* 2407 * 2408 * 82547 workaround to avoid controller hang in half-duplex environment. 2409 * The workaround is to avoid queuing a large packet that would span 2410 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers 2411 * in this case. We do that only when FIFO is quiescent. 2412 * 2413 **********************************************************************/ 2414static void 2415em_82547_move_tail(void *arg) 2416{ 2417 struct adapter *adapter = arg; 2418 struct e1000_tx_desc *tx_desc; 2419 u16 hw_tdt, sw_tdt, length = 0; 2420 bool eop = 0; 2421 2422 EM_TX_LOCK_ASSERT(adapter); 2423 2424 hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0)); 2425 sw_tdt = adapter->next_avail_tx_desc; 2426 2427 while (hw_tdt != sw_tdt) { 2428 tx_desc = &adapter->tx_desc_base[hw_tdt]; 2429 length += tx_desc->lower.flags.length; 2430 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP; 2431 if (++hw_tdt == adapter->num_tx_desc) 2432 hw_tdt = 0; 2433 2434 if (eop) { 2435 if (em_82547_fifo_workaround(adapter, length)) { 2436 adapter->tx_fifo_wrk_cnt++; 2437 callout_reset(&adapter->tx_fifo_timer, 1, 2438 em_82547_move_tail, adapter); 2439 break; 2440 } 2441 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt); 2442 em_82547_update_fifo_head(adapter, length); 2443 length = 0; 2444 } 2445 } 2446} 2447 2448static int 2449em_82547_fifo_workaround(struct adapter *adapter, int len) 2450{ 2451 int fifo_space, fifo_pkt_len; 2452 2453 fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2454 2455 if (adapter->link_duplex == HALF_DUPLEX) { 2456 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head; 2457 2458 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) { 2459 if (em_82547_tx_fifo_reset(adapter)) 2460 return (0); 2461 else 2462 return (1); 2463 } 2464 } 2465 2466 return (0); 2467} 2468 2469static void 2470em_82547_update_fifo_head(struct adapter *adapter, int len) 2471{ 2472 int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2473 2474 /* tx_fifo_head is always 16 byte aligned */ 2475 adapter->tx_fifo_head += fifo_pkt_len; 2476 if (adapter->tx_fifo_head >= adapter->tx_fifo_size) { 2477 adapter->tx_fifo_head -= adapter->tx_fifo_size; 2478 } 2479} 2480 2481 2482static int 2483em_82547_tx_fifo_reset(struct adapter *adapter) 2484{ 2485 u32 tctl; 2486 2487 if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) == 2488 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) && 2489 (E1000_READ_REG(&adapter->hw, E1000_TDFT) == 2490 E1000_READ_REG(&adapter->hw, E1000_TDFH)) && 2491 (E1000_READ_REG(&adapter->hw, E1000_TDFTS) == 2492 E1000_READ_REG(&adapter->hw, E1000_TDFHS)) && 2493 (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) { 2494 /* Disable TX unit */ 2495 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 2496 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, 2497 tctl & ~E1000_TCTL_EN); 2498 2499 /* Reset FIFO pointers */ 2500 E1000_WRITE_REG(&adapter->hw, E1000_TDFT, 2501 adapter->tx_head_addr); 2502 E1000_WRITE_REG(&adapter->hw, E1000_TDFH, 2503 adapter->tx_head_addr); 2504 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS, 2505 adapter->tx_head_addr); 2506 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS, 2507 adapter->tx_head_addr); 2508 2509 /* Re-enable TX unit */ 2510 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 2511 E1000_WRITE_FLUSH(&adapter->hw); 2512 2513 adapter->tx_fifo_head = 0; 2514 adapter->tx_fifo_reset_cnt++; 2515 2516 return (TRUE); 2517 } 2518 else { 2519 return (FALSE); 2520 } 2521} 2522 2523static void 2524em_set_promisc(struct adapter *adapter) 2525{ 2526 struct ifnet *ifp = adapter->ifp; 2527 u32 reg_rctl; 2528 2529 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2530 2531 if (ifp->if_flags & IFF_PROMISC) { 2532 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2533 /* Turn this on if you want to see bad packets */ 2534 if (em_debug_sbp) 2535 reg_rctl |= E1000_RCTL_SBP; 2536 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2537 } else if (ifp->if_flags & IFF_ALLMULTI) { 2538 reg_rctl |= E1000_RCTL_MPE; 2539 reg_rctl &= ~E1000_RCTL_UPE; 2540 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2541 } 2542} 2543 2544static void 2545em_disable_promisc(struct adapter *adapter) 2546{ 2547 u32 reg_rctl; 2548 2549 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2550 2551 reg_rctl &= (~E1000_RCTL_UPE); 2552 reg_rctl &= (~E1000_RCTL_MPE); 2553 reg_rctl &= (~E1000_RCTL_SBP); 2554 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2555} 2556 2557 2558/********************************************************************* 2559 * Multicast Update 2560 * 2561 * This routine is called whenever multicast address list is updated. 2562 * 2563 **********************************************************************/ 2564 2565static void 2566em_set_multi(struct adapter *adapter) 2567{ 2568 struct ifnet *ifp = adapter->ifp; 2569 struct ifmultiaddr *ifma; 2570 u32 reg_rctl = 0; 2571 u8 *mta; /* Multicast array memory */ 2572 int mcnt = 0; 2573 2574 IOCTL_DEBUGOUT("em_set_multi: begin"); 2575 2576 if (adapter->hw.mac.type == e1000_82542 && 2577 adapter->hw.revision_id == E1000_REVISION_2) { 2578 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2579 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2580 e1000_pci_clear_mwi(&adapter->hw); 2581 reg_rctl |= E1000_RCTL_RST; 2582 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2583 msec_delay(5); 2584 } 2585 2586 /* Allocate temporary memory to setup array */ 2587 mta = malloc(sizeof(u8) * 2588 (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES), 2589 M_DEVBUF, M_NOWAIT | M_ZERO); 2590 if (mta == NULL) 2591 panic("em_set_multi memory failure\n"); 2592 2593 IF_ADDR_LOCK(ifp); 2594 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2595 if (ifma->ifma_addr->sa_family != AF_LINK) 2596 continue; 2597 2598 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 2599 break; 2600 2601 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2602 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 2603 mcnt++; 2604 } 2605 IF_ADDR_UNLOCK(ifp); 2606 2607 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 2608 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2609 reg_rctl |= E1000_RCTL_MPE; 2610 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2611 } else 2612 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 2613 2614 if (adapter->hw.mac.type == e1000_82542 && 2615 adapter->hw.revision_id == E1000_REVISION_2) { 2616 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2617 reg_rctl &= ~E1000_RCTL_RST; 2618 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2619 msec_delay(5); 2620 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2621 e1000_pci_set_mwi(&adapter->hw); 2622 } 2623 free(mta, M_DEVBUF); 2624} 2625 2626 2627/********************************************************************* 2628 * Timer routine 2629 * 2630 * This routine checks for link status and updates statistics. 2631 * 2632 **********************************************************************/ 2633 2634static void 2635em_local_timer(void *arg) 2636{ 2637 struct adapter *adapter = arg; 2638 struct ifnet *ifp = adapter->ifp; 2639 2640 EM_CORE_LOCK_ASSERT(adapter); 2641 2642 taskqueue_enqueue(adapter->tq, 2643 &adapter->rxtx_task); 2644 em_update_link_status(adapter); 2645 em_update_stats_counters(adapter); 2646 2647 /* Reset LAA into RAR[0] on 82571 */ 2648 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE) 2649 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 2650 2651 if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 2652 em_print_hw_stats(adapter); 2653 2654 em_smartspeed(adapter); 2655 2656 /* 2657 * Each second we check the watchdog to 2658 * protect against hardware hangs. 2659 */ 2660 em_watchdog(adapter); 2661 2662 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 2663 2664} 2665 2666static void 2667em_update_link_status(struct adapter *adapter) 2668{ 2669 struct e1000_hw *hw = &adapter->hw; 2670 struct ifnet *ifp = adapter->ifp; 2671 device_t dev = adapter->dev; 2672 u32 link_check = 0; 2673 2674 /* Get the cached link value or read phy for real */ 2675 switch (hw->phy.media_type) { 2676 case e1000_media_type_copper: 2677 if (hw->mac.get_link_status) { 2678 /* Do the work to read phy */ 2679 e1000_check_for_link(hw); 2680 link_check = !hw->mac.get_link_status; 2681 if (link_check) /* ESB2 fix */ 2682 e1000_cfg_on_link_up(hw); 2683 } else 2684 link_check = TRUE; 2685 break; 2686 case e1000_media_type_fiber: 2687 e1000_check_for_link(hw); 2688 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2689 E1000_STATUS_LU); 2690 break; 2691 case e1000_media_type_internal_serdes: 2692 e1000_check_for_link(hw); 2693 link_check = adapter->hw.mac.serdes_has_link; 2694 break; 2695 default: 2696 case e1000_media_type_unknown: 2697 break; 2698 } 2699 2700 /* Now check for a transition */ 2701 if (link_check && (adapter->link_active == 0)) { 2702 e1000_get_speed_and_duplex(hw, &adapter->link_speed, 2703 &adapter->link_duplex); 2704 /* Check if we must disable SPEED_MODE bit on PCI-E */ 2705 if ((adapter->link_speed != SPEED_1000) && 2706 ((hw->mac.type == e1000_82571) || 2707 (hw->mac.type == e1000_82572))) { 2708 int tarc0; 2709 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2710 tarc0 &= ~SPEED_MODE_BIT; 2711 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2712 } 2713 if (bootverbose) 2714 device_printf(dev, "Link is up %d Mbps %s\n", 2715 adapter->link_speed, 2716 ((adapter->link_duplex == FULL_DUPLEX) ? 2717 "Full Duplex" : "Half Duplex")); 2718 adapter->link_active = 1; 2719 adapter->smartspeed = 0; 2720 ifp->if_baudrate = adapter->link_speed * 1000000; 2721 if_link_state_change(ifp, LINK_STATE_UP); 2722 } else if (!link_check && (adapter->link_active == 1)) { 2723 ifp->if_baudrate = adapter->link_speed = 0; 2724 adapter->link_duplex = 0; 2725 if (bootverbose) 2726 device_printf(dev, "Link is Down\n"); 2727 adapter->link_active = 0; 2728 /* Link down, disable watchdog */ 2729 adapter->watchdog_timer = FALSE; 2730 if_link_state_change(ifp, LINK_STATE_DOWN); 2731 } 2732} 2733 2734/********************************************************************* 2735 * 2736 * This routine disables all traffic on the adapter by issuing a 2737 * global reset on the MAC and deallocates TX/RX buffers. 2738 * 2739 * This routine should always be called with BOTH the CORE 2740 * and TX locks. 2741 **********************************************************************/ 2742 2743static void 2744em_stop(void *arg) 2745{ 2746 struct adapter *adapter = arg; 2747 struct ifnet *ifp = adapter->ifp; 2748 2749 EM_CORE_LOCK_ASSERT(adapter); 2750 EM_TX_LOCK_ASSERT(adapter); 2751 2752 INIT_DEBUGOUT("em_stop: begin"); 2753 2754 em_disable_intr(adapter); 2755 callout_stop(&adapter->timer); 2756 callout_stop(&adapter->tx_fifo_timer); 2757 2758 /* Tell the stack that the interface is no longer active */ 2759 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2760 2761#ifdef EM_TIMESYNC 2762 /* Disable IEEE 1588 Time hardware */ 2763 if ((adapter->hw.mac.type == e1000_82574) || 2764 (adapter->hw.mac.type == e1000_ich10lan)) 2765 em_tsync_disable(adapter); 2766#endif 2767 2768 e1000_reset_hw(&adapter->hw); 2769 if (adapter->hw.mac.type >= e1000_82544) 2770 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2771} 2772 2773 2774/********************************************************************* 2775 * 2776 * Determine hardware revision. 2777 * 2778 **********************************************************************/ 2779static void 2780em_identify_hardware(struct adapter *adapter) 2781{ 2782 device_t dev = adapter->dev; 2783 2784 /* Make sure our PCI config space has the necessary stuff set */ 2785 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2786 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2787 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2788 device_printf(dev, "Memory Access and/or Bus Master bits " 2789 "were not set!\n"); 2790 adapter->hw.bus.pci_cmd_word |= 2791 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2792 pci_write_config(dev, PCIR_COMMAND, 2793 adapter->hw.bus.pci_cmd_word, 2); 2794 } 2795 2796 /* Save off the information about this board */ 2797 adapter->hw.vendor_id = pci_get_vendor(dev); 2798 adapter->hw.device_id = pci_get_device(dev); 2799 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2800 adapter->hw.subsystem_vendor_id = 2801 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2802 adapter->hw.subsystem_device_id = 2803 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2804 2805 /* Do Shared Code Init and Setup */ 2806 if (e1000_set_mac_type(&adapter->hw)) { 2807 device_printf(dev, "Setup init failure\n"); 2808 return; 2809 } 2810} 2811 2812static int 2813em_allocate_pci_resources(struct adapter *adapter) 2814{ 2815 device_t dev = adapter->dev; 2816 int val, rid, error = E1000_SUCCESS; 2817 2818 rid = PCIR_BAR(0); 2819 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2820 &rid, RF_ACTIVE); 2821 if (adapter->memory == NULL) { 2822 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2823 return (ENXIO); 2824 } 2825 adapter->osdep.mem_bus_space_tag = 2826 rman_get_bustag(adapter->memory); 2827 adapter->osdep.mem_bus_space_handle = 2828 rman_get_bushandle(adapter->memory); 2829 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2830 2831 /* Only older adapters use IO mapping */ 2832 if ((adapter->hw.mac.type > e1000_82543) && 2833 (adapter->hw.mac.type < e1000_82571)) { 2834 /* Figure our where our IO BAR is ? */ 2835 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { 2836 val = pci_read_config(dev, rid, 4); 2837 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { 2838 adapter->io_rid = rid; 2839 break; 2840 } 2841 rid += 4; 2842 /* check for 64bit BAR */ 2843 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) 2844 rid += 4; 2845 } 2846 if (rid >= PCIR_CIS) { 2847 device_printf(dev, "Unable to locate IO BAR\n"); 2848 return (ENXIO); 2849 } 2850 adapter->ioport = bus_alloc_resource_any(dev, 2851 SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); 2852 if (adapter->ioport == NULL) { 2853 device_printf(dev, "Unable to allocate bus resource: " 2854 "ioport\n"); 2855 return (ENXIO); 2856 } 2857 adapter->hw.io_base = 0; 2858 adapter->osdep.io_bus_space_tag = 2859 rman_get_bustag(adapter->ioport); 2860 adapter->osdep.io_bus_space_handle = 2861 rman_get_bushandle(adapter->ioport); 2862 } 2863 2864 /* 2865 ** Init the resource arrays 2866 ** used by MSIX setup 2867 */ 2868 for (int i = 0; i < 3; i++) { 2869 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 2870 adapter->tag[i] = NULL; 2871 adapter->res[i] = NULL; 2872 } 2873 2874 /* 2875 * Setup MSI/X or MSI if PCI Express 2876 */ 2877 if (em_enable_msi) 2878 adapter->msi = em_setup_msix(adapter); 2879 2880 adapter->hw.back = &adapter->osdep; 2881 2882 return (error); 2883} 2884 2885/********************************************************************* 2886 * 2887 * Setup the Legacy or MSI Interrupt handler 2888 * 2889 **********************************************************************/ 2890int 2891em_allocate_legacy(struct adapter *adapter) 2892{ 2893 device_t dev = adapter->dev; 2894 int error; 2895 2896 /* Manually turn off all interrupts */ 2897 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2898 2899 /* Legacy RID is 0 */ 2900 if (adapter->msi == 0) 2901 adapter->rid[0] = 0; 2902 2903 /* We allocate a single interrupt resource */ 2904 adapter->res[0] = bus_alloc_resource_any(dev, 2905 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); 2906 if (adapter->res[0] == NULL) { 2907 device_printf(dev, "Unable to allocate bus resource: " 2908 "interrupt\n"); 2909 return (ENXIO); 2910 } 2911 2912#ifdef EM_LEGACY_IRQ 2913 /* We do Legacy setup */ 2914 if ((error = bus_setup_intr(dev, adapter->res[0], 2915#if __FreeBSD_version > 700000 2916 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter, 2917#else /* 6.X */ 2918 INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter, 2919#endif 2920 &adapter->tag[0])) != 0) { 2921 device_printf(dev, "Failed to register interrupt handler"); 2922 return (error); 2923 } 2924 2925#else /* FAST_IRQ */ 2926 /* 2927 * Try allocating a fast interrupt and the associated deferred 2928 * processing contexts. 2929 */ 2930 TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter); 2931 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2932 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2933 taskqueue_thread_enqueue, &adapter->tq); 2934 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2935 device_get_nameunit(adapter->dev)); 2936#if __FreeBSD_version < 700000 2937 if ((error = bus_setup_intr(dev, adapter->res[0], 2938 INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter, 2939#else 2940 if ((error = bus_setup_intr(dev, adapter->res[0], 2941 INTR_TYPE_NET, em_irq_fast, NULL, adapter, 2942#endif 2943 &adapter->tag[0])) != 0) { 2944 device_printf(dev, "Failed to register fast interrupt " 2945 "handler: %d\n", error); 2946 taskqueue_free(adapter->tq); 2947 adapter->tq = NULL; 2948 return (error); 2949 } 2950#endif /* EM_LEGACY_IRQ */ 2951 2952 return (0); 2953} 2954 2955/********************************************************************* 2956 * 2957 * Setup the MSIX Interrupt handlers 2958 * This is not really Multiqueue, rather 2959 * its just multiple interrupt vectors. 2960 * 2961 **********************************************************************/ 2962int 2963em_allocate_msix(struct adapter *adapter) 2964{ 2965 device_t dev = adapter->dev; 2966 int error; 2967 2968 /* Make sure all interrupts are disabled */ 2969 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2970 2971 /* First get the resources */ 2972 for (int i = 0; i < adapter->msi; i++) { 2973 adapter->res[i] = bus_alloc_resource_any(dev, 2974 SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE); 2975 if (adapter->res[i] == NULL) { 2976 device_printf(dev, 2977 "Unable to allocate bus resource: " 2978 "MSIX Interrupt\n"); 2979 return (ENXIO); 2980 } 2981 } 2982 2983 /* 2984 * Now allocate deferred processing contexts. 2985 */ 2986 TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter); 2987 TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter); 2988 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2989 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2990 taskqueue_thread_enqueue, &adapter->tq); 2991 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2992 device_get_nameunit(adapter->dev)); 2993 2994 /* 2995 * And setup the interrupt handlers 2996 */ 2997 2998 /* First slot to RX */ 2999 if ((error = bus_setup_intr(dev, adapter->res[0], 3000#if __FreeBSD_version > 700000 3001 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, adapter, 3002#else /* 6.X */ 3003 INTR_TYPE_NET | INTR_MPSAFE, em_msix_rx, adapter, 3004#endif 3005 &adapter->tag[0])) != 0) { 3006 device_printf(dev, "Failed to register RX handler"); 3007 return (error); 3008 } 3009 3010 /* Next TX */ 3011 if ((error = bus_setup_intr(dev, adapter->res[1], 3012#if __FreeBSD_version > 700000 3013 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, adapter, 3014#else /* 6.X */ 3015 INTR_TYPE_NET | INTR_MPSAFE, em_msix_tx, adapter, 3016#endif 3017 &adapter->tag[1])) != 0) { 3018 device_printf(dev, "Failed to register TX handler"); 3019 return (error); 3020 } 3021 3022 /* And Link */ 3023 if ((error = bus_setup_intr(dev, adapter->res[2], 3024#if __FreeBSD_version > 700000 3025 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_link, adapter, 3026#else /* 6.X */ 3027 INTR_TYPE_NET | INTR_MPSAFE, em_msix_link, adapter, 3028#endif 3029 &adapter->tag[2])) != 0) { 3030 device_printf(dev, "Failed to register TX handler"); 3031 return (error); 3032 } 3033 3034 return (0); 3035} 3036 3037 3038static void 3039em_free_pci_resources(struct adapter *adapter) 3040{ 3041 device_t dev = adapter->dev; 3042 3043 /* Make sure the for loop below runs once */ 3044 if (adapter->msi == 0) 3045 adapter->msi = 1; 3046 3047 /* 3048 * First release all the interrupt resources: 3049 * notice that since these are just kept 3050 * in an array we can do the same logic 3051 * whether its MSIX or just legacy. 3052 */ 3053 for (int i = 0; i < adapter->msi; i++) { 3054 if (adapter->tag[i] != NULL) { 3055 bus_teardown_intr(dev, adapter->res[i], 3056 adapter->tag[i]); 3057 adapter->tag[i] = NULL; 3058 } 3059 if (adapter->res[i] != NULL) { 3060 bus_release_resource(dev, SYS_RES_IRQ, 3061 adapter->rid[i], adapter->res[i]); 3062 } 3063 } 3064 3065 if (adapter->msi) 3066 pci_release_msi(dev); 3067 3068 if (adapter->msix != NULL) 3069 bus_release_resource(dev, SYS_RES_MEMORY, 3070 PCIR_BAR(EM_MSIX_BAR), adapter->msix); 3071 3072 if (adapter->memory != NULL) 3073 bus_release_resource(dev, SYS_RES_MEMORY, 3074 PCIR_BAR(0), adapter->memory); 3075 3076 if (adapter->flash != NULL) 3077 bus_release_resource(dev, SYS_RES_MEMORY, 3078 EM_FLASH, adapter->flash); 3079 3080 if (adapter->ioport != NULL) 3081 bus_release_resource(dev, SYS_RES_IOPORT, 3082 adapter->io_rid, adapter->ioport); 3083} 3084 3085/* 3086 * Setup MSI or MSI/X 3087 */ 3088static int 3089em_setup_msix(struct adapter *adapter) 3090{ 3091 device_t dev = adapter->dev; 3092 int val = 0; 3093 3094 if (adapter->hw.mac.type < e1000_82571) 3095 return (0); 3096 3097 /* Setup MSI/X for Hartwell */ 3098 if (adapter->hw.mac.type == e1000_82574) { 3099 /* Map the MSIX BAR */ 3100 int rid = PCIR_BAR(EM_MSIX_BAR); 3101 adapter->msix = bus_alloc_resource_any(dev, 3102 SYS_RES_MEMORY, &rid, RF_ACTIVE); 3103 if (!adapter->msix) { 3104 /* May not be enabled */ 3105 device_printf(adapter->dev, 3106 "Unable to map MSIX table \n"); 3107 goto msi; 3108 } 3109 val = pci_msix_count(dev); 3110 /* 3111 ** 82574 can be configured for 5 but 3112 ** we limit use to 3. 3113 */ 3114 if (val > 3) val = 3; 3115 if ((val) && pci_alloc_msix(dev, &val) == 0) { 3116 device_printf(adapter->dev,"Using MSIX interrupts\n"); 3117 return (val); 3118 } 3119 } 3120msi: 3121 val = pci_msi_count(dev); 3122 if (val == 1 && pci_alloc_msi(dev, &val) == 0) { 3123 adapter->msi = 1; 3124 device_printf(adapter->dev,"Using MSI interrupt\n"); 3125 return (val); 3126 } 3127 return (0); 3128} 3129 3130/********************************************************************* 3131 * 3132 * Initialize the hardware to a configuration 3133 * as specified by the adapter structure. 3134 * 3135 **********************************************************************/ 3136static int 3137em_hardware_init(struct adapter *adapter) 3138{ 3139 device_t dev = adapter->dev; 3140 u16 rx_buffer_size; 3141 3142 INIT_DEBUGOUT("em_hardware_init: begin"); 3143 3144 /* Issue a global reset */ 3145 e1000_reset_hw(&adapter->hw); 3146 3147 /* Get control from any management/hw control */ 3148 if (((adapter->hw.mac.type == e1000_82573) || 3149 (adapter->hw.mac.type == e1000_ich8lan) || 3150 (adapter->hw.mac.type == e1000_ich10lan) || 3151 (adapter->hw.mac.type == e1000_ich9lan)) && 3152 e1000_check_mng_mode(&adapter->hw)) 3153 em_get_hw_control(adapter); 3154 3155 /* When hardware is reset, fifo_head is also reset */ 3156 adapter->tx_fifo_head = 0; 3157 3158 /* Set up smart power down as default off on newer adapters. */ 3159 if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 || 3160 adapter->hw.mac.type == e1000_82572)) { 3161 u16 phy_tmp = 0; 3162 3163 /* Speed up time to link by disabling smart power down. */ 3164 e1000_read_phy_reg(&adapter->hw, 3165 IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 3166 phy_tmp &= ~IGP02E1000_PM_SPD; 3167 e1000_write_phy_reg(&adapter->hw, 3168 IGP02E1000_PHY_POWER_MGMT, phy_tmp); 3169 } 3170 3171 /* 3172 * These parameters control the automatic generation (Tx) and 3173 * response (Rx) to Ethernet PAUSE frames. 3174 * - High water mark should allow for at least two frames to be 3175 * received after sending an XOFF. 3176 * - Low water mark works best when it is very near the high water mark. 3177 * This allows the receiver to restart by sending XON when it has 3178 * drained a bit. Here we use an arbitary value of 1500 which will 3179 * restart after one full frame is pulled from the buffer. There 3180 * could be several smaller frames in the buffer and if so they will 3181 * not trigger the XON until their total number reduces the buffer 3182 * by 1500. 3183 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 3184 */ 3185 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 3186 0xffff) << 10 ); 3187 3188 adapter->hw.fc.high_water = rx_buffer_size - 3189 roundup2(adapter->max_frame_size, 1024); 3190 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; 3191 3192 if (adapter->hw.mac.type == e1000_80003es2lan) 3193 adapter->hw.fc.pause_time = 0xFFFF; 3194 else 3195 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME; 3196 adapter->hw.fc.send_xon = TRUE; 3197 adapter->hw.fc.requested_mode = e1000_fc_full; 3198 3199 if (e1000_init_hw(&adapter->hw) < 0) { 3200 device_printf(dev, "Hardware Initialization Failed\n"); 3201 return (EIO); 3202 } 3203 3204 e1000_check_for_link(&adapter->hw); 3205 3206 return (0); 3207} 3208 3209/********************************************************************* 3210 * 3211 * Setup networking device structure and register an interface. 3212 * 3213 **********************************************************************/ 3214static void 3215em_setup_interface(device_t dev, struct adapter *adapter) 3216{ 3217 struct ifnet *ifp; 3218 3219 INIT_DEBUGOUT("em_setup_interface: begin"); 3220 3221 ifp = adapter->ifp = if_alloc(IFT_ETHER); 3222 if (ifp == NULL) 3223 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 3224 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 3225 ifp->if_mtu = ETHERMTU; 3226 ifp->if_init = em_init; 3227 ifp->if_softc = adapter; 3228 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 3229 ifp->if_ioctl = em_ioctl; 3230 ifp->if_start = em_start; 3231 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 3232 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 3233 IFQ_SET_READY(&ifp->if_snd); 3234 3235 ether_ifattach(ifp, adapter->hw.mac.addr); 3236 3237 ifp->if_capabilities = ifp->if_capenable = 0; 3238 3239#ifdef IFNET_BUF_RING 3240 ifp->if_transmit = em_transmit; 3241 ifp->if_qflush = em_qflush; 3242 adapter->br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK, &adapter->tx_mtx); 3243#endif 3244 if (adapter->hw.mac.type >= e1000_82543) { 3245 int version_cap; 3246#if __FreeBSD_version < 700000 3247 version_cap = IFCAP_HWCSUM; 3248#else 3249 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 3250#endif 3251 ifp->if_capabilities |= version_cap; 3252 ifp->if_capenable |= version_cap; 3253 } 3254 3255#if __FreeBSD_version >= 700000 3256 /* Identify TSO capable adapters */ 3257 if ((adapter->hw.mac.type > e1000_82544) && 3258 (adapter->hw.mac.type != e1000_82547)) 3259 ifp->if_capabilities |= IFCAP_TSO4; 3260 /* 3261 * By default only enable on PCI-E, this 3262 * can be overriden by ifconfig. 3263 */ 3264 if (adapter->hw.mac.type >= e1000_82571) 3265 ifp->if_capenable |= IFCAP_TSO4; 3266#endif 3267 3268 /* 3269 * Tell the upper layer(s) we support long frames. 3270 */ 3271 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 3272 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 3273 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 3274 3275#ifdef DEVICE_POLLING 3276 ifp->if_capabilities |= IFCAP_POLLING; 3277#endif 3278 3279 /* 3280 * Specify the media types supported by this adapter and register 3281 * callbacks to update media and link information 3282 */ 3283 ifmedia_init(&adapter->media, IFM_IMASK, 3284 em_media_change, em_media_status); 3285 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3286 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 3287 u_char fiber_type = IFM_1000_SX; /* default type */ 3288 3289 if (adapter->hw.mac.type == e1000_82545) 3290 fiber_type = IFM_1000_LX; 3291 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 3292 0, NULL); 3293 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); 3294 } else { 3295 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 3296 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 3297 0, NULL); 3298 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 3299 0, NULL); 3300 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 3301 0, NULL); 3302 if (adapter->hw.phy.type != e1000_phy_ife) { 3303 ifmedia_add(&adapter->media, 3304 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 3305 ifmedia_add(&adapter->media, 3306 IFM_ETHER | IFM_1000_T, 0, NULL); 3307 } 3308 } 3309 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 3310 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 3311} 3312 3313 3314/********************************************************************* 3315 * 3316 * Workaround for SmartSpeed on 82541 and 82547 controllers 3317 * 3318 **********************************************************************/ 3319static void 3320em_smartspeed(struct adapter *adapter) 3321{ 3322 u16 phy_tmp; 3323 3324 if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || 3325 adapter->hw.mac.autoneg == 0 || 3326 (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 3327 return; 3328 3329 if (adapter->smartspeed == 0) { 3330 /* If Master/Slave config fault is asserted twice, 3331 * we assume back-to-back */ 3332 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 3333 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 3334 return; 3335 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 3336 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 3337 e1000_read_phy_reg(&adapter->hw, 3338 PHY_1000T_CTRL, &phy_tmp); 3339 if(phy_tmp & CR_1000T_MS_ENABLE) { 3340 phy_tmp &= ~CR_1000T_MS_ENABLE; 3341 e1000_write_phy_reg(&adapter->hw, 3342 PHY_1000T_CTRL, phy_tmp); 3343 adapter->smartspeed++; 3344 if(adapter->hw.mac.autoneg && 3345 !e1000_phy_setup_autoneg(&adapter->hw) && 3346 !e1000_read_phy_reg(&adapter->hw, 3347 PHY_CONTROL, &phy_tmp)) { 3348 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3349 MII_CR_RESTART_AUTO_NEG); 3350 e1000_write_phy_reg(&adapter->hw, 3351 PHY_CONTROL, phy_tmp); 3352 } 3353 } 3354 } 3355 return; 3356 } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { 3357 /* If still no link, perhaps using 2/3 pair cable */ 3358 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); 3359 phy_tmp |= CR_1000T_MS_ENABLE; 3360 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); 3361 if(adapter->hw.mac.autoneg && 3362 !e1000_phy_setup_autoneg(&adapter->hw) && 3363 !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { 3364 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3365 MII_CR_RESTART_AUTO_NEG); 3366 e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); 3367 } 3368 } 3369 /* Restart process after EM_SMARTSPEED_MAX iterations */ 3370 if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) 3371 adapter->smartspeed = 0; 3372} 3373 3374 3375/* 3376 * Manage DMA'able memory. 3377 */ 3378static void 3379em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3380{ 3381 if (error) 3382 return; 3383 *(bus_addr_t *) arg = segs[0].ds_addr; 3384} 3385 3386static int 3387em_dma_malloc(struct adapter *adapter, bus_size_t size, 3388 struct em_dma_alloc *dma, int mapflags) 3389{ 3390 int error; 3391 3392#if __FreeBSD_version >= 700000 3393 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 3394#else 3395 error = bus_dma_tag_create(NULL, /* parent */ 3396#endif 3397 EM_DBA_ALIGN, 0, /* alignment, bounds */ 3398 BUS_SPACE_MAXADDR, /* lowaddr */ 3399 BUS_SPACE_MAXADDR, /* highaddr */ 3400 NULL, NULL, /* filter, filterarg */ 3401 size, /* maxsize */ 3402 1, /* nsegments */ 3403 size, /* maxsegsize */ 3404 0, /* flags */ 3405 NULL, /* lockfunc */ 3406 NULL, /* lockarg */ 3407 &dma->dma_tag); 3408 if (error) { 3409 device_printf(adapter->dev, 3410 "%s: bus_dma_tag_create failed: %d\n", 3411 __func__, error); 3412 goto fail_0; 3413 } 3414 3415 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 3416 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); 3417 if (error) { 3418 device_printf(adapter->dev, 3419 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 3420 __func__, (uintmax_t)size, error); 3421 goto fail_2; 3422 } 3423 3424 dma->dma_paddr = 0; 3425 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3426 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 3427 if (error || dma->dma_paddr == 0) { 3428 device_printf(adapter->dev, 3429 "%s: bus_dmamap_load failed: %d\n", 3430 __func__, error); 3431 goto fail_3; 3432 } 3433 3434 return (0); 3435 3436fail_3: 3437 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3438fail_2: 3439 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3440 bus_dma_tag_destroy(dma->dma_tag); 3441fail_0: 3442 dma->dma_map = NULL; 3443 dma->dma_tag = NULL; 3444 3445 return (error); 3446} 3447 3448static void 3449em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) 3450{ 3451 if (dma->dma_tag == NULL) 3452 return; 3453 if (dma->dma_map != NULL) { 3454 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3455 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3456 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3457 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3458 dma->dma_map = NULL; 3459 } 3460 bus_dma_tag_destroy(dma->dma_tag); 3461 dma->dma_tag = NULL; 3462} 3463 3464 3465/********************************************************************* 3466 * 3467 * Allocate memory for tx_buffer structures. The tx_buffer stores all 3468 * the information needed to transmit a packet on the wire. 3469 * 3470 **********************************************************************/ 3471static int 3472em_allocate_transmit_structures(struct adapter *adapter) 3473{ 3474 device_t dev = adapter->dev; 3475 struct em_buffer *tx_buffer; 3476 int error; 3477 3478 /* 3479 * Create DMA tags for tx descriptors 3480 */ 3481#if __FreeBSD_version >= 700000 3482 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 3483#else 3484 if ((error = bus_dma_tag_create(NULL, /* parent */ 3485#endif 3486 1, 0, /* alignment, bounds */ 3487 BUS_SPACE_MAXADDR, /* lowaddr */ 3488 BUS_SPACE_MAXADDR, /* highaddr */ 3489 NULL, NULL, /* filter, filterarg */ 3490 EM_TSO_SIZE, /* maxsize */ 3491 EM_MAX_SCATTER, /* nsegments */ 3492 EM_TSO_SEG_SIZE, /* maxsegsize */ 3493 0, /* flags */ 3494 NULL, /* lockfunc */ 3495 NULL, /* lockarg */ 3496 &adapter->txtag)) != 0) { 3497 device_printf(dev, "Unable to allocate TX DMA tag\n"); 3498 goto fail; 3499 } 3500 3501 adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) * 3502 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 3503 if (adapter->tx_buffer_area == NULL) { 3504 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 3505 error = ENOMEM; 3506 goto fail; 3507 } 3508 3509 /* Create the descriptor buffer dma maps */ 3510 for (int i = 0; i < adapter->num_tx_desc; i++) { 3511 tx_buffer = &adapter->tx_buffer_area[i]; 3512 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map); 3513 if (error != 0) { 3514 device_printf(dev, "Unable to create TX DMA map\n"); 3515 goto fail; 3516 } 3517 tx_buffer->next_eop = -1; 3518 } 3519 3520 return (0); 3521fail: 3522 em_free_transmit_structures(adapter); 3523 return (error); 3524} 3525 3526/********************************************************************* 3527 * 3528 * (Re)Initialize transmit structures. 3529 * 3530 **********************************************************************/ 3531static void 3532em_setup_transmit_structures(struct adapter *adapter) 3533{ 3534 struct em_buffer *tx_buffer; 3535 3536 /* Clear the old ring contents */ 3537 bzero(adapter->tx_desc_base, 3538 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); 3539 3540 /* Free any existing TX buffers */ 3541 for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3542 tx_buffer = &adapter->tx_buffer_area[i]; 3543 bus_dmamap_sync(adapter->txtag, tx_buffer->map, 3544 BUS_DMASYNC_POSTWRITE); 3545 bus_dmamap_unload(adapter->txtag, tx_buffer->map); 3546 m_freem(tx_buffer->m_head); 3547 tx_buffer->m_head = NULL; 3548 tx_buffer->next_eop = -1; 3549 } 3550 3551 /* Reset state */ 3552 adapter->next_avail_tx_desc = 0; 3553 adapter->next_tx_to_clean = 0; 3554 adapter->num_tx_desc_avail = adapter->num_tx_desc; 3555 3556 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 3557 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3558 3559 return; 3560} 3561 3562/********************************************************************* 3563 * 3564 * Enable transmit unit. 3565 * 3566 **********************************************************************/ 3567static void 3568em_initialize_transmit_unit(struct adapter *adapter) 3569{ 3570 u32 tctl, tarc, tipg = 0; 3571 u64 bus_addr; 3572 3573 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 3574 /* Setup the Base and Length of the Tx Descriptor Ring */ 3575 bus_addr = adapter->txdma.dma_paddr; 3576 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0), 3577 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3578 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0), 3579 (u32)(bus_addr >> 32)); 3580 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0), 3581 (u32)bus_addr); 3582 /* Setup the HW Tx Head and Tail descriptor pointers */ 3583 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0); 3584 E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0); 3585 3586 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3587 E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)), 3588 E1000_READ_REG(&adapter->hw, E1000_TDLEN(0))); 3589 3590 /* Set the default values for the Tx Inter Packet Gap timer */ 3591 switch (adapter->hw.mac.type) { 3592 case e1000_82542: 3593 tipg = DEFAULT_82542_TIPG_IPGT; 3594 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3595 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3596 break; 3597 case e1000_80003es2lan: 3598 tipg = DEFAULT_82543_TIPG_IPGR1; 3599 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 3600 E1000_TIPG_IPGR2_SHIFT; 3601 break; 3602 default: 3603 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3604 (adapter->hw.phy.media_type == 3605 e1000_media_type_internal_serdes)) 3606 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 3607 else 3608 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 3609 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3610 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3611 } 3612 3613 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); 3614 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); 3615 if(adapter->hw.mac.type >= e1000_82540) 3616 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 3617 adapter->tx_abs_int_delay.value); 3618 3619 if ((adapter->hw.mac.type == e1000_82571) || 3620 (adapter->hw.mac.type == e1000_82572)) { 3621 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3622 tarc |= SPEED_MODE_BIT; 3623 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3624 } else if (adapter->hw.mac.type == e1000_80003es2lan) { 3625 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3626 tarc |= 1; 3627 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3628 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 3629 tarc |= 1; 3630 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 3631 } 3632 3633 /* Program the Transmit Control Register */ 3634 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 3635 tctl &= ~E1000_TCTL_CT; 3636 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3637 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3638 3639 if (adapter->hw.mac.type >= e1000_82571) 3640 tctl |= E1000_TCTL_MULR; 3641 3642 /* This write will effectively turn on the transmit unit. */ 3643 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 3644 3645 /* Setup Transmit Descriptor Base Settings */ 3646 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 3647 3648 if (adapter->tx_int_delay.value > 0) 3649 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3650} 3651 3652/********************************************************************* 3653 * 3654 * Free all transmit related data structures. 3655 * 3656 **********************************************************************/ 3657static void 3658em_free_transmit_structures(struct adapter *adapter) 3659{ 3660 struct em_buffer *tx_buffer; 3661 3662 INIT_DEBUGOUT("free_transmit_structures: begin"); 3663 3664 if (adapter->tx_buffer_area != NULL) { 3665 for (int i = 0; i < adapter->num_tx_desc; i++) { 3666 tx_buffer = &adapter->tx_buffer_area[i]; 3667 if (tx_buffer->m_head != NULL) { 3668 bus_dmamap_sync(adapter->txtag, tx_buffer->map, 3669 BUS_DMASYNC_POSTWRITE); 3670 bus_dmamap_unload(adapter->txtag, 3671 tx_buffer->map); 3672 m_freem(tx_buffer->m_head); 3673 tx_buffer->m_head = NULL; 3674 } else if (tx_buffer->map != NULL) 3675 bus_dmamap_unload(adapter->txtag, 3676 tx_buffer->map); 3677 if (tx_buffer->map != NULL) { 3678 bus_dmamap_destroy(adapter->txtag, 3679 tx_buffer->map); 3680 tx_buffer->map = NULL; 3681 } 3682 } 3683 } 3684 if (adapter->tx_buffer_area != NULL) { 3685 free(adapter->tx_buffer_area, M_DEVBUF); 3686 adapter->tx_buffer_area = NULL; 3687 } 3688 if (adapter->txtag != NULL) { 3689 bus_dma_tag_destroy(adapter->txtag); 3690 adapter->txtag = NULL; 3691 } 3692} 3693 3694/********************************************************************* 3695 * 3696 * The offload context needs to be set when we transfer the first 3697 * packet of a particular protocol (TCP/UDP). This routine has been 3698 * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) 3699 * 3700 **********************************************************************/ 3701static void 3702em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp, 3703 u32 *txd_upper, u32 *txd_lower) 3704{ 3705 struct e1000_context_desc *TXD; 3706 struct em_buffer *tx_buffer; 3707 struct ether_vlan_header *eh; 3708 struct ip *ip = NULL; 3709 struct ip6_hdr *ip6; 3710 struct tcp_hdr *th; 3711 int curr_txd, ehdrlen; 3712 u32 cmd, hdr_len, ip_hlen; 3713 u16 etype; 3714 u8 ipproto; 3715 3716 cmd = hdr_len = ipproto = 0; 3717 /* Setup checksum offload context. */ 3718 curr_txd = adapter->next_avail_tx_desc; 3719 tx_buffer = &adapter->tx_buffer_area[curr_txd]; 3720 TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; 3721 3722 /* 3723 * Determine where frame payload starts. 3724 * Jump over vlan headers if already present, 3725 * helpful for QinQ too. 3726 */ 3727 eh = mtod(mp, struct ether_vlan_header *); 3728 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3729 etype = ntohs(eh->evl_proto); 3730 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3731 } else { 3732 etype = ntohs(eh->evl_encap_proto); 3733 ehdrlen = ETHER_HDR_LEN; 3734 } 3735 3736 /* 3737 * We only support TCP/UDP for IPv4 and IPv6 for the moment. 3738 * TODO: Support SCTP too when it hits the tree. 3739 */ 3740 switch (etype) { 3741 case ETHERTYPE_IP: 3742 ip = (struct ip *)(mp->m_data + ehdrlen); 3743 ip_hlen = ip->ip_hl << 2; 3744 3745 /* Setup of IP header checksum. */ 3746 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 3747 /* 3748 * Start offset for header checksum calculation. 3749 * End offset for header checksum calculation. 3750 * Offset of place to put the checksum. 3751 */ 3752 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3753 TXD->lower_setup.ip_fields.ipcse = 3754 htole16(ehdrlen + ip_hlen); 3755 TXD->lower_setup.ip_fields.ipcso = 3756 ehdrlen + offsetof(struct ip, ip_sum); 3757 cmd |= E1000_TXD_CMD_IP; 3758 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 3759 } 3760 3761 if (mp->m_len < ehdrlen + ip_hlen) 3762 return; /* failure */ 3763 3764 hdr_len = ehdrlen + ip_hlen; 3765 ipproto = ip->ip_p; 3766 3767 break; 3768 case ETHERTYPE_IPV6: 3769 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3770 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ 3771 3772 if (mp->m_len < ehdrlen + ip_hlen) 3773 return; /* failure */ 3774 3775 /* IPv6 doesn't have a header checksum. */ 3776 3777 hdr_len = ehdrlen + ip_hlen; 3778 ipproto = ip6->ip6_nxt; 3779 3780 break; 3781#ifdef EM_TIMESYNC 3782 case ETHERTYPE_IEEE1588: 3783 *txd_upper |= E1000_TXD_EXTCMD_TSTAMP; 3784 break; 3785#endif 3786 default: 3787 *txd_upper = 0; 3788 *txd_lower = 0; 3789 return; 3790 } 3791 3792 switch (ipproto) { 3793 case IPPROTO_TCP: 3794 if (mp->m_pkthdr.csum_flags & CSUM_TCP) { 3795 /* 3796 * Start offset for payload checksum calculation. 3797 * End offset for payload checksum calculation. 3798 * Offset of place to put the checksum. 3799 */ 3800 th = (struct tcp_hdr *)(mp->m_data + hdr_len); 3801 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3802 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3803 TXD->upper_setup.tcp_fields.tucso = 3804 hdr_len + offsetof(struct tcphdr, th_sum); 3805 cmd |= E1000_TXD_CMD_TCP; 3806 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3807 } 3808 break; 3809 case IPPROTO_UDP: 3810 { 3811#ifdef EM_TIMESYNC 3812 void *hdr = (caddr_t) ip + ip_hlen; 3813 struct udphdr *uh = (struct udphdr *)hdr; 3814 3815 if (uh->uh_dport == htons(TSYNC_PORT)) { 3816 *txd_upper |= E1000_TXD_EXTCMD_TSTAMP; 3817 IOCTL_DEBUGOUT("@@@ Sending Event Packet\n"); 3818 } 3819#endif 3820 if (mp->m_pkthdr.csum_flags & CSUM_UDP) { 3821 /* 3822 * Start offset for header checksum calculation. 3823 * End offset for header checksum calculation. 3824 * Offset of place to put the checksum. 3825 */ 3826 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3827 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3828 TXD->upper_setup.tcp_fields.tucso = 3829 hdr_len + offsetof(struct udphdr, uh_sum); 3830 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3831 } 3832 /* Fall Thru */ 3833 } 3834 default: 3835 break; 3836 } 3837 3838#ifdef EM_TIMESYNC 3839 /* 3840 ** We might be here just for TIMESYNC 3841 ** which means we don't need the context 3842 ** descriptor. 3843 */ 3844 if (!mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) 3845 return; 3846#endif 3847 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 3848 E1000_TXD_DTYP_D; /* Data descr */ 3849 TXD->tcp_seg_setup.data = htole32(0); 3850 TXD->cmd_and_length = 3851 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); 3852 tx_buffer->m_head = NULL; 3853 tx_buffer->next_eop = -1; 3854 3855 if (++curr_txd == adapter->num_tx_desc) 3856 curr_txd = 0; 3857 3858 adapter->num_tx_desc_avail--; 3859 adapter->next_avail_tx_desc = curr_txd; 3860} 3861 3862 3863#if __FreeBSD_version >= 700000 3864/********************************************************************** 3865 * 3866 * Setup work for hardware segmentation offload (TSO) 3867 * 3868 **********************************************************************/ 3869static bool 3870em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper, 3871 u32 *txd_lower) 3872{ 3873 struct e1000_context_desc *TXD; 3874 struct em_buffer *tx_buffer; 3875 struct ether_vlan_header *eh; 3876 struct ip *ip; 3877 struct ip6_hdr *ip6; 3878 struct tcphdr *th; 3879 int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6; 3880 u16 etype; 3881 3882 /* 3883 * This function could/should be extended to support IP/IPv6 3884 * fragmentation as well. But as they say, one step at a time. 3885 */ 3886 3887 /* 3888 * Determine where frame payload starts. 3889 * Jump over vlan headers if already present, 3890 * helpful for QinQ too. 3891 */ 3892 eh = mtod(mp, struct ether_vlan_header *); 3893 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3894 etype = ntohs(eh->evl_proto); 3895 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3896 } else { 3897 etype = ntohs(eh->evl_encap_proto); 3898 ehdrlen = ETHER_HDR_LEN; 3899 } 3900 3901 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3902 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3903 return FALSE; /* -1 */ 3904 3905 /* 3906 * We only support TCP for IPv4 and IPv6 (notyet) for the moment. 3907 * TODO: Support SCTP too when it hits the tree. 3908 */ 3909 switch (etype) { 3910 case ETHERTYPE_IP: 3911 isip6 = 0; 3912 ip = (struct ip *)(mp->m_data + ehdrlen); 3913 if (ip->ip_p != IPPROTO_TCP) 3914 return FALSE; /* 0 */ 3915 ip->ip_len = 0; 3916 ip->ip_sum = 0; 3917 ip_hlen = ip->ip_hl << 2; 3918 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3919 return FALSE; /* -1 */ 3920 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3921#if 1 3922 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3923 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3924#else 3925 th->th_sum = mp->m_pkthdr.csum_data; 3926#endif 3927 break; 3928 case ETHERTYPE_IPV6: 3929 isip6 = 1; 3930 return FALSE; /* Not supported yet. */ 3931 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3932 if (ip6->ip6_nxt != IPPROTO_TCP) 3933 return FALSE; /* 0 */ 3934 ip6->ip6_plen = 0; 3935 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */ 3936 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3937 return FALSE; /* -1 */ 3938 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 3939#if 0 3940 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst, 3941 htons(IPPROTO_TCP)); /* XXX: function notyet. */ 3942#else 3943 th->th_sum = mp->m_pkthdr.csum_data; 3944#endif 3945 break; 3946 default: 3947 return FALSE; 3948 } 3949 hdr_len = ehdrlen + ip_hlen + (th->th_off << 2); 3950 3951 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 3952 E1000_TXD_DTYP_D | /* Data descr type */ 3953 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 3954 3955 /* IP and/or TCP header checksum calculation and insertion. */ 3956 *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) | 3957 E1000_TXD_POPTS_TXSM) << 8; 3958 3959 curr_txd = adapter->next_avail_tx_desc; 3960 tx_buffer = &adapter->tx_buffer_area[curr_txd]; 3961 TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; 3962 3963 /* IPv6 doesn't have a header checksum. */ 3964 if (!isip6) { 3965 /* 3966 * Start offset for header checksum calculation. 3967 * End offset for header checksum calculation. 3968 * Offset of place put the checksum. 3969 */ 3970 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3971 TXD->lower_setup.ip_fields.ipcse = 3972 htole16(ehdrlen + ip_hlen - 1); 3973 TXD->lower_setup.ip_fields.ipcso = 3974 ehdrlen + offsetof(struct ip, ip_sum); 3975 } 3976 /* 3977 * Start offset for payload checksum calculation. 3978 * End offset for payload checksum calculation. 3979 * Offset of place to put the checksum. 3980 */ 3981 TXD->upper_setup.tcp_fields.tucss = 3982 ehdrlen + ip_hlen; 3983 TXD->upper_setup.tcp_fields.tucse = 0; 3984 TXD->upper_setup.tcp_fields.tucso = 3985 ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum); 3986 /* 3987 * Payload size per packet w/o any headers. 3988 * Length of all headers up to payload. 3989 */ 3990 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); 3991 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 3992 3993 TXD->cmd_and_length = htole32(adapter->txd_cmd | 3994 E1000_TXD_CMD_DEXT | /* Extended descr */ 3995 E1000_TXD_CMD_TSE | /* TSE context */ 3996 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */ 3997 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 3998 (mp->m_pkthdr.len - (hdr_len))); /* Total len */ 3999 4000 tx_buffer->m_head = NULL; 4001 tx_buffer->next_eop = -1; 4002 4003 if (++curr_txd == adapter->num_tx_desc) 4004 curr_txd = 0; 4005 4006 adapter->num_tx_desc_avail--; 4007 adapter->next_avail_tx_desc = curr_txd; 4008 adapter->tx_tso = TRUE; 4009 4010 return TRUE; 4011} 4012 4013#endif /* __FreeBSD_version >= 700000 */ 4014 4015/********************************************************************** 4016 * 4017 * Examine each tx_buffer in the used queue. If the hardware is done 4018 * processing the packet then free associated resources. The 4019 * tx_buffer is put back on the free queue. 4020 * 4021 **********************************************************************/ 4022static void 4023em_txeof(struct adapter *adapter) 4024{ 4025 int first, last, done, num_avail; 4026 struct em_buffer *tx_buffer; 4027 struct e1000_tx_desc *tx_desc, *eop_desc; 4028 struct ifnet *ifp = adapter->ifp; 4029 4030 EM_TX_LOCK_ASSERT(adapter); 4031 4032 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) 4033 return; 4034 4035 num_avail = adapter->num_tx_desc_avail; 4036 first = adapter->next_tx_to_clean; 4037 tx_desc = &adapter->tx_desc_base[first]; 4038 tx_buffer = &adapter->tx_buffer_area[first]; 4039 last = tx_buffer->next_eop; 4040 eop_desc = &adapter->tx_desc_base[last]; 4041 4042 /* 4043 * What this does is get the index of the 4044 * first descriptor AFTER the EOP of the 4045 * first packet, that way we can do the 4046 * simple comparison on the inner while loop. 4047 */ 4048 if (++last == adapter->num_tx_desc) 4049 last = 0; 4050 done = last; 4051 4052 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 4053 BUS_DMASYNC_POSTREAD); 4054 4055 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 4056 /* We clean the range of the packet */ 4057 while (first != done) { 4058 tx_desc->upper.data = 0; 4059 tx_desc->lower.data = 0; 4060 tx_desc->buffer_addr = 0; 4061 num_avail++; 4062 4063 if (tx_buffer->m_head) { 4064 ifp->if_opackets++; 4065 bus_dmamap_sync(adapter->txtag, 4066 tx_buffer->map, 4067 BUS_DMASYNC_POSTWRITE); 4068 bus_dmamap_unload(adapter->txtag, 4069 tx_buffer->map); 4070 4071 m_freem(tx_buffer->m_head); 4072 tx_buffer->m_head = NULL; 4073 } 4074 tx_buffer->next_eop = -1; 4075 4076 if (++first == adapter->num_tx_desc) 4077 first = 0; 4078 4079 tx_buffer = &adapter->tx_buffer_area[first]; 4080 tx_desc = &adapter->tx_desc_base[first]; 4081 } 4082 /* See if we can continue to the next packet */ 4083 last = tx_buffer->next_eop; 4084 if (last != -1) { 4085 eop_desc = &adapter->tx_desc_base[last]; 4086 /* Get new done point */ 4087 if (++last == adapter->num_tx_desc) last = 0; 4088 done = last; 4089 } else 4090 break; 4091 } 4092 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 4093 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4094 4095 adapter->next_tx_to_clean = first; 4096 4097 /* 4098 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack 4099 * that it is OK to send packets. 4100 * If there are no pending descriptors, clear the timeout. Otherwise, 4101 * if some descriptors have been freed, restart the timeout. 4102 */ 4103 if (num_avail > EM_TX_CLEANUP_THRESHOLD) { 4104 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 4105 /* All clean, turn off the timer */ 4106 if (num_avail == adapter->num_tx_desc) { 4107 adapter->watchdog_timer = 0; 4108 } else 4109 /* Some cleaned, reset the timer */ 4110 if (num_avail != adapter->num_tx_desc_avail) 4111 adapter->watchdog_timer = EM_TX_TIMEOUT; 4112 } 4113 adapter->num_tx_desc_avail = num_avail; 4114 return; 4115} 4116 4117/********************************************************************* 4118 * 4119 * When Link is lost sometimes there is work still in the TX ring 4120 * which will result in a watchdog, rather than allow that do an 4121 * attempted cleanup and then reinit here. Note that this has been 4122 * seens mostly with fiber adapters. 4123 * 4124 **********************************************************************/ 4125static void 4126em_tx_purge(struct adapter *adapter) 4127{ 4128 if ((!adapter->link_active) && (adapter->watchdog_timer)) { 4129 EM_TX_LOCK(adapter); 4130 em_txeof(adapter); 4131 EM_TX_UNLOCK(adapter); 4132 if (adapter->watchdog_timer) { /* Still not clean? */ 4133 adapter->watchdog_timer = 0; 4134 em_init_locked(adapter); 4135 } 4136 } 4137} 4138 4139/********************************************************************* 4140 * 4141 * Get a buffer from system mbuf buffer pool. 4142 * 4143 **********************************************************************/ 4144static int 4145em_get_buf(struct adapter *adapter, int i) 4146{ 4147 struct mbuf *m; 4148 bus_dma_segment_t segs[1]; 4149 bus_dmamap_t map; 4150 struct em_buffer *rx_buffer; 4151 int error, nsegs; 4152 4153 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 4154 if (m == NULL) { 4155 adapter->mbuf_cluster_failed++; 4156 return (ENOBUFS); 4157 } 4158 m->m_len = m->m_pkthdr.len = MCLBYTES; 4159 4160 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 4161 m_adj(m, ETHER_ALIGN); 4162 4163 /* 4164 * Using memory from the mbuf cluster pool, invoke the 4165 * bus_dma machinery to arrange the memory mapping. 4166 */ 4167 error = bus_dmamap_load_mbuf_sg(adapter->rxtag, 4168 adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT); 4169 if (error != 0) { 4170 m_free(m); 4171 return (error); 4172 } 4173 4174 /* If nsegs is wrong then the stack is corrupt. */ 4175 KASSERT(nsegs == 1, ("Too many segments returned!")); 4176 4177 rx_buffer = &adapter->rx_buffer_area[i]; 4178 if (rx_buffer->m_head != NULL) 4179 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 4180 4181 map = rx_buffer->map; 4182 rx_buffer->map = adapter->rx_sparemap; 4183 adapter->rx_sparemap = map; 4184 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD); 4185 rx_buffer->m_head = m; 4186 4187 adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr); 4188 return (0); 4189} 4190 4191/********************************************************************* 4192 * 4193 * Allocate memory for rx_buffer structures. Since we use one 4194 * rx_buffer per received packet, the maximum number of rx_buffer's 4195 * that we'll need is equal to the number of receive descriptors 4196 * that we've allocated. 4197 * 4198 **********************************************************************/ 4199static int 4200em_allocate_receive_structures(struct adapter *adapter) 4201{ 4202 device_t dev = adapter->dev; 4203 struct em_buffer *rx_buffer; 4204 int i, error; 4205 4206 adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) * 4207 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 4208 if (adapter->rx_buffer_area == NULL) { 4209 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 4210 return (ENOMEM); 4211 } 4212 4213#if __FreeBSD_version >= 700000 4214 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 4215#else 4216 error = bus_dma_tag_create(NULL, /* parent */ 4217#endif 4218 1, 0, /* alignment, bounds */ 4219 BUS_SPACE_MAXADDR, /* lowaddr */ 4220 BUS_SPACE_MAXADDR, /* highaddr */ 4221 NULL, NULL, /* filter, filterarg */ 4222 MCLBYTES, /* maxsize */ 4223 1, /* nsegments */ 4224 MCLBYTES, /* maxsegsize */ 4225 0, /* flags */ 4226 NULL, /* lockfunc */ 4227 NULL, /* lockarg */ 4228 &adapter->rxtag); 4229 if (error) { 4230 device_printf(dev, "%s: bus_dma_tag_create failed %d\n", 4231 __func__, error); 4232 goto fail; 4233 } 4234 4235 /* Create the spare map (used by getbuf) */ 4236 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, 4237 &adapter->rx_sparemap); 4238 if (error) { 4239 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 4240 __func__, error); 4241 goto fail; 4242 } 4243 4244 rx_buffer = adapter->rx_buffer_area; 4245 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4246 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, 4247 &rx_buffer->map); 4248 if (error) { 4249 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 4250 __func__, error); 4251 goto fail; 4252 } 4253 } 4254 4255 return (0); 4256 4257fail: 4258 em_free_receive_structures(adapter); 4259 return (error); 4260} 4261 4262/********************************************************************* 4263 * 4264 * (Re)initialize receive structures. 4265 * 4266 **********************************************************************/ 4267static int 4268em_setup_receive_structures(struct adapter *adapter) 4269{ 4270 struct em_buffer *rx_buffer; 4271 int i, error; 4272 4273 /* Reset descriptor ring */ 4274 bzero(adapter->rx_desc_base, 4275 (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc); 4276 4277 /* Free current RX buffers. */ 4278 rx_buffer = adapter->rx_buffer_area; 4279 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4280 if (rx_buffer->m_head != NULL) { 4281 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, 4282 BUS_DMASYNC_POSTREAD); 4283 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 4284 m_freem(rx_buffer->m_head); 4285 rx_buffer->m_head = NULL; 4286 } 4287 } 4288 4289 /* Allocate new ones. */ 4290 for (i = 0; i < adapter->num_rx_desc; i++) { 4291 error = em_get_buf(adapter, i); 4292 if (error) 4293 return (error); 4294 } 4295 4296 /* Setup our descriptor pointers */ 4297 adapter->next_rx_desc_to_check = 0; 4298 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4299 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4300 4301 return (0); 4302} 4303 4304/********************************************************************* 4305 * 4306 * Enable receive unit. 4307 * 4308 **********************************************************************/ 4309#define MAX_INTS_PER_SEC 8000 4310#define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256) 4311 4312static void 4313em_initialize_receive_unit(struct adapter *adapter) 4314{ 4315 struct ifnet *ifp = adapter->ifp; 4316 u64 bus_addr; 4317 u32 rctl, rxcsum; 4318 4319 INIT_DEBUGOUT("em_initialize_receive_unit: begin"); 4320 4321 /* 4322 * Make sure receives are disabled while setting 4323 * up the descriptor ring 4324 */ 4325 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4326 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4327 4328 if (adapter->hw.mac.type >= e1000_82540) { 4329 E1000_WRITE_REG(&adapter->hw, E1000_RADV, 4330 adapter->rx_abs_int_delay.value); 4331 /* 4332 * Set the interrupt throttling rate. Value is calculated 4333 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 4334 */ 4335 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR); 4336 } 4337 4338 /* 4339 ** When using MSIX interrupts we need to throttle 4340 ** using the EITR register (82574 only) 4341 */ 4342 if (adapter->msix) 4343 for (int i = 0; i < 4; i++) 4344 E1000_WRITE_REG(&adapter->hw, 4345 E1000_EITR_82574(i), DEFAULT_ITR); 4346 4347 /* Disable accelerated ackknowledge */ 4348 if (adapter->hw.mac.type == e1000_82574) 4349 E1000_WRITE_REG(&adapter->hw, 4350 E1000_RFCTL, E1000_RFCTL_ACK_DIS); 4351 4352 /* Setup the Base and Length of the Rx Descriptor Ring */ 4353 bus_addr = adapter->rxdma.dma_paddr; 4354 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0), 4355 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 4356 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0), 4357 (u32)(bus_addr >> 32)); 4358 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0), 4359 (u32)bus_addr); 4360 4361 /* Setup the Receive Control Register */ 4362 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4363 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 4364 E1000_RCTL_RDMTS_HALF | 4365 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4366 4367 /* Make sure VLAN Filters are off */ 4368 rctl &= ~E1000_RCTL_VFE; 4369 4370 if (e1000_tbi_sbp_enabled_82543(&adapter->hw)) 4371 rctl |= E1000_RCTL_SBP; 4372 else 4373 rctl &= ~E1000_RCTL_SBP; 4374 4375 switch (adapter->rx_buffer_len) { 4376 default: 4377 case 2048: 4378 rctl |= E1000_RCTL_SZ_2048; 4379 break; 4380 case 4096: 4381 rctl |= E1000_RCTL_SZ_4096 | 4382 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4383 break; 4384 case 8192: 4385 rctl |= E1000_RCTL_SZ_8192 | 4386 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4387 break; 4388 case 16384: 4389 rctl |= E1000_RCTL_SZ_16384 | 4390 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4391 break; 4392 } 4393 4394 if (ifp->if_mtu > ETHERMTU) 4395 rctl |= E1000_RCTL_LPE; 4396 else 4397 rctl &= ~E1000_RCTL_LPE; 4398 4399 /* Enable 82543 Receive Checksum Offload for TCP and UDP */ 4400 if ((adapter->hw.mac.type >= e1000_82543) && 4401 (ifp->if_capenable & IFCAP_RXCSUM)) { 4402 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); 4403 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 4404 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); 4405 } 4406 4407 /* 4408 ** XXX TEMPORARY WORKAROUND: on some systems with 82573 4409 ** long latencies are observed, like Lenovo X60. This 4410 ** change eliminates the problem, but since having positive 4411 ** values in RDTR is a known source of problems on other 4412 ** platforms another solution is being sought. 4413 */ 4414 if (adapter->hw.mac.type == e1000_82573) 4415 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20); 4416 4417 /* Enable Receives */ 4418 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4419 4420 /* 4421 * Setup the HW Rx Head and 4422 * Tail Descriptor Pointers 4423 */ 4424 E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0); 4425 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1); 4426 4427 return; 4428} 4429 4430/********************************************************************* 4431 * 4432 * Free receive related data structures. 4433 * 4434 **********************************************************************/ 4435static void 4436em_free_receive_structures(struct adapter *adapter) 4437{ 4438 struct em_buffer *rx_buffer; 4439 int i; 4440 4441 INIT_DEBUGOUT("free_receive_structures: begin"); 4442 4443 if (adapter->rx_sparemap) { 4444 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap); 4445 adapter->rx_sparemap = NULL; 4446 } 4447 4448 /* Cleanup any existing buffers */ 4449 if (adapter->rx_buffer_area != NULL) { 4450 rx_buffer = adapter->rx_buffer_area; 4451 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4452 if (rx_buffer->m_head != NULL) { 4453 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, 4454 BUS_DMASYNC_POSTREAD); 4455 bus_dmamap_unload(adapter->rxtag, 4456 rx_buffer->map); 4457 m_freem(rx_buffer->m_head); 4458 rx_buffer->m_head = NULL; 4459 } else if (rx_buffer->map != NULL) 4460 bus_dmamap_unload(adapter->rxtag, 4461 rx_buffer->map); 4462 if (rx_buffer->map != NULL) { 4463 bus_dmamap_destroy(adapter->rxtag, 4464 rx_buffer->map); 4465 rx_buffer->map = NULL; 4466 } 4467 } 4468 } 4469 4470 if (adapter->rx_buffer_area != NULL) { 4471 free(adapter->rx_buffer_area, M_DEVBUF); 4472 adapter->rx_buffer_area = NULL; 4473 } 4474 4475 if (adapter->rxtag != NULL) { 4476 bus_dma_tag_destroy(adapter->rxtag); 4477 adapter->rxtag = NULL; 4478 } 4479} 4480 4481/********************************************************************* 4482 * 4483 * This routine executes in interrupt context. It replenishes 4484 * the mbufs in the descriptor and sends data which has been 4485 * dma'ed into host memory to upper layer. 4486 * 4487 * We loop at most count times if count is > 0, or until done if 4488 * count < 0. 4489 * 4490 *********************************************************************/ 4491static int 4492em_rxeof(struct adapter *adapter, int count) 4493{ 4494 struct ifnet *ifp = adapter->ifp;; 4495 struct mbuf *mp; 4496 u8 status, accept_frame = 0, eop = 0; 4497 u16 len, desc_len, prev_len_adj; 4498 int i; 4499 struct e1000_rx_desc *current_desc; 4500 4501 EM_RX_LOCK(adapter); 4502 i = adapter->next_rx_desc_to_check; 4503 current_desc = &adapter->rx_desc_base[i]; 4504 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4505 BUS_DMASYNC_POSTREAD); 4506 4507 if (!((current_desc->status) & E1000_RXD_STAT_DD)) { 4508 EM_RX_UNLOCK(adapter); 4509 return (0); 4510 } 4511 4512 while ((current_desc->status & E1000_RXD_STAT_DD) && 4513 (count != 0) && 4514 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4515 struct mbuf *m = NULL; 4516 4517 mp = adapter->rx_buffer_area[i].m_head; 4518 /* 4519 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT 4520 * needs to access the last received byte in the mbuf. 4521 */ 4522 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map, 4523 BUS_DMASYNC_POSTREAD); 4524 4525 accept_frame = 1; 4526 prev_len_adj = 0; 4527 desc_len = le16toh(current_desc->length); 4528 status = current_desc->status; 4529 if (status & E1000_RXD_STAT_EOP) { 4530 count--; 4531 eop = 1; 4532 if (desc_len < ETHER_CRC_LEN) { 4533 len = 0; 4534 prev_len_adj = ETHER_CRC_LEN - desc_len; 4535 } else 4536 len = desc_len - ETHER_CRC_LEN; 4537 } else { 4538 eop = 0; 4539 len = desc_len; 4540 } 4541 4542 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 4543 u8 last_byte; 4544 u32 pkt_len = desc_len; 4545 4546 if (adapter->fmp != NULL) 4547 pkt_len += adapter->fmp->m_pkthdr.len; 4548 4549 last_byte = *(mtod(mp, caddr_t) + desc_len - 1); 4550 if (TBI_ACCEPT(&adapter->hw, status, 4551 current_desc->errors, pkt_len, last_byte, 4552 adapter->min_frame_size, adapter->max_frame_size)) { 4553 e1000_tbi_adjust_stats_82543(&adapter->hw, 4554 &adapter->stats, pkt_len, 4555 adapter->hw.mac.addr, 4556 adapter->max_frame_size); 4557 if (len > 0) 4558 len--; 4559 } else 4560 accept_frame = 0; 4561 } 4562 4563 if (accept_frame) { 4564 if (em_get_buf(adapter, i) != 0) { 4565 ifp->if_iqdrops++; 4566 goto discard; 4567 } 4568 4569 /* Assign correct length to the current fragment */ 4570 mp->m_len = len; 4571 4572 if (adapter->fmp == NULL) { 4573 mp->m_pkthdr.len = len; 4574 adapter->fmp = mp; /* Store the first mbuf */ 4575 adapter->lmp = mp; 4576 } else { 4577 /* Chain mbuf's together */ 4578 mp->m_flags &= ~M_PKTHDR; 4579 /* 4580 * Adjust length of previous mbuf in chain if 4581 * we received less than 4 bytes in the last 4582 * descriptor. 4583 */ 4584 if (prev_len_adj > 0) { 4585 adapter->lmp->m_len -= prev_len_adj; 4586 adapter->fmp->m_pkthdr.len -= 4587 prev_len_adj; 4588 } 4589 adapter->lmp->m_next = mp; 4590 adapter->lmp = adapter->lmp->m_next; 4591 adapter->fmp->m_pkthdr.len += len; 4592 } 4593 4594 if (eop) { 4595 adapter->fmp->m_pkthdr.rcvif = ifp; 4596 ifp->if_ipackets++; 4597 em_receive_checksum(adapter, current_desc, 4598 adapter->fmp); 4599#ifndef __NO_STRICT_ALIGNMENT 4600 if (adapter->max_frame_size > 4601 (MCLBYTES - ETHER_ALIGN) && 4602 em_fixup_rx(adapter) != 0) 4603 goto skip; 4604#endif 4605 if (status & E1000_RXD_STAT_VP) { 4606#if __FreeBSD_version < 700000 4607 VLAN_INPUT_TAG_NEW(ifp, adapter->fmp, 4608 (le16toh(current_desc->special) & 4609 E1000_RXD_SPC_VLAN_MASK)); 4610#else 4611 adapter->fmp->m_pkthdr.ether_vtag = 4612 (le16toh(current_desc->special) & 4613 E1000_RXD_SPC_VLAN_MASK); 4614 adapter->fmp->m_flags |= M_VLANTAG; 4615#endif 4616 } 4617#ifndef __NO_STRICT_ALIGNMENT 4618skip: 4619#endif 4620 m = adapter->fmp; 4621 adapter->fmp = NULL; 4622 adapter->lmp = NULL; 4623 } 4624 } else { 4625 ifp->if_ierrors++; 4626discard: 4627 /* Reuse loaded DMA map and just update mbuf chain */ 4628 mp = adapter->rx_buffer_area[i].m_head; 4629 mp->m_len = mp->m_pkthdr.len = MCLBYTES; 4630 mp->m_data = mp->m_ext.ext_buf; 4631 mp->m_next = NULL; 4632 if (adapter->max_frame_size <= 4633 (MCLBYTES - ETHER_ALIGN)) 4634 m_adj(mp, ETHER_ALIGN); 4635 if (adapter->fmp != NULL) { 4636 m_freem(adapter->fmp); 4637 adapter->fmp = NULL; 4638 adapter->lmp = NULL; 4639 } 4640 m = NULL; 4641 } 4642 4643 /* Zero out the receive descriptors status. */ 4644 current_desc->status = 0; 4645 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4646 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4647 4648 /* Advance our pointers to the next descriptor. */ 4649 if (++i == adapter->num_rx_desc) 4650 i = 0; 4651 if (m != NULL) { 4652 adapter->next_rx_desc_to_check = i; 4653 /* Unlock for call into stack */ 4654 EM_RX_UNLOCK(adapter); 4655 (*ifp->if_input)(ifp, m); 4656 EM_RX_LOCK(adapter); 4657 i = adapter->next_rx_desc_to_check; 4658 } 4659 current_desc = &adapter->rx_desc_base[i]; 4660 } 4661 adapter->next_rx_desc_to_check = i; 4662 4663 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ 4664 if (--i < 0) 4665 i = adapter->num_rx_desc - 1; 4666 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); 4667 EM_RX_UNLOCK(adapter); 4668 if (!((current_desc->status) & E1000_RXD_STAT_DD)) 4669 return (0); 4670 4671 return (1); 4672} 4673 4674#ifndef __NO_STRICT_ALIGNMENT 4675/* 4676 * When jumbo frames are enabled we should realign entire payload on 4677 * architecures with strict alignment. This is serious design mistake of 8254x 4678 * as it nullifies DMA operations. 8254x just allows RX buffer size to be 4679 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its 4680 * payload. On architecures without strict alignment restrictions 8254x still 4681 * performs unaligned memory access which would reduce the performance too. 4682 * To avoid copying over an entire frame to align, we allocate a new mbuf and 4683 * copy ethernet header to the new mbuf. The new mbuf is prepended into the 4684 * existing mbuf chain. 4685 * 4686 * Be aware, best performance of the 8254x is achived only when jumbo frame is 4687 * not used at all on architectures with strict alignment. 4688 */ 4689static int 4690em_fixup_rx(struct adapter *adapter) 4691{ 4692 struct mbuf *m, *n; 4693 int error; 4694 4695 error = 0; 4696 m = adapter->fmp; 4697 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { 4698 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); 4699 m->m_data += ETHER_HDR_LEN; 4700 } else { 4701 MGETHDR(n, M_DONTWAIT, MT_DATA); 4702 if (n != NULL) { 4703 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); 4704 m->m_data += ETHER_HDR_LEN; 4705 m->m_len -= ETHER_HDR_LEN; 4706 n->m_len = ETHER_HDR_LEN; 4707 M_MOVE_PKTHDR(n, m); 4708 n->m_next = m; 4709 adapter->fmp = n; 4710 } else { 4711 adapter->dropped_pkts++; 4712 m_freem(adapter->fmp); 4713 adapter->fmp = NULL; 4714 error = ENOMEM; 4715 } 4716 } 4717 4718 return (error); 4719} 4720#endif 4721 4722/********************************************************************* 4723 * 4724 * Verify that the hardware indicated that the checksum is valid. 4725 * Inform the stack about the status of checksum so that stack 4726 * doesn't spend time verifying the checksum. 4727 * 4728 *********************************************************************/ 4729static void 4730em_receive_checksum(struct adapter *adapter, 4731 struct e1000_rx_desc *rx_desc, struct mbuf *mp) 4732{ 4733 /* 82543 or newer only */ 4734 if ((adapter->hw.mac.type < e1000_82543) || 4735 /* Ignore Checksum bit is set */ 4736 (rx_desc->status & E1000_RXD_STAT_IXSM)) { 4737 mp->m_pkthdr.csum_flags = 0; 4738 return; 4739 } 4740 4741 if (rx_desc->status & E1000_RXD_STAT_IPCS) { 4742 /* Did it pass? */ 4743 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { 4744 /* IP Checksum Good */ 4745 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4746 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4747 4748 } else { 4749 mp->m_pkthdr.csum_flags = 0; 4750 } 4751 } 4752 4753 if (rx_desc->status & E1000_RXD_STAT_TCPCS) { 4754 /* Did it pass? */ 4755 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { 4756 mp->m_pkthdr.csum_flags |= 4757 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4758 mp->m_pkthdr.csum_data = htons(0xffff); 4759 } 4760 } 4761} 4762 4763/* 4764 * This routine is run via an vlan 4765 * config EVENT 4766 */ 4767static void 4768em_register_vlan(void *unused, struct ifnet *ifp, u16 vtag) 4769{ 4770 struct adapter *adapter = ifp->if_softc; 4771 u32 ctrl, rctl, index, vfta; 4772 4773 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4774 ctrl |= E1000_CTRL_VME; 4775 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4776 4777 /* Setup for Hardware Filter */ 4778 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4779 rctl |= E1000_RCTL_VFE; 4780 rctl &= ~E1000_RCTL_CFIEN; 4781 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4782 4783 /* Make entry in the hardware filter table */ 4784 index = ((vtag >> 5) & 0x7F); 4785 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index); 4786 vfta |= (1 << (vtag & 0x1F)); 4787 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta); 4788 4789 /* Update the frame size */ 4790 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4791 adapter->max_frame_size + VLAN_TAG_SIZE); 4792 4793} 4794 4795/* 4796 * This routine is run via an vlan 4797 * unconfig EVENT 4798 */ 4799static void 4800em_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag) 4801{ 4802 struct adapter *adapter = ifp->if_softc; 4803 u32 index, vfta; 4804 4805 /* Remove entry in the hardware filter table */ 4806 index = ((vtag >> 5) & 0x7F); 4807 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index); 4808 vfta &= ~(1 << (vtag & 0x1F)); 4809 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta); 4810 /* Have all vlans unregistered? */ 4811 if (adapter->ifp->if_vlantrunk == NULL) { 4812 u32 rctl; 4813 /* Turn off the filter table */ 4814 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4815 rctl &= ~E1000_RCTL_VFE; 4816 rctl |= E1000_RCTL_CFIEN; 4817 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4818 /* Reset the frame size */ 4819 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4820 adapter->max_frame_size); 4821 } 4822} 4823 4824static void 4825em_enable_intr(struct adapter *adapter) 4826{ 4827 struct e1000_hw *hw = &adapter->hw; 4828 u32 ims_mask = IMS_ENABLE_MASK; 4829 4830 if (adapter->msix) { 4831 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); 4832 ims_mask |= EM_MSIX_MASK; 4833 } 4834 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 4835} 4836 4837static void 4838em_disable_intr(struct adapter *adapter) 4839{ 4840 struct e1000_hw *hw = &adapter->hw; 4841 4842 if (adapter->msix) 4843 E1000_WRITE_REG(hw, EM_EIAC, 0); 4844 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 4845} 4846 4847/* 4848 * Bit of a misnomer, what this really means is 4849 * to enable OS management of the system... aka 4850 * to disable special hardware management features 4851 */ 4852static void 4853em_init_manageability(struct adapter *adapter) 4854{ 4855 /* A shared code workaround */ 4856#define E1000_82542_MANC2H E1000_MANC2H 4857 if (adapter->has_manage) { 4858 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4859 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4860 4861 /* disable hardware interception of ARP */ 4862 manc &= ~(E1000_MANC_ARP_EN); 4863 4864 /* enable receiving management packets to the host */ 4865 if (adapter->hw.mac.type >= e1000_82571) { 4866 manc |= E1000_MANC_EN_MNG2HOST; 4867#define E1000_MNG2HOST_PORT_623 (1 << 5) 4868#define E1000_MNG2HOST_PORT_664 (1 << 6) 4869 manc2h |= E1000_MNG2HOST_PORT_623; 4870 manc2h |= E1000_MNG2HOST_PORT_664; 4871 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4872 } 4873 4874 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4875 } 4876} 4877 4878/* 4879 * Give control back to hardware management 4880 * controller if there is one. 4881 */ 4882static void 4883em_release_manageability(struct adapter *adapter) 4884{ 4885 if (adapter->has_manage) { 4886 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4887 4888 /* re-enable hardware interception of ARP */ 4889 manc |= E1000_MANC_ARP_EN; 4890 4891 if (adapter->hw.mac.type >= e1000_82571) 4892 manc &= ~E1000_MANC_EN_MNG2HOST; 4893 4894 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4895 } 4896} 4897 4898/* 4899 * em_get_hw_control sets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4900 * For ASF and Pass Through versions of f/w this means that 4901 * the driver is loaded. For AMT version (only with 82573) 4902 * of the f/w this means that the network i/f is open. 4903 * 4904 */ 4905static void 4906em_get_hw_control(struct adapter *adapter) 4907{ 4908 u32 ctrl_ext, swsm; 4909 4910 /* Let firmware know the driver has taken over */ 4911 switch (adapter->hw.mac.type) { 4912 case e1000_82573: 4913 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4914 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4915 swsm | E1000_SWSM_DRV_LOAD); 4916 break; 4917 case e1000_82571: 4918 case e1000_82572: 4919 case e1000_80003es2lan: 4920 case e1000_ich8lan: 4921 case e1000_ich9lan: 4922 case e1000_ich10lan: 4923 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4924 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4925 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4926 break; 4927 default: 4928 break; 4929 } 4930} 4931 4932/* 4933 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4934 * For ASF and Pass Through versions of f/w this means that the 4935 * driver is no longer loaded. For AMT version (only with 82573) i 4936 * of the f/w this means that the network i/f is closed. 4937 * 4938 */ 4939static void 4940em_release_hw_control(struct adapter *adapter) 4941{ 4942 u32 ctrl_ext, swsm; 4943 4944 /* Let firmware taken over control of h/w */ 4945 switch (adapter->hw.mac.type) { 4946 case e1000_82573: 4947 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4948 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4949 swsm & ~E1000_SWSM_DRV_LOAD); 4950 break; 4951 case e1000_82571: 4952 case e1000_82572: 4953 case e1000_80003es2lan: 4954 case e1000_ich8lan: 4955 case e1000_ich9lan: 4956 case e1000_ich10lan: 4957 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4958 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4959 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4960 break; 4961 default: 4962 break; 4963 4964 } 4965} 4966 4967static int 4968em_is_valid_ether_addr(u8 *addr) 4969{ 4970 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4971 4972 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4973 return (FALSE); 4974 } 4975 4976 return (TRUE); 4977} 4978 4979/* 4980 * Enable PCI Wake On Lan capability 4981 */ 4982void 4983em_enable_wakeup(device_t dev) 4984{ 4985 u16 cap, status; 4986 u8 id; 4987 4988 /* First find the capabilities pointer*/ 4989 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 4990 /* Read the PM Capabilities */ 4991 id = pci_read_config(dev, cap, 1); 4992 if (id != PCIY_PMG) /* Something wrong */ 4993 return; 4994 /* OK, we have the power capabilities, so 4995 now get the status register */ 4996 cap += PCIR_POWER_STATUS; 4997 status = pci_read_config(dev, cap, 2); 4998 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4999 pci_write_config(dev, cap, status, 2); 5000 return; 5001} 5002 5003 5004/********************************************************************* 5005* 82544 Coexistence issue workaround. 5006* There are 2 issues. 5007* 1. Transmit Hang issue. 5008* To detect this issue, following equation can be used... 5009* SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 5010* If SUM[3:0] is in between 1 to 4, we will have this issue. 5011* 5012* 2. DAC issue. 5013* To detect this issue, following equation can be used... 5014* SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 5015* If SUM[3:0] is in between 9 to c, we will have this issue. 5016* 5017* 5018* WORKAROUND: 5019* Make sure we do not have ending address 5020* as 1,2,3,4(Hang) or 9,a,b,c (DAC) 5021* 5022*************************************************************************/ 5023static u32 5024em_fill_descriptors (bus_addr_t address, u32 length, 5025 PDESC_ARRAY desc_array) 5026{ 5027 u32 safe_terminator; 5028 5029 /* Since issue is sensitive to length and address.*/ 5030 /* Let us first check the address...*/ 5031 if (length <= 4) { 5032 desc_array->descriptor[0].address = address; 5033 desc_array->descriptor[0].length = length; 5034 desc_array->elements = 1; 5035 return (desc_array->elements); 5036 } 5037 safe_terminator = (u32)((((u32)address & 0x7) + 5038 (length & 0xF)) & 0xF); 5039 /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */ 5040 if (safe_terminator == 0 || 5041 (safe_terminator > 4 && 5042 safe_terminator < 9) || 5043 (safe_terminator > 0xC && 5044 safe_terminator <= 0xF)) { 5045 desc_array->descriptor[0].address = address; 5046 desc_array->descriptor[0].length = length; 5047 desc_array->elements = 1; 5048 return (desc_array->elements); 5049 } 5050 5051 desc_array->descriptor[0].address = address; 5052 desc_array->descriptor[0].length = length - 4; 5053 desc_array->descriptor[1].address = address + (length - 4); 5054 desc_array->descriptor[1].length = 4; 5055 desc_array->elements = 2; 5056 return (desc_array->elements); 5057} 5058 5059/********************************************************************** 5060 * 5061 * Update the board statistics counters. 5062 * 5063 **********************************************************************/ 5064static void 5065em_update_stats_counters(struct adapter *adapter) 5066{ 5067 struct ifnet *ifp; 5068 5069 if(adapter->hw.phy.media_type == e1000_media_type_copper || 5070 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 5071 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 5072 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 5073 } 5074 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 5075 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 5076 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 5077 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 5078 5079 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 5080 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 5081 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 5082 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 5083 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 5084 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 5085 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 5086 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 5087 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 5088 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 5089 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 5090 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 5091 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 5092 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 5093 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 5094 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 5095 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 5096 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 5097 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 5098 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 5099 5100 /* For the 64-bit byte counters the low dword must be read first. */ 5101 /* Both registers clear on the read of the high dword */ 5102 5103 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 5104 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 5105 5106 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 5107 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 5108 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 5109 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 5110 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 5111 5112 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 5113 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 5114 5115 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 5116 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 5117 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 5118 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 5119 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 5120 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 5121 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 5122 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 5123 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 5124 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 5125 5126 if (adapter->hw.mac.type >= e1000_82543) { 5127 adapter->stats.algnerrc += 5128 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 5129 adapter->stats.rxerrc += 5130 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 5131 adapter->stats.tncrs += 5132 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 5133 adapter->stats.cexterr += 5134 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 5135 adapter->stats.tsctc += 5136 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 5137 adapter->stats.tsctfc += 5138 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 5139 } 5140 ifp = adapter->ifp; 5141 5142 ifp->if_collisions = adapter->stats.colc; 5143 5144 /* Rx Errors */ 5145 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 5146 adapter->stats.crcerrs + adapter->stats.algnerrc + 5147 adapter->stats.ruc + adapter->stats.roc + 5148 adapter->stats.mpc + adapter->stats.cexterr; 5149 5150 /* Tx Errors */ 5151 ifp->if_oerrors = adapter->stats.ecol + 5152 adapter->stats.latecol + adapter->watchdog_events; 5153} 5154 5155 5156/********************************************************************** 5157 * 5158 * This routine is called only when em_display_debug_stats is enabled. 5159 * This routine provides a way to take a look at important statistics 5160 * maintained by the driver and hardware. 5161 * 5162 **********************************************************************/ 5163static void 5164em_print_debug_info(struct adapter *adapter) 5165{ 5166 device_t dev = adapter->dev; 5167 u8 *hw_addr = adapter->hw.hw_addr; 5168 5169 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 5170 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 5171 E1000_READ_REG(&adapter->hw, E1000_CTRL), 5172 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 5173 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 5174 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 5175 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 5176 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 5177 adapter->hw.fc.high_water, 5178 adapter->hw.fc.low_water); 5179 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 5180 E1000_READ_REG(&adapter->hw, E1000_TIDV), 5181 E1000_READ_REG(&adapter->hw, E1000_TADV)); 5182 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 5183 E1000_READ_REG(&adapter->hw, E1000_RDTR), 5184 E1000_READ_REG(&adapter->hw, E1000_RADV)); 5185 device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n", 5186 (long long)adapter->tx_fifo_wrk_cnt, 5187 (long long)adapter->tx_fifo_reset_cnt); 5188 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 5189 E1000_READ_REG(&adapter->hw, E1000_TDH(0)), 5190 E1000_READ_REG(&adapter->hw, E1000_TDT(0))); 5191 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 5192 E1000_READ_REG(&adapter->hw, E1000_RDH(0)), 5193 E1000_READ_REG(&adapter->hw, E1000_RDT(0))); 5194 device_printf(dev, "Num Tx descriptors avail = %d\n", 5195 adapter->num_tx_desc_avail); 5196 device_printf(dev, "Tx Descriptors not avail1 = %ld\n", 5197 adapter->no_tx_desc_avail1); 5198 device_printf(dev, "Tx Descriptors not avail2 = %ld\n", 5199 adapter->no_tx_desc_avail2); 5200 device_printf(dev, "Std mbuf failed = %ld\n", 5201 adapter->mbuf_alloc_failed); 5202 device_printf(dev, "Std mbuf cluster failed = %ld\n", 5203 adapter->mbuf_cluster_failed); 5204 device_printf(dev, "Driver dropped packets = %ld\n", 5205 adapter->dropped_pkts); 5206 device_printf(dev, "Driver tx dma failure in encap = %ld\n", 5207 adapter->no_tx_dma_setup); 5208} 5209 5210static void 5211em_print_hw_stats(struct adapter *adapter) 5212{ 5213 device_t dev = adapter->dev; 5214 5215 device_printf(dev, "Excessive collisions = %lld\n", 5216 (long long)adapter->stats.ecol); 5217#if (DEBUG_HW > 0) /* Dont output these errors normally */ 5218 device_printf(dev, "Symbol errors = %lld\n", 5219 (long long)adapter->stats.symerrs); 5220#endif 5221 device_printf(dev, "Sequence errors = %lld\n", 5222 (long long)adapter->stats.sec); 5223 device_printf(dev, "Defer count = %lld\n", 5224 (long long)adapter->stats.dc); 5225 device_printf(dev, "Missed Packets = %lld\n", 5226 (long long)adapter->stats.mpc); 5227 device_printf(dev, "Receive No Buffers = %lld\n", 5228 (long long)adapter->stats.rnbc); 5229 /* RLEC is inaccurate on some hardware, calculate our own. */ 5230 device_printf(dev, "Receive Length Errors = %lld\n", 5231 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 5232 device_printf(dev, "Receive errors = %lld\n", 5233 (long long)adapter->stats.rxerrc); 5234 device_printf(dev, "Crc errors = %lld\n", 5235 (long long)adapter->stats.crcerrs); 5236 device_printf(dev, "Alignment errors = %lld\n", 5237 (long long)adapter->stats.algnerrc); 5238 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 5239 (long long)adapter->stats.cexterr); 5240 device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); 5241 device_printf(dev, "watchdog timeouts = %ld\n", 5242 adapter->watchdog_events); 5243 device_printf(dev, "RX MSIX IRQ = %ld TX MSIX IRQ = %ld" 5244 " LINK MSIX IRQ = %ld\n", adapter->rx_irq, 5245 adapter->tx_irq , adapter->link_irq); 5246 device_printf(dev, "XON Rcvd = %lld\n", 5247 (long long)adapter->stats.xonrxc); 5248 device_printf(dev, "XON Xmtd = %lld\n", 5249 (long long)adapter->stats.xontxc); 5250 device_printf(dev, "XOFF Rcvd = %lld\n", 5251 (long long)adapter->stats.xoffrxc); 5252 device_printf(dev, "XOFF Xmtd = %lld\n", 5253 (long long)adapter->stats.xofftxc); 5254 device_printf(dev, "Good Packets Rcvd = %lld\n", 5255 (long long)adapter->stats.gprc); 5256 device_printf(dev, "Good Packets Xmtd = %lld\n", 5257 (long long)adapter->stats.gptc); 5258 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 5259 (long long)adapter->stats.tsctc); 5260 device_printf(dev, "TSO Contexts Failed = %lld\n", 5261 (long long)adapter->stats.tsctfc); 5262} 5263 5264/********************************************************************** 5265 * 5266 * This routine provides a way to dump out the adapter eeprom, 5267 * often a useful debug/service tool. This only dumps the first 5268 * 32 words, stuff that matters is in that extent. 5269 * 5270 **********************************************************************/ 5271static void 5272em_print_nvm_info(struct adapter *adapter) 5273{ 5274 u16 eeprom_data; 5275 int i, j, row = 0; 5276 5277 /* Its a bit crude, but it gets the job done */ 5278 printf("\nInterface EEPROM Dump:\n"); 5279 printf("Offset\n0x0000 "); 5280 for (i = 0, j = 0; i < 32; i++, j++) { 5281 if (j == 8) { /* Make the offset block */ 5282 j = 0; ++row; 5283 printf("\n0x00%x0 ",row); 5284 } 5285 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 5286 printf("%04x ", eeprom_data); 5287 } 5288 printf("\n"); 5289} 5290 5291static int 5292em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 5293{ 5294 struct adapter *adapter; 5295 int error; 5296 int result; 5297 5298 result = -1; 5299 error = sysctl_handle_int(oidp, &result, 0, req); 5300 5301 if (error || !req->newptr) 5302 return (error); 5303 5304 if (result == 1) { 5305 adapter = (struct adapter *)arg1; 5306 em_print_debug_info(adapter); 5307 } 5308 /* 5309 * This value will cause a hex dump of the 5310 * first 32 16-bit words of the EEPROM to 5311 * the screen. 5312 */ 5313 if (result == 2) { 5314 adapter = (struct adapter *)arg1; 5315 em_print_nvm_info(adapter); 5316 } 5317 5318 return (error); 5319} 5320 5321 5322static int 5323em_sysctl_stats(SYSCTL_HANDLER_ARGS) 5324{ 5325 struct adapter *adapter; 5326 int error; 5327 int result; 5328 5329 result = -1; 5330 error = sysctl_handle_int(oidp, &result, 0, req); 5331 5332 if (error || !req->newptr) 5333 return (error); 5334 5335 if (result == 1) { 5336 adapter = (struct adapter *)arg1; 5337 em_print_hw_stats(adapter); 5338 } 5339 5340 return (error); 5341} 5342 5343static int 5344em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 5345{ 5346 struct em_int_delay_info *info; 5347 struct adapter *adapter; 5348 u32 regval; 5349 int error; 5350 int usecs; 5351 int ticks; 5352 5353 info = (struct em_int_delay_info *)arg1; 5354 usecs = info->value; 5355 error = sysctl_handle_int(oidp, &usecs, 0, req); 5356 if (error != 0 || req->newptr == NULL) 5357 return (error); 5358 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 5359 return (EINVAL); 5360 info->value = usecs; 5361 ticks = EM_USECS_TO_TICKS(usecs); 5362 5363 adapter = info->adapter; 5364 5365 EM_CORE_LOCK(adapter); 5366 regval = E1000_READ_OFFSET(&adapter->hw, info->offset); 5367 regval = (regval & ~0xffff) | (ticks & 0xffff); 5368 /* Handle a few special cases. */ 5369 switch (info->offset) { 5370 case E1000_RDTR: 5371 break; 5372 case E1000_TIDV: 5373 if (ticks == 0) { 5374 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; 5375 /* Don't write 0 into the TIDV register. */ 5376 regval++; 5377 } else 5378 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 5379 break; 5380 } 5381 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); 5382 EM_CORE_UNLOCK(adapter); 5383 return (0); 5384} 5385 5386static void 5387em_add_int_delay_sysctl(struct adapter *adapter, const char *name, 5388 const char *description, struct em_int_delay_info *info, 5389 int offset, int value) 5390{ 5391 info->adapter = adapter; 5392 info->offset = offset; 5393 info->value = value; 5394 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), 5395 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5396 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, 5397 info, 0, em_sysctl_int_delay, "I", description); 5398} 5399 5400#ifndef EM_LEGACY_IRQ 5401static void 5402em_add_rx_process_limit(struct adapter *adapter, const char *name, 5403 const char *description, int *limit, int value) 5404{ 5405 *limit = value; 5406 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5407 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5408 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5409} 5410#endif 5411 5412#ifdef EM_TIMESYNC 5413/* 5414 * Initialize the Time Sync Feature 5415 */ 5416static int 5417em_tsync_init(struct adapter *adapter) 5418{ 5419 device_t dev = adapter->dev; 5420 u32 tx_ctl, rx_ctl; 5421 5422 5423 E1000_WRITE_REG(&adapter->hw, E1000_TIMINCA, (1<<24) | 5424 20833/PICOSECS_PER_TICK); 5425 5426 adapter->last_stamp = E1000_READ_REG(&adapter->hw, E1000_SYSTIML); 5427 adapter->last_stamp |= (u64)E1000_READ_REG(&adapter->hw, 5428 E1000_SYSTIMH) << 32ULL; 5429 5430 /* Enable the TX side */ 5431 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5432 tx_ctl |= 0x10; 5433 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl); 5434 E1000_WRITE_FLUSH(&adapter->hw); 5435 5436 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5437 if ((tx_ctl & 0x10) == 0) { 5438 device_printf(dev, "Failed to enable TX timestamping\n"); 5439 return (ENXIO); 5440 } 5441 5442 /* Enable RX */ 5443 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5444 rx_ctl |= 0x10; /* Enable the feature */ 5445 rx_ctl |= 0x0a; /* This value turns on Ver 1 and 2 */ 5446 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl); 5447 5448 /* 5449 * Ethertype Stamping (Ethertype = 0x88F7) 5450 */ 5451 E1000_WRITE_REG(&adapter->hw, E1000_RXMTRL, htonl(0x440088f7)); 5452 5453 /* 5454 * Source Port Queue Filter Setup: 5455 * this is for UDP port filtering 5456 */ 5457 E1000_WRITE_REG(&adapter->hw, E1000_RXUDP, htons(TSYNC_PORT)); 5458 /* Protocol = UDP, enable Timestamp, and filter on source/protocol */ 5459 5460 E1000_WRITE_FLUSH(&adapter->hw); 5461 5462 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5463 if ((rx_ctl & 0x10) == 0) { 5464 device_printf(dev, "Failed to enable RX timestamping\n"); 5465 return (ENXIO); 5466 } 5467 5468 device_printf(dev, "IEEE 1588 Precision Time Protocol enabled\n"); 5469 5470 return (0); 5471} 5472 5473/* 5474 * Disable the Time Sync Feature 5475 */ 5476static void 5477em_tsync_disable(struct adapter *adapter) 5478{ 5479 u32 tx_ctl, rx_ctl; 5480 5481 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5482 tx_ctl &= ~0x10; 5483 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl); 5484 E1000_WRITE_FLUSH(&adapter->hw); 5485 5486 /* Invalidate TX Timestamp */ 5487 E1000_READ_REG(&adapter->hw, E1000_TXSTMPH); 5488 5489 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5490 if (tx_ctl & 0x10) 5491 HW_DEBUGOUT("Failed to disable TX timestamping\n"); 5492 5493 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5494 rx_ctl &= ~0x10; 5495 5496 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl); 5497 E1000_WRITE_FLUSH(&adapter->hw); 5498 5499 /* Invalidate RX Timestamp */ 5500 E1000_READ_REG(&adapter->hw, E1000_RXSATRH); 5501 5502 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5503 if (rx_ctl & 0x10) 5504 HW_DEBUGOUT("Failed to disable RX timestamping\n"); 5505 5506 return; 5507} 5508#endif /* EM_TIMESYNC */ 5509