if_em.c revision 191580
1/****************************************************************************** 2 3 Copyright (c) 2001-2009, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ 33/*$FreeBSD: head/sys/dev/e1000/if_em.c 191580 2009-04-27 18:35:06Z jfv $*/ 34 35#ifdef HAVE_KERNEL_OPTION_HEADERS 36#include "opt_device_polling.h" 37#include "opt_inet.h" 38#endif 39 40#include <sys/param.h> 41#include <sys/systm.h> 42#include <sys/buf_ring.h> 43#include <sys/bus.h> 44#include <sys/endian.h> 45#include <sys/kernel.h> 46#include <sys/kthread.h> 47#include <sys/malloc.h> 48#include <sys/mbuf.h> 49#include <sys/module.h> 50#include <sys/rman.h> 51#include <sys/socket.h> 52#include <sys/sockio.h> 53#include <sys/sysctl.h> 54#include <sys/taskqueue.h> 55#include <sys/eventhandler.h> 56#ifdef EM_TIMESYNC 57#include <sys/ioccom.h> 58#include <sys/time.h> 59#endif 60#include <machine/bus.h> 61#include <machine/resource.h> 62 63#include <net/bpf.h> 64#include <net/ethernet.h> 65#include <net/if.h> 66#include <net/if_arp.h> 67#include <net/if_dl.h> 68#include <net/if_media.h> 69 70#include <net/if_types.h> 71#include <net/if_vlan_var.h> 72 73#include <netinet/in_systm.h> 74#include <netinet/in.h> 75#include <netinet/if_ether.h> 76#include <netinet/ip.h> 77#include <netinet/ip6.h> 78#include <netinet/tcp.h> 79#include <netinet/udp.h> 80 81#include <machine/in_cksum.h> 82#include <dev/pci/pcivar.h> 83#include <dev/pci/pcireg.h> 84 85#include "e1000_api.h" 86#include "e1000_82571.h" 87#include "if_em.h" 88 89/********************************************************************* 90 * Set this to one to display debug statistics 91 *********************************************************************/ 92int em_display_debug_stats = 0; 93 94/********************************************************************* 95 * Driver version: 96 *********************************************************************/ 97char em_driver_version[] = "6.9.9"; 98 99 100/********************************************************************* 101 * PCI Device ID Table 102 * 103 * Used by probe to select devices to load on 104 * Last field stores an index into e1000_strings 105 * Last entry must be all 0s 106 * 107 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 108 *********************************************************************/ 109 110static em_vendor_info_t em_vendor_info_array[] = 111{ 112 /* Intel(R) PRO/1000 Network Connection */ 113 { 0x8086, E1000_DEV_ID_82540EM, PCI_ANY_ID, PCI_ANY_ID, 0}, 114 { 0x8086, E1000_DEV_ID_82540EM_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 115 { 0x8086, E1000_DEV_ID_82540EP, PCI_ANY_ID, PCI_ANY_ID, 0}, 116 { 0x8086, E1000_DEV_ID_82540EP_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 117 { 0x8086, E1000_DEV_ID_82540EP_LP, PCI_ANY_ID, PCI_ANY_ID, 0}, 118 119 { 0x8086, E1000_DEV_ID_82541EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 120 { 0x8086, E1000_DEV_ID_82541ER, PCI_ANY_ID, PCI_ANY_ID, 0}, 121 { 0x8086, E1000_DEV_ID_82541ER_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 122 { 0x8086, E1000_DEV_ID_82541EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 123 { 0x8086, E1000_DEV_ID_82541GI, PCI_ANY_ID, PCI_ANY_ID, 0}, 124 { 0x8086, E1000_DEV_ID_82541GI_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 125 { 0x8086, E1000_DEV_ID_82541GI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 126 127 { 0x8086, E1000_DEV_ID_82542, PCI_ANY_ID, PCI_ANY_ID, 0}, 128 129 { 0x8086, E1000_DEV_ID_82543GC_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 130 { 0x8086, E1000_DEV_ID_82543GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 131 132 { 0x8086, E1000_DEV_ID_82544EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 133 { 0x8086, E1000_DEV_ID_82544EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82544GC_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_82544GC_LOM, PCI_ANY_ID, PCI_ANY_ID, 0}, 136 137 { 0x8086, E1000_DEV_ID_82545EM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 138 { 0x8086, E1000_DEV_ID_82545EM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 139 { 0x8086, E1000_DEV_ID_82545GM_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 140 { 0x8086, E1000_DEV_ID_82545GM_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 141 { 0x8086, E1000_DEV_ID_82545GM_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 142 143 { 0x8086, E1000_DEV_ID_82546EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 144 { 0x8086, E1000_DEV_ID_82546EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 145 { 0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 146 { 0x8086, E1000_DEV_ID_82546GB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 147 { 0x8086, E1000_DEV_ID_82546GB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 148 { 0x8086, E1000_DEV_ID_82546GB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 149 { 0x8086, E1000_DEV_ID_82546GB_PCIE, PCI_ANY_ID, PCI_ANY_ID, 0}, 150 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 151 { 0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, 152 PCI_ANY_ID, PCI_ANY_ID, 0}, 153 154 { 0x8086, E1000_DEV_ID_82547EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 155 { 0x8086, E1000_DEV_ID_82547EI_MOBILE, PCI_ANY_ID, PCI_ANY_ID, 0}, 156 { 0x8086, E1000_DEV_ID_82547GI, PCI_ANY_ID, PCI_ANY_ID, 0}, 157 158 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 159 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 160 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 161 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 162 PCI_ANY_ID, PCI_ANY_ID, 0}, 163 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 164 PCI_ANY_ID, PCI_ANY_ID, 0}, 165 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 166 PCI_ANY_ID, PCI_ANY_ID, 0}, 167 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 168 PCI_ANY_ID, PCI_ANY_ID, 0}, 169 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 170 PCI_ANY_ID, PCI_ANY_ID, 0}, 171 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 172 PCI_ANY_ID, PCI_ANY_ID, 0}, 173 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 174 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 175 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 176 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 177 178 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, 179 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 180 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, 181 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 182 PCI_ANY_ID, PCI_ANY_ID, 0}, 183 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 184 PCI_ANY_ID, PCI_ANY_ID, 0}, 185 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 186 PCI_ANY_ID, PCI_ANY_ID, 0}, 187 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 188 PCI_ANY_ID, PCI_ANY_ID, 0}, 189 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 190 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 191 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 192 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 193 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 194 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 195 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 196 197 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 198 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 199 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 200 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 201 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 202 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 203 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 204 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 205 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, 206 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, 207 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 208 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 209 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 210 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 211 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 212 /* required last entry */ 213 { 0, 0, 0, 0, 0} 214}; 215 216/********************************************************************* 217 * Table of branding strings for all supported NICs. 218 *********************************************************************/ 219 220static char *em_strings[] = { 221 "Intel(R) PRO/1000 Network Connection" 222}; 223 224/********************************************************************* 225 * Function prototypes 226 *********************************************************************/ 227static int em_probe(device_t); 228static int em_attach(device_t); 229static int em_detach(device_t); 230static int em_shutdown(device_t); 231static int em_suspend(device_t); 232static int em_resume(device_t); 233static void em_start(struct ifnet *); 234static void em_start_locked(struct ifnet *ifp); 235static int em_ioctl(struct ifnet *, u_long, caddr_t); 236static void em_watchdog(struct adapter *); 237static void em_init(void *); 238static void em_init_locked(struct adapter *); 239static void em_stop(void *); 240static void em_media_status(struct ifnet *, struct ifmediareq *); 241static int em_media_change(struct ifnet *); 242static void em_identify_hardware(struct adapter *); 243static int em_allocate_pci_resources(struct adapter *); 244static int em_allocate_legacy(struct adapter *adapter); 245static int em_allocate_msix(struct adapter *adapter); 246static int em_setup_msix(struct adapter *); 247static void em_free_pci_resources(struct adapter *); 248static void em_local_timer(void *); 249static int em_hardware_init(struct adapter *); 250static void em_setup_interface(device_t, struct adapter *); 251static void em_setup_transmit_structures(struct adapter *); 252static void em_initialize_transmit_unit(struct adapter *); 253static int em_setup_receive_structures(struct adapter *); 254static void em_initialize_receive_unit(struct adapter *); 255static void em_enable_intr(struct adapter *); 256static void em_disable_intr(struct adapter *); 257static void em_free_transmit_structures(struct adapter *); 258static void em_free_receive_structures(struct adapter *); 259static void em_update_stats_counters(struct adapter *); 260static void em_txeof(struct adapter *); 261static void em_tx_purge(struct adapter *); 262static int em_allocate_receive_structures(struct adapter *); 263static int em_allocate_transmit_structures(struct adapter *); 264static int em_rxeof(struct adapter *, int); 265#ifndef __NO_STRICT_ALIGNMENT 266static int em_fixup_rx(struct adapter *); 267#endif 268static void em_receive_checksum(struct adapter *, struct e1000_rx_desc *, 269 struct mbuf *); 270static void em_transmit_checksum_setup(struct adapter *, struct mbuf *, 271 u32 *, u32 *); 272#if __FreeBSD_version >= 700000 273static bool em_tso_setup(struct adapter *, struct mbuf *, 274 u32 *, u32 *); 275#endif /* FreeBSD_version >= 700000 */ 276static void em_set_promisc(struct adapter *); 277static void em_disable_promisc(struct adapter *); 278static void em_set_multi(struct adapter *); 279static void em_print_hw_stats(struct adapter *); 280static void em_update_link_status(struct adapter *); 281static int em_get_buf(struct adapter *, int); 282static void em_register_vlan(void *, struct ifnet *, u16); 283static void em_unregister_vlan(void *, struct ifnet *, u16); 284static int em_xmit(struct adapter *, struct mbuf **); 285static void em_smartspeed(struct adapter *); 286static int em_82547_fifo_workaround(struct adapter *, int); 287static void em_82547_update_fifo_head(struct adapter *, int); 288static int em_82547_tx_fifo_reset(struct adapter *); 289static void em_82547_move_tail(void *); 290static int em_dma_malloc(struct adapter *, bus_size_t, 291 struct em_dma_alloc *, int); 292static void em_dma_free(struct adapter *, struct em_dma_alloc *); 293static void em_print_debug_info(struct adapter *); 294static void em_print_nvm_info(struct adapter *); 295static int em_is_valid_ether_addr(u8 *); 296static int em_sysctl_stats(SYSCTL_HANDLER_ARGS); 297static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 298static u32 em_fill_descriptors (bus_addr_t address, u32 length, 299 PDESC_ARRAY desc_array); 300static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 301static void em_add_int_delay_sysctl(struct adapter *, const char *, 302 const char *, struct em_int_delay_info *, int, int); 303/* Management and WOL Support */ 304static void em_init_manageability(struct adapter *); 305static void em_release_manageability(struct adapter *); 306static void em_get_hw_control(struct adapter *); 307static void em_release_hw_control(struct adapter *); 308static void em_enable_wakeup(device_t); 309 310#ifdef EM_TIMESYNC 311/* Precision Time sync support */ 312static int em_tsync_init(struct adapter *); 313static void em_tsync_disable(struct adapter *); 314#endif 315 316#ifdef EM_LEGACY_IRQ 317static void em_intr(void *); 318#else /* FAST IRQ */ 319#if __FreeBSD_version < 700000 320static void em_irq_fast(void *); 321#else 322static int em_irq_fast(void *); 323#endif 324 325/* MSIX handlers */ 326static void em_msix_tx(void *); 327static void em_msix_rx(void *); 328static void em_msix_link(void *); 329static void em_handle_rx(void *context, int pending); 330static void em_handle_tx(void *context, int pending); 331 332static void em_handle_rxtx(void *context, int pending); 333static void em_handle_link(void *context, int pending); 334static void em_add_rx_process_limit(struct adapter *, const char *, 335 const char *, int *, int); 336#endif /* ~EM_LEGACY_IRQ */ 337 338#ifdef DEVICE_POLLING 339static poll_handler_t em_poll; 340#endif /* POLLING */ 341 342/********************************************************************* 343 * FreeBSD Device Interface Entry Points 344 *********************************************************************/ 345 346static device_method_t em_methods[] = { 347 /* Device interface */ 348 DEVMETHOD(device_probe, em_probe), 349 DEVMETHOD(device_attach, em_attach), 350 DEVMETHOD(device_detach, em_detach), 351 DEVMETHOD(device_shutdown, em_shutdown), 352 DEVMETHOD(device_suspend, em_suspend), 353 DEVMETHOD(device_resume, em_resume), 354 {0, 0} 355}; 356 357static driver_t em_driver = { 358 "em", em_methods, sizeof(struct adapter), 359}; 360 361static devclass_t em_devclass; 362DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); 363MODULE_DEPEND(em, pci, 1, 1, 1); 364MODULE_DEPEND(em, ether, 1, 1, 1); 365 366/********************************************************************* 367 * Tunable default values. 368 *********************************************************************/ 369 370#define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 371#define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 372#define M_TSO_LEN 66 373 374/* Allow common code without TSO */ 375#ifndef CSUM_TSO 376#define CSUM_TSO 0 377#endif 378 379static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 380static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 381static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 382static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 383static int em_rxd = EM_DEFAULT_RXD; 384static int em_txd = EM_DEFAULT_TXD; 385static int em_smart_pwr_down = FALSE; 386/* Controls whether promiscuous also shows bad packets */ 387static int em_debug_sbp = FALSE; 388/* Local switch for MSI/MSIX */ 389static int em_enable_msi = TRUE; 390 391TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); 392TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); 393TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); 394TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); 395TUNABLE_INT("hw.em.rxd", &em_rxd); 396TUNABLE_INT("hw.em.txd", &em_txd); 397TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); 398TUNABLE_INT("hw.em.sbp", &em_debug_sbp); 399TUNABLE_INT("hw.em.enable_msi", &em_enable_msi); 400 401#ifndef EM_LEGACY_IRQ 402/* How many packets rxeof tries to clean at a time */ 403static int em_rx_process_limit = 100; 404TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); 405#endif 406 407/* Global used in WOL setup with multiport cards */ 408static int global_quad_port_a = 0; 409 410/********************************************************************* 411 * Device identification routine 412 * 413 * em_probe determines if the driver should be loaded on 414 * adapter based on PCI vendor/device id of the adapter. 415 * 416 * return BUS_PROBE_DEFAULT on success, positive on failure 417 *********************************************************************/ 418 419static int 420em_probe(device_t dev) 421{ 422 char adapter_name[60]; 423 u16 pci_vendor_id = 0; 424 u16 pci_device_id = 0; 425 u16 pci_subvendor_id = 0; 426 u16 pci_subdevice_id = 0; 427 em_vendor_info_t *ent; 428 429 INIT_DEBUGOUT("em_probe: begin"); 430 431 pci_vendor_id = pci_get_vendor(dev); 432 if (pci_vendor_id != EM_VENDOR_ID) 433 return (ENXIO); 434 435 pci_device_id = pci_get_device(dev); 436 pci_subvendor_id = pci_get_subvendor(dev); 437 pci_subdevice_id = pci_get_subdevice(dev); 438 439 ent = em_vendor_info_array; 440 while (ent->vendor_id != 0) { 441 if ((pci_vendor_id == ent->vendor_id) && 442 (pci_device_id == ent->device_id) && 443 444 ((pci_subvendor_id == ent->subvendor_id) || 445 (ent->subvendor_id == PCI_ANY_ID)) && 446 447 ((pci_subdevice_id == ent->subdevice_id) || 448 (ent->subdevice_id == PCI_ANY_ID))) { 449 sprintf(adapter_name, "%s %s", 450 em_strings[ent->index], 451 em_driver_version); 452 device_set_desc_copy(dev, adapter_name); 453 return (BUS_PROBE_DEFAULT); 454 } 455 ent++; 456 } 457 458 return (ENXIO); 459} 460 461/********************************************************************* 462 * Device initialization routine 463 * 464 * The attach entry point is called when the driver is being loaded. 465 * This routine identifies the type of hardware, allocates all resources 466 * and initializes the hardware. 467 * 468 * return 0 on success, positive on failure 469 *********************************************************************/ 470 471static int 472em_attach(device_t dev) 473{ 474 struct adapter *adapter; 475 int tsize, rsize; 476 int error = 0; 477 u16 eeprom_data, device_id; 478 479 INIT_DEBUGOUT("em_attach: begin"); 480 481 adapter = device_get_softc(dev); 482 adapter->dev = adapter->osdep.dev = dev; 483 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 484 EM_TX_LOCK_INIT(adapter, device_get_nameunit(dev)); 485 EM_RX_LOCK_INIT(adapter, device_get_nameunit(dev)); 486 487 /* SYSCTL stuff */ 488 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 489 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 490 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 491 em_sysctl_debug_info, "I", "Debug Information"); 492 493 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 494 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 495 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 496 em_sysctl_stats, "I", "Statistics"); 497 498 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 499 callout_init_mtx(&adapter->tx_fifo_timer, &adapter->tx_mtx, 0); 500 501 /* Determine hardware and mac info */ 502 em_identify_hardware(adapter); 503 504 /* Setup PCI resources */ 505 if (em_allocate_pci_resources(adapter)) { 506 device_printf(dev, "Allocation of PCI resources failed\n"); 507 error = ENXIO; 508 goto err_pci; 509 } 510 511 /* 512 ** For ICH8 and family we need to 513 ** map the flash memory, and this 514 ** must happen after the MAC is 515 ** identified 516 */ 517 if ((adapter->hw.mac.type == e1000_ich8lan) || 518 (adapter->hw.mac.type == e1000_ich9lan) || 519 (adapter->hw.mac.type == e1000_ich10lan)) { 520 int rid = EM_BAR_TYPE_FLASH; 521 adapter->flash = bus_alloc_resource_any(dev, 522 SYS_RES_MEMORY, &rid, RF_ACTIVE); 523 if (adapter->flash == NULL) { 524 device_printf(dev, "Mapping of Flash failed\n"); 525 error = ENXIO; 526 goto err_pci; 527 } 528 /* This is used in the shared code */ 529 adapter->hw.flash_address = (u8 *)adapter->flash; 530 adapter->osdep.flash_bus_space_tag = 531 rman_get_bustag(adapter->flash); 532 adapter->osdep.flash_bus_space_handle = 533 rman_get_bushandle(adapter->flash); 534 } 535 536 /* Do Shared Code initialization */ 537 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 538 device_printf(dev, "Setup of Shared code failed\n"); 539 error = ENXIO; 540 goto err_pci; 541 } 542 543 e1000_get_bus_info(&adapter->hw); 544 545 /* Set up some sysctls for the tunable interrupt delays */ 546 em_add_int_delay_sysctl(adapter, "rx_int_delay", 547 "receive interrupt delay in usecs", &adapter->rx_int_delay, 548 E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt); 549 em_add_int_delay_sysctl(adapter, "tx_int_delay", 550 "transmit interrupt delay in usecs", &adapter->tx_int_delay, 551 E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt); 552 if (adapter->hw.mac.type >= e1000_82540) { 553 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay", 554 "receive interrupt delay limit in usecs", 555 &adapter->rx_abs_int_delay, 556 E1000_REGISTER(&adapter->hw, E1000_RADV), 557 em_rx_abs_int_delay_dflt); 558 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay", 559 "transmit interrupt delay limit in usecs", 560 &adapter->tx_abs_int_delay, 561 E1000_REGISTER(&adapter->hw, E1000_TADV), 562 em_tx_abs_int_delay_dflt); 563 } 564 565#ifndef EM_LEGACY_IRQ 566 /* Sysctls for limiting the amount of work done in the taskqueue */ 567 em_add_rx_process_limit(adapter, "rx_processing_limit", 568 "max number of rx packets to process", &adapter->rx_process_limit, 569 em_rx_process_limit); 570#endif 571 572 /* 573 * Validate number of transmit and receive descriptors. It 574 * must not exceed hardware maximum, and must be multiple 575 * of E1000_DBA_ALIGN. 576 */ 577 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || 578 (adapter->hw.mac.type >= e1000_82544 && em_txd > EM_MAX_TXD) || 579 (adapter->hw.mac.type < e1000_82544 && em_txd > EM_MAX_TXD_82543) || 580 (em_txd < EM_MIN_TXD)) { 581 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 582 EM_DEFAULT_TXD, em_txd); 583 adapter->num_tx_desc = EM_DEFAULT_TXD; 584 } else 585 adapter->num_tx_desc = em_txd; 586 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || 587 (adapter->hw.mac.type >= e1000_82544 && em_rxd > EM_MAX_RXD) || 588 (adapter->hw.mac.type < e1000_82544 && em_rxd > EM_MAX_RXD_82543) || 589 (em_rxd < EM_MIN_RXD)) { 590 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 591 EM_DEFAULT_RXD, em_rxd); 592 adapter->num_rx_desc = EM_DEFAULT_RXD; 593 } else 594 adapter->num_rx_desc = em_rxd; 595 596 adapter->hw.mac.autoneg = DO_AUTO_NEG; 597 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 598 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 599 adapter->rx_buffer_len = 2048; 600 601 e1000_init_script_state_82541(&adapter->hw, TRUE); 602 e1000_set_tbi_compatibility_82543(&adapter->hw, TRUE); 603 604 /* Copper options */ 605 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 606 adapter->hw.phy.mdix = AUTO_ALL_MODES; 607 adapter->hw.phy.disable_polarity_correction = FALSE; 608 adapter->hw.phy.ms_type = EM_MASTER_SLAVE; 609 } 610 611 /* 612 * Set the frame limits assuming 613 * standard ethernet sized frames. 614 */ 615 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 616 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 617 618 /* 619 * This controls when hardware reports transmit completion 620 * status. 621 */ 622 adapter->hw.mac.report_tx_early = 1; 623 624 tsize = roundup2(adapter->num_tx_desc * sizeof(struct e1000_tx_desc), 625 EM_DBA_ALIGN); 626 627 /* Allocate Transmit Descriptor ring */ 628 if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) { 629 device_printf(dev, "Unable to allocate tx_desc memory\n"); 630 error = ENOMEM; 631 goto err_tx_desc; 632 } 633 adapter->tx_desc_base = 634 (struct e1000_tx_desc *)adapter->txdma.dma_vaddr; 635 636 rsize = roundup2(adapter->num_rx_desc * sizeof(struct e1000_rx_desc), 637 EM_DBA_ALIGN); 638 639 /* Allocate Receive Descriptor ring */ 640 if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) { 641 device_printf(dev, "Unable to allocate rx_desc memory\n"); 642 error = ENOMEM; 643 goto err_rx_desc; 644 } 645 adapter->rx_desc_base = 646 (struct e1000_rx_desc *)adapter->rxdma.dma_vaddr; 647 648 /* 649 ** Start from a known state, this is 650 ** important in reading the nvm and 651 ** mac from that. 652 */ 653 e1000_reset_hw(&adapter->hw); 654 655 /* Make sure we have a good EEPROM before we read from it */ 656 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 657 /* 658 ** Some PCI-E parts fail the first check due to 659 ** the link being in sleep state, call it again, 660 ** if it fails a second time its a real issue. 661 */ 662 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 663 device_printf(dev, 664 "The EEPROM Checksum Is Not Valid\n"); 665 error = EIO; 666 goto err_hw_init; 667 } 668 } 669 670 /* Copy the permanent MAC address out of the EEPROM */ 671 if (e1000_read_mac_addr(&adapter->hw) < 0) { 672 device_printf(dev, "EEPROM read error while reading MAC" 673 " address\n"); 674 error = EIO; 675 goto err_hw_init; 676 } 677 678 if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) { 679 device_printf(dev, "Invalid MAC address\n"); 680 error = EIO; 681 goto err_hw_init; 682 } 683 684 /* Initialize the hardware */ 685 if (em_hardware_init(adapter)) { 686 device_printf(dev, "Unable to initialize the hardware\n"); 687 error = EIO; 688 goto err_hw_init; 689 } 690 691 /* Allocate transmit descriptors and buffers */ 692 if (em_allocate_transmit_structures(adapter)) { 693 device_printf(dev, "Could not setup transmit structures\n"); 694 error = ENOMEM; 695 goto err_tx_struct; 696 } 697 698 /* Allocate receive descriptors and buffers */ 699 if (em_allocate_receive_structures(adapter)) { 700 device_printf(dev, "Could not setup receive structures\n"); 701 error = ENOMEM; 702 goto err_rx_struct; 703 } 704 705 /* 706 ** Do interrupt configuration 707 */ 708 if (adapter->msi > 1) /* Do MSI/X */ 709 error = em_allocate_msix(adapter); 710 else /* MSI or Legacy */ 711 error = em_allocate_legacy(adapter); 712 if (error) 713 goto err_rx_struct; 714 715 /* Setup OS specific network interface */ 716 em_setup_interface(dev, adapter); 717 718 /* Initialize statistics */ 719 em_update_stats_counters(adapter); 720 721 adapter->hw.mac.get_link_status = 1; 722 em_update_link_status(adapter); 723 724 /* Indicate SOL/IDER usage */ 725 if (e1000_check_reset_block(&adapter->hw)) 726 device_printf(dev, 727 "PHY reset is blocked due to SOL/IDER session.\n"); 728 729 /* Determine if we have to control management hardware */ 730 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 731 732 /* 733 * Setup Wake-on-Lan 734 */ 735 switch (adapter->hw.mac.type) { 736 737 case e1000_82542: 738 case e1000_82543: 739 break; 740 case e1000_82546: 741 case e1000_82546_rev_3: 742 case e1000_82571: 743 case e1000_80003es2lan: 744 if (adapter->hw.bus.func == 1) 745 e1000_read_nvm(&adapter->hw, 746 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 747 else 748 e1000_read_nvm(&adapter->hw, 749 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 750 eeprom_data &= EM_EEPROM_APME; 751 break; 752 default: 753 /* APME bit in EEPROM is mapped to WUC.APME */ 754 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & 755 E1000_WUC_APME; 756 break; 757 } 758 if (eeprom_data) 759 adapter->wol = E1000_WUFC_MAG; 760 /* 761 * We have the eeprom settings, now apply the special cases 762 * where the eeprom may be wrong or the board won't support 763 * wake on lan on a particular port 764 */ 765 device_id = pci_get_device(dev); 766 switch (device_id) { 767 case E1000_DEV_ID_82546GB_PCIE: 768 adapter->wol = 0; 769 break; 770 case E1000_DEV_ID_82546EB_FIBER: 771 case E1000_DEV_ID_82546GB_FIBER: 772 case E1000_DEV_ID_82571EB_FIBER: 773 /* Wake events only supported on port A for dual fiber 774 * regardless of eeprom setting */ 775 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 776 E1000_STATUS_FUNC_1) 777 adapter->wol = 0; 778 break; 779 case E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3: 780 case E1000_DEV_ID_82571EB_QUAD_COPPER: 781 case E1000_DEV_ID_82571EB_QUAD_FIBER: 782 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 783 /* if quad port adapter, disable WoL on all but port A */ 784 if (global_quad_port_a != 0) 785 adapter->wol = 0; 786 /* Reset for multiple quad port adapters */ 787 if (++global_quad_port_a == 4) 788 global_quad_port_a = 0; 789 break; 790 } 791 792 /* Do we need workaround for 82544 PCI-X adapter? */ 793 if (adapter->hw.bus.type == e1000_bus_type_pcix && 794 adapter->hw.mac.type == e1000_82544) 795 adapter->pcix_82544 = TRUE; 796 else 797 adapter->pcix_82544 = FALSE; 798 799 /* Register for VLAN events */ 800 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 801 em_register_vlan, 0, EVENTHANDLER_PRI_FIRST); 802 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 803 em_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST); 804 805 /* Tell the stack that the interface is not active */ 806 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 807 808 INIT_DEBUGOUT("em_attach: end"); 809 810 return (0); 811 812err_rx_struct: 813 em_free_transmit_structures(adapter); 814err_tx_struct: 815err_hw_init: 816 em_release_hw_control(adapter); 817 em_dma_free(adapter, &adapter->rxdma); 818err_rx_desc: 819 em_dma_free(adapter, &adapter->txdma); 820err_tx_desc: 821err_pci: 822 em_free_pci_resources(adapter); 823 EM_TX_LOCK_DESTROY(adapter); 824 EM_RX_LOCK_DESTROY(adapter); 825 EM_CORE_LOCK_DESTROY(adapter); 826 827 return (error); 828} 829 830/********************************************************************* 831 * Device removal routine 832 * 833 * The detach entry point is called when the driver is being removed. 834 * This routine stops the adapter and deallocates all the resources 835 * that were allocated for driver operation. 836 * 837 * return 0 on success, positive on failure 838 *********************************************************************/ 839 840static int 841em_detach(device_t dev) 842{ 843 struct adapter *adapter = device_get_softc(dev); 844 struct ifnet *ifp = adapter->ifp; 845 846 INIT_DEBUGOUT("em_detach: begin"); 847 848 /* Make sure VLANS are not using driver */ 849#if __FreeBSD_version >= 700000 850 if (adapter->ifp->if_vlantrunk != NULL) { 851#else 852 if (adapter->ifp->if_nvlans != 0) { 853#endif 854 device_printf(dev,"Vlan in use, detach first\n"); 855 return (EBUSY); 856 } 857 858#ifdef DEVICE_POLLING 859 if (ifp->if_capenable & IFCAP_POLLING) 860 ether_poll_deregister(ifp); 861#endif 862 863 EM_CORE_LOCK(adapter); 864 EM_TX_LOCK(adapter); 865 adapter->in_detach = 1; 866 em_stop(adapter); 867 e1000_phy_hw_reset(&adapter->hw); 868 869 em_release_manageability(adapter); 870 871 if (((adapter->hw.mac.type == e1000_82573) || 872 (adapter->hw.mac.type == e1000_ich8lan) || 873 (adapter->hw.mac.type == e1000_ich10lan) || 874 (adapter->hw.mac.type == e1000_ich9lan)) && 875 e1000_check_mng_mode(&adapter->hw)) 876 em_release_hw_control(adapter); 877 878 if (adapter->wol) { 879 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 880 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 881 em_enable_wakeup(dev); 882 } 883 884 EM_TX_UNLOCK(adapter); 885 EM_CORE_UNLOCK(adapter); 886 887 /* Unregister VLAN events */ 888 if (adapter->vlan_attach != NULL) 889 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 890 if (adapter->vlan_detach != NULL) 891 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 892 893 ether_ifdetach(adapter->ifp); 894 callout_drain(&adapter->timer); 895 callout_drain(&adapter->tx_fifo_timer); 896 897 em_free_pci_resources(adapter); 898 bus_generic_detach(dev); 899 if_free(ifp); 900 901#ifdef IFNET_BUF_RING 902 drbr_free(adapter->br, M_DEVBUF); 903#endif 904 em_free_transmit_structures(adapter); 905 em_free_receive_structures(adapter); 906 907 /* Free Transmit Descriptor ring */ 908 if (adapter->tx_desc_base) { 909 em_dma_free(adapter, &adapter->txdma); 910 adapter->tx_desc_base = NULL; 911 } 912 913 /* Free Receive Descriptor ring */ 914 if (adapter->rx_desc_base) { 915 em_dma_free(adapter, &adapter->rxdma); 916 adapter->rx_desc_base = NULL; 917 } 918 919 EM_TX_LOCK_DESTROY(adapter); 920 EM_RX_LOCK_DESTROY(adapter); 921 EM_CORE_LOCK_DESTROY(adapter); 922 923 return (0); 924} 925 926/********************************************************************* 927 * 928 * Shutdown entry point 929 * 930 **********************************************************************/ 931 932static int 933em_shutdown(device_t dev) 934{ 935 return em_suspend(dev); 936} 937 938/* 939 * Suspend/resume device methods. 940 */ 941static int 942em_suspend(device_t dev) 943{ 944 struct adapter *adapter = device_get_softc(dev); 945 946 EM_CORE_LOCK(adapter); 947 948 EM_TX_LOCK(adapter); 949 em_stop(adapter); 950 EM_TX_UNLOCK(adapter); 951 952 em_release_manageability(adapter); 953 954 if (((adapter->hw.mac.type == e1000_82573) || 955 (adapter->hw.mac.type == e1000_ich8lan) || 956 (adapter->hw.mac.type == e1000_ich10lan) || 957 (adapter->hw.mac.type == e1000_ich9lan)) && 958 e1000_check_mng_mode(&adapter->hw)) 959 em_release_hw_control(adapter); 960 961 if (adapter->wol) { 962 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 963 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 964 em_enable_wakeup(dev); 965 } 966 967 EM_CORE_UNLOCK(adapter); 968 969 return bus_generic_suspend(dev); 970} 971 972static int 973em_resume(device_t dev) 974{ 975 struct adapter *adapter = device_get_softc(dev); 976 struct ifnet *ifp = adapter->ifp; 977 978 EM_CORE_LOCK(adapter); 979 em_init_locked(adapter); 980 em_init_manageability(adapter); 981 EM_CORE_UNLOCK(adapter); 982 em_start(ifp); 983 984 return bus_generic_resume(dev); 985} 986 987 988/********************************************************************* 989 * Transmit entry point 990 * 991 * em_start is called by the stack to initiate a transmit. 992 * The driver will remain in this routine as long as there are 993 * packets to transmit and transmit resources are available. 994 * In case resources are not available stack is notified and 995 * the packet is requeued. 996 **********************************************************************/ 997 998#ifdef IFNET_BUF_RING 999static int 1000em_transmit_locked(struct ifnet *ifp, struct mbuf *m) 1001{ 1002 struct adapter *adapter = ifp->if_softc; 1003 int error; 1004 1005 EM_TX_LOCK_ASSERT(adapter); 1006 if (((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 1007 IFF_DRV_RUNNING) 1008 || (!adapter->link_active)) { 1009 error = drbr_enqueue(ifp, adapter->br, m); 1010 return (error); 1011 } 1012 1013 if (ADAPTER_RING_EMPTY(adapter) && 1014 (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) { 1015 if (em_xmit(adapter, &m)) { 1016 if (m && (error = drbr_enqueue(ifp, adapter->br, m)) != 0) 1017 return (error); 1018 } else { 1019 /* 1020 ** Send a copy of the frame to the BPF 1021 ** listener and set the watchdog on. 1022 */ 1023 ETHER_BPF_MTAP(ifp, m); 1024 adapter->watchdog_timer = EM_TX_TIMEOUT; 1025 } 1026 } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0) 1027 return (error); 1028 1029 if (!ADAPTER_RING_EMPTY(adapter)) 1030 em_start_locked(ifp); 1031 1032 return (0); 1033} 1034 1035static int 1036em_transmit(struct ifnet *ifp, struct mbuf *m) 1037{ 1038 1039 struct adapter *adapter = ifp->if_softc; 1040 int error = 0; 1041 1042 if(EM_TX_TRYLOCK(adapter)) { 1043 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1044 error = em_transmit_locked(ifp, m); 1045 EM_TX_UNLOCK(adapter); 1046 } else 1047 error = drbr_enqueue(ifp, adapter->br, m); 1048 1049 return (error); 1050} 1051 1052static void 1053em_qflush(struct ifnet *ifp) 1054{ 1055 struct mbuf *m; 1056 struct adapter *adapter = (struct adapter *)ifp->if_softc; 1057 1058 EM_TX_LOCK(adapter); 1059 while ((m = buf_ring_dequeue_sc(adapter->br)) != NULL) 1060 m_freem(m); 1061 if_qflush(ifp); 1062 EM_TX_UNLOCK(adapter); 1063} 1064 1065static void 1066em_start_locked(struct ifnet *ifp) 1067{ 1068 struct adapter *adapter = ifp->if_softc; 1069 struct mbuf *m_head; 1070 1071 EM_TX_LOCK_ASSERT(adapter); 1072 1073 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 1074 IFF_DRV_RUNNING) 1075 return; 1076 if (!adapter->link_active) 1077 return; 1078 1079 while ((adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD) 1080 && (!ADAPTER_RING_EMPTY(adapter))) { 1081 1082 m_head = drbr_dequeue(ifp, adapter->br); 1083 if (m_head == NULL) 1084 break; 1085 /* 1086 * Encapsulation can modify our pointer, and or make it 1087 * NULL on failure. In that event, we can't requeue. 1088 */ 1089 if (em_xmit(adapter, &m_head)) { 1090 if (m_head == NULL) 1091 break; 1092 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1093 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 1094 break; 1095 } 1096 1097 /* Send a copy of the frame to the BPF listener */ 1098 ETHER_BPF_MTAP(ifp, m_head); 1099 1100 /* Set timeout in case hardware has problems transmitting. */ 1101 adapter->watchdog_timer = EM_TX_TIMEOUT; 1102 } 1103 if ((adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)) 1104 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1105 1106} 1107#else 1108static void 1109em_start_locked(struct ifnet *ifp) 1110{ 1111 struct adapter *adapter = ifp->if_softc; 1112 struct mbuf *m_head; 1113 1114 EM_TX_LOCK_ASSERT(adapter); 1115 1116 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 1117 IFF_DRV_RUNNING) 1118 return; 1119 if (!adapter->link_active) 1120 return; 1121 1122 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 1123 1124 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 1125 if (m_head == NULL) 1126 break; 1127 /* 1128 * Encapsulation can modify our pointer, and or make it 1129 * NULL on failure. In that event, we can't requeue. 1130 */ 1131 if (em_xmit(adapter, &m_head)) { 1132 if (m_head == NULL) 1133 break; 1134 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1135 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 1136 break; 1137 } 1138 1139 /* Send a copy of the frame to the BPF listener */ 1140 ETHER_BPF_MTAP(ifp, m_head); 1141 1142 /* Set timeout in case hardware has problems transmitting. */ 1143 adapter->watchdog_timer = EM_TX_TIMEOUT; 1144 } 1145} 1146 1147#endif 1148 1149static void 1150em_start(struct ifnet *ifp) 1151{ 1152 struct adapter *adapter = ifp->if_softc; 1153 1154 EM_TX_LOCK(adapter); 1155 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1156 em_start_locked(ifp); 1157 EM_TX_UNLOCK(adapter); 1158} 1159 1160/********************************************************************* 1161 * Ioctl entry point 1162 * 1163 * em_ioctl is called when the user wants to configure the 1164 * interface. 1165 * 1166 * return 0 on success, positive on failure 1167 **********************************************************************/ 1168 1169static int 1170em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 1171{ 1172 struct adapter *adapter = ifp->if_softc; 1173 struct ifreq *ifr = (struct ifreq *)data; 1174#ifdef INET 1175 struct ifaddr *ifa = (struct ifaddr *)data; 1176#endif 1177 int error = 0; 1178 1179 if (adapter->in_detach) 1180 return (error); 1181 1182 switch (command) { 1183 case SIOCSIFADDR: 1184#ifdef INET 1185 if (ifa->ifa_addr->sa_family == AF_INET) { 1186 /* 1187 * XXX 1188 * Since resetting hardware takes a very long time 1189 * and results in link renegotiation we only 1190 * initialize the hardware only when it is absolutely 1191 * required. 1192 */ 1193 ifp->if_flags |= IFF_UP; 1194 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1195 EM_CORE_LOCK(adapter); 1196 em_init_locked(adapter); 1197 EM_CORE_UNLOCK(adapter); 1198 } 1199 arp_ifinit(ifp, ifa); 1200 } else 1201#endif 1202 error = ether_ioctl(ifp, command, data); 1203 break; 1204 case SIOCSIFMTU: 1205 { 1206 int max_frame_size; 1207 u16 eeprom_data = 0; 1208 1209 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 1210 1211 EM_CORE_LOCK(adapter); 1212 switch (adapter->hw.mac.type) { 1213 case e1000_82573: 1214 /* 1215 * 82573 only supports jumbo frames 1216 * if ASPM is disabled. 1217 */ 1218 e1000_read_nvm(&adapter->hw, 1219 NVM_INIT_3GIO_3, 1, &eeprom_data); 1220 if (eeprom_data & NVM_WORD1A_ASPM_MASK) { 1221 max_frame_size = ETHER_MAX_LEN; 1222 break; 1223 } 1224 /* Allow Jumbo frames - fall thru */ 1225 case e1000_82571: 1226 case e1000_82572: 1227 case e1000_ich9lan: 1228 case e1000_ich10lan: 1229 case e1000_82574: 1230 case e1000_80003es2lan: /* Limit Jumbo Frame size */ 1231 max_frame_size = 9234; 1232 break; 1233 /* Adapters that do not support jumbo frames */ 1234 case e1000_82542: 1235 case e1000_ich8lan: 1236 max_frame_size = ETHER_MAX_LEN; 1237 break; 1238 default: 1239 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1240 } 1241 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1242 ETHER_CRC_LEN) { 1243 EM_CORE_UNLOCK(adapter); 1244 error = EINVAL; 1245 break; 1246 } 1247 1248 ifp->if_mtu = ifr->ifr_mtu; 1249 adapter->max_frame_size = 1250 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1251 em_init_locked(adapter); 1252 EM_CORE_UNLOCK(adapter); 1253 break; 1254 } 1255 case SIOCSIFFLAGS: 1256 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1257 SIOCSIFFLAGS (Set Interface Flags)"); 1258 EM_CORE_LOCK(adapter); 1259 if (ifp->if_flags & IFF_UP) { 1260 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1261 if ((ifp->if_flags ^ adapter->if_flags) & 1262 (IFF_PROMISC | IFF_ALLMULTI)) { 1263 em_disable_promisc(adapter); 1264 em_set_promisc(adapter); 1265 } 1266 } else 1267 em_init_locked(adapter); 1268 } else 1269 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1270 EM_TX_LOCK(adapter); 1271 em_stop(adapter); 1272 EM_TX_UNLOCK(adapter); 1273 } 1274 adapter->if_flags = ifp->if_flags; 1275 EM_CORE_UNLOCK(adapter); 1276 break; 1277 case SIOCADDMULTI: 1278 case SIOCDELMULTI: 1279 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1280 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1281 EM_CORE_LOCK(adapter); 1282 em_disable_intr(adapter); 1283 em_set_multi(adapter); 1284 if (adapter->hw.mac.type == e1000_82542 && 1285 adapter->hw.revision_id == E1000_REVISION_2) { 1286 em_initialize_receive_unit(adapter); 1287 } 1288#ifdef DEVICE_POLLING 1289 if (!(ifp->if_capenable & IFCAP_POLLING)) 1290#endif 1291 em_enable_intr(adapter); 1292 EM_CORE_UNLOCK(adapter); 1293 } 1294 break; 1295 case SIOCSIFMEDIA: 1296 /* Check SOL/IDER usage */ 1297 EM_CORE_LOCK(adapter); 1298 if (e1000_check_reset_block(&adapter->hw)) { 1299 EM_CORE_UNLOCK(adapter); 1300 device_printf(adapter->dev, "Media change is" 1301 " blocked due to SOL/IDER session.\n"); 1302 break; 1303 } 1304 EM_CORE_UNLOCK(adapter); 1305 case SIOCGIFMEDIA: 1306 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1307 SIOCxIFMEDIA (Get/Set Interface Media)"); 1308 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1309 break; 1310 case SIOCSIFCAP: 1311 { 1312 int mask, reinit; 1313 1314 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1315 reinit = 0; 1316 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1317#ifdef DEVICE_POLLING 1318 if (mask & IFCAP_POLLING) { 1319 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1320 error = ether_poll_register(em_poll, ifp); 1321 if (error) 1322 return (error); 1323 EM_CORE_LOCK(adapter); 1324 em_disable_intr(adapter); 1325 ifp->if_capenable |= IFCAP_POLLING; 1326 EM_CORE_UNLOCK(adapter); 1327 } else { 1328 error = ether_poll_deregister(ifp); 1329 /* Enable interrupt even in error case */ 1330 EM_CORE_LOCK(adapter); 1331 em_enable_intr(adapter); 1332 ifp->if_capenable &= ~IFCAP_POLLING; 1333 EM_CORE_UNLOCK(adapter); 1334 } 1335 } 1336#endif 1337 if (mask & IFCAP_HWCSUM) { 1338 ifp->if_capenable ^= IFCAP_HWCSUM; 1339 reinit = 1; 1340 } 1341#if __FreeBSD_version >= 700000 1342 if (mask & IFCAP_TSO4) { 1343 ifp->if_capenable ^= IFCAP_TSO4; 1344 reinit = 1; 1345 } 1346#endif 1347 1348 if (mask & IFCAP_VLAN_HWTAGGING) { 1349 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1350 reinit = 1; 1351 } 1352 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1353 em_init(adapter); 1354#if __FreeBSD_version >= 700000 1355 VLAN_CAPABILITIES(ifp); 1356#endif 1357 break; 1358 } 1359 1360#ifdef EM_TIMESYNC 1361 /* 1362 ** IOCTL support for Precision Time (IEEE 1588) Support 1363 */ 1364 case EM_TIMESYNC_READTS: 1365 { 1366 u32 rx_ctl, tx_ctl; 1367 struct em_tsync_read *tdata; 1368 1369 tdata = (struct em_tsync_read *) ifr->ifr_data; 1370 1371 IOCTL_DEBUGOUT("Reading Timestamp\n"); 1372 1373 if (tdata->read_current_time) { 1374 getnanotime(&tdata->system_time); 1375 tdata->network_time = E1000_READ_REG(&adapter->hw, E1000_SYSTIML); 1376 tdata->network_time |= 1377 (u64)E1000_READ_REG(&adapter->hw, E1000_SYSTIMH ) << 32; 1378 } 1379 1380 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 1381 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 1382 1383 IOCTL_DEBUGOUT1("RX_CTL value = %u\n", rx_ctl); 1384 IOCTL_DEBUGOUT1("TX_CTL value = %u\n", tx_ctl); 1385 1386 if (rx_ctl & 0x1) { 1387 IOCTL_DEBUGOUT("RX timestamp is valid\n"); 1388 u32 tmp; 1389 unsigned char *tmp_cp; 1390 1391 tdata->rx_valid = 1; 1392 tdata->rx_stamp = E1000_READ_REG(&adapter->hw, E1000_RXSTMPL); 1393 tdata->rx_stamp |= (u64)E1000_READ_REG(&adapter->hw, 1394 E1000_RXSTMPH) << 32; 1395 1396 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRL); 1397 tmp_cp = (unsigned char *) &tmp; 1398 tdata->srcid[0] = tmp_cp[0]; 1399 tdata->srcid[1] = tmp_cp[1]; 1400 tdata->srcid[2] = tmp_cp[2]; 1401 tdata->srcid[3] = tmp_cp[3]; 1402 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRH); 1403 tmp_cp = (unsigned char *) &tmp; 1404 tdata->srcid[4] = tmp_cp[0]; 1405 tdata->srcid[5] = tmp_cp[1]; 1406 tdata->seqid = tmp >> 16; 1407 tdata->seqid = htons(tdata->seqid); 1408 } else 1409 tdata->rx_valid = 0; 1410 1411 if (tx_ctl & 0x1) { 1412 IOCTL_DEBUGOUT("TX timestamp is valid\n"); 1413 tdata->tx_valid = 1; 1414 tdata->tx_stamp = E1000_READ_REG(&adapter->hw, E1000_TXSTMPL); 1415 tdata->tx_stamp |= (u64) E1000_READ_REG(&adapter->hw, 1416 E1000_TXSTMPH) << 32; 1417 } else 1418 tdata->tx_valid = 0; 1419 1420 return (0); 1421 } 1422#endif /* EM_TIMESYNC */ 1423 1424 default: 1425 error = ether_ioctl(ifp, command, data); 1426 break; 1427 } 1428 1429 return (error); 1430} 1431 1432/********************************************************************* 1433 * Watchdog timer: 1434 * 1435 * This routine is called from the local timer every second. 1436 * As long as transmit descriptors are being cleaned the value 1437 * is non-zero and we do nothing. Reaching 0 indicates a tx hang 1438 * and we then reset the device. 1439 * 1440 **********************************************************************/ 1441 1442static void 1443em_watchdog(struct adapter *adapter) 1444{ 1445 1446 EM_CORE_LOCK_ASSERT(adapter); 1447 1448 /* 1449 ** The timer is set to 5 every time start queues a packet. 1450 ** Then txeof keeps resetting it as long as it cleans at 1451 ** least one descriptor. 1452 ** Finally, anytime all descriptors are clean the timer is 1453 ** set to 0. 1454 */ 1455 EM_TX_LOCK(adapter); 1456 if ((adapter->watchdog_timer == 0) || (--adapter->watchdog_timer)) { 1457 EM_TX_UNLOCK(adapter); 1458 return; 1459 } 1460 1461 /* If we are in this routine because of pause frames, then 1462 * don't reset the hardware. 1463 */ 1464 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 1465 E1000_STATUS_TXOFF) { 1466 adapter->watchdog_timer = EM_TX_TIMEOUT; 1467 EM_TX_UNLOCK(adapter); 1468 return; 1469 } 1470 1471 if (e1000_check_for_link(&adapter->hw) == 0) 1472 device_printf(adapter->dev, "watchdog timeout -- resetting\n"); 1473 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1474 adapter->watchdog_events++; 1475 EM_TX_UNLOCK(adapter); 1476 1477 em_init_locked(adapter); 1478} 1479 1480/********************************************************************* 1481 * Init entry point 1482 * 1483 * This routine is used in two ways. It is used by the stack as 1484 * init entry point in network interface structure. It is also used 1485 * by the driver as a hw/sw initialization routine to get to a 1486 * consistent state. 1487 * 1488 * return 0 on success, positive on failure 1489 **********************************************************************/ 1490 1491static void 1492em_init_locked(struct adapter *adapter) 1493{ 1494 struct ifnet *ifp = adapter->ifp; 1495 device_t dev = adapter->dev; 1496 u32 pba; 1497 1498 INIT_DEBUGOUT("em_init: begin"); 1499 1500 EM_CORE_LOCK_ASSERT(adapter); 1501 1502 EM_TX_LOCK(adapter); 1503 em_stop(adapter); 1504 EM_TX_UNLOCK(adapter); 1505 1506 /* 1507 * Packet Buffer Allocation (PBA) 1508 * Writing PBA sets the receive portion of the buffer 1509 * the remainder is used for the transmit buffer. 1510 * 1511 * Devices before the 82547 had a Packet Buffer of 64K. 1512 * Default allocation: PBA=48K for Rx, leaving 16K for Tx. 1513 * After the 82547 the buffer was reduced to 40K. 1514 * Default allocation: PBA=30K for Rx, leaving 10K for Tx. 1515 * Note: default does not leave enough room for Jumbo Frame >10k. 1516 */ 1517 switch (adapter->hw.mac.type) { 1518 case e1000_82547: 1519 case e1000_82547_rev_2: /* 82547: Total Packet Buffer is 40K */ 1520 if (adapter->max_frame_size > 8192) 1521 pba = E1000_PBA_22K; /* 22K for Rx, 18K for Tx */ 1522 else 1523 pba = E1000_PBA_30K; /* 30K for Rx, 10K for Tx */ 1524 adapter->tx_fifo_head = 0; 1525 adapter->tx_head_addr = pba << EM_TX_HEAD_ADDR_SHIFT; 1526 adapter->tx_fifo_size = 1527 (E1000_PBA_40K - pba) << EM_PBA_BYTES_SHIFT; 1528 break; 1529 /* Total Packet Buffer on these is 48K */ 1530 case e1000_82571: 1531 case e1000_82572: 1532 case e1000_80003es2lan: 1533 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1534 break; 1535 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1536 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1537 break; 1538 case e1000_82574: 1539 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1540 break; 1541 case e1000_ich9lan: 1542 case e1000_ich10lan: 1543#define E1000_PBA_10K 0x000A 1544 pba = E1000_PBA_10K; 1545 break; 1546 case e1000_ich8lan: 1547 pba = E1000_PBA_8K; 1548 break; 1549 default: 1550 /* Devices before 82547 had a Packet Buffer of 64K. */ 1551 if (adapter->max_frame_size > 8192) 1552 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1553 else 1554 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1555 } 1556 1557 INIT_DEBUGOUT1("em_init: pba=%dK",pba); 1558 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1559 1560 /* Get the latest mac address, User can use a LAA */ 1561 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1562 ETHER_ADDR_LEN); 1563 1564 /* Put the address into the Receive Address Array */ 1565 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1566 1567 /* 1568 * With the 82571 adapter, RAR[0] may be overwritten 1569 * when the other port is reset, we make a duplicate 1570 * in RAR[14] for that eventuality, this assures 1571 * the interface continues to function. 1572 */ 1573 if (adapter->hw.mac.type == e1000_82571) { 1574 e1000_set_laa_state_82571(&adapter->hw, TRUE); 1575 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 1576 E1000_RAR_ENTRIES - 1); 1577 } 1578 1579 /* Initialize the hardware */ 1580 if (em_hardware_init(adapter)) { 1581 device_printf(dev, "Unable to initialize the hardware\n"); 1582 return; 1583 } 1584 em_update_link_status(adapter); 1585 1586 /* Setup VLAN support, basic and offload if available */ 1587 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1588 1589 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) && 1590 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) { 1591 u32 ctrl; 1592 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1593 ctrl |= E1000_CTRL_VME; 1594 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1595 } 1596 1597 1598 /* Set hardware offload abilities */ 1599 ifp->if_hwassist = 0; 1600 if (adapter->hw.mac.type >= e1000_82543) { 1601 if (ifp->if_capenable & IFCAP_TXCSUM) 1602 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1603#if __FreeBSD_version >= 700000 1604 if (ifp->if_capenable & IFCAP_TSO4) 1605 ifp->if_hwassist |= CSUM_TSO; 1606#endif 1607 } 1608 1609 /* Configure for OS presence */ 1610 em_init_manageability(adapter); 1611 1612 /* Prepare transmit descriptors and buffers */ 1613 em_setup_transmit_structures(adapter); 1614 em_initialize_transmit_unit(adapter); 1615 1616 /* Setup Multicast table */ 1617 em_set_multi(adapter); 1618 1619 /* Prepare receive descriptors and buffers */ 1620 if (em_setup_receive_structures(adapter)) { 1621 device_printf(dev, "Could not setup receive structures\n"); 1622 EM_TX_LOCK(adapter); 1623 em_stop(adapter); 1624 EM_TX_UNLOCK(adapter); 1625 return; 1626 } 1627 em_initialize_receive_unit(adapter); 1628 1629 /* Don't lose promiscuous settings */ 1630 em_set_promisc(adapter); 1631 1632 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1633 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1634 1635 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1636 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1637 1638 /* MSI/X configuration for 82574 */ 1639 if (adapter->hw.mac.type == e1000_82574) { 1640 int tmp; 1641 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 1642 tmp |= E1000_CTRL_EXT_PBA_CLR; 1643 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); 1644 /* 1645 ** Set the IVAR - interrupt vector routing. 1646 ** Each nibble represents a vector, high bit 1647 ** is enable, other 3 bits are the MSIX table 1648 ** entry, we map RXQ0 to 0, TXQ0 to 1, and 1649 ** Link (other) to 2, hence the magic number. 1650 */ 1651 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, 0x800A0908); 1652 } 1653 1654#ifdef DEVICE_POLLING 1655 /* 1656 * Only enable interrupts if we are not polling, make sure 1657 * they are off otherwise. 1658 */ 1659 if (ifp->if_capenable & IFCAP_POLLING) 1660 em_disable_intr(adapter); 1661 else 1662#endif /* DEVICE_POLLING */ 1663 em_enable_intr(adapter); 1664 1665#ifdef EM_TIMESYNC 1666 /* Initializae IEEE 1588 Precision Time hardware */ 1667 if ((adapter->hw.mac.type == e1000_82574) || 1668 (adapter->hw.mac.type == e1000_ich10lan)) 1669 em_tsync_init(adapter); 1670#endif 1671 1672 /* Don't reset the phy next time init gets called */ 1673 adapter->hw.phy.reset_disable = TRUE; 1674} 1675 1676static void 1677em_init(void *arg) 1678{ 1679 struct adapter *adapter = arg; 1680 1681 EM_CORE_LOCK(adapter); 1682 em_init_locked(adapter); 1683 EM_CORE_UNLOCK(adapter); 1684} 1685 1686 1687#ifdef DEVICE_POLLING 1688/********************************************************************* 1689 * 1690 * Legacy polling routine 1691 * 1692 *********************************************************************/ 1693static void 1694em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1695{ 1696 struct adapter *adapter = ifp->if_softc; 1697 u32 reg_icr; 1698 1699 EM_CORE_LOCK(adapter); 1700 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1701 EM_CORE_UNLOCK(adapter); 1702 return; 1703 } 1704 1705 if (cmd == POLL_AND_CHECK_STATUS) { 1706 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1707 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1708 callout_stop(&adapter->timer); 1709 adapter->hw.mac.get_link_status = 1; 1710 em_update_link_status(adapter); 1711 callout_reset(&adapter->timer, hz, 1712 em_local_timer, adapter); 1713 } 1714 } 1715 EM_CORE_UNLOCK(adapter); 1716 1717 em_rxeof(adapter, count); 1718 1719 EM_TX_LOCK(adapter); 1720 em_txeof(adapter); 1721 1722 if (!ADAPTER_RING_EMPTY(adapter)) 1723 em_start_locked(ifp); 1724 EM_TX_UNLOCK(adapter); 1725} 1726#endif /* DEVICE_POLLING */ 1727 1728#ifdef EM_LEGACY_IRQ 1729/********************************************************************* 1730 * 1731 * Legacy Interrupt Service routine 1732 * 1733 *********************************************************************/ 1734 1735static void 1736em_intr(void *arg) 1737{ 1738 struct adapter *adapter = arg; 1739 struct ifnet *ifp = adapter->ifp; 1740 u32 reg_icr; 1741 1742 1743 if (ifp->if_capenable & IFCAP_POLLING) 1744 return; 1745 1746 EM_CORE_LOCK(adapter); 1747 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1748 if ((reg_icr == 0xffffffff) || (reg_icr == 0)|| 1749 (adapter->hw.mac.type >= e1000_82571 && 1750 (reg_icr & E1000_ICR_INT_ASSERTED) == 0)) 1751 goto out; 1752 1753 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 1754 goto out; 1755 1756 EM_TX_LOCK(adapter); 1757 em_txeof(adapter); 1758 em_rxeof(adapter, -1); 1759 em_txeof(adapter); 1760 EM_TX_UNLOCK(adapter); 1761 1762 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1763 callout_stop(&adapter->timer); 1764 adapter->hw.mac.get_link_status = 1; 1765 em_update_link_status(adapter); 1766 /* Deal with TX cruft when link lost */ 1767 em_tx_purge(adapter); 1768 callout_reset(&adapter->timer, hz, 1769 em_local_timer, adapter); 1770 } 1771 1772 if (reg_icr & E1000_ICR_RXO) 1773 adapter->rx_overruns++; 1774out: 1775 EM_CORE_UNLOCK(adapter); 1776 if (ifp->if_drv_flags & IFF_DRV_RUNNING && 1777 !ADAPTER_RING_EMPTY(adapter)) 1778 em_start(ifp); 1779} 1780 1781#else /* EM_FAST_IRQ, then fast interrupt routines only */ 1782 1783static void 1784em_handle_link(void *context, int pending) 1785{ 1786 struct adapter *adapter = context; 1787 struct ifnet *ifp = adapter->ifp; 1788 1789 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1790 return; 1791 1792 EM_CORE_LOCK(adapter); 1793 callout_stop(&adapter->timer); 1794 em_update_link_status(adapter); 1795 /* Deal with TX cruft when link lost */ 1796 em_tx_purge(adapter); 1797 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1798 EM_CORE_UNLOCK(adapter); 1799} 1800 1801 1802/* Combined RX/TX handler, used by Legacy and MSI */ 1803static void 1804em_handle_rxtx(void *context, int pending) 1805{ 1806 struct adapter *adapter = context; 1807 struct ifnet *ifp = adapter->ifp; 1808 1809 1810 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1811 if (em_rxeof(adapter, adapter->rx_process_limit) != 0) 1812 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1813 EM_TX_LOCK(adapter); 1814 em_txeof(adapter); 1815 1816 if (!ADAPTER_RING_EMPTY(adapter)) 1817 em_start_locked(ifp); 1818 EM_TX_UNLOCK(adapter); 1819 } 1820 1821 em_enable_intr(adapter); 1822} 1823 1824/********************************************************************* 1825 * 1826 * Fast Legacy/MSI Combined Interrupt Service routine 1827 * 1828 *********************************************************************/ 1829#if __FreeBSD_version < 700000 1830#define FILTER_STRAY 1831#define FILTER_HANDLED 1832static void 1833#else 1834static int 1835#endif 1836em_irq_fast(void *arg) 1837{ 1838 struct adapter *adapter = arg; 1839 struct ifnet *ifp; 1840 u32 reg_icr; 1841 1842 ifp = adapter->ifp; 1843 1844 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1845 1846 /* Hot eject? */ 1847 if (reg_icr == 0xffffffff) 1848 return FILTER_STRAY; 1849 1850 /* Definitely not our interrupt. */ 1851 if (reg_icr == 0x0) 1852 return FILTER_STRAY; 1853 1854 /* 1855 * Starting with the 82571 chip, bit 31 should be used to 1856 * determine whether the interrupt belongs to us. 1857 */ 1858 if (adapter->hw.mac.type >= e1000_82571 && 1859 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1860 return FILTER_STRAY; 1861 1862 /* 1863 * Mask interrupts until the taskqueue is finished running. This is 1864 * cheap, just assume that it is needed. This also works around the 1865 * MSI message reordering errata on certain systems. 1866 */ 1867 em_disable_intr(adapter); 1868 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1869 1870 /* Link status change */ 1871 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1872 adapter->hw.mac.get_link_status = 1; 1873 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1874 } 1875 1876 if (reg_icr & E1000_ICR_RXO) 1877 adapter->rx_overruns++; 1878 return FILTER_HANDLED; 1879} 1880 1881/********************************************************************* 1882 * 1883 * MSIX Interrupt Service Routines 1884 * 1885 **********************************************************************/ 1886#define EM_MSIX_TX 0x00040000 1887#define EM_MSIX_RX 0x00010000 1888#define EM_MSIX_LINK 0x00100000 1889 1890static void 1891em_msix_tx(void *arg) 1892{ 1893 struct adapter *adapter = arg; 1894 struct ifnet *ifp = adapter->ifp; 1895 1896 ++adapter->tx_irq; 1897 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1898 EM_TX_LOCK(adapter); 1899 em_txeof(adapter); 1900 EM_TX_UNLOCK(adapter); 1901 taskqueue_enqueue(adapter->tq, &adapter->tx_task); 1902 } 1903 /* Reenable this interrupt */ 1904 E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_TX); 1905 return; 1906} 1907 1908/********************************************************************* 1909 * 1910 * MSIX RX Interrupt Service routine 1911 * 1912 **********************************************************************/ 1913 1914static void 1915em_msix_rx(void *arg) 1916{ 1917 struct adapter *adapter = arg; 1918 struct ifnet *ifp = adapter->ifp; 1919 1920 ++adapter->rx_irq; 1921 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && 1922 (em_rxeof(adapter, adapter->rx_process_limit) != 0)) 1923 taskqueue_enqueue(adapter->tq, &adapter->rx_task); 1924 /* Reenable this interrupt */ 1925 E1000_WRITE_REG(&adapter->hw, E1000_IMS, EM_MSIX_RX); 1926 return; 1927} 1928 1929/********************************************************************* 1930 * 1931 * MSIX Link Fast Interrupt Service routine 1932 * 1933 **********************************************************************/ 1934 1935static void 1936em_msix_link(void *arg) 1937{ 1938 struct adapter *adapter = arg; 1939 u32 reg_icr; 1940 1941 ++adapter->link_irq; 1942 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1943 1944 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1945 adapter->hw.mac.get_link_status = 1; 1946 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1947 } 1948 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1949 EM_MSIX_LINK | E1000_IMS_LSC); 1950 return; 1951} 1952 1953static void 1954em_handle_rx(void *context, int pending) 1955{ 1956 struct adapter *adapter = context; 1957 struct ifnet *ifp = adapter->ifp; 1958 1959 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && 1960 (em_rxeof(adapter, adapter->rx_process_limit) != 0)) 1961 taskqueue_enqueue(adapter->tq, &adapter->rx_task); 1962 1963} 1964 1965static void 1966em_handle_tx(void *context, int pending) 1967{ 1968 struct adapter *adapter = context; 1969 struct ifnet *ifp = adapter->ifp; 1970 1971 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1972#ifdef IFNET_BUF_RING 1973 if (!EM_TX_TRYLOCK(adapter)) 1974 return; 1975#else 1976 EM_TX_LOCK(adapter); 1977#endif 1978 1979 em_txeof(adapter); 1980 if (!ADAPTER_RING_EMPTY(adapter)) 1981 em_start_locked(ifp); 1982 EM_TX_UNLOCK(adapter); 1983 } 1984} 1985#endif /* EM_FAST_IRQ */ 1986 1987/********************************************************************* 1988 * 1989 * Media Ioctl callback 1990 * 1991 * This routine is called whenever the user queries the status of 1992 * the interface using ifconfig. 1993 * 1994 **********************************************************************/ 1995static void 1996em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1997{ 1998 struct adapter *adapter = ifp->if_softc; 1999 u_char fiber_type = IFM_1000_SX; 2000 2001 INIT_DEBUGOUT("em_media_status: begin"); 2002 2003 EM_CORE_LOCK(adapter); 2004 em_update_link_status(adapter); 2005 2006 ifmr->ifm_status = IFM_AVALID; 2007 ifmr->ifm_active = IFM_ETHER; 2008 2009 if (!adapter->link_active) { 2010 EM_CORE_UNLOCK(adapter); 2011 return; 2012 } 2013 2014 ifmr->ifm_status |= IFM_ACTIVE; 2015 2016 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2017 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2018 if (adapter->hw.mac.type == e1000_82545) 2019 fiber_type = IFM_1000_LX; 2020 ifmr->ifm_active |= fiber_type | IFM_FDX; 2021 } else { 2022 switch (adapter->link_speed) { 2023 case 10: 2024 ifmr->ifm_active |= IFM_10_T; 2025 break; 2026 case 100: 2027 ifmr->ifm_active |= IFM_100_TX; 2028 break; 2029 case 1000: 2030 ifmr->ifm_active |= IFM_1000_T; 2031 break; 2032 } 2033 if (adapter->link_duplex == FULL_DUPLEX) 2034 ifmr->ifm_active |= IFM_FDX; 2035 else 2036 ifmr->ifm_active |= IFM_HDX; 2037 } 2038 EM_CORE_UNLOCK(adapter); 2039} 2040 2041/********************************************************************* 2042 * 2043 * Media Ioctl callback 2044 * 2045 * This routine is called when the user changes speed/duplex using 2046 * media/mediopt option with ifconfig. 2047 * 2048 **********************************************************************/ 2049static int 2050em_media_change(struct ifnet *ifp) 2051{ 2052 struct adapter *adapter = ifp->if_softc; 2053 struct ifmedia *ifm = &adapter->media; 2054 2055 INIT_DEBUGOUT("em_media_change: begin"); 2056 2057 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 2058 return (EINVAL); 2059 2060 EM_CORE_LOCK(adapter); 2061 switch (IFM_SUBTYPE(ifm->ifm_media)) { 2062 case IFM_AUTO: 2063 adapter->hw.mac.autoneg = DO_AUTO_NEG; 2064 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 2065 break; 2066 case IFM_1000_LX: 2067 case IFM_1000_SX: 2068 case IFM_1000_T: 2069 adapter->hw.mac.autoneg = DO_AUTO_NEG; 2070 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 2071 break; 2072 case IFM_100_TX: 2073 adapter->hw.mac.autoneg = FALSE; 2074 adapter->hw.phy.autoneg_advertised = 0; 2075 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2076 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 2077 else 2078 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 2079 break; 2080 case IFM_10_T: 2081 adapter->hw.mac.autoneg = FALSE; 2082 adapter->hw.phy.autoneg_advertised = 0; 2083 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 2084 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 2085 else 2086 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 2087 break; 2088 default: 2089 device_printf(adapter->dev, "Unsupported media type\n"); 2090 } 2091 2092 /* As the speed/duplex settings my have changed we need to 2093 * reset the PHY. 2094 */ 2095 adapter->hw.phy.reset_disable = FALSE; 2096 2097 em_init_locked(adapter); 2098 EM_CORE_UNLOCK(adapter); 2099 2100 return (0); 2101} 2102 2103/********************************************************************* 2104 * 2105 * This routine maps the mbufs to tx descriptors. 2106 * 2107 * return 0 on success, positive on failure 2108 **********************************************************************/ 2109 2110static int 2111em_xmit(struct adapter *adapter, struct mbuf **m_headp) 2112{ 2113 bus_dma_segment_t segs[EM_MAX_SCATTER]; 2114 bus_dmamap_t map; 2115 struct em_buffer *tx_buffer, *tx_buffer_mapped; 2116 struct e1000_tx_desc *ctxd = NULL; 2117 struct mbuf *m_head; 2118 u32 txd_upper, txd_lower, txd_used, txd_saved; 2119 int nsegs, i, j, first, last = 0; 2120 int error, do_tso, tso_desc = 0; 2121#if __FreeBSD_version < 700000 2122 struct m_tag *mtag; 2123#endif 2124 m_head = *m_headp; 2125 txd_upper = txd_lower = txd_used = txd_saved = 0; 2126 2127#if __FreeBSD_version >= 700000 2128 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); 2129#else 2130 do_tso = 0; 2131#endif 2132 2133 /* 2134 * Force a cleanup if number of TX descriptors 2135 * available hits the threshold 2136 */ 2137 if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { 2138 em_txeof(adapter); 2139 /* Now do we at least have a minimal? */ 2140 if (adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD) { 2141 adapter->no_tx_desc_avail1++; 2142 return (ENOBUFS); 2143 } 2144 } 2145 2146 2147 /* 2148 * TSO workaround: 2149 * If an mbuf is only header we need 2150 * to pull 4 bytes of data into it. 2151 */ 2152 if (do_tso && (m_head->m_len <= M_TSO_LEN)) { 2153 m_head = m_pullup(m_head, M_TSO_LEN + 4); 2154 *m_headp = m_head; 2155 if (m_head == NULL) 2156 return (ENOBUFS); 2157 } 2158 2159 /* 2160 * Map the packet for DMA 2161 * 2162 * Capture the first descriptor index, 2163 * this descriptor will have the index 2164 * of the EOP which is the only one that 2165 * now gets a DONE bit writeback. 2166 */ 2167 first = adapter->next_avail_tx_desc; 2168 tx_buffer = &adapter->tx_buffer_area[first]; 2169 tx_buffer_mapped = tx_buffer; 2170 map = tx_buffer->map; 2171 2172 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, 2173 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 2174 2175 /* 2176 * There are two types of errors we can (try) to handle: 2177 * - EFBIG means the mbuf chain was too long and bus_dma ran 2178 * out of segments. Defragment the mbuf chain and try again. 2179 * - ENOMEM means bus_dma could not obtain enough bounce buffers 2180 * at this point in time. Defer sending and try again later. 2181 * All other errors, in particular EINVAL, are fatal and prevent the 2182 * mbuf chain from ever going through. Drop it and report error. 2183 */ 2184 if (error == EFBIG) { 2185 struct mbuf *m; 2186 2187 m = m_defrag(*m_headp, M_DONTWAIT); 2188 if (m == NULL) { 2189 adapter->mbuf_alloc_failed++; 2190 m_freem(*m_headp); 2191 *m_headp = NULL; 2192 return (ENOBUFS); 2193 } 2194 *m_headp = m; 2195 2196 /* Try it again */ 2197 error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, 2198 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 2199 2200 if (error) { 2201 adapter->no_tx_dma_setup++; 2202 m_freem(*m_headp); 2203 *m_headp = NULL; 2204 return (error); 2205 } 2206 } else if (error != 0) { 2207 adapter->no_tx_dma_setup++; 2208 return (error); 2209 } 2210 2211 /* 2212 * TSO Hardware workaround, if this packet is not 2213 * TSO, and is only a single descriptor long, and 2214 * it follows a TSO burst, then we need to add a 2215 * sentinel descriptor to prevent premature writeback. 2216 */ 2217 if ((do_tso == 0) && (adapter->tx_tso == TRUE)) { 2218 if (nsegs == 1) 2219 tso_desc = TRUE; 2220 adapter->tx_tso = FALSE; 2221 } 2222 2223 if (nsegs > (adapter->num_tx_desc_avail - 2)) { 2224 adapter->no_tx_desc_avail2++; 2225 bus_dmamap_unload(adapter->txtag, map); 2226 return (ENOBUFS); 2227 } 2228 m_head = *m_headp; 2229 2230 /* Do hardware assists */ 2231#if __FreeBSD_version >= 700000 2232 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 2233 error = em_tso_setup(adapter, m_head, &txd_upper, &txd_lower); 2234 if (error != TRUE) 2235 return (ENXIO); /* something foobar */ 2236 /* we need to make a final sentinel transmit desc */ 2237 tso_desc = TRUE; 2238 } else 2239#endif 2240#ifndef EM_TIMESYNC 2241 /* 2242 ** Timesync needs to check the packet header 2243 ** so call checksum code to do so, but don't 2244 ** penalize the code if not defined. 2245 */ 2246 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) 2247#endif 2248 em_transmit_checksum_setup(adapter, m_head, 2249 &txd_upper, &txd_lower); 2250 2251 i = adapter->next_avail_tx_desc; 2252 if (adapter->pcix_82544) 2253 txd_saved = i; 2254 2255 /* Set up our transmit descriptors */ 2256 for (j = 0; j < nsegs; j++) { 2257 bus_size_t seg_len; 2258 bus_addr_t seg_addr; 2259 /* If adapter is 82544 and on PCIX bus */ 2260 if(adapter->pcix_82544) { 2261 DESC_ARRAY desc_array; 2262 u32 array_elements, counter; 2263 /* 2264 * Check the Address and Length combination and 2265 * split the data accordingly 2266 */ 2267 array_elements = em_fill_descriptors(segs[j].ds_addr, 2268 segs[j].ds_len, &desc_array); 2269 for (counter = 0; counter < array_elements; counter++) { 2270 if (txd_used == adapter->num_tx_desc_avail) { 2271 adapter->next_avail_tx_desc = txd_saved; 2272 adapter->no_tx_desc_avail2++; 2273 bus_dmamap_unload(adapter->txtag, map); 2274 return (ENOBUFS); 2275 } 2276 tx_buffer = &adapter->tx_buffer_area[i]; 2277 ctxd = &adapter->tx_desc_base[i]; 2278 ctxd->buffer_addr = htole64( 2279 desc_array.descriptor[counter].address); 2280 ctxd->lower.data = htole32( 2281 (adapter->txd_cmd | txd_lower | (u16) 2282 desc_array.descriptor[counter].length)); 2283 ctxd->upper.data = 2284 htole32((txd_upper)); 2285 last = i; 2286 if (++i == adapter->num_tx_desc) 2287 i = 0; 2288 tx_buffer->m_head = NULL; 2289 tx_buffer->next_eop = -1; 2290 txd_used++; 2291 } 2292 } else { 2293 tx_buffer = &adapter->tx_buffer_area[i]; 2294 ctxd = &adapter->tx_desc_base[i]; 2295 seg_addr = segs[j].ds_addr; 2296 seg_len = segs[j].ds_len; 2297 /* 2298 ** TSO Workaround: 2299 ** If this is the last descriptor, we want to 2300 ** split it so we have a small final sentinel 2301 */ 2302 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { 2303 seg_len -= 4; 2304 ctxd->buffer_addr = htole64(seg_addr); 2305 ctxd->lower.data = htole32( 2306 adapter->txd_cmd | txd_lower | seg_len); 2307 ctxd->upper.data = 2308 htole32(txd_upper); 2309 if (++i == adapter->num_tx_desc) 2310 i = 0; 2311 /* Now make the sentinel */ 2312 ++txd_used; /* using an extra txd */ 2313 ctxd = &adapter->tx_desc_base[i]; 2314 tx_buffer = &adapter->tx_buffer_area[i]; 2315 ctxd->buffer_addr = 2316 htole64(seg_addr + seg_len); 2317 ctxd->lower.data = htole32( 2318 adapter->txd_cmd | txd_lower | 4); 2319 ctxd->upper.data = 2320 htole32(txd_upper); 2321 last = i; 2322 if (++i == adapter->num_tx_desc) 2323 i = 0; 2324 } else { 2325 ctxd->buffer_addr = htole64(seg_addr); 2326 ctxd->lower.data = htole32( 2327 adapter->txd_cmd | txd_lower | seg_len); 2328 ctxd->upper.data = 2329 htole32(txd_upper); 2330 last = i; 2331 if (++i == adapter->num_tx_desc) 2332 i = 0; 2333 } 2334 tx_buffer->m_head = NULL; 2335 tx_buffer->next_eop = -1; 2336 } 2337 } 2338 2339 adapter->next_avail_tx_desc = i; 2340 if (adapter->pcix_82544) 2341 adapter->num_tx_desc_avail -= txd_used; 2342 else { 2343 adapter->num_tx_desc_avail -= nsegs; 2344 if (tso_desc) /* TSO used an extra for sentinel */ 2345 adapter->num_tx_desc_avail -= txd_used; 2346 } 2347 2348 /* 2349 ** Handle VLAN tag, this is the 2350 ** biggest difference between 2351 ** 6.x and 7 2352 */ 2353#if __FreeBSD_version < 700000 2354 /* Find out if we are in vlan mode. */ 2355 mtag = VLAN_OUTPUT_TAG(ifp, m_head); 2356 if (mtag != NULL) { 2357 ctxd->upper.fields.special = 2358 htole16(VLAN_TAG_VALUE(mtag)); 2359#else /* FreeBSD 7 */ 2360 if (m_head->m_flags & M_VLANTAG) { 2361 /* Set the vlan id. */ 2362 ctxd->upper.fields.special = 2363 htole16(m_head->m_pkthdr.ether_vtag); 2364#endif 2365 /* Tell hardware to add tag */ 2366 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); 2367 } 2368 2369 tx_buffer->m_head = m_head; 2370 tx_buffer_mapped->map = tx_buffer->map; 2371 tx_buffer->map = map; 2372 bus_dmamap_sync(adapter->txtag, map, BUS_DMASYNC_PREWRITE); 2373 2374 /* 2375 * Last Descriptor of Packet 2376 * needs End Of Packet (EOP) 2377 * and Report Status (RS) 2378 */ 2379 ctxd->lower.data |= 2380 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 2381 /* 2382 * Keep track in the first buffer which 2383 * descriptor will be written back 2384 */ 2385 tx_buffer = &adapter->tx_buffer_area[first]; 2386 tx_buffer->next_eop = last; 2387 2388 /* 2389 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 2390 * that this frame is available to transmit. 2391 */ 2392 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 2393 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2394 if (adapter->hw.mac.type == e1000_82547 && 2395 adapter->link_duplex == HALF_DUPLEX) 2396 em_82547_move_tail(adapter); 2397 else { 2398 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), i); 2399 if (adapter->hw.mac.type == e1000_82547) 2400 em_82547_update_fifo_head(adapter, 2401 m_head->m_pkthdr.len); 2402 } 2403 2404#ifdef EM_TIMESYNC 2405 if (ctxd->upper.data & E1000_TXD_EXTCMD_TSTAMP) { 2406 HW_DEBUGOUT( "@@@ Timestamp bit is set in transmit descriptor\n" ); 2407 } 2408#endif 2409 return (0); 2410} 2411 2412/********************************************************************* 2413 * 2414 * 82547 workaround to avoid controller hang in half-duplex environment. 2415 * The workaround is to avoid queuing a large packet that would span 2416 * the internal Tx FIFO ring boundary. We need to reset the FIFO pointers 2417 * in this case. We do that only when FIFO is quiescent. 2418 * 2419 **********************************************************************/ 2420static void 2421em_82547_move_tail(void *arg) 2422{ 2423 struct adapter *adapter = arg; 2424 struct e1000_tx_desc *tx_desc; 2425 u16 hw_tdt, sw_tdt, length = 0; 2426 bool eop = 0; 2427 2428 EM_TX_LOCK_ASSERT(adapter); 2429 2430 hw_tdt = E1000_READ_REG(&adapter->hw, E1000_TDT(0)); 2431 sw_tdt = adapter->next_avail_tx_desc; 2432 2433 while (hw_tdt != sw_tdt) { 2434 tx_desc = &adapter->tx_desc_base[hw_tdt]; 2435 length += tx_desc->lower.flags.length; 2436 eop = tx_desc->lower.data & E1000_TXD_CMD_EOP; 2437 if (++hw_tdt == adapter->num_tx_desc) 2438 hw_tdt = 0; 2439 2440 if (eop) { 2441 if (em_82547_fifo_workaround(adapter, length)) { 2442 adapter->tx_fifo_wrk_cnt++; 2443 callout_reset(&adapter->tx_fifo_timer, 1, 2444 em_82547_move_tail, adapter); 2445 break; 2446 } 2447 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), hw_tdt); 2448 em_82547_update_fifo_head(adapter, length); 2449 length = 0; 2450 } 2451 } 2452} 2453 2454static int 2455em_82547_fifo_workaround(struct adapter *adapter, int len) 2456{ 2457 int fifo_space, fifo_pkt_len; 2458 2459 fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2460 2461 if (adapter->link_duplex == HALF_DUPLEX) { 2462 fifo_space = adapter->tx_fifo_size - adapter->tx_fifo_head; 2463 2464 if (fifo_pkt_len >= (EM_82547_PKT_THRESH + fifo_space)) { 2465 if (em_82547_tx_fifo_reset(adapter)) 2466 return (0); 2467 else 2468 return (1); 2469 } 2470 } 2471 2472 return (0); 2473} 2474 2475static void 2476em_82547_update_fifo_head(struct adapter *adapter, int len) 2477{ 2478 int fifo_pkt_len = roundup2(len + EM_FIFO_HDR, EM_FIFO_HDR); 2479 2480 /* tx_fifo_head is always 16 byte aligned */ 2481 adapter->tx_fifo_head += fifo_pkt_len; 2482 if (adapter->tx_fifo_head >= adapter->tx_fifo_size) { 2483 adapter->tx_fifo_head -= adapter->tx_fifo_size; 2484 } 2485} 2486 2487 2488static int 2489em_82547_tx_fifo_reset(struct adapter *adapter) 2490{ 2491 u32 tctl; 2492 2493 if ((E1000_READ_REG(&adapter->hw, E1000_TDT(0)) == 2494 E1000_READ_REG(&adapter->hw, E1000_TDH(0))) && 2495 (E1000_READ_REG(&adapter->hw, E1000_TDFT) == 2496 E1000_READ_REG(&adapter->hw, E1000_TDFH)) && 2497 (E1000_READ_REG(&adapter->hw, E1000_TDFTS) == 2498 E1000_READ_REG(&adapter->hw, E1000_TDFHS)) && 2499 (E1000_READ_REG(&adapter->hw, E1000_TDFPC) == 0)) { 2500 /* Disable TX unit */ 2501 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 2502 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, 2503 tctl & ~E1000_TCTL_EN); 2504 2505 /* Reset FIFO pointers */ 2506 E1000_WRITE_REG(&adapter->hw, E1000_TDFT, 2507 adapter->tx_head_addr); 2508 E1000_WRITE_REG(&adapter->hw, E1000_TDFH, 2509 adapter->tx_head_addr); 2510 E1000_WRITE_REG(&adapter->hw, E1000_TDFTS, 2511 adapter->tx_head_addr); 2512 E1000_WRITE_REG(&adapter->hw, E1000_TDFHS, 2513 adapter->tx_head_addr); 2514 2515 /* Re-enable TX unit */ 2516 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 2517 E1000_WRITE_FLUSH(&adapter->hw); 2518 2519 adapter->tx_fifo_head = 0; 2520 adapter->tx_fifo_reset_cnt++; 2521 2522 return (TRUE); 2523 } 2524 else { 2525 return (FALSE); 2526 } 2527} 2528 2529static void 2530em_set_promisc(struct adapter *adapter) 2531{ 2532 struct ifnet *ifp = adapter->ifp; 2533 u32 reg_rctl; 2534 2535 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2536 2537 if (ifp->if_flags & IFF_PROMISC) { 2538 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 2539 /* Turn this on if you want to see bad packets */ 2540 if (em_debug_sbp) 2541 reg_rctl |= E1000_RCTL_SBP; 2542 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2543 } else if (ifp->if_flags & IFF_ALLMULTI) { 2544 reg_rctl |= E1000_RCTL_MPE; 2545 reg_rctl &= ~E1000_RCTL_UPE; 2546 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2547 } 2548} 2549 2550static void 2551em_disable_promisc(struct adapter *adapter) 2552{ 2553 u32 reg_rctl; 2554 2555 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2556 2557 reg_rctl &= (~E1000_RCTL_UPE); 2558 reg_rctl &= (~E1000_RCTL_MPE); 2559 reg_rctl &= (~E1000_RCTL_SBP); 2560 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2561} 2562 2563 2564/********************************************************************* 2565 * Multicast Update 2566 * 2567 * This routine is called whenever multicast address list is updated. 2568 * 2569 **********************************************************************/ 2570 2571static void 2572em_set_multi(struct adapter *adapter) 2573{ 2574 struct ifnet *ifp = adapter->ifp; 2575 struct ifmultiaddr *ifma; 2576 u32 reg_rctl = 0; 2577 u8 *mta; /* Multicast array memory */ 2578 int mcnt = 0; 2579 2580 IOCTL_DEBUGOUT("em_set_multi: begin"); 2581 2582 if (adapter->hw.mac.type == e1000_82542 && 2583 adapter->hw.revision_id == E1000_REVISION_2) { 2584 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2585 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2586 e1000_pci_clear_mwi(&adapter->hw); 2587 reg_rctl |= E1000_RCTL_RST; 2588 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2589 msec_delay(5); 2590 } 2591 2592 /* Allocate temporary memory to setup array */ 2593 mta = malloc(sizeof(u8) * 2594 (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES), 2595 M_DEVBUF, M_NOWAIT | M_ZERO); 2596 if (mta == NULL) 2597 panic("em_set_multi memory failure\n"); 2598 2599 IF_ADDR_LOCK(ifp); 2600 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 2601 if (ifma->ifma_addr->sa_family != AF_LINK) 2602 continue; 2603 2604 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 2605 break; 2606 2607 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 2608 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 2609 mcnt++; 2610 } 2611 IF_ADDR_UNLOCK(ifp); 2612 2613 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 2614 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2615 reg_rctl |= E1000_RCTL_MPE; 2616 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2617 } else 2618 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 2619 2620 if (adapter->hw.mac.type == e1000_82542 && 2621 adapter->hw.revision_id == E1000_REVISION_2) { 2622 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2623 reg_rctl &= ~E1000_RCTL_RST; 2624 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2625 msec_delay(5); 2626 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2627 e1000_pci_set_mwi(&adapter->hw); 2628 } 2629 free(mta, M_DEVBUF); 2630} 2631 2632 2633/********************************************************************* 2634 * Timer routine 2635 * 2636 * This routine checks for link status and updates statistics. 2637 * 2638 **********************************************************************/ 2639 2640static void 2641em_local_timer(void *arg) 2642{ 2643 struct adapter *adapter = arg; 2644 struct ifnet *ifp = adapter->ifp; 2645 2646 EM_CORE_LOCK_ASSERT(adapter); 2647 2648 taskqueue_enqueue(adapter->tq, 2649 &adapter->rxtx_task); 2650 em_update_link_status(adapter); 2651 em_update_stats_counters(adapter); 2652 2653 /* Reset LAA into RAR[0] on 82571 */ 2654 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE) 2655 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 2656 2657 if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 2658 em_print_hw_stats(adapter); 2659 2660 em_smartspeed(adapter); 2661 2662 /* 2663 * Each second we check the watchdog to 2664 * protect against hardware hangs. 2665 */ 2666 em_watchdog(adapter); 2667 2668 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 2669 2670} 2671 2672static void 2673em_update_link_status(struct adapter *adapter) 2674{ 2675 struct e1000_hw *hw = &adapter->hw; 2676 struct ifnet *ifp = adapter->ifp; 2677 device_t dev = adapter->dev; 2678 u32 link_check = 0; 2679 2680 /* Get the cached link value or read phy for real */ 2681 switch (hw->phy.media_type) { 2682 case e1000_media_type_copper: 2683 if (hw->mac.get_link_status) { 2684 /* Do the work to read phy */ 2685 e1000_check_for_link(hw); 2686 link_check = !hw->mac.get_link_status; 2687 if (link_check) /* ESB2 fix */ 2688 e1000_cfg_on_link_up(hw); 2689 } else 2690 link_check = TRUE; 2691 break; 2692 case e1000_media_type_fiber: 2693 e1000_check_for_link(hw); 2694 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2695 E1000_STATUS_LU); 2696 break; 2697 case e1000_media_type_internal_serdes: 2698 e1000_check_for_link(hw); 2699 link_check = adapter->hw.mac.serdes_has_link; 2700 break; 2701 default: 2702 case e1000_media_type_unknown: 2703 break; 2704 } 2705 2706 /* Now check for a transition */ 2707 if (link_check && (adapter->link_active == 0)) { 2708 e1000_get_speed_and_duplex(hw, &adapter->link_speed, 2709 &adapter->link_duplex); 2710 /* Check if we must disable SPEED_MODE bit on PCI-E */ 2711 if ((adapter->link_speed != SPEED_1000) && 2712 ((hw->mac.type == e1000_82571) || 2713 (hw->mac.type == e1000_82572))) { 2714 int tarc0; 2715 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2716 tarc0 &= ~SPEED_MODE_BIT; 2717 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2718 } 2719 if (bootverbose) 2720 device_printf(dev, "Link is up %d Mbps %s\n", 2721 adapter->link_speed, 2722 ((adapter->link_duplex == FULL_DUPLEX) ? 2723 "Full Duplex" : "Half Duplex")); 2724 adapter->link_active = 1; 2725 adapter->smartspeed = 0; 2726 ifp->if_baudrate = adapter->link_speed * 1000000; 2727 if_link_state_change(ifp, LINK_STATE_UP); 2728 } else if (!link_check && (adapter->link_active == 1)) { 2729 ifp->if_baudrate = adapter->link_speed = 0; 2730 adapter->link_duplex = 0; 2731 if (bootverbose) 2732 device_printf(dev, "Link is Down\n"); 2733 adapter->link_active = 0; 2734 /* Link down, disable watchdog */ 2735 adapter->watchdog_timer = FALSE; 2736 if_link_state_change(ifp, LINK_STATE_DOWN); 2737 } 2738} 2739 2740/********************************************************************* 2741 * 2742 * This routine disables all traffic on the adapter by issuing a 2743 * global reset on the MAC and deallocates TX/RX buffers. 2744 * 2745 * This routine should always be called with BOTH the CORE 2746 * and TX locks. 2747 **********************************************************************/ 2748 2749static void 2750em_stop(void *arg) 2751{ 2752 struct adapter *adapter = arg; 2753 struct ifnet *ifp = adapter->ifp; 2754 2755 EM_CORE_LOCK_ASSERT(adapter); 2756 EM_TX_LOCK_ASSERT(adapter); 2757 2758 INIT_DEBUGOUT("em_stop: begin"); 2759 2760 em_disable_intr(adapter); 2761 callout_stop(&adapter->timer); 2762 callout_stop(&adapter->tx_fifo_timer); 2763 2764 /* Tell the stack that the interface is no longer active */ 2765 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2766 2767#ifdef EM_TIMESYNC 2768 /* Disable IEEE 1588 Time hardware */ 2769 if ((adapter->hw.mac.type == e1000_82574) || 2770 (adapter->hw.mac.type == e1000_ich10lan)) 2771 em_tsync_disable(adapter); 2772#endif 2773 2774 e1000_reset_hw(&adapter->hw); 2775 if (adapter->hw.mac.type >= e1000_82544) 2776 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2777} 2778 2779 2780/********************************************************************* 2781 * 2782 * Determine hardware revision. 2783 * 2784 **********************************************************************/ 2785static void 2786em_identify_hardware(struct adapter *adapter) 2787{ 2788 device_t dev = adapter->dev; 2789 2790 /* Make sure our PCI config space has the necessary stuff set */ 2791 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2792 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2793 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2794 device_printf(dev, "Memory Access and/or Bus Master bits " 2795 "were not set!\n"); 2796 adapter->hw.bus.pci_cmd_word |= 2797 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2798 pci_write_config(dev, PCIR_COMMAND, 2799 adapter->hw.bus.pci_cmd_word, 2); 2800 } 2801 2802 /* Save off the information about this board */ 2803 adapter->hw.vendor_id = pci_get_vendor(dev); 2804 adapter->hw.device_id = pci_get_device(dev); 2805 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2806 adapter->hw.subsystem_vendor_id = 2807 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2808 adapter->hw.subsystem_device_id = 2809 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2810 2811 /* Do Shared Code Init and Setup */ 2812 if (e1000_set_mac_type(&adapter->hw)) { 2813 device_printf(dev, "Setup init failure\n"); 2814 return; 2815 } 2816} 2817 2818static int 2819em_allocate_pci_resources(struct adapter *adapter) 2820{ 2821 device_t dev = adapter->dev; 2822 int val, rid, error = E1000_SUCCESS; 2823 2824 rid = PCIR_BAR(0); 2825 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2826 &rid, RF_ACTIVE); 2827 if (adapter->memory == NULL) { 2828 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2829 return (ENXIO); 2830 } 2831 adapter->osdep.mem_bus_space_tag = 2832 rman_get_bustag(adapter->memory); 2833 adapter->osdep.mem_bus_space_handle = 2834 rman_get_bushandle(adapter->memory); 2835 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2836 2837 /* Only older adapters use IO mapping */ 2838 if ((adapter->hw.mac.type > e1000_82543) && 2839 (adapter->hw.mac.type < e1000_82571)) { 2840 /* Figure our where our IO BAR is ? */ 2841 for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { 2842 val = pci_read_config(dev, rid, 4); 2843 if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { 2844 adapter->io_rid = rid; 2845 break; 2846 } 2847 rid += 4; 2848 /* check for 64bit BAR */ 2849 if (EM_BAR_MEM_TYPE(val) == EM_BAR_MEM_TYPE_64BIT) 2850 rid += 4; 2851 } 2852 if (rid >= PCIR_CIS) { 2853 device_printf(dev, "Unable to locate IO BAR\n"); 2854 return (ENXIO); 2855 } 2856 adapter->ioport = bus_alloc_resource_any(dev, 2857 SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); 2858 if (adapter->ioport == NULL) { 2859 device_printf(dev, "Unable to allocate bus resource: " 2860 "ioport\n"); 2861 return (ENXIO); 2862 } 2863 adapter->hw.io_base = 0; 2864 adapter->osdep.io_bus_space_tag = 2865 rman_get_bustag(adapter->ioport); 2866 adapter->osdep.io_bus_space_handle = 2867 rman_get_bushandle(adapter->ioport); 2868 } 2869 2870 /* 2871 ** Init the resource arrays 2872 ** used by MSIX setup 2873 */ 2874 for (int i = 0; i < 3; i++) { 2875 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 2876 adapter->tag[i] = NULL; 2877 adapter->res[i] = NULL; 2878 } 2879 2880 /* 2881 * Setup MSI/X or MSI if PCI Express 2882 */ 2883 if (em_enable_msi) 2884 adapter->msi = em_setup_msix(adapter); 2885 2886 adapter->hw.back = &adapter->osdep; 2887 2888 return (error); 2889} 2890 2891/********************************************************************* 2892 * 2893 * Setup the Legacy or MSI Interrupt handler 2894 * 2895 **********************************************************************/ 2896int 2897em_allocate_legacy(struct adapter *adapter) 2898{ 2899 device_t dev = adapter->dev; 2900 int error; 2901 2902 /* Manually turn off all interrupts */ 2903 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2904 2905 /* Legacy RID is 0 */ 2906 if (adapter->msi == 0) 2907 adapter->rid[0] = 0; 2908 2909 /* We allocate a single interrupt resource */ 2910 adapter->res[0] = bus_alloc_resource_any(dev, 2911 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); 2912 if (adapter->res[0] == NULL) { 2913 device_printf(dev, "Unable to allocate bus resource: " 2914 "interrupt\n"); 2915 return (ENXIO); 2916 } 2917 2918#ifdef EM_LEGACY_IRQ 2919 /* We do Legacy setup */ 2920 if ((error = bus_setup_intr(dev, adapter->res[0], 2921#if __FreeBSD_version > 700000 2922 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_intr, adapter, 2923#else /* 6.X */ 2924 INTR_TYPE_NET | INTR_MPSAFE, em_intr, adapter, 2925#endif 2926 &adapter->tag[0])) != 0) { 2927 device_printf(dev, "Failed to register interrupt handler"); 2928 return (error); 2929 } 2930 2931#else /* FAST_IRQ */ 2932 /* 2933 * Try allocating a fast interrupt and the associated deferred 2934 * processing contexts. 2935 */ 2936 TASK_INIT(&adapter->rxtx_task, 0, em_handle_rxtx, adapter); 2937 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2938 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2939 taskqueue_thread_enqueue, &adapter->tq); 2940 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2941 device_get_nameunit(adapter->dev)); 2942#if __FreeBSD_version < 700000 2943 if ((error = bus_setup_intr(dev, adapter->res[0], 2944 INTR_TYPE_NET | INTR_FAST, em_irq_fast, adapter, 2945#else 2946 if ((error = bus_setup_intr(dev, adapter->res[0], 2947 INTR_TYPE_NET, em_irq_fast, NULL, adapter, 2948#endif 2949 &adapter->tag[0])) != 0) { 2950 device_printf(dev, "Failed to register fast interrupt " 2951 "handler: %d\n", error); 2952 taskqueue_free(adapter->tq); 2953 adapter->tq = NULL; 2954 return (error); 2955 } 2956#endif /* EM_LEGACY_IRQ */ 2957 2958 return (0); 2959} 2960 2961/********************************************************************* 2962 * 2963 * Setup the MSIX Interrupt handlers 2964 * This is not really Multiqueue, rather 2965 * its just multiple interrupt vectors. 2966 * 2967 **********************************************************************/ 2968int 2969em_allocate_msix(struct adapter *adapter) 2970{ 2971 device_t dev = adapter->dev; 2972 int error; 2973 2974 /* Make sure all interrupts are disabled */ 2975 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2976 2977 /* First get the resources */ 2978 for (int i = 0; i < adapter->msi; i++) { 2979 adapter->res[i] = bus_alloc_resource_any(dev, 2980 SYS_RES_IRQ, &adapter->rid[i], RF_ACTIVE); 2981 if (adapter->res[i] == NULL) { 2982 device_printf(dev, 2983 "Unable to allocate bus resource: " 2984 "MSIX Interrupt\n"); 2985 return (ENXIO); 2986 } 2987 } 2988 2989 /* 2990 * Now allocate deferred processing contexts. 2991 */ 2992 TASK_INIT(&adapter->rx_task, 0, em_handle_rx, adapter); 2993 TASK_INIT(&adapter->tx_task, 0, em_handle_tx, adapter); 2994 /* 2995 * Handle compatibility for msi case for deferral due to 2996 * trylock failure 2997 */ 2998 TASK_INIT(&adapter->rxtx_task, 0, em_handle_tx, adapter); 2999 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 3000 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 3001 taskqueue_thread_enqueue, &adapter->tq); 3002 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 3003 device_get_nameunit(adapter->dev)); 3004 3005 /* 3006 * And setup the interrupt handlers 3007 */ 3008 3009 /* First slot to RX */ 3010 if ((error = bus_setup_intr(dev, adapter->res[0], 3011#if __FreeBSD_version > 700000 3012 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, adapter, 3013#else /* 6.X */ 3014 INTR_TYPE_NET | INTR_MPSAFE, em_msix_rx, adapter, 3015#endif 3016 &adapter->tag[0])) != 0) { 3017 device_printf(dev, "Failed to register RX handler"); 3018 return (error); 3019 } 3020 3021 /* Next TX */ 3022 if ((error = bus_setup_intr(dev, adapter->res[1], 3023#if __FreeBSD_version > 700000 3024 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, adapter, 3025#else /* 6.X */ 3026 INTR_TYPE_NET | INTR_MPSAFE, em_msix_tx, adapter, 3027#endif 3028 &adapter->tag[1])) != 0) { 3029 device_printf(dev, "Failed to register TX handler"); 3030 return (error); 3031 } 3032 3033 /* And Link */ 3034 if ((error = bus_setup_intr(dev, adapter->res[2], 3035#if __FreeBSD_version > 700000 3036 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_link, adapter, 3037#else /* 6.X */ 3038 INTR_TYPE_NET | INTR_MPSAFE, em_msix_link, adapter, 3039#endif 3040 &adapter->tag[2])) != 0) { 3041 device_printf(dev, "Failed to register TX handler"); 3042 return (error); 3043 } 3044 3045 return (0); 3046} 3047 3048 3049static void 3050em_free_pci_resources(struct adapter *adapter) 3051{ 3052 device_t dev = adapter->dev; 3053 3054 /* Make sure the for loop below runs once */ 3055 if (adapter->msi == 0) 3056 adapter->msi = 1; 3057 3058 /* 3059 * First release all the interrupt resources: 3060 * notice that since these are just kept 3061 * in an array we can do the same logic 3062 * whether its MSIX or just legacy. 3063 */ 3064 for (int i = 0; i < adapter->msi; i++) { 3065 if (adapter->tag[i] != NULL) { 3066 bus_teardown_intr(dev, adapter->res[i], 3067 adapter->tag[i]); 3068 adapter->tag[i] = NULL; 3069 } 3070 if (adapter->res[i] != NULL) { 3071 bus_release_resource(dev, SYS_RES_IRQ, 3072 adapter->rid[i], adapter->res[i]); 3073 } 3074 } 3075 3076 if (adapter->msi) 3077 pci_release_msi(dev); 3078 3079 if (adapter->msix != NULL) 3080 bus_release_resource(dev, SYS_RES_MEMORY, 3081 PCIR_BAR(EM_MSIX_BAR), adapter->msix); 3082 3083 if (adapter->memory != NULL) 3084 bus_release_resource(dev, SYS_RES_MEMORY, 3085 PCIR_BAR(0), adapter->memory); 3086 3087 if (adapter->flash != NULL) 3088 bus_release_resource(dev, SYS_RES_MEMORY, 3089 EM_FLASH, adapter->flash); 3090 3091 if (adapter->ioport != NULL) 3092 bus_release_resource(dev, SYS_RES_IOPORT, 3093 adapter->io_rid, adapter->ioport); 3094} 3095 3096/* 3097 * Setup MSI or MSI/X 3098 */ 3099static int 3100em_setup_msix(struct adapter *adapter) 3101{ 3102 device_t dev = adapter->dev; 3103 int val = 0; 3104 3105 if (adapter->hw.mac.type < e1000_82571) 3106 return (0); 3107 3108 /* Setup MSI/X for Hartwell */ 3109 if (adapter->hw.mac.type == e1000_82574) { 3110 /* Map the MSIX BAR */ 3111 int rid = PCIR_BAR(EM_MSIX_BAR); 3112 adapter->msix = bus_alloc_resource_any(dev, 3113 SYS_RES_MEMORY, &rid, RF_ACTIVE); 3114 if (!adapter->msix) { 3115 /* May not be enabled */ 3116 device_printf(adapter->dev, 3117 "Unable to map MSIX table \n"); 3118 goto msi; 3119 } 3120 val = pci_msix_count(dev); 3121 /* 3122 ** 82574 can be configured for 5 but 3123 ** we limit use to 3. 3124 */ 3125 if (val > 3) val = 3; 3126 if ((val) && pci_alloc_msix(dev, &val) == 0) { 3127 device_printf(adapter->dev,"Using MSIX interrupts\n"); 3128 return (val); 3129 } 3130 } 3131msi: 3132 val = pci_msi_count(dev); 3133 if (val == 1 && pci_alloc_msi(dev, &val) == 0) { 3134 adapter->msi = 1; 3135 device_printf(adapter->dev,"Using MSI interrupt\n"); 3136 return (val); 3137 } 3138 return (0); 3139} 3140 3141/********************************************************************* 3142 * 3143 * Initialize the hardware to a configuration 3144 * as specified by the adapter structure. 3145 * 3146 **********************************************************************/ 3147static int 3148em_hardware_init(struct adapter *adapter) 3149{ 3150 device_t dev = adapter->dev; 3151 u16 rx_buffer_size; 3152 3153 INIT_DEBUGOUT("em_hardware_init: begin"); 3154 3155 /* Issue a global reset */ 3156 e1000_reset_hw(&adapter->hw); 3157 3158 /* Get control from any management/hw control */ 3159 if (((adapter->hw.mac.type == e1000_82573) || 3160 (adapter->hw.mac.type == e1000_ich8lan) || 3161 (adapter->hw.mac.type == e1000_ich10lan) || 3162 (adapter->hw.mac.type == e1000_ich9lan)) && 3163 e1000_check_mng_mode(&adapter->hw)) 3164 em_get_hw_control(adapter); 3165 3166 /* When hardware is reset, fifo_head is also reset */ 3167 adapter->tx_fifo_head = 0; 3168 3169 /* Set up smart power down as default off on newer adapters. */ 3170 if (!em_smart_pwr_down && (adapter->hw.mac.type == e1000_82571 || 3171 adapter->hw.mac.type == e1000_82572)) { 3172 u16 phy_tmp = 0; 3173 3174 /* Speed up time to link by disabling smart power down. */ 3175 e1000_read_phy_reg(&adapter->hw, 3176 IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 3177 phy_tmp &= ~IGP02E1000_PM_SPD; 3178 e1000_write_phy_reg(&adapter->hw, 3179 IGP02E1000_PHY_POWER_MGMT, phy_tmp); 3180 } 3181 3182 /* 3183 * These parameters control the automatic generation (Tx) and 3184 * response (Rx) to Ethernet PAUSE frames. 3185 * - High water mark should allow for at least two frames to be 3186 * received after sending an XOFF. 3187 * - Low water mark works best when it is very near the high water mark. 3188 * This allows the receiver to restart by sending XON when it has 3189 * drained a bit. Here we use an arbitary value of 1500 which will 3190 * restart after one full frame is pulled from the buffer. There 3191 * could be several smaller frames in the buffer and if so they will 3192 * not trigger the XON until their total number reduces the buffer 3193 * by 1500. 3194 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 3195 */ 3196 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 3197 0xffff) << 10 ); 3198 3199 adapter->hw.fc.high_water = rx_buffer_size - 3200 roundup2(adapter->max_frame_size, 1024); 3201 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; 3202 3203 if (adapter->hw.mac.type == e1000_80003es2lan) 3204 adapter->hw.fc.pause_time = 0xFFFF; 3205 else 3206 adapter->hw.fc.pause_time = EM_FC_PAUSE_TIME; 3207 adapter->hw.fc.send_xon = TRUE; 3208 adapter->hw.fc.requested_mode = e1000_fc_full; 3209 3210 if (e1000_init_hw(&adapter->hw) < 0) { 3211 device_printf(dev, "Hardware Initialization Failed\n"); 3212 return (EIO); 3213 } 3214 3215 e1000_check_for_link(&adapter->hw); 3216 3217 return (0); 3218} 3219 3220/********************************************************************* 3221 * 3222 * Setup networking device structure and register an interface. 3223 * 3224 **********************************************************************/ 3225static void 3226em_setup_interface(device_t dev, struct adapter *adapter) 3227{ 3228 struct ifnet *ifp; 3229 3230 INIT_DEBUGOUT("em_setup_interface: begin"); 3231 3232 ifp = adapter->ifp = if_alloc(IFT_ETHER); 3233 if (ifp == NULL) 3234 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 3235 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 3236 ifp->if_mtu = ETHERMTU; 3237 ifp->if_init = em_init; 3238 ifp->if_softc = adapter; 3239 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 3240 ifp->if_ioctl = em_ioctl; 3241 ifp->if_start = em_start; 3242 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 3243 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 3244 IFQ_SET_READY(&ifp->if_snd); 3245 3246 ether_ifattach(ifp, adapter->hw.mac.addr); 3247 3248 ifp->if_capabilities = ifp->if_capenable = 0; 3249 3250#ifdef IFNET_BUF_RING 3251 ifp->if_transmit = em_transmit; 3252 ifp->if_qflush = em_qflush; 3253 adapter->br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK, &adapter->tx_mtx); 3254#endif 3255 if (adapter->hw.mac.type >= e1000_82543) { 3256 int version_cap; 3257#if __FreeBSD_version < 700000 3258 version_cap = IFCAP_HWCSUM; 3259#else 3260 version_cap = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 3261#endif 3262 ifp->if_capabilities |= version_cap; 3263 ifp->if_capenable |= version_cap; 3264 } 3265 3266#if __FreeBSD_version >= 700000 3267 /* Identify TSO capable adapters */ 3268 if ((adapter->hw.mac.type > e1000_82544) && 3269 (adapter->hw.mac.type != e1000_82547)) 3270 ifp->if_capabilities |= IFCAP_TSO4; 3271 /* 3272 * By default only enable on PCI-E, this 3273 * can be overriden by ifconfig. 3274 */ 3275 if (adapter->hw.mac.type >= e1000_82571) 3276 ifp->if_capenable |= IFCAP_TSO4; 3277#endif 3278 3279 /* 3280 * Tell the upper layer(s) we support long frames. 3281 */ 3282 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 3283 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 3284 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 3285 3286#ifdef DEVICE_POLLING 3287 ifp->if_capabilities |= IFCAP_POLLING; 3288#endif 3289 3290 /* 3291 * Specify the media types supported by this adapter and register 3292 * callbacks to update media and link information 3293 */ 3294 ifmedia_init(&adapter->media, IFM_IMASK, 3295 em_media_change, em_media_status); 3296 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3297 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 3298 u_char fiber_type = IFM_1000_SX; /* default type */ 3299 3300 if (adapter->hw.mac.type == e1000_82545) 3301 fiber_type = IFM_1000_LX; 3302 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 3303 0, NULL); 3304 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); 3305 } else { 3306 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 3307 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 3308 0, NULL); 3309 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 3310 0, NULL); 3311 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 3312 0, NULL); 3313 if (adapter->hw.phy.type != e1000_phy_ife) { 3314 ifmedia_add(&adapter->media, 3315 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 3316 ifmedia_add(&adapter->media, 3317 IFM_ETHER | IFM_1000_T, 0, NULL); 3318 } 3319 } 3320 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 3321 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 3322} 3323 3324 3325/********************************************************************* 3326 * 3327 * Workaround for SmartSpeed on 82541 and 82547 controllers 3328 * 3329 **********************************************************************/ 3330static void 3331em_smartspeed(struct adapter *adapter) 3332{ 3333 u16 phy_tmp; 3334 3335 if (adapter->link_active || (adapter->hw.phy.type != e1000_phy_igp) || 3336 adapter->hw.mac.autoneg == 0 || 3337 (adapter->hw.phy.autoneg_advertised & ADVERTISE_1000_FULL) == 0) 3338 return; 3339 3340 if (adapter->smartspeed == 0) { 3341 /* If Master/Slave config fault is asserted twice, 3342 * we assume back-to-back */ 3343 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 3344 if (!(phy_tmp & SR_1000T_MS_CONFIG_FAULT)) 3345 return; 3346 e1000_read_phy_reg(&adapter->hw, PHY_1000T_STATUS, &phy_tmp); 3347 if (phy_tmp & SR_1000T_MS_CONFIG_FAULT) { 3348 e1000_read_phy_reg(&adapter->hw, 3349 PHY_1000T_CTRL, &phy_tmp); 3350 if(phy_tmp & CR_1000T_MS_ENABLE) { 3351 phy_tmp &= ~CR_1000T_MS_ENABLE; 3352 e1000_write_phy_reg(&adapter->hw, 3353 PHY_1000T_CTRL, phy_tmp); 3354 adapter->smartspeed++; 3355 if(adapter->hw.mac.autoneg && 3356 !e1000_phy_setup_autoneg(&adapter->hw) && 3357 !e1000_read_phy_reg(&adapter->hw, 3358 PHY_CONTROL, &phy_tmp)) { 3359 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3360 MII_CR_RESTART_AUTO_NEG); 3361 e1000_write_phy_reg(&adapter->hw, 3362 PHY_CONTROL, phy_tmp); 3363 } 3364 } 3365 } 3366 return; 3367 } else if(adapter->smartspeed == EM_SMARTSPEED_DOWNSHIFT) { 3368 /* If still no link, perhaps using 2/3 pair cable */ 3369 e1000_read_phy_reg(&adapter->hw, PHY_1000T_CTRL, &phy_tmp); 3370 phy_tmp |= CR_1000T_MS_ENABLE; 3371 e1000_write_phy_reg(&adapter->hw, PHY_1000T_CTRL, phy_tmp); 3372 if(adapter->hw.mac.autoneg && 3373 !e1000_phy_setup_autoneg(&adapter->hw) && 3374 !e1000_read_phy_reg(&adapter->hw, PHY_CONTROL, &phy_tmp)) { 3375 phy_tmp |= (MII_CR_AUTO_NEG_EN | 3376 MII_CR_RESTART_AUTO_NEG); 3377 e1000_write_phy_reg(&adapter->hw, PHY_CONTROL, phy_tmp); 3378 } 3379 } 3380 /* Restart process after EM_SMARTSPEED_MAX iterations */ 3381 if(adapter->smartspeed++ == EM_SMARTSPEED_MAX) 3382 adapter->smartspeed = 0; 3383} 3384 3385 3386/* 3387 * Manage DMA'able memory. 3388 */ 3389static void 3390em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 3391{ 3392 if (error) 3393 return; 3394 *(bus_addr_t *) arg = segs[0].ds_addr; 3395} 3396 3397static int 3398em_dma_malloc(struct adapter *adapter, bus_size_t size, 3399 struct em_dma_alloc *dma, int mapflags) 3400{ 3401 int error; 3402 3403#if __FreeBSD_version >= 700000 3404 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 3405#else 3406 error = bus_dma_tag_create(NULL, /* parent */ 3407#endif 3408 EM_DBA_ALIGN, 0, /* alignment, bounds */ 3409 BUS_SPACE_MAXADDR, /* lowaddr */ 3410 BUS_SPACE_MAXADDR, /* highaddr */ 3411 NULL, NULL, /* filter, filterarg */ 3412 size, /* maxsize */ 3413 1, /* nsegments */ 3414 size, /* maxsegsize */ 3415 0, /* flags */ 3416 NULL, /* lockfunc */ 3417 NULL, /* lockarg */ 3418 &dma->dma_tag); 3419 if (error) { 3420 device_printf(adapter->dev, 3421 "%s: bus_dma_tag_create failed: %d\n", 3422 __func__, error); 3423 goto fail_0; 3424 } 3425 3426 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 3427 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); 3428 if (error) { 3429 device_printf(adapter->dev, 3430 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 3431 __func__, (uintmax_t)size, error); 3432 goto fail_2; 3433 } 3434 3435 dma->dma_paddr = 0; 3436 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 3437 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 3438 if (error || dma->dma_paddr == 0) { 3439 device_printf(adapter->dev, 3440 "%s: bus_dmamap_load failed: %d\n", 3441 __func__, error); 3442 goto fail_3; 3443 } 3444 3445 return (0); 3446 3447fail_3: 3448 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3449fail_2: 3450 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3451 bus_dma_tag_destroy(dma->dma_tag); 3452fail_0: 3453 dma->dma_map = NULL; 3454 dma->dma_tag = NULL; 3455 3456 return (error); 3457} 3458 3459static void 3460em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) 3461{ 3462 if (dma->dma_tag == NULL) 3463 return; 3464 if (dma->dma_map != NULL) { 3465 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 3466 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3467 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 3468 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 3469 dma->dma_map = NULL; 3470 } 3471 bus_dma_tag_destroy(dma->dma_tag); 3472 dma->dma_tag = NULL; 3473} 3474 3475 3476/********************************************************************* 3477 * 3478 * Allocate memory for tx_buffer structures. The tx_buffer stores all 3479 * the information needed to transmit a packet on the wire. 3480 * 3481 **********************************************************************/ 3482static int 3483em_allocate_transmit_structures(struct adapter *adapter) 3484{ 3485 device_t dev = adapter->dev; 3486 struct em_buffer *tx_buffer; 3487 int error; 3488 3489 /* 3490 * Create DMA tags for tx descriptors 3491 */ 3492#if __FreeBSD_version >= 700000 3493 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 3494#else 3495 if ((error = bus_dma_tag_create(NULL, /* parent */ 3496#endif 3497 1, 0, /* alignment, bounds */ 3498 BUS_SPACE_MAXADDR, /* lowaddr */ 3499 BUS_SPACE_MAXADDR, /* highaddr */ 3500 NULL, NULL, /* filter, filterarg */ 3501 EM_TSO_SIZE, /* maxsize */ 3502 EM_MAX_SCATTER, /* nsegments */ 3503 EM_TSO_SEG_SIZE, /* maxsegsize */ 3504 0, /* flags */ 3505 NULL, /* lockfunc */ 3506 NULL, /* lockarg */ 3507 &adapter->txtag)) != 0) { 3508 device_printf(dev, "Unable to allocate TX DMA tag\n"); 3509 goto fail; 3510 } 3511 3512 adapter->tx_buffer_area = malloc(sizeof(struct em_buffer) * 3513 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 3514 if (adapter->tx_buffer_area == NULL) { 3515 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 3516 error = ENOMEM; 3517 goto fail; 3518 } 3519 3520 /* Create the descriptor buffer dma maps */ 3521 for (int i = 0; i < adapter->num_tx_desc; i++) { 3522 tx_buffer = &adapter->tx_buffer_area[i]; 3523 error = bus_dmamap_create(adapter->txtag, 0, &tx_buffer->map); 3524 if (error != 0) { 3525 device_printf(dev, "Unable to create TX DMA map\n"); 3526 goto fail; 3527 } 3528 tx_buffer->next_eop = -1; 3529 } 3530 3531 return (0); 3532fail: 3533 em_free_transmit_structures(adapter); 3534 return (error); 3535} 3536 3537/********************************************************************* 3538 * 3539 * (Re)Initialize transmit structures. 3540 * 3541 **********************************************************************/ 3542static void 3543em_setup_transmit_structures(struct adapter *adapter) 3544{ 3545 struct em_buffer *tx_buffer; 3546 3547 /* Clear the old ring contents */ 3548 bzero(adapter->tx_desc_base, 3549 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); 3550 3551 /* Free any existing TX buffers */ 3552 for (int i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3553 tx_buffer = &adapter->tx_buffer_area[i]; 3554 bus_dmamap_sync(adapter->txtag, tx_buffer->map, 3555 BUS_DMASYNC_POSTWRITE); 3556 bus_dmamap_unload(adapter->txtag, tx_buffer->map); 3557 m_freem(tx_buffer->m_head); 3558 tx_buffer->m_head = NULL; 3559 tx_buffer->next_eop = -1; 3560 } 3561 3562 /* Reset state */ 3563 adapter->next_avail_tx_desc = 0; 3564 adapter->next_tx_to_clean = 0; 3565 adapter->num_tx_desc_avail = adapter->num_tx_desc; 3566 3567 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 3568 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3569 3570 return; 3571} 3572 3573/********************************************************************* 3574 * 3575 * Enable transmit unit. 3576 * 3577 **********************************************************************/ 3578static void 3579em_initialize_transmit_unit(struct adapter *adapter) 3580{ 3581 u32 tctl, tarc, tipg = 0; 3582 u64 bus_addr; 3583 3584 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 3585 /* Setup the Base and Length of the Tx Descriptor Ring */ 3586 bus_addr = adapter->txdma.dma_paddr; 3587 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(0), 3588 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3589 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(0), 3590 (u32)(bus_addr >> 32)); 3591 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(0), 3592 (u32)bus_addr); 3593 /* Setup the HW Tx Head and Tail descriptor pointers */ 3594 E1000_WRITE_REG(&adapter->hw, E1000_TDT(0), 0); 3595 E1000_WRITE_REG(&adapter->hw, E1000_TDH(0), 0); 3596 3597 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3598 E1000_READ_REG(&adapter->hw, E1000_TDBAL(0)), 3599 E1000_READ_REG(&adapter->hw, E1000_TDLEN(0))); 3600 3601 /* Set the default values for the Tx Inter Packet Gap timer */ 3602 switch (adapter->hw.mac.type) { 3603 case e1000_82542: 3604 tipg = DEFAULT_82542_TIPG_IPGT; 3605 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3606 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3607 break; 3608 case e1000_80003es2lan: 3609 tipg = DEFAULT_82543_TIPG_IPGR1; 3610 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 3611 E1000_TIPG_IPGR2_SHIFT; 3612 break; 3613 default: 3614 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3615 (adapter->hw.phy.media_type == 3616 e1000_media_type_internal_serdes)) 3617 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 3618 else 3619 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 3620 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3621 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3622 } 3623 3624 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); 3625 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); 3626 if(adapter->hw.mac.type >= e1000_82540) 3627 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 3628 adapter->tx_abs_int_delay.value); 3629 3630 if ((adapter->hw.mac.type == e1000_82571) || 3631 (adapter->hw.mac.type == e1000_82572)) { 3632 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3633 tarc |= SPEED_MODE_BIT; 3634 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3635 } else if (adapter->hw.mac.type == e1000_80003es2lan) { 3636 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3637 tarc |= 1; 3638 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3639 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 3640 tarc |= 1; 3641 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 3642 } 3643 3644 /* Program the Transmit Control Register */ 3645 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 3646 tctl &= ~E1000_TCTL_CT; 3647 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3648 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3649 3650 if (adapter->hw.mac.type >= e1000_82571) 3651 tctl |= E1000_TCTL_MULR; 3652 3653 /* This write will effectively turn on the transmit unit. */ 3654 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 3655 3656 /* Setup Transmit Descriptor Base Settings */ 3657 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 3658 3659 if (adapter->tx_int_delay.value > 0) 3660 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3661} 3662 3663/********************************************************************* 3664 * 3665 * Free all transmit related data structures. 3666 * 3667 **********************************************************************/ 3668static void 3669em_free_transmit_structures(struct adapter *adapter) 3670{ 3671 struct em_buffer *tx_buffer; 3672 3673 INIT_DEBUGOUT("free_transmit_structures: begin"); 3674 3675 if (adapter->tx_buffer_area != NULL) { 3676 for (int i = 0; i < adapter->num_tx_desc; i++) { 3677 tx_buffer = &adapter->tx_buffer_area[i]; 3678 if (tx_buffer->m_head != NULL) { 3679 bus_dmamap_sync(adapter->txtag, tx_buffer->map, 3680 BUS_DMASYNC_POSTWRITE); 3681 bus_dmamap_unload(adapter->txtag, 3682 tx_buffer->map); 3683 m_freem(tx_buffer->m_head); 3684 tx_buffer->m_head = NULL; 3685 } else if (tx_buffer->map != NULL) 3686 bus_dmamap_unload(adapter->txtag, 3687 tx_buffer->map); 3688 if (tx_buffer->map != NULL) { 3689 bus_dmamap_destroy(adapter->txtag, 3690 tx_buffer->map); 3691 tx_buffer->map = NULL; 3692 } 3693 } 3694 } 3695 if (adapter->tx_buffer_area != NULL) { 3696 free(adapter->tx_buffer_area, M_DEVBUF); 3697 adapter->tx_buffer_area = NULL; 3698 } 3699 if (adapter->txtag != NULL) { 3700 bus_dma_tag_destroy(adapter->txtag); 3701 adapter->txtag = NULL; 3702 } 3703} 3704 3705/********************************************************************* 3706 * 3707 * The offload context needs to be set when we transfer the first 3708 * packet of a particular protocol (TCP/UDP). This routine has been 3709 * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) 3710 * 3711 **********************************************************************/ 3712static void 3713em_transmit_checksum_setup(struct adapter *adapter, struct mbuf *mp, 3714 u32 *txd_upper, u32 *txd_lower) 3715{ 3716 struct e1000_context_desc *TXD; 3717 struct em_buffer *tx_buffer; 3718 struct ether_vlan_header *eh; 3719 struct ip *ip = NULL; 3720 struct ip6_hdr *ip6; 3721 struct tcp_hdr *th; 3722 int curr_txd, ehdrlen; 3723 u32 cmd, hdr_len, ip_hlen; 3724 u16 etype; 3725 u8 ipproto; 3726 3727 cmd = hdr_len = ipproto = 0; 3728 /* Setup checksum offload context. */ 3729 curr_txd = adapter->next_avail_tx_desc; 3730 tx_buffer = &adapter->tx_buffer_area[curr_txd]; 3731 TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; 3732 3733 /* 3734 * Determine where frame payload starts. 3735 * Jump over vlan headers if already present, 3736 * helpful for QinQ too. 3737 */ 3738 eh = mtod(mp, struct ether_vlan_header *); 3739 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3740 etype = ntohs(eh->evl_proto); 3741 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3742 } else { 3743 etype = ntohs(eh->evl_encap_proto); 3744 ehdrlen = ETHER_HDR_LEN; 3745 } 3746 3747 /* 3748 * We only support TCP/UDP for IPv4 and IPv6 for the moment. 3749 * TODO: Support SCTP too when it hits the tree. 3750 */ 3751 switch (etype) { 3752 case ETHERTYPE_IP: 3753 ip = (struct ip *)(mp->m_data + ehdrlen); 3754 ip_hlen = ip->ip_hl << 2; 3755 3756 /* Setup of IP header checksum. */ 3757 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 3758 /* 3759 * Start offset for header checksum calculation. 3760 * End offset for header checksum calculation. 3761 * Offset of place to put the checksum. 3762 */ 3763 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3764 TXD->lower_setup.ip_fields.ipcse = 3765 htole16(ehdrlen + ip_hlen); 3766 TXD->lower_setup.ip_fields.ipcso = 3767 ehdrlen + offsetof(struct ip, ip_sum); 3768 cmd |= E1000_TXD_CMD_IP; 3769 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 3770 } 3771 3772 if (mp->m_len < ehdrlen + ip_hlen) 3773 return; /* failure */ 3774 3775 hdr_len = ehdrlen + ip_hlen; 3776 ipproto = ip->ip_p; 3777 3778 break; 3779 case ETHERTYPE_IPV6: 3780 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3781 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ 3782 3783 if (mp->m_len < ehdrlen + ip_hlen) 3784 return; /* failure */ 3785 3786 /* IPv6 doesn't have a header checksum. */ 3787 3788 hdr_len = ehdrlen + ip_hlen; 3789 ipproto = ip6->ip6_nxt; 3790 3791 break; 3792#ifdef EM_TIMESYNC 3793 case ETHERTYPE_IEEE1588: 3794 *txd_upper |= E1000_TXD_EXTCMD_TSTAMP; 3795 break; 3796#endif 3797 default: 3798 *txd_upper = 0; 3799 *txd_lower = 0; 3800 return; 3801 } 3802 3803 switch (ipproto) { 3804 case IPPROTO_TCP: 3805 if (mp->m_pkthdr.csum_flags & CSUM_TCP) { 3806 /* 3807 * Start offset for payload checksum calculation. 3808 * End offset for payload checksum calculation. 3809 * Offset of place to put the checksum. 3810 */ 3811 th = (struct tcp_hdr *)(mp->m_data + hdr_len); 3812 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3813 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3814 TXD->upper_setup.tcp_fields.tucso = 3815 hdr_len + offsetof(struct tcphdr, th_sum); 3816 cmd |= E1000_TXD_CMD_TCP; 3817 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3818 } 3819 break; 3820 case IPPROTO_UDP: 3821 { 3822#ifdef EM_TIMESYNC 3823 void *hdr = (caddr_t) ip + ip_hlen; 3824 struct udphdr *uh = (struct udphdr *)hdr; 3825 3826 if (uh->uh_dport == htons(TSYNC_PORT)) { 3827 *txd_upper |= E1000_TXD_EXTCMD_TSTAMP; 3828 IOCTL_DEBUGOUT("@@@ Sending Event Packet\n"); 3829 } 3830#endif 3831 if (mp->m_pkthdr.csum_flags & CSUM_UDP) { 3832 /* 3833 * Start offset for header checksum calculation. 3834 * End offset for header checksum calculation. 3835 * Offset of place to put the checksum. 3836 */ 3837 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3838 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3839 TXD->upper_setup.tcp_fields.tucso = 3840 hdr_len + offsetof(struct udphdr, uh_sum); 3841 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3842 } 3843 /* Fall Thru */ 3844 } 3845 default: 3846 break; 3847 } 3848 3849#ifdef EM_TIMESYNC 3850 /* 3851 ** We might be here just for TIMESYNC 3852 ** which means we don't need the context 3853 ** descriptor. 3854 */ 3855 if (!mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) 3856 return; 3857#endif 3858 *txd_lower = E1000_TXD_CMD_DEXT | /* Extended descr type */ 3859 E1000_TXD_DTYP_D; /* Data descr */ 3860 TXD->tcp_seg_setup.data = htole32(0); 3861 TXD->cmd_and_length = 3862 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); 3863 tx_buffer->m_head = NULL; 3864 tx_buffer->next_eop = -1; 3865 3866 if (++curr_txd == adapter->num_tx_desc) 3867 curr_txd = 0; 3868 3869 adapter->num_tx_desc_avail--; 3870 adapter->next_avail_tx_desc = curr_txd; 3871} 3872 3873 3874#if __FreeBSD_version >= 700000 3875/********************************************************************** 3876 * 3877 * Setup work for hardware segmentation offload (TSO) 3878 * 3879 **********************************************************************/ 3880static bool 3881em_tso_setup(struct adapter *adapter, struct mbuf *mp, u32 *txd_upper, 3882 u32 *txd_lower) 3883{ 3884 struct e1000_context_desc *TXD; 3885 struct em_buffer *tx_buffer; 3886 struct ether_vlan_header *eh; 3887 struct ip *ip; 3888 struct ip6_hdr *ip6; 3889 struct tcphdr *th; 3890 int curr_txd, ehdrlen, hdr_len, ip_hlen, isip6; 3891 u16 etype; 3892 3893 /* 3894 * This function could/should be extended to support IP/IPv6 3895 * fragmentation as well. But as they say, one step at a time. 3896 */ 3897 3898 /* 3899 * Determine where frame payload starts. 3900 * Jump over vlan headers if already present, 3901 * helpful for QinQ too. 3902 */ 3903 eh = mtod(mp, struct ether_vlan_header *); 3904 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3905 etype = ntohs(eh->evl_proto); 3906 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3907 } else { 3908 etype = ntohs(eh->evl_encap_proto); 3909 ehdrlen = ETHER_HDR_LEN; 3910 } 3911 3912 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3913 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3914 return FALSE; /* -1 */ 3915 3916 /* 3917 * We only support TCP for IPv4 and IPv6 (notyet) for the moment. 3918 * TODO: Support SCTP too when it hits the tree. 3919 */ 3920 switch (etype) { 3921 case ETHERTYPE_IP: 3922 isip6 = 0; 3923 ip = (struct ip *)(mp->m_data + ehdrlen); 3924 if (ip->ip_p != IPPROTO_TCP) 3925 return FALSE; /* 0 */ 3926 ip->ip_len = 0; 3927 ip->ip_sum = 0; 3928 ip_hlen = ip->ip_hl << 2; 3929 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3930 return FALSE; /* -1 */ 3931 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3932#if 1 3933 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3934 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3935#else 3936 th->th_sum = mp->m_pkthdr.csum_data; 3937#endif 3938 break; 3939 case ETHERTYPE_IPV6: 3940 isip6 = 1; 3941 return FALSE; /* Not supported yet. */ 3942 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3943 if (ip6->ip6_nxt != IPPROTO_TCP) 3944 return FALSE; /* 0 */ 3945 ip6->ip6_plen = 0; 3946 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */ 3947 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3948 return FALSE; /* -1 */ 3949 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 3950#if 0 3951 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst, 3952 htons(IPPROTO_TCP)); /* XXX: function notyet. */ 3953#else 3954 th->th_sum = mp->m_pkthdr.csum_data; 3955#endif 3956 break; 3957 default: 3958 return FALSE; 3959 } 3960 hdr_len = ehdrlen + ip_hlen + (th->th_off << 2); 3961 3962 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 3963 E1000_TXD_DTYP_D | /* Data descr type */ 3964 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 3965 3966 /* IP and/or TCP header checksum calculation and insertion. */ 3967 *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) | 3968 E1000_TXD_POPTS_TXSM) << 8; 3969 3970 curr_txd = adapter->next_avail_tx_desc; 3971 tx_buffer = &adapter->tx_buffer_area[curr_txd]; 3972 TXD = (struct e1000_context_desc *) &adapter->tx_desc_base[curr_txd]; 3973 3974 /* IPv6 doesn't have a header checksum. */ 3975 if (!isip6) { 3976 /* 3977 * Start offset for header checksum calculation. 3978 * End offset for header checksum calculation. 3979 * Offset of place put the checksum. 3980 */ 3981 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3982 TXD->lower_setup.ip_fields.ipcse = 3983 htole16(ehdrlen + ip_hlen - 1); 3984 TXD->lower_setup.ip_fields.ipcso = 3985 ehdrlen + offsetof(struct ip, ip_sum); 3986 } 3987 /* 3988 * Start offset for payload checksum calculation. 3989 * End offset for payload checksum calculation. 3990 * Offset of place to put the checksum. 3991 */ 3992 TXD->upper_setup.tcp_fields.tucss = 3993 ehdrlen + ip_hlen; 3994 TXD->upper_setup.tcp_fields.tucse = 0; 3995 TXD->upper_setup.tcp_fields.tucso = 3996 ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum); 3997 /* 3998 * Payload size per packet w/o any headers. 3999 * Length of all headers up to payload. 4000 */ 4001 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); 4002 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 4003 4004 TXD->cmd_and_length = htole32(adapter->txd_cmd | 4005 E1000_TXD_CMD_DEXT | /* Extended descr */ 4006 E1000_TXD_CMD_TSE | /* TSE context */ 4007 (isip6 ? 0 : E1000_TXD_CMD_IP) | /* Do IP csum */ 4008 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 4009 (mp->m_pkthdr.len - (hdr_len))); /* Total len */ 4010 4011 tx_buffer->m_head = NULL; 4012 tx_buffer->next_eop = -1; 4013 4014 if (++curr_txd == adapter->num_tx_desc) 4015 curr_txd = 0; 4016 4017 adapter->num_tx_desc_avail--; 4018 adapter->next_avail_tx_desc = curr_txd; 4019 adapter->tx_tso = TRUE; 4020 4021 return TRUE; 4022} 4023 4024#endif /* __FreeBSD_version >= 700000 */ 4025 4026/********************************************************************** 4027 * 4028 * Examine each tx_buffer in the used queue. If the hardware is done 4029 * processing the packet then free associated resources. The 4030 * tx_buffer is put back on the free queue. 4031 * 4032 **********************************************************************/ 4033static void 4034em_txeof(struct adapter *adapter) 4035{ 4036 int first, last, done, num_avail; 4037 u32 cleaned = 0; 4038 struct em_buffer *tx_buffer; 4039 struct e1000_tx_desc *tx_desc, *eop_desc; 4040 struct ifnet *ifp = adapter->ifp; 4041 4042 EM_TX_LOCK_ASSERT(adapter); 4043 4044 if (adapter->num_tx_desc_avail == adapter->num_tx_desc) 4045 return; 4046 4047 num_avail = adapter->num_tx_desc_avail; 4048 first = adapter->next_tx_to_clean; 4049 tx_desc = &adapter->tx_desc_base[first]; 4050 tx_buffer = &adapter->tx_buffer_area[first]; 4051 last = tx_buffer->next_eop; 4052 eop_desc = &adapter->tx_desc_base[last]; 4053 4054 /* 4055 * What this does is get the index of the 4056 * first descriptor AFTER the EOP of the 4057 * first packet, that way we can do the 4058 * simple comparison on the inner while loop. 4059 */ 4060 if (++last == adapter->num_tx_desc) 4061 last = 0; 4062 done = last; 4063 4064 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 4065 BUS_DMASYNC_POSTREAD); 4066 4067 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 4068 /* We clean the range of the packet */ 4069 while (first != done) { 4070 tx_desc->upper.data = 0; 4071 tx_desc->lower.data = 0; 4072 tx_desc->buffer_addr = 0; 4073 ++num_avail; ++cleaned; 4074 4075 if (tx_buffer->m_head) { 4076 ifp->if_opackets++; 4077 bus_dmamap_sync(adapter->txtag, 4078 tx_buffer->map, 4079 BUS_DMASYNC_POSTWRITE); 4080 bus_dmamap_unload(adapter->txtag, 4081 tx_buffer->map); 4082 4083 m_freem(tx_buffer->m_head); 4084 tx_buffer->m_head = NULL; 4085 } 4086 tx_buffer->next_eop = -1; 4087 4088 if (++first == adapter->num_tx_desc) 4089 first = 0; 4090 4091 tx_buffer = &adapter->tx_buffer_area[first]; 4092 tx_desc = &adapter->tx_desc_base[first]; 4093 } 4094 /* See if we can continue to the next packet */ 4095 last = tx_buffer->next_eop; 4096 if (last != -1) { 4097 eop_desc = &adapter->tx_desc_base[last]; 4098 /* Get new done point */ 4099 if (++last == adapter->num_tx_desc) last = 0; 4100 done = last; 4101 } else 4102 break; 4103 } 4104 bus_dmamap_sync(adapter->txdma.dma_tag, adapter->txdma.dma_map, 4105 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4106 4107 adapter->next_tx_to_clean = first; 4108 4109 /* 4110 * If we have enough room, clear IFF_DRV_OACTIVE to 4111 * tell the stack that it is OK to send packets. 4112 * If there are no pending descriptors, clear the timeout. 4113 */ 4114 if (num_avail > EM_TX_CLEANUP_THRESHOLD) { 4115 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 4116 if (num_avail == adapter->num_tx_desc) { 4117 adapter->watchdog_timer = 0; 4118 adapter->num_tx_desc_avail = num_avail; 4119 return; 4120 } 4121 } 4122 4123 /* If any descriptors cleaned, reset the watchdog */ 4124 if (cleaned) 4125 adapter->watchdog_timer = EM_TX_TIMEOUT; 4126 adapter->num_tx_desc_avail = num_avail; 4127 return; 4128} 4129 4130/********************************************************************* 4131 * 4132 * When Link is lost sometimes there is work still in the TX ring 4133 * which will result in a watchdog, rather than allow that do an 4134 * attempted cleanup and then reinit here. Note that this has been 4135 * seens mostly with fiber adapters. 4136 * 4137 **********************************************************************/ 4138static void 4139em_tx_purge(struct adapter *adapter) 4140{ 4141 if ((!adapter->link_active) && (adapter->watchdog_timer)) { 4142 EM_TX_LOCK(adapter); 4143 em_txeof(adapter); 4144 EM_TX_UNLOCK(adapter); 4145 if (adapter->watchdog_timer) { /* Still not clean? */ 4146 adapter->watchdog_timer = 0; 4147 em_init_locked(adapter); 4148 } 4149 } 4150} 4151 4152/********************************************************************* 4153 * 4154 * Get a buffer from system mbuf buffer pool. 4155 * 4156 **********************************************************************/ 4157static int 4158em_get_buf(struct adapter *adapter, int i) 4159{ 4160 struct mbuf *m; 4161 bus_dma_segment_t segs[1]; 4162 bus_dmamap_t map; 4163 struct em_buffer *rx_buffer; 4164 int error, nsegs; 4165 4166 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 4167 if (m == NULL) { 4168 adapter->mbuf_cluster_failed++; 4169 return (ENOBUFS); 4170 } 4171 m->m_len = m->m_pkthdr.len = MCLBYTES; 4172 4173 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 4174 m_adj(m, ETHER_ALIGN); 4175 4176 /* 4177 * Using memory from the mbuf cluster pool, invoke the 4178 * bus_dma machinery to arrange the memory mapping. 4179 */ 4180 error = bus_dmamap_load_mbuf_sg(adapter->rxtag, 4181 adapter->rx_sparemap, m, segs, &nsegs, BUS_DMA_NOWAIT); 4182 if (error != 0) { 4183 m_free(m); 4184 return (error); 4185 } 4186 4187 /* If nsegs is wrong then the stack is corrupt. */ 4188 KASSERT(nsegs == 1, ("Too many segments returned!")); 4189 4190 rx_buffer = &adapter->rx_buffer_area[i]; 4191 if (rx_buffer->m_head != NULL) 4192 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 4193 4194 map = rx_buffer->map; 4195 rx_buffer->map = adapter->rx_sparemap; 4196 adapter->rx_sparemap = map; 4197 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, BUS_DMASYNC_PREREAD); 4198 rx_buffer->m_head = m; 4199 4200 adapter->rx_desc_base[i].buffer_addr = htole64(segs[0].ds_addr); 4201 return (0); 4202} 4203 4204/********************************************************************* 4205 * 4206 * Allocate memory for rx_buffer structures. Since we use one 4207 * rx_buffer per received packet, the maximum number of rx_buffer's 4208 * that we'll need is equal to the number of receive descriptors 4209 * that we've allocated. 4210 * 4211 **********************************************************************/ 4212static int 4213em_allocate_receive_structures(struct adapter *adapter) 4214{ 4215 device_t dev = adapter->dev; 4216 struct em_buffer *rx_buffer; 4217 int i, error; 4218 4219 adapter->rx_buffer_area = malloc(sizeof(struct em_buffer) * 4220 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 4221 if (adapter->rx_buffer_area == NULL) { 4222 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 4223 return (ENOMEM); 4224 } 4225 4226#if __FreeBSD_version >= 700000 4227 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 4228#else 4229 error = bus_dma_tag_create(NULL, /* parent */ 4230#endif 4231 1, 0, /* alignment, bounds */ 4232 BUS_SPACE_MAXADDR, /* lowaddr */ 4233 BUS_SPACE_MAXADDR, /* highaddr */ 4234 NULL, NULL, /* filter, filterarg */ 4235 MCLBYTES, /* maxsize */ 4236 1, /* nsegments */ 4237 MCLBYTES, /* maxsegsize */ 4238 0, /* flags */ 4239 NULL, /* lockfunc */ 4240 NULL, /* lockarg */ 4241 &adapter->rxtag); 4242 if (error) { 4243 device_printf(dev, "%s: bus_dma_tag_create failed %d\n", 4244 __func__, error); 4245 goto fail; 4246 } 4247 4248 /* Create the spare map (used by getbuf) */ 4249 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, 4250 &adapter->rx_sparemap); 4251 if (error) { 4252 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 4253 __func__, error); 4254 goto fail; 4255 } 4256 4257 rx_buffer = adapter->rx_buffer_area; 4258 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4259 error = bus_dmamap_create(adapter->rxtag, BUS_DMA_NOWAIT, 4260 &rx_buffer->map); 4261 if (error) { 4262 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 4263 __func__, error); 4264 goto fail; 4265 } 4266 } 4267 4268 return (0); 4269 4270fail: 4271 em_free_receive_structures(adapter); 4272 return (error); 4273} 4274 4275/********************************************************************* 4276 * 4277 * (Re)initialize receive structures. 4278 * 4279 **********************************************************************/ 4280static int 4281em_setup_receive_structures(struct adapter *adapter) 4282{ 4283 struct em_buffer *rx_buffer; 4284 int i, error; 4285 4286 /* Reset descriptor ring */ 4287 bzero(adapter->rx_desc_base, 4288 (sizeof(struct e1000_rx_desc)) * adapter->num_rx_desc); 4289 4290 /* Free current RX buffers. */ 4291 rx_buffer = adapter->rx_buffer_area; 4292 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4293 if (rx_buffer->m_head != NULL) { 4294 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, 4295 BUS_DMASYNC_POSTREAD); 4296 bus_dmamap_unload(adapter->rxtag, rx_buffer->map); 4297 m_freem(rx_buffer->m_head); 4298 rx_buffer->m_head = NULL; 4299 } 4300 } 4301 4302 /* Allocate new ones. */ 4303 for (i = 0; i < adapter->num_rx_desc; i++) { 4304 error = em_get_buf(adapter, i); 4305 if (error) 4306 return (error); 4307 } 4308 4309 /* Setup our descriptor pointers */ 4310 adapter->next_rx_desc_to_check = 0; 4311 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4312 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4313 4314 return (0); 4315} 4316 4317/********************************************************************* 4318 * 4319 * Enable receive unit. 4320 * 4321 **********************************************************************/ 4322#define MAX_INTS_PER_SEC 8000 4323#define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256) 4324 4325static void 4326em_initialize_receive_unit(struct adapter *adapter) 4327{ 4328 struct ifnet *ifp = adapter->ifp; 4329 u64 bus_addr; 4330 u32 rctl, rxcsum; 4331 4332 INIT_DEBUGOUT("em_initialize_receive_unit: begin"); 4333 4334 /* 4335 * Make sure receives are disabled while setting 4336 * up the descriptor ring 4337 */ 4338 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4339 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 4340 4341 if (adapter->hw.mac.type >= e1000_82540) { 4342 E1000_WRITE_REG(&adapter->hw, E1000_RADV, 4343 adapter->rx_abs_int_delay.value); 4344 /* 4345 * Set the interrupt throttling rate. Value is calculated 4346 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 4347 */ 4348 E1000_WRITE_REG(&adapter->hw, E1000_ITR, DEFAULT_ITR); 4349 } 4350 4351 /* 4352 ** When using MSIX interrupts we need to throttle 4353 ** using the EITR register (82574 only) 4354 */ 4355 if (adapter->msix) 4356 for (int i = 0; i < 4; i++) 4357 E1000_WRITE_REG(&adapter->hw, 4358 E1000_EITR_82574(i), DEFAULT_ITR); 4359 4360 /* Disable accelerated ackknowledge */ 4361 if (adapter->hw.mac.type == e1000_82574) 4362 E1000_WRITE_REG(&adapter->hw, 4363 E1000_RFCTL, E1000_RFCTL_ACK_DIS); 4364 4365 /* Setup the Base and Length of the Rx Descriptor Ring */ 4366 bus_addr = adapter->rxdma.dma_paddr; 4367 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(0), 4368 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 4369 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(0), 4370 (u32)(bus_addr >> 32)); 4371 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(0), 4372 (u32)bus_addr); 4373 4374 /* Setup the Receive Control Register */ 4375 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4376 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 4377 E1000_RCTL_RDMTS_HALF | 4378 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4379 4380 /* Make sure VLAN Filters are off */ 4381 rctl &= ~E1000_RCTL_VFE; 4382 4383 if (e1000_tbi_sbp_enabled_82543(&adapter->hw)) 4384 rctl |= E1000_RCTL_SBP; 4385 else 4386 rctl &= ~E1000_RCTL_SBP; 4387 4388 switch (adapter->rx_buffer_len) { 4389 default: 4390 case 2048: 4391 rctl |= E1000_RCTL_SZ_2048; 4392 break; 4393 case 4096: 4394 rctl |= E1000_RCTL_SZ_4096 | 4395 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4396 break; 4397 case 8192: 4398 rctl |= E1000_RCTL_SZ_8192 | 4399 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4400 break; 4401 case 16384: 4402 rctl |= E1000_RCTL_SZ_16384 | 4403 E1000_RCTL_BSEX | E1000_RCTL_LPE; 4404 break; 4405 } 4406 4407 if (ifp->if_mtu > ETHERMTU) 4408 rctl |= E1000_RCTL_LPE; 4409 else 4410 rctl &= ~E1000_RCTL_LPE; 4411 4412 /* Enable 82543 Receive Checksum Offload for TCP and UDP */ 4413 if ((adapter->hw.mac.type >= e1000_82543) && 4414 (ifp->if_capenable & IFCAP_RXCSUM)) { 4415 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); 4416 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 4417 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); 4418 } 4419 4420 /* 4421 ** XXX TEMPORARY WORKAROUND: on some systems with 82573 4422 ** long latencies are observed, like Lenovo X60. This 4423 ** change eliminates the problem, but since having positive 4424 ** values in RDTR is a known source of problems on other 4425 ** platforms another solution is being sought. 4426 */ 4427 if (adapter->hw.mac.type == e1000_82573) 4428 E1000_WRITE_REG(&adapter->hw, E1000_RDTR, 0x20); 4429 4430 /* Enable Receives */ 4431 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4432 4433 /* 4434 * Setup the HW Rx Head and 4435 * Tail Descriptor Pointers 4436 */ 4437 E1000_WRITE_REG(&adapter->hw, E1000_RDH(0), 0); 4438 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), adapter->num_rx_desc - 1); 4439 4440 return; 4441} 4442 4443/********************************************************************* 4444 * 4445 * Free receive related data structures. 4446 * 4447 **********************************************************************/ 4448static void 4449em_free_receive_structures(struct adapter *adapter) 4450{ 4451 struct em_buffer *rx_buffer; 4452 int i; 4453 4454 INIT_DEBUGOUT("free_receive_structures: begin"); 4455 4456 if (adapter->rx_sparemap) { 4457 bus_dmamap_destroy(adapter->rxtag, adapter->rx_sparemap); 4458 adapter->rx_sparemap = NULL; 4459 } 4460 4461 /* Cleanup any existing buffers */ 4462 if (adapter->rx_buffer_area != NULL) { 4463 rx_buffer = adapter->rx_buffer_area; 4464 for (i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 4465 if (rx_buffer->m_head != NULL) { 4466 bus_dmamap_sync(adapter->rxtag, rx_buffer->map, 4467 BUS_DMASYNC_POSTREAD); 4468 bus_dmamap_unload(adapter->rxtag, 4469 rx_buffer->map); 4470 m_freem(rx_buffer->m_head); 4471 rx_buffer->m_head = NULL; 4472 } else if (rx_buffer->map != NULL) 4473 bus_dmamap_unload(adapter->rxtag, 4474 rx_buffer->map); 4475 if (rx_buffer->map != NULL) { 4476 bus_dmamap_destroy(adapter->rxtag, 4477 rx_buffer->map); 4478 rx_buffer->map = NULL; 4479 } 4480 } 4481 } 4482 4483 if (adapter->rx_buffer_area != NULL) { 4484 free(adapter->rx_buffer_area, M_DEVBUF); 4485 adapter->rx_buffer_area = NULL; 4486 } 4487 4488 if (adapter->rxtag != NULL) { 4489 bus_dma_tag_destroy(adapter->rxtag); 4490 adapter->rxtag = NULL; 4491 } 4492} 4493 4494/********************************************************************* 4495 * 4496 * This routine executes in interrupt context. It replenishes 4497 * the mbufs in the descriptor and sends data which has been 4498 * dma'ed into host memory to upper layer. 4499 * 4500 * We loop at most count times if count is > 0, or until done if 4501 * count < 0. 4502 * 4503 *********************************************************************/ 4504static int 4505em_rxeof(struct adapter *adapter, int count) 4506{ 4507 struct ifnet *ifp = adapter->ifp;; 4508 struct mbuf *mp; 4509 u8 status, accept_frame = 0, eop = 0; 4510 u16 len, desc_len, prev_len_adj; 4511 int i; 4512 struct e1000_rx_desc *current_desc; 4513 4514 EM_RX_LOCK(adapter); 4515 i = adapter->next_rx_desc_to_check; 4516 current_desc = &adapter->rx_desc_base[i]; 4517 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4518 BUS_DMASYNC_POSTREAD); 4519 4520 if (!((current_desc->status) & E1000_RXD_STAT_DD)) { 4521 EM_RX_UNLOCK(adapter); 4522 return (0); 4523 } 4524 4525 while ((current_desc->status & E1000_RXD_STAT_DD) && 4526 (count != 0) && 4527 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 4528 struct mbuf *m = NULL; 4529 4530 mp = adapter->rx_buffer_area[i].m_head; 4531 /* 4532 * Can't defer bus_dmamap_sync(9) because TBI_ACCEPT 4533 * needs to access the last received byte in the mbuf. 4534 */ 4535 bus_dmamap_sync(adapter->rxtag, adapter->rx_buffer_area[i].map, 4536 BUS_DMASYNC_POSTREAD); 4537 4538 accept_frame = 1; 4539 prev_len_adj = 0; 4540 desc_len = le16toh(current_desc->length); 4541 status = current_desc->status; 4542 if (status & E1000_RXD_STAT_EOP) { 4543 count--; 4544 eop = 1; 4545 if (desc_len < ETHER_CRC_LEN) { 4546 len = 0; 4547 prev_len_adj = ETHER_CRC_LEN - desc_len; 4548 } else 4549 len = desc_len - ETHER_CRC_LEN; 4550 } else { 4551 eop = 0; 4552 len = desc_len; 4553 } 4554 4555 if (current_desc->errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 4556 u8 last_byte; 4557 u32 pkt_len = desc_len; 4558 4559 if (adapter->fmp != NULL) 4560 pkt_len += adapter->fmp->m_pkthdr.len; 4561 4562 last_byte = *(mtod(mp, caddr_t) + desc_len - 1); 4563 if (TBI_ACCEPT(&adapter->hw, status, 4564 current_desc->errors, pkt_len, last_byte, 4565 adapter->min_frame_size, adapter->max_frame_size)) { 4566 e1000_tbi_adjust_stats_82543(&adapter->hw, 4567 &adapter->stats, pkt_len, 4568 adapter->hw.mac.addr, 4569 adapter->max_frame_size); 4570 if (len > 0) 4571 len--; 4572 } else 4573 accept_frame = 0; 4574 } 4575 4576 if (accept_frame) { 4577 if (em_get_buf(adapter, i) != 0) { 4578 ifp->if_iqdrops++; 4579 goto discard; 4580 } 4581 4582 /* Assign correct length to the current fragment */ 4583 mp->m_len = len; 4584 4585 if (adapter->fmp == NULL) { 4586 mp->m_pkthdr.len = len; 4587 adapter->fmp = mp; /* Store the first mbuf */ 4588 adapter->lmp = mp; 4589 } else { 4590 /* Chain mbuf's together */ 4591 mp->m_flags &= ~M_PKTHDR; 4592 /* 4593 * Adjust length of previous mbuf in chain if 4594 * we received less than 4 bytes in the last 4595 * descriptor. 4596 */ 4597 if (prev_len_adj > 0) { 4598 adapter->lmp->m_len -= prev_len_adj; 4599 adapter->fmp->m_pkthdr.len -= 4600 prev_len_adj; 4601 } 4602 adapter->lmp->m_next = mp; 4603 adapter->lmp = adapter->lmp->m_next; 4604 adapter->fmp->m_pkthdr.len += len; 4605 } 4606 4607 if (eop) { 4608 adapter->fmp->m_pkthdr.rcvif = ifp; 4609 ifp->if_ipackets++; 4610 em_receive_checksum(adapter, current_desc, 4611 adapter->fmp); 4612#ifndef __NO_STRICT_ALIGNMENT 4613 if (adapter->max_frame_size > 4614 (MCLBYTES - ETHER_ALIGN) && 4615 em_fixup_rx(adapter) != 0) 4616 goto skip; 4617#endif 4618 if (status & E1000_RXD_STAT_VP) { 4619#if __FreeBSD_version < 700000 4620 VLAN_INPUT_TAG_NEW(ifp, adapter->fmp, 4621 (le16toh(current_desc->special) & 4622 E1000_RXD_SPC_VLAN_MASK)); 4623#else 4624 adapter->fmp->m_pkthdr.ether_vtag = 4625 (le16toh(current_desc->special) & 4626 E1000_RXD_SPC_VLAN_MASK); 4627 adapter->fmp->m_flags |= M_VLANTAG; 4628#endif 4629 } 4630#ifndef __NO_STRICT_ALIGNMENT 4631skip: 4632#endif 4633 m = adapter->fmp; 4634 adapter->fmp = NULL; 4635 adapter->lmp = NULL; 4636 } 4637 } else { 4638 ifp->if_ierrors++; 4639discard: 4640 /* Reuse loaded DMA map and just update mbuf chain */ 4641 mp = adapter->rx_buffer_area[i].m_head; 4642 mp->m_len = mp->m_pkthdr.len = MCLBYTES; 4643 mp->m_data = mp->m_ext.ext_buf; 4644 mp->m_next = NULL; 4645 if (adapter->max_frame_size <= 4646 (MCLBYTES - ETHER_ALIGN)) 4647 m_adj(mp, ETHER_ALIGN); 4648 if (adapter->fmp != NULL) { 4649 m_freem(adapter->fmp); 4650 adapter->fmp = NULL; 4651 adapter->lmp = NULL; 4652 } 4653 m = NULL; 4654 } 4655 4656 /* Zero out the receive descriptors status. */ 4657 current_desc->status = 0; 4658 bus_dmamap_sync(adapter->rxdma.dma_tag, adapter->rxdma.dma_map, 4659 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4660 4661 /* Advance our pointers to the next descriptor. */ 4662 if (++i == adapter->num_rx_desc) 4663 i = 0; 4664 if (m != NULL) { 4665 adapter->next_rx_desc_to_check = i; 4666 /* Unlock for call into stack */ 4667 EM_RX_UNLOCK(adapter); 4668 (*ifp->if_input)(ifp, m); 4669 EM_RX_LOCK(adapter); 4670 i = adapter->next_rx_desc_to_check; 4671 } 4672 current_desc = &adapter->rx_desc_base[i]; 4673 } 4674 adapter->next_rx_desc_to_check = i; 4675 4676 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ 4677 if (--i < 0) 4678 i = adapter->num_rx_desc - 1; 4679 E1000_WRITE_REG(&adapter->hw, E1000_RDT(0), i); 4680 EM_RX_UNLOCK(adapter); 4681 if (!((current_desc->status) & E1000_RXD_STAT_DD)) 4682 return (0); 4683 4684 return (1); 4685} 4686 4687#ifndef __NO_STRICT_ALIGNMENT 4688/* 4689 * When jumbo frames are enabled we should realign entire payload on 4690 * architecures with strict alignment. This is serious design mistake of 8254x 4691 * as it nullifies DMA operations. 8254x just allows RX buffer size to be 4692 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its 4693 * payload. On architecures without strict alignment restrictions 8254x still 4694 * performs unaligned memory access which would reduce the performance too. 4695 * To avoid copying over an entire frame to align, we allocate a new mbuf and 4696 * copy ethernet header to the new mbuf. The new mbuf is prepended into the 4697 * existing mbuf chain. 4698 * 4699 * Be aware, best performance of the 8254x is achived only when jumbo frame is 4700 * not used at all on architectures with strict alignment. 4701 */ 4702static int 4703em_fixup_rx(struct adapter *adapter) 4704{ 4705 struct mbuf *m, *n; 4706 int error; 4707 4708 error = 0; 4709 m = adapter->fmp; 4710 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { 4711 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); 4712 m->m_data += ETHER_HDR_LEN; 4713 } else { 4714 MGETHDR(n, M_DONTWAIT, MT_DATA); 4715 if (n != NULL) { 4716 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); 4717 m->m_data += ETHER_HDR_LEN; 4718 m->m_len -= ETHER_HDR_LEN; 4719 n->m_len = ETHER_HDR_LEN; 4720 M_MOVE_PKTHDR(n, m); 4721 n->m_next = m; 4722 adapter->fmp = n; 4723 } else { 4724 adapter->dropped_pkts++; 4725 m_freem(adapter->fmp); 4726 adapter->fmp = NULL; 4727 error = ENOMEM; 4728 } 4729 } 4730 4731 return (error); 4732} 4733#endif 4734 4735/********************************************************************* 4736 * 4737 * Verify that the hardware indicated that the checksum is valid. 4738 * Inform the stack about the status of checksum so that stack 4739 * doesn't spend time verifying the checksum. 4740 * 4741 *********************************************************************/ 4742static void 4743em_receive_checksum(struct adapter *adapter, 4744 struct e1000_rx_desc *rx_desc, struct mbuf *mp) 4745{ 4746 /* 82543 or newer only */ 4747 if ((adapter->hw.mac.type < e1000_82543) || 4748 /* Ignore Checksum bit is set */ 4749 (rx_desc->status & E1000_RXD_STAT_IXSM)) { 4750 mp->m_pkthdr.csum_flags = 0; 4751 return; 4752 } 4753 4754 if (rx_desc->status & E1000_RXD_STAT_IPCS) { 4755 /* Did it pass? */ 4756 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { 4757 /* IP Checksum Good */ 4758 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4759 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4760 4761 } else { 4762 mp->m_pkthdr.csum_flags = 0; 4763 } 4764 } 4765 4766 if (rx_desc->status & E1000_RXD_STAT_TCPCS) { 4767 /* Did it pass? */ 4768 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { 4769 mp->m_pkthdr.csum_flags |= 4770 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4771 mp->m_pkthdr.csum_data = htons(0xffff); 4772 } 4773 } 4774} 4775 4776/* 4777 * This routine is run via an vlan 4778 * config EVENT 4779 */ 4780static void 4781em_register_vlan(void *unused, struct ifnet *ifp, u16 vtag) 4782{ 4783 struct adapter *adapter = ifp->if_softc; 4784 u32 ctrl, rctl, index, vfta; 4785 4786 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4787 ctrl |= E1000_CTRL_VME; 4788 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4789 4790 /* Setup for Hardware Filter */ 4791 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4792 rctl |= E1000_RCTL_VFE; 4793 rctl &= ~E1000_RCTL_CFIEN; 4794 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4795 4796 /* Make entry in the hardware filter table */ 4797 index = ((vtag >> 5) & 0x7F); 4798 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index); 4799 vfta |= (1 << (vtag & 0x1F)); 4800 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta); 4801 4802 /* Update the frame size */ 4803 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4804 adapter->max_frame_size + VLAN_TAG_SIZE); 4805 4806} 4807 4808/* 4809 * This routine is run via an vlan 4810 * unconfig EVENT 4811 */ 4812static void 4813em_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag) 4814{ 4815 struct adapter *adapter = ifp->if_softc; 4816 u32 index, vfta; 4817 4818 /* Remove entry in the hardware filter table */ 4819 index = ((vtag >> 5) & 0x7F); 4820 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index); 4821 vfta &= ~(1 << (vtag & 0x1F)); 4822 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta); 4823 /* Have all vlans unregistered? */ 4824 if (adapter->ifp->if_vlantrunk == NULL) { 4825 u32 rctl; 4826 /* Turn off the filter table */ 4827 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4828 rctl &= ~E1000_RCTL_VFE; 4829 rctl |= E1000_RCTL_CFIEN; 4830 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4831 /* Reset the frame size */ 4832 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4833 adapter->max_frame_size); 4834 } 4835} 4836 4837static void 4838em_enable_intr(struct adapter *adapter) 4839{ 4840 struct e1000_hw *hw = &adapter->hw; 4841 u32 ims_mask = IMS_ENABLE_MASK; 4842 4843 if (adapter->msix) { 4844 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); 4845 ims_mask |= EM_MSIX_MASK; 4846 } 4847 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 4848} 4849 4850static void 4851em_disable_intr(struct adapter *adapter) 4852{ 4853 struct e1000_hw *hw = &adapter->hw; 4854 4855 if (adapter->msix) 4856 E1000_WRITE_REG(hw, EM_EIAC, 0); 4857 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 4858} 4859 4860/* 4861 * Bit of a misnomer, what this really means is 4862 * to enable OS management of the system... aka 4863 * to disable special hardware management features 4864 */ 4865static void 4866em_init_manageability(struct adapter *adapter) 4867{ 4868 /* A shared code workaround */ 4869#define E1000_82542_MANC2H E1000_MANC2H 4870 if (adapter->has_manage) { 4871 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4872 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4873 4874 /* disable hardware interception of ARP */ 4875 manc &= ~(E1000_MANC_ARP_EN); 4876 4877 /* enable receiving management packets to the host */ 4878 if (adapter->hw.mac.type >= e1000_82571) { 4879 manc |= E1000_MANC_EN_MNG2HOST; 4880#define E1000_MNG2HOST_PORT_623 (1 << 5) 4881#define E1000_MNG2HOST_PORT_664 (1 << 6) 4882 manc2h |= E1000_MNG2HOST_PORT_623; 4883 manc2h |= E1000_MNG2HOST_PORT_664; 4884 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4885 } 4886 4887 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4888 } 4889} 4890 4891/* 4892 * Give control back to hardware management 4893 * controller if there is one. 4894 */ 4895static void 4896em_release_manageability(struct adapter *adapter) 4897{ 4898 if (adapter->has_manage) { 4899 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4900 4901 /* re-enable hardware interception of ARP */ 4902 manc |= E1000_MANC_ARP_EN; 4903 4904 if (adapter->hw.mac.type >= e1000_82571) 4905 manc &= ~E1000_MANC_EN_MNG2HOST; 4906 4907 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4908 } 4909} 4910 4911/* 4912 * em_get_hw_control sets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4913 * For ASF and Pass Through versions of f/w this means that 4914 * the driver is loaded. For AMT version (only with 82573) 4915 * of the f/w this means that the network i/f is open. 4916 * 4917 */ 4918static void 4919em_get_hw_control(struct adapter *adapter) 4920{ 4921 u32 ctrl_ext, swsm; 4922 4923 /* Let firmware know the driver has taken over */ 4924 switch (adapter->hw.mac.type) { 4925 case e1000_82573: 4926 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4927 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4928 swsm | E1000_SWSM_DRV_LOAD); 4929 break; 4930 case e1000_82571: 4931 case e1000_82572: 4932 case e1000_80003es2lan: 4933 case e1000_ich8lan: 4934 case e1000_ich9lan: 4935 case e1000_ich10lan: 4936 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4937 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4938 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4939 break; 4940 default: 4941 break; 4942 } 4943} 4944 4945/* 4946 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4947 * For ASF and Pass Through versions of f/w this means that the 4948 * driver is no longer loaded. For AMT version (only with 82573) i 4949 * of the f/w this means that the network i/f is closed. 4950 * 4951 */ 4952static void 4953em_release_hw_control(struct adapter *adapter) 4954{ 4955 u32 ctrl_ext, swsm; 4956 4957 /* Let firmware taken over control of h/w */ 4958 switch (adapter->hw.mac.type) { 4959 case e1000_82573: 4960 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4961 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4962 swsm & ~E1000_SWSM_DRV_LOAD); 4963 break; 4964 case e1000_82571: 4965 case e1000_82572: 4966 case e1000_80003es2lan: 4967 case e1000_ich8lan: 4968 case e1000_ich9lan: 4969 case e1000_ich10lan: 4970 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4971 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4972 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4973 break; 4974 default: 4975 break; 4976 4977 } 4978} 4979 4980static int 4981em_is_valid_ether_addr(u8 *addr) 4982{ 4983 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4984 4985 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4986 return (FALSE); 4987 } 4988 4989 return (TRUE); 4990} 4991 4992/* 4993 * Enable PCI Wake On Lan capability 4994 */ 4995void 4996em_enable_wakeup(device_t dev) 4997{ 4998 u16 cap, status; 4999 u8 id; 5000 5001 /* First find the capabilities pointer*/ 5002 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 5003 /* Read the PM Capabilities */ 5004 id = pci_read_config(dev, cap, 1); 5005 if (id != PCIY_PMG) /* Something wrong */ 5006 return; 5007 /* OK, we have the power capabilities, so 5008 now get the status register */ 5009 cap += PCIR_POWER_STATUS; 5010 status = pci_read_config(dev, cap, 2); 5011 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 5012 pci_write_config(dev, cap, status, 2); 5013 return; 5014} 5015 5016 5017/********************************************************************* 5018* 82544 Coexistence issue workaround. 5019* There are 2 issues. 5020* 1. Transmit Hang issue. 5021* To detect this issue, following equation can be used... 5022* SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 5023* If SUM[3:0] is in between 1 to 4, we will have this issue. 5024* 5025* 2. DAC issue. 5026* To detect this issue, following equation can be used... 5027* SIZE[3:0] + ADDR[2:0] = SUM[3:0]. 5028* If SUM[3:0] is in between 9 to c, we will have this issue. 5029* 5030* 5031* WORKAROUND: 5032* Make sure we do not have ending address 5033* as 1,2,3,4(Hang) or 9,a,b,c (DAC) 5034* 5035*************************************************************************/ 5036static u32 5037em_fill_descriptors (bus_addr_t address, u32 length, 5038 PDESC_ARRAY desc_array) 5039{ 5040 u32 safe_terminator; 5041 5042 /* Since issue is sensitive to length and address.*/ 5043 /* Let us first check the address...*/ 5044 if (length <= 4) { 5045 desc_array->descriptor[0].address = address; 5046 desc_array->descriptor[0].length = length; 5047 desc_array->elements = 1; 5048 return (desc_array->elements); 5049 } 5050 safe_terminator = (u32)((((u32)address & 0x7) + 5051 (length & 0xF)) & 0xF); 5052 /* if it does not fall between 0x1 to 0x4 and 0x9 to 0xC then return */ 5053 if (safe_terminator == 0 || 5054 (safe_terminator > 4 && 5055 safe_terminator < 9) || 5056 (safe_terminator > 0xC && 5057 safe_terminator <= 0xF)) { 5058 desc_array->descriptor[0].address = address; 5059 desc_array->descriptor[0].length = length; 5060 desc_array->elements = 1; 5061 return (desc_array->elements); 5062 } 5063 5064 desc_array->descriptor[0].address = address; 5065 desc_array->descriptor[0].length = length - 4; 5066 desc_array->descriptor[1].address = address + (length - 4); 5067 desc_array->descriptor[1].length = 4; 5068 desc_array->elements = 2; 5069 return (desc_array->elements); 5070} 5071 5072/********************************************************************** 5073 * 5074 * Update the board statistics counters. 5075 * 5076 **********************************************************************/ 5077static void 5078em_update_stats_counters(struct adapter *adapter) 5079{ 5080 struct ifnet *ifp; 5081 5082 if(adapter->hw.phy.media_type == e1000_media_type_copper || 5083 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 5084 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 5085 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 5086 } 5087 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 5088 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 5089 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 5090 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 5091 5092 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 5093 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 5094 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 5095 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 5096 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 5097 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 5098 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 5099 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 5100 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 5101 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 5102 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 5103 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 5104 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 5105 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 5106 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 5107 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 5108 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 5109 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 5110 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 5111 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 5112 5113 /* For the 64-bit byte counters the low dword must be read first. */ 5114 /* Both registers clear on the read of the high dword */ 5115 5116 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 5117 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 5118 5119 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 5120 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 5121 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 5122 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 5123 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 5124 5125 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 5126 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 5127 5128 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 5129 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 5130 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 5131 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 5132 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 5133 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 5134 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 5135 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 5136 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 5137 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 5138 5139 if (adapter->hw.mac.type >= e1000_82543) { 5140 adapter->stats.algnerrc += 5141 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 5142 adapter->stats.rxerrc += 5143 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 5144 adapter->stats.tncrs += 5145 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 5146 adapter->stats.cexterr += 5147 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 5148 adapter->stats.tsctc += 5149 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 5150 adapter->stats.tsctfc += 5151 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 5152 } 5153 ifp = adapter->ifp; 5154 5155 ifp->if_collisions = adapter->stats.colc; 5156 5157 /* Rx Errors */ 5158 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 5159 adapter->stats.crcerrs + adapter->stats.algnerrc + 5160 adapter->stats.ruc + adapter->stats.roc + 5161 adapter->stats.mpc + adapter->stats.cexterr; 5162 5163 /* Tx Errors */ 5164 ifp->if_oerrors = adapter->stats.ecol + 5165 adapter->stats.latecol + adapter->watchdog_events; 5166} 5167 5168 5169/********************************************************************** 5170 * 5171 * This routine is called only when em_display_debug_stats is enabled. 5172 * This routine provides a way to take a look at important statistics 5173 * maintained by the driver and hardware. 5174 * 5175 **********************************************************************/ 5176static void 5177em_print_debug_info(struct adapter *adapter) 5178{ 5179 device_t dev = adapter->dev; 5180 u8 *hw_addr = adapter->hw.hw_addr; 5181 5182 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 5183 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 5184 E1000_READ_REG(&adapter->hw, E1000_CTRL), 5185 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 5186 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 5187 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 5188 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 5189 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 5190 adapter->hw.fc.high_water, 5191 adapter->hw.fc.low_water); 5192 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 5193 E1000_READ_REG(&adapter->hw, E1000_TIDV), 5194 E1000_READ_REG(&adapter->hw, E1000_TADV)); 5195 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 5196 E1000_READ_REG(&adapter->hw, E1000_RDTR), 5197 E1000_READ_REG(&adapter->hw, E1000_RADV)); 5198 device_printf(dev, "fifo workaround = %lld, fifo_reset_count = %lld\n", 5199 (long long)adapter->tx_fifo_wrk_cnt, 5200 (long long)adapter->tx_fifo_reset_cnt); 5201 device_printf(dev, "hw tdh = %d, hw tdt = %d\n", 5202 E1000_READ_REG(&adapter->hw, E1000_TDH(0)), 5203 E1000_READ_REG(&adapter->hw, E1000_TDT(0))); 5204 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 5205 E1000_READ_REG(&adapter->hw, E1000_RDH(0)), 5206 E1000_READ_REG(&adapter->hw, E1000_RDT(0))); 5207 device_printf(dev, "Num Tx descriptors avail = %d\n", 5208 adapter->num_tx_desc_avail); 5209 device_printf(dev, "Tx Descriptors not avail1 = %ld\n", 5210 adapter->no_tx_desc_avail1); 5211 device_printf(dev, "Tx Descriptors not avail2 = %ld\n", 5212 adapter->no_tx_desc_avail2); 5213 device_printf(dev, "Std mbuf failed = %ld\n", 5214 adapter->mbuf_alloc_failed); 5215 device_printf(dev, "Std mbuf cluster failed = %ld\n", 5216 adapter->mbuf_cluster_failed); 5217 device_printf(dev, "Driver dropped packets = %ld\n", 5218 adapter->dropped_pkts); 5219 device_printf(dev, "Driver tx dma failure in encap = %ld\n", 5220 adapter->no_tx_dma_setup); 5221} 5222 5223static void 5224em_print_hw_stats(struct adapter *adapter) 5225{ 5226 device_t dev = adapter->dev; 5227 5228 device_printf(dev, "Excessive collisions = %lld\n", 5229 (long long)adapter->stats.ecol); 5230#if (DEBUG_HW > 0) /* Dont output these errors normally */ 5231 device_printf(dev, "Symbol errors = %lld\n", 5232 (long long)adapter->stats.symerrs); 5233#endif 5234 device_printf(dev, "Sequence errors = %lld\n", 5235 (long long)adapter->stats.sec); 5236 device_printf(dev, "Defer count = %lld\n", 5237 (long long)adapter->stats.dc); 5238 device_printf(dev, "Missed Packets = %lld\n", 5239 (long long)adapter->stats.mpc); 5240 device_printf(dev, "Receive No Buffers = %lld\n", 5241 (long long)adapter->stats.rnbc); 5242 /* RLEC is inaccurate on some hardware, calculate our own. */ 5243 device_printf(dev, "Receive Length Errors = %lld\n", 5244 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 5245 device_printf(dev, "Receive errors = %lld\n", 5246 (long long)adapter->stats.rxerrc); 5247 device_printf(dev, "Crc errors = %lld\n", 5248 (long long)adapter->stats.crcerrs); 5249 device_printf(dev, "Alignment errors = %lld\n", 5250 (long long)adapter->stats.algnerrc); 5251 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 5252 (long long)adapter->stats.cexterr); 5253 device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); 5254 device_printf(dev, "watchdog timeouts = %ld\n", 5255 adapter->watchdog_events); 5256 device_printf(dev, "RX MSIX IRQ = %ld TX MSIX IRQ = %ld" 5257 " LINK MSIX IRQ = %ld\n", adapter->rx_irq, 5258 adapter->tx_irq , adapter->link_irq); 5259 device_printf(dev, "XON Rcvd = %lld\n", 5260 (long long)adapter->stats.xonrxc); 5261 device_printf(dev, "XON Xmtd = %lld\n", 5262 (long long)adapter->stats.xontxc); 5263 device_printf(dev, "XOFF Rcvd = %lld\n", 5264 (long long)adapter->stats.xoffrxc); 5265 device_printf(dev, "XOFF Xmtd = %lld\n", 5266 (long long)adapter->stats.xofftxc); 5267 device_printf(dev, "Good Packets Rcvd = %lld\n", 5268 (long long)adapter->stats.gprc); 5269 device_printf(dev, "Good Packets Xmtd = %lld\n", 5270 (long long)adapter->stats.gptc); 5271 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 5272 (long long)adapter->stats.tsctc); 5273 device_printf(dev, "TSO Contexts Failed = %lld\n", 5274 (long long)adapter->stats.tsctfc); 5275} 5276 5277/********************************************************************** 5278 * 5279 * This routine provides a way to dump out the adapter eeprom, 5280 * often a useful debug/service tool. This only dumps the first 5281 * 32 words, stuff that matters is in that extent. 5282 * 5283 **********************************************************************/ 5284static void 5285em_print_nvm_info(struct adapter *adapter) 5286{ 5287 u16 eeprom_data; 5288 int i, j, row = 0; 5289 5290 /* Its a bit crude, but it gets the job done */ 5291 printf("\nInterface EEPROM Dump:\n"); 5292 printf("Offset\n0x0000 "); 5293 for (i = 0, j = 0; i < 32; i++, j++) { 5294 if (j == 8) { /* Make the offset block */ 5295 j = 0; ++row; 5296 printf("\n0x00%x0 ",row); 5297 } 5298 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 5299 printf("%04x ", eeprom_data); 5300 } 5301 printf("\n"); 5302} 5303 5304static int 5305em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 5306{ 5307 struct adapter *adapter; 5308 int error; 5309 int result; 5310 5311 result = -1; 5312 error = sysctl_handle_int(oidp, &result, 0, req); 5313 5314 if (error || !req->newptr) 5315 return (error); 5316 5317 if (result == 1) { 5318 adapter = (struct adapter *)arg1; 5319 em_print_debug_info(adapter); 5320 } 5321 /* 5322 * This value will cause a hex dump of the 5323 * first 32 16-bit words of the EEPROM to 5324 * the screen. 5325 */ 5326 if (result == 2) { 5327 adapter = (struct adapter *)arg1; 5328 em_print_nvm_info(adapter); 5329 } 5330 5331 return (error); 5332} 5333 5334 5335static int 5336em_sysctl_stats(SYSCTL_HANDLER_ARGS) 5337{ 5338 struct adapter *adapter; 5339 int error; 5340 int result; 5341 5342 result = -1; 5343 error = sysctl_handle_int(oidp, &result, 0, req); 5344 5345 if (error || !req->newptr) 5346 return (error); 5347 5348 if (result == 1) { 5349 adapter = (struct adapter *)arg1; 5350 em_print_hw_stats(adapter); 5351 } 5352 5353 return (error); 5354} 5355 5356static int 5357em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 5358{ 5359 struct em_int_delay_info *info; 5360 struct adapter *adapter; 5361 u32 regval; 5362 int error; 5363 int usecs; 5364 int ticks; 5365 5366 info = (struct em_int_delay_info *)arg1; 5367 usecs = info->value; 5368 error = sysctl_handle_int(oidp, &usecs, 0, req); 5369 if (error != 0 || req->newptr == NULL) 5370 return (error); 5371 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 5372 return (EINVAL); 5373 info->value = usecs; 5374 ticks = EM_USECS_TO_TICKS(usecs); 5375 5376 adapter = info->adapter; 5377 5378 EM_CORE_LOCK(adapter); 5379 regval = E1000_READ_OFFSET(&adapter->hw, info->offset); 5380 regval = (regval & ~0xffff) | (ticks & 0xffff); 5381 /* Handle a few special cases. */ 5382 switch (info->offset) { 5383 case E1000_RDTR: 5384 break; 5385 case E1000_TIDV: 5386 if (ticks == 0) { 5387 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; 5388 /* Don't write 0 into the TIDV register. */ 5389 regval++; 5390 } else 5391 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 5392 break; 5393 } 5394 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); 5395 EM_CORE_UNLOCK(adapter); 5396 return (0); 5397} 5398 5399static void 5400em_add_int_delay_sysctl(struct adapter *adapter, const char *name, 5401 const char *description, struct em_int_delay_info *info, 5402 int offset, int value) 5403{ 5404 info->adapter = adapter; 5405 info->offset = offset; 5406 info->value = value; 5407 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), 5408 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5409 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, 5410 info, 0, em_sysctl_int_delay, "I", description); 5411} 5412 5413#ifndef EM_LEGACY_IRQ 5414static void 5415em_add_rx_process_limit(struct adapter *adapter, const char *name, 5416 const char *description, int *limit, int value) 5417{ 5418 *limit = value; 5419 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5420 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5421 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5422} 5423#endif 5424 5425#ifdef EM_TIMESYNC 5426/* 5427 * Initialize the Time Sync Feature 5428 */ 5429static int 5430em_tsync_init(struct adapter *adapter) 5431{ 5432 device_t dev = adapter->dev; 5433 u32 tx_ctl, rx_ctl; 5434 5435 5436 E1000_WRITE_REG(&adapter->hw, E1000_TIMINCA, (1<<24) | 5437 20833/PICOSECS_PER_TICK); 5438 5439 adapter->last_stamp = E1000_READ_REG(&adapter->hw, E1000_SYSTIML); 5440 adapter->last_stamp |= (u64)E1000_READ_REG(&adapter->hw, 5441 E1000_SYSTIMH) << 32ULL; 5442 5443 /* Enable the TX side */ 5444 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5445 tx_ctl |= 0x10; 5446 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl); 5447 E1000_WRITE_FLUSH(&adapter->hw); 5448 5449 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5450 if ((tx_ctl & 0x10) == 0) { 5451 device_printf(dev, "Failed to enable TX timestamping\n"); 5452 return (ENXIO); 5453 } 5454 5455 /* Enable RX */ 5456 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5457 rx_ctl |= 0x10; /* Enable the feature */ 5458 rx_ctl |= 0x0a; /* This value turns on Ver 1 and 2 */ 5459 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl); 5460 5461 /* 5462 * Ethertype Stamping (Ethertype = 0x88F7) 5463 */ 5464 E1000_WRITE_REG(&adapter->hw, E1000_RXMTRL, htonl(0x440088f7)); 5465 5466 /* 5467 * Source Port Queue Filter Setup: 5468 * this is for UDP port filtering 5469 */ 5470 E1000_WRITE_REG(&adapter->hw, E1000_RXUDP, htons(TSYNC_PORT)); 5471 /* Protocol = UDP, enable Timestamp, and filter on source/protocol */ 5472 5473 E1000_WRITE_FLUSH(&adapter->hw); 5474 5475 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5476 if ((rx_ctl & 0x10) == 0) { 5477 device_printf(dev, "Failed to enable RX timestamping\n"); 5478 return (ENXIO); 5479 } 5480 5481 device_printf(dev, "IEEE 1588 Precision Time Protocol enabled\n"); 5482 5483 return (0); 5484} 5485 5486/* 5487 * Disable the Time Sync Feature 5488 */ 5489static void 5490em_tsync_disable(struct adapter *adapter) 5491{ 5492 u32 tx_ctl, rx_ctl; 5493 5494 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5495 tx_ctl &= ~0x10; 5496 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl); 5497 E1000_WRITE_FLUSH(&adapter->hw); 5498 5499 /* Invalidate TX Timestamp */ 5500 E1000_READ_REG(&adapter->hw, E1000_TXSTMPH); 5501 5502 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 5503 if (tx_ctl & 0x10) 5504 HW_DEBUGOUT("Failed to disable TX timestamping\n"); 5505 5506 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5507 rx_ctl &= ~0x10; 5508 5509 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl); 5510 E1000_WRITE_FLUSH(&adapter->hw); 5511 5512 /* Invalidate RX Timestamp */ 5513 E1000_READ_REG(&adapter->hw, E1000_RXSATRH); 5514 5515 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 5516 if (rx_ctl & 0x10) 5517 HW_DEBUGOUT("Failed to disable RX timestamping\n"); 5518 5519 return; 5520} 5521#endif /* EM_TIMESYNC */ 5522