if_em.c revision 205884
1/****************************************************************************** 2 3 Copyright (c) 2001-2010, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ 33/*$FreeBSD: head/sys/dev/e1000/if_em.c 205884 2010-03-30 17:05:24Z jfv $*/ 34 35#ifdef HAVE_KERNEL_OPTION_HEADERS 36#include "opt_device_polling.h" 37#include "opt_inet.h" 38#endif 39 40#include <sys/param.h> 41#include <sys/systm.h> 42#if __FreeBSD_version >= 800000 43#include <sys/buf_ring.h> 44#endif 45#include <sys/bus.h> 46#include <sys/endian.h> 47#include <sys/kernel.h> 48#include <sys/kthread.h> 49#include <sys/malloc.h> 50#include <sys/mbuf.h> 51#include <sys/module.h> 52#include <sys/rman.h> 53#include <sys/socket.h> 54#include <sys/sockio.h> 55#include <sys/sysctl.h> 56#include <sys/taskqueue.h> 57#include <sys/eventhandler.h> 58#include <machine/bus.h> 59#include <machine/resource.h> 60 61#include <net/bpf.h> 62#include <net/ethernet.h> 63#include <net/if.h> 64#include <net/if_arp.h> 65#include <net/if_dl.h> 66#include <net/if_media.h> 67 68#include <net/if_types.h> 69#include <net/if_vlan_var.h> 70 71#include <netinet/in_systm.h> 72#include <netinet/in.h> 73#include <netinet/if_ether.h> 74#include <netinet/ip.h> 75#include <netinet/ip6.h> 76#include <netinet/tcp.h> 77#include <netinet/udp.h> 78 79#include <machine/in_cksum.h> 80#include <dev/pci/pcivar.h> 81#include <dev/pci/pcireg.h> 82 83#include "e1000_api.h" 84#include "e1000_82571.h" 85#include "if_em.h" 86 87/********************************************************************* 88 * Set this to one to display debug statistics 89 *********************************************************************/ 90int em_display_debug_stats = 0; 91 92/********************************************************************* 93 * Driver version: 94 *********************************************************************/ 95char em_driver_version[] = "7.0.0"; 96 97 98/********************************************************************* 99 * PCI Device ID Table 100 * 101 * Used by probe to select devices to load on 102 * Last field stores an index into e1000_strings 103 * Last entry must be all 0s 104 * 105 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 106 *********************************************************************/ 107 108static em_vendor_info_t em_vendor_info_array[] = 109{ 110 /* Intel(R) PRO/1000 Network Connection */ 111 { 0x8086, E1000_DEV_ID_82571EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 112 { 0x8086, E1000_DEV_ID_82571EB_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 113 { 0x8086, E1000_DEV_ID_82571EB_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 114 { 0x8086, E1000_DEV_ID_82571EB_SERDES_DUAL, 115 PCI_ANY_ID, PCI_ANY_ID, 0}, 116 { 0x8086, E1000_DEV_ID_82571EB_SERDES_QUAD, 117 PCI_ANY_ID, PCI_ANY_ID, 0}, 118 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER, 119 PCI_ANY_ID, PCI_ANY_ID, 0}, 120 { 0x8086, E1000_DEV_ID_82571EB_QUAD_COPPER_LP, 121 PCI_ANY_ID, PCI_ANY_ID, 0}, 122 { 0x8086, E1000_DEV_ID_82571EB_QUAD_FIBER, 123 PCI_ANY_ID, PCI_ANY_ID, 0}, 124 { 0x8086, E1000_DEV_ID_82571PT_QUAD_COPPER, 125 PCI_ANY_ID, PCI_ANY_ID, 0}, 126 { 0x8086, E1000_DEV_ID_82572EI_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 127 { 0x8086, E1000_DEV_ID_82572EI_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 128 { 0x8086, E1000_DEV_ID_82572EI_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 129 { 0x8086, E1000_DEV_ID_82572EI, PCI_ANY_ID, PCI_ANY_ID, 0}, 130 131 { 0x8086, E1000_DEV_ID_82573E, PCI_ANY_ID, PCI_ANY_ID, 0}, 132 { 0x8086, E1000_DEV_ID_82573E_IAMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 133 { 0x8086, E1000_DEV_ID_82573L, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82583V, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_SPT, 136 PCI_ANY_ID, PCI_ANY_ID, 0}, 137 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_SPT, 138 PCI_ANY_ID, PCI_ANY_ID, 0}, 139 { 0x8086, E1000_DEV_ID_80003ES2LAN_COPPER_DPT, 140 PCI_ANY_ID, PCI_ANY_ID, 0}, 141 { 0x8086, E1000_DEV_ID_80003ES2LAN_SERDES_DPT, 142 PCI_ANY_ID, PCI_ANY_ID, 0}, 143 { 0x8086, E1000_DEV_ID_ICH8_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 144 { 0x8086, E1000_DEV_ID_ICH8_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 145 { 0x8086, E1000_DEV_ID_ICH8_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 146 { 0x8086, E1000_DEV_ID_ICH8_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 147 { 0x8086, E1000_DEV_ID_ICH8_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 148 { 0x8086, E1000_DEV_ID_ICH8_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 149 { 0x8086, E1000_DEV_ID_ICH8_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 150 { 0x8086, E1000_DEV_ID_ICH8_82567V_3, PCI_ANY_ID, PCI_ANY_ID, 0}, 151 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 152 { 0x8086, E1000_DEV_ID_ICH9_IGP_AMT, PCI_ANY_ID, PCI_ANY_ID, 0}, 153 { 0x8086, E1000_DEV_ID_ICH9_IGP_C, PCI_ANY_ID, PCI_ANY_ID, 0}, 154 { 0x8086, E1000_DEV_ID_ICH9_IGP_M, PCI_ANY_ID, PCI_ANY_ID, 0}, 155 { 0x8086, E1000_DEV_ID_ICH9_IGP_M_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 156 { 0x8086, E1000_DEV_ID_ICH9_IFE, PCI_ANY_ID, PCI_ANY_ID, 0}, 157 { 0x8086, E1000_DEV_ID_ICH9_IFE_GT, PCI_ANY_ID, PCI_ANY_ID, 0}, 158 { 0x8086, E1000_DEV_ID_ICH9_IFE_G, PCI_ANY_ID, PCI_ANY_ID, 0}, 159 { 0x8086, E1000_DEV_ID_ICH9_BM, PCI_ANY_ID, PCI_ANY_ID, 0}, 160 { 0x8086, E1000_DEV_ID_82574L, PCI_ANY_ID, PCI_ANY_ID, 0}, 161 { 0x8086, E1000_DEV_ID_82574LA, PCI_ANY_ID, PCI_ANY_ID, 0}, 162 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 163 { 0x8086, E1000_DEV_ID_ICH10_R_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 164 { 0x8086, E1000_DEV_ID_ICH10_R_BM_V, PCI_ANY_ID, PCI_ANY_ID, 0}, 165 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 166 { 0x8086, E1000_DEV_ID_ICH10_D_BM_LF, PCI_ANY_ID, PCI_ANY_ID, 0}, 167 { 0x8086, E1000_DEV_ID_PCH_M_HV_LM, PCI_ANY_ID, PCI_ANY_ID, 0}, 168 { 0x8086, E1000_DEV_ID_PCH_M_HV_LC, PCI_ANY_ID, PCI_ANY_ID, 0}, 169 { 0x8086, E1000_DEV_ID_PCH_D_HV_DM, PCI_ANY_ID, PCI_ANY_ID, 0}, 170 { 0x8086, E1000_DEV_ID_PCH_D_HV_DC, PCI_ANY_ID, PCI_ANY_ID, 0}, 171 /* required last entry */ 172 { 0, 0, 0, 0, 0} 173}; 174 175/********************************************************************* 176 * Table of branding strings for all supported NICs. 177 *********************************************************************/ 178 179static char *em_strings[] = { 180 "Intel(R) PRO/1000 Network Connection" 181}; 182 183/********************************************************************* 184 * Function prototypes 185 *********************************************************************/ 186static int em_probe(device_t); 187static int em_attach(device_t); 188static int em_detach(device_t); 189static int em_shutdown(device_t); 190static int em_suspend(device_t); 191static int em_resume(device_t); 192static void em_start(struct ifnet *); 193static void em_start_locked(struct ifnet *, struct tx_ring *); 194#if __FreeBSD_version >= 800000 195static int em_mq_start(struct ifnet *, struct mbuf *); 196static int em_mq_start_locked(struct ifnet *, 197 struct tx_ring *, struct mbuf *); 198static void em_qflush(struct ifnet *); 199#endif 200static int em_ioctl(struct ifnet *, u_long, caddr_t); 201static void em_init(void *); 202static void em_init_locked(struct adapter *); 203static void em_stop(void *); 204static void em_media_status(struct ifnet *, struct ifmediareq *); 205static int em_media_change(struct ifnet *); 206static void em_identify_hardware(struct adapter *); 207static int em_allocate_pci_resources(struct adapter *); 208static int em_allocate_legacy(struct adapter *); 209static int em_allocate_msix(struct adapter *); 210static int em_allocate_queues(struct adapter *); 211static int em_setup_msix(struct adapter *); 212static void em_free_pci_resources(struct adapter *); 213static void em_local_timer(void *); 214static void em_reset(struct adapter *); 215static void em_setup_interface(device_t, struct adapter *); 216 217static void em_setup_transmit_structures(struct adapter *); 218static void em_initialize_transmit_unit(struct adapter *); 219static int em_allocate_transmit_buffers(struct tx_ring *); 220static void em_free_transmit_structures(struct adapter *); 221static void em_free_transmit_buffers(struct tx_ring *); 222 223static int em_setup_receive_structures(struct adapter *); 224static int em_allocate_receive_buffers(struct rx_ring *); 225static void em_initialize_receive_unit(struct adapter *); 226static void em_free_receive_structures(struct adapter *); 227static void em_free_receive_buffers(struct rx_ring *); 228 229static void em_enable_intr(struct adapter *); 230static void em_disable_intr(struct adapter *); 231static void em_update_stats_counters(struct adapter *); 232static bool em_txeof(struct tx_ring *); 233static int em_rxeof(struct rx_ring *, int); 234#ifndef __NO_STRICT_ALIGNMENT 235static int em_fixup_rx(struct rx_ring *); 236#endif 237static void em_receive_checksum(struct e1000_rx_desc *, struct mbuf *); 238static void em_transmit_checksum_setup(struct tx_ring *, struct mbuf *, 239 u32 *, u32 *); 240static bool em_tso_setup(struct tx_ring *, struct mbuf *, u32 *, u32 *); 241static void em_set_promisc(struct adapter *); 242static void em_disable_promisc(struct adapter *); 243static void em_set_multi(struct adapter *); 244static void em_print_hw_stats(struct adapter *); 245static void em_update_link_status(struct adapter *); 246static void em_refresh_mbufs(struct rx_ring *, int); 247static void em_register_vlan(void *, struct ifnet *, u16); 248static void em_unregister_vlan(void *, struct ifnet *, u16); 249static void em_setup_vlan_hw_support(struct adapter *); 250static int em_xmit(struct tx_ring *, struct mbuf **); 251static int em_dma_malloc(struct adapter *, bus_size_t, 252 struct em_dma_alloc *, int); 253static void em_dma_free(struct adapter *, struct em_dma_alloc *); 254static void em_print_debug_info(struct adapter *); 255static void em_print_nvm_info(struct adapter *); 256static int em_is_valid_ether_addr(u8 *); 257static int em_sysctl_stats(SYSCTL_HANDLER_ARGS); 258static int em_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 259static int em_sysctl_int_delay(SYSCTL_HANDLER_ARGS); 260static void em_add_int_delay_sysctl(struct adapter *, const char *, 261 const char *, struct em_int_delay_info *, int, int); 262/* Management and WOL Support */ 263static void em_init_manageability(struct adapter *); 264static void em_release_manageability(struct adapter *); 265static void em_get_hw_control(struct adapter *); 266static void em_release_hw_control(struct adapter *); 267static void em_get_wakeup(device_t); 268static void em_enable_wakeup(device_t); 269static int em_enable_phy_wakeup(struct adapter *); 270 271static int em_irq_fast(void *); 272 273/* MSIX handlers */ 274static void em_msix_tx(void *); 275static void em_msix_rx(void *); 276static void em_msix_link(void *); 277static void em_handle_tx(void *context, int pending); 278static void em_handle_rx(void *context, int pending); 279static void em_handle_link(void *context, int pending); 280 281static void em_add_rx_process_limit(struct adapter *, const char *, 282 const char *, int *, int); 283 284#ifdef DEVICE_POLLING 285static poll_handler_t em_poll; 286#endif /* POLLING */ 287 288/********************************************************************* 289 * FreeBSD Device Interface Entry Points 290 *********************************************************************/ 291 292static device_method_t em_methods[] = { 293 /* Device interface */ 294 DEVMETHOD(device_probe, em_probe), 295 DEVMETHOD(device_attach, em_attach), 296 DEVMETHOD(device_detach, em_detach), 297 DEVMETHOD(device_shutdown, em_shutdown), 298 DEVMETHOD(device_suspend, em_suspend), 299 DEVMETHOD(device_resume, em_resume), 300 {0, 0} 301}; 302 303static driver_t em_driver = { 304 "em", em_methods, sizeof(struct adapter), 305}; 306 307devclass_t em_devclass; 308DRIVER_MODULE(em, pci, em_driver, em_devclass, 0, 0); 309MODULE_DEPEND(em, pci, 1, 1, 1); 310MODULE_DEPEND(em, ether, 1, 1, 1); 311 312/********************************************************************* 313 * Tunable default values. 314 *********************************************************************/ 315 316#define EM_TICKS_TO_USECS(ticks) ((1024 * (ticks) + 500) / 1000) 317#define EM_USECS_TO_TICKS(usecs) ((1000 * (usecs) + 512) / 1024) 318#define M_TSO_LEN 66 319 320/* Allow common code without TSO */ 321#ifndef CSUM_TSO 322#define CSUM_TSO 0 323#endif 324 325static int em_tx_int_delay_dflt = EM_TICKS_TO_USECS(EM_TIDV); 326static int em_rx_int_delay_dflt = EM_TICKS_TO_USECS(EM_RDTR); 327TUNABLE_INT("hw.em.tx_int_delay", &em_tx_int_delay_dflt); 328TUNABLE_INT("hw.em.rx_int_delay", &em_rx_int_delay_dflt); 329 330static int em_tx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_TADV); 331static int em_rx_abs_int_delay_dflt = EM_TICKS_TO_USECS(EM_RADV); 332TUNABLE_INT("hw.em.tx_abs_int_delay", &em_tx_abs_int_delay_dflt); 333TUNABLE_INT("hw.em.rx_abs_int_delay", &em_rx_abs_int_delay_dflt); 334 335static int em_rxd = EM_DEFAULT_RXD; 336static int em_txd = EM_DEFAULT_TXD; 337TUNABLE_INT("hw.em.rxd", &em_rxd); 338TUNABLE_INT("hw.em.txd", &em_txd); 339 340static int em_smart_pwr_down = FALSE; 341TUNABLE_INT("hw.em.smart_pwr_down", &em_smart_pwr_down); 342 343/* Controls whether promiscuous also shows bad packets */ 344static int em_debug_sbp = FALSE; 345TUNABLE_INT("hw.em.sbp", &em_debug_sbp); 346 347/* Local controls for MSI/MSIX */ 348static int em_enable_msix = TRUE; 349static int em_msix_queues = 2; /* for 82574, can be 1 or 2 */ 350TUNABLE_INT("hw.em.enable_msix", &em_enable_msix); 351TUNABLE_INT("hw.em.msix_queues", &em_msix_queues); 352 353/* How many packets rxeof tries to clean at a time */ 354static int em_rx_process_limit = 100; 355TUNABLE_INT("hw.em.rx_process_limit", &em_rx_process_limit); 356 357/* Flow control setting - default to FULL */ 358static int em_fc_setting = e1000_fc_full; 359TUNABLE_INT("hw.em.fc_setting", &em_fc_setting); 360 361/* 362** Shadow VFTA table, this is needed because 363** the real vlan filter table gets cleared during 364** a soft reset and the driver needs to be able 365** to repopulate it. 366*/ 367static u32 em_shadow_vfta[EM_VFTA_SIZE]; 368 369/* Global used in WOL setup with multiport cards */ 370static int global_quad_port_a = 0; 371 372/********************************************************************* 373 * Device identification routine 374 * 375 * em_probe determines if the driver should be loaded on 376 * adapter based on PCI vendor/device id of the adapter. 377 * 378 * return BUS_PROBE_DEFAULT on success, positive on failure 379 *********************************************************************/ 380 381static int 382em_probe(device_t dev) 383{ 384 char adapter_name[60]; 385 u16 pci_vendor_id = 0; 386 u16 pci_device_id = 0; 387 u16 pci_subvendor_id = 0; 388 u16 pci_subdevice_id = 0; 389 em_vendor_info_t *ent; 390 391 INIT_DEBUGOUT("em_probe: begin"); 392 393 pci_vendor_id = pci_get_vendor(dev); 394 if (pci_vendor_id != EM_VENDOR_ID) 395 return (ENXIO); 396 397 pci_device_id = pci_get_device(dev); 398 pci_subvendor_id = pci_get_subvendor(dev); 399 pci_subdevice_id = pci_get_subdevice(dev); 400 401 ent = em_vendor_info_array; 402 while (ent->vendor_id != 0) { 403 if ((pci_vendor_id == ent->vendor_id) && 404 (pci_device_id == ent->device_id) && 405 406 ((pci_subvendor_id == ent->subvendor_id) || 407 (ent->subvendor_id == PCI_ANY_ID)) && 408 409 ((pci_subdevice_id == ent->subdevice_id) || 410 (ent->subdevice_id == PCI_ANY_ID))) { 411 sprintf(adapter_name, "%s %s", 412 em_strings[ent->index], 413 em_driver_version); 414 device_set_desc_copy(dev, adapter_name); 415 return (BUS_PROBE_DEFAULT); 416 } 417 ent++; 418 } 419 420 return (ENXIO); 421} 422 423/********************************************************************* 424 * Device initialization routine 425 * 426 * The attach entry point is called when the driver is being loaded. 427 * This routine identifies the type of hardware, allocates all resources 428 * and initializes the hardware. 429 * 430 * return 0 on success, positive on failure 431 *********************************************************************/ 432 433static int 434em_attach(device_t dev) 435{ 436 struct adapter *adapter; 437 int error = 0; 438 439 INIT_DEBUGOUT("em_attach: begin"); 440 441 adapter = device_get_softc(dev); 442 adapter->dev = adapter->osdep.dev = dev; 443 EM_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 444 445 /* SYSCTL stuff */ 446 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 447 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 448 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 449 em_sysctl_debug_info, "I", "Debug Information"); 450 451 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 452 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 453 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 454 em_sysctl_stats, "I", "Statistics"); 455 456 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 457 458 /* Determine hardware and mac info */ 459 em_identify_hardware(adapter); 460 461 /* Setup PCI resources */ 462 if (em_allocate_pci_resources(adapter)) { 463 device_printf(dev, "Allocation of PCI resources failed\n"); 464 error = ENXIO; 465 goto err_pci; 466 } 467 468 /* 469 ** For ICH8 and family we need to 470 ** map the flash memory, and this 471 ** must happen after the MAC is 472 ** identified 473 */ 474 if ((adapter->hw.mac.type == e1000_ich8lan) || 475 (adapter->hw.mac.type == e1000_pchlan) || 476 (adapter->hw.mac.type == e1000_ich9lan) || 477 (adapter->hw.mac.type == e1000_ich10lan)) { 478 int rid = EM_BAR_TYPE_FLASH; 479 adapter->flash = bus_alloc_resource_any(dev, 480 SYS_RES_MEMORY, &rid, RF_ACTIVE); 481 if (adapter->flash == NULL) { 482 device_printf(dev, "Mapping of Flash failed\n"); 483 error = ENXIO; 484 goto err_pci; 485 } 486 /* This is used in the shared code */ 487 adapter->hw.flash_address = (u8 *)adapter->flash; 488 adapter->osdep.flash_bus_space_tag = 489 rman_get_bustag(adapter->flash); 490 adapter->osdep.flash_bus_space_handle = 491 rman_get_bushandle(adapter->flash); 492 } 493 494 /* Do Shared Code initialization */ 495 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 496 device_printf(dev, "Setup of Shared code failed\n"); 497 error = ENXIO; 498 goto err_pci; 499 } 500 501 e1000_get_bus_info(&adapter->hw); 502 503 /* Set up some sysctls for the tunable interrupt delays */ 504 em_add_int_delay_sysctl(adapter, "rx_int_delay", 505 "receive interrupt delay in usecs", &adapter->rx_int_delay, 506 E1000_REGISTER(&adapter->hw, E1000_RDTR), em_rx_int_delay_dflt); 507 em_add_int_delay_sysctl(adapter, "tx_int_delay", 508 "transmit interrupt delay in usecs", &adapter->tx_int_delay, 509 E1000_REGISTER(&adapter->hw, E1000_TIDV), em_tx_int_delay_dflt); 510 em_add_int_delay_sysctl(adapter, "rx_abs_int_delay", 511 "receive interrupt delay limit in usecs", 512 &adapter->rx_abs_int_delay, 513 E1000_REGISTER(&adapter->hw, E1000_RADV), 514 em_rx_abs_int_delay_dflt); 515 em_add_int_delay_sysctl(adapter, "tx_abs_int_delay", 516 "transmit interrupt delay limit in usecs", 517 &adapter->tx_abs_int_delay, 518 E1000_REGISTER(&adapter->hw, E1000_TADV), 519 em_tx_abs_int_delay_dflt); 520 521 /* Sysctls for limiting the amount of work done in the taskqueue */ 522 em_add_rx_process_limit(adapter, "rx_processing_limit", 523 "max number of rx packets to process", &adapter->rx_process_limit, 524 em_rx_process_limit); 525 526 /* 527 * Validate number of transmit and receive descriptors. It 528 * must not exceed hardware maximum, and must be multiple 529 * of E1000_DBA_ALIGN. 530 */ 531 if (((em_txd * sizeof(struct e1000_tx_desc)) % EM_DBA_ALIGN) != 0 || 532 (em_txd > EM_MAX_TXD) || (em_txd < EM_MIN_TXD)) { 533 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 534 EM_DEFAULT_TXD, em_txd); 535 adapter->num_tx_desc = EM_DEFAULT_TXD; 536 } else 537 adapter->num_tx_desc = em_txd; 538 539 if (((em_rxd * sizeof(struct e1000_rx_desc)) % EM_DBA_ALIGN) != 0 || 540 (em_rxd > EM_MAX_RXD) || (em_rxd < EM_MIN_RXD)) { 541 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 542 EM_DEFAULT_RXD, em_rxd); 543 adapter->num_rx_desc = EM_DEFAULT_RXD; 544 } else 545 adapter->num_rx_desc = em_rxd; 546 547 adapter->hw.mac.autoneg = DO_AUTO_NEG; 548 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 549 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 550 551 /* Copper options */ 552 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 553 adapter->hw.phy.mdix = AUTO_ALL_MODES; 554 adapter->hw.phy.disable_polarity_correction = FALSE; 555 adapter->hw.phy.ms_type = EM_MASTER_SLAVE; 556 } 557 558 /* 559 * Set the frame limits assuming 560 * standard ethernet sized frames. 561 */ 562 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 563 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 564 565 /* 566 * This controls when hardware reports transmit completion 567 * status. 568 */ 569 adapter->hw.mac.report_tx_early = 1; 570 571 /* 572 ** Get queue/ring memory 573 */ 574 if (em_allocate_queues(adapter)) { 575 error = ENOMEM; 576 goto err_pci; 577 } 578 579 /* 580 ** Start from a known state, this is 581 ** important in reading the nvm and 582 ** mac from that. 583 */ 584 e1000_reset_hw(&adapter->hw); 585 586 /* Make sure we have a good EEPROM before we read from it */ 587 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 588 /* 589 ** Some PCI-E parts fail the first check due to 590 ** the link being in sleep state, call it again, 591 ** if it fails a second time its a real issue. 592 */ 593 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 594 device_printf(dev, 595 "The EEPROM Checksum Is Not Valid\n"); 596 error = EIO; 597 goto err_late; 598 } 599 } 600 601 /* Copy the permanent MAC address out of the EEPROM */ 602 if (e1000_read_mac_addr(&adapter->hw) < 0) { 603 device_printf(dev, "EEPROM read error while reading MAC" 604 " address\n"); 605 error = EIO; 606 goto err_late; 607 } 608 609 if (!em_is_valid_ether_addr(adapter->hw.mac.addr)) { 610 device_printf(dev, "Invalid MAC address\n"); 611 error = EIO; 612 goto err_late; 613 } 614 615 /* 616 ** Do interrupt configuration 617 */ 618 if (adapter->msix > 1) /* Do MSIX */ 619 error = em_allocate_msix(adapter); 620 else /* MSI or Legacy */ 621 error = em_allocate_legacy(adapter); 622 if (error) 623 goto err_late; 624 625 /* 626 * Get Wake-on-Lan and Management info for later use 627 */ 628 em_get_wakeup(dev); 629 630 /* Setup OS specific network interface */ 631 em_setup_interface(dev, adapter); 632 633 em_reset(adapter); 634 635 /* Initialize statistics */ 636 em_update_stats_counters(adapter); 637 638 adapter->hw.mac.get_link_status = 1; 639 em_update_link_status(adapter); 640 641 /* Indicate SOL/IDER usage */ 642 if (e1000_check_reset_block(&adapter->hw)) 643 device_printf(dev, 644 "PHY reset is blocked due to SOL/IDER session.\n"); 645 646 /* Register for VLAN events */ 647 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 648 em_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 649 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 650 em_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 651 652 /* Non-AMT based hardware can now take control from firmware */ 653 if (adapter->has_manage && !adapter->has_amt) 654 em_get_hw_control(adapter); 655 656 /* Tell the stack that the interface is not active */ 657 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 658 659 INIT_DEBUGOUT("em_attach: end"); 660 661 return (0); 662 663err_late: 664 em_free_transmit_structures(adapter); 665 em_free_receive_structures(adapter); 666 em_release_hw_control(adapter); 667err_pci: 668 em_free_pci_resources(adapter); 669 EM_CORE_LOCK_DESTROY(adapter); 670 671 return (error); 672} 673 674/********************************************************************* 675 * Device removal routine 676 * 677 * The detach entry point is called when the driver is being removed. 678 * This routine stops the adapter and deallocates all the resources 679 * that were allocated for driver operation. 680 * 681 * return 0 on success, positive on failure 682 *********************************************************************/ 683 684static int 685em_detach(device_t dev) 686{ 687 struct adapter *adapter = device_get_softc(dev); 688 struct ifnet *ifp = adapter->ifp; 689 690 INIT_DEBUGOUT("em_detach: begin"); 691 692 /* Make sure VLANS are not using driver */ 693 if (adapter->ifp->if_vlantrunk != NULL) { 694 device_printf(dev,"Vlan in use, detach first\n"); 695 return (EBUSY); 696 } 697 698#ifdef DEVICE_POLLING 699 if (ifp->if_capenable & IFCAP_POLLING) 700 ether_poll_deregister(ifp); 701#endif 702 703 EM_CORE_LOCK(adapter); 704 adapter->in_detach = 1; 705 em_stop(adapter); 706 EM_CORE_UNLOCK(adapter); 707 EM_CORE_LOCK_DESTROY(adapter); 708 709 e1000_phy_hw_reset(&adapter->hw); 710 711 em_release_manageability(adapter); 712 em_release_hw_control(adapter); 713 714 /* Unregister VLAN events */ 715 if (adapter->vlan_attach != NULL) 716 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 717 if (adapter->vlan_detach != NULL) 718 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 719 720 ether_ifdetach(adapter->ifp); 721 callout_drain(&adapter->timer); 722 723 em_free_pci_resources(adapter); 724 bus_generic_detach(dev); 725 if_free(ifp); 726 727 em_free_transmit_structures(adapter); 728 em_free_receive_structures(adapter); 729 730 em_release_hw_control(adapter); 731 732 return (0); 733} 734 735/********************************************************************* 736 * 737 * Shutdown entry point 738 * 739 **********************************************************************/ 740 741static int 742em_shutdown(device_t dev) 743{ 744 return em_suspend(dev); 745} 746 747/* 748 * Suspend/resume device methods. 749 */ 750static int 751em_suspend(device_t dev) 752{ 753 struct adapter *adapter = device_get_softc(dev); 754 755 EM_CORE_LOCK(adapter); 756 757 em_release_manageability(adapter); 758 em_release_hw_control(adapter); 759 em_enable_wakeup(dev); 760 761 EM_CORE_UNLOCK(adapter); 762 763 return bus_generic_suspend(dev); 764} 765 766static int 767em_resume(device_t dev) 768{ 769 struct adapter *adapter = device_get_softc(dev); 770 struct ifnet *ifp = adapter->ifp; 771 772 EM_CORE_LOCK(adapter); 773 em_init_locked(adapter); 774 em_init_manageability(adapter); 775 EM_CORE_UNLOCK(adapter); 776 em_start(ifp); 777 778 return bus_generic_resume(dev); 779} 780 781 782/********************************************************************* 783 * Transmit entry point 784 * 785 * em_start is called by the stack to initiate a transmit. 786 * The driver will remain in this routine as long as there are 787 * packets to transmit and transmit resources are available. 788 * In case resources are not available stack is notified and 789 * the packet is requeued. 790 **********************************************************************/ 791 792#if __FreeBSD_version >= 800000 793static int 794em_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 795{ 796 struct adapter *adapter = txr->adapter; 797 struct mbuf *next; 798 int err = 0, enq = 0; 799 800 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 801 IFF_DRV_RUNNING || adapter->link_active == 0) { 802 if (m != NULL) 803 err = drbr_enqueue(ifp, txr->br, m); 804 return (err); 805 } 806 807 enq = 0; 808 if (m == NULL) 809 next = drbr_dequeue(ifp, txr->br); 810 else 811 next = m; 812 813 /* Process the queue */ 814 while (next != NULL) { 815 if ((err = em_xmit(txr, &next)) != 0) { 816 if (next != NULL) 817 err = drbr_enqueue(ifp, txr->br, next); 818 break; 819 } 820 enq++; 821 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 822 ETHER_BPF_MTAP(ifp, next); 823 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 824 break; 825 next = drbr_dequeue(ifp, txr->br); 826 } 827 828 if (enq > 0) { 829 /* Set the watchdog */ 830 txr->watchdog_check = TRUE; 831 } 832 return (err); 833} 834 835/* 836** Multiqueue capable stack interface, this is not 837** yet truely multiqueue, but that is coming... 838*/ 839static int 840em_mq_start(struct ifnet *ifp, struct mbuf *m) 841{ 842 struct adapter *adapter = ifp->if_softc; 843 struct tx_ring *txr; 844 int i, error = 0; 845 846 /* Which queue to use */ 847 if ((m->m_flags & M_FLOWID) != 0) 848 i = m->m_pkthdr.flowid % adapter->num_queues; 849 else 850 i = curcpu % adapter->num_queues; 851 852 txr = &adapter->tx_rings[i]; 853 854 if (EM_TX_TRYLOCK(txr)) { 855 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 856 error = em_mq_start_locked(ifp, txr, m); 857 EM_TX_UNLOCK(txr); 858 } else 859 error = drbr_enqueue(ifp, txr->br, m); 860 861 return (error); 862} 863 864/* 865** Flush all ring buffers 866*/ 867static void 868em_qflush(struct ifnet *ifp) 869{ 870 struct adapter *adapter = ifp->if_softc; 871 struct tx_ring *txr = adapter->tx_rings; 872 struct mbuf *m; 873 874 for (int i = 0; i < adapter->num_queues; i++, txr++) { 875 EM_TX_LOCK(txr); 876 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 877 m_freem(m); 878 EM_TX_UNLOCK(txr); 879 } 880 if_qflush(ifp); 881} 882 883#endif /* FreeBSD_version */ 884 885static void 886em_start_locked(struct ifnet *ifp, struct tx_ring *txr) 887{ 888 struct adapter *adapter = ifp->if_softc; 889 struct mbuf *m_head; 890 891 EM_TX_LOCK_ASSERT(txr); 892 893 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 894 IFF_DRV_RUNNING) 895 return; 896 897 if (!adapter->link_active) 898 return; 899 900 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 901 902 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 903 if (m_head == NULL) 904 break; 905 /* 906 * Encapsulation can modify our pointer, and or make it 907 * NULL on failure. In that event, we can't requeue. 908 */ 909 if (em_xmit(txr, &m_head)) { 910 if (m_head == NULL) 911 break; 912 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 913 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 914 break; 915 } 916 917 /* Send a copy of the frame to the BPF listener */ 918 ETHER_BPF_MTAP(ifp, m_head); 919 920 /* Set timeout in case hardware has problems transmitting. */ 921 txr->watchdog_check = TRUE; 922 } 923 924 return; 925} 926 927static void 928em_start(struct ifnet *ifp) 929{ 930 struct adapter *adapter = ifp->if_softc; 931 struct tx_ring *txr = adapter->tx_rings; 932 933 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 934 EM_TX_LOCK(txr); 935 em_start_locked(ifp, txr); 936 EM_TX_UNLOCK(txr); 937 } 938 return; 939} 940 941/********************************************************************* 942 * Ioctl entry point 943 * 944 * em_ioctl is called when the user wants to configure the 945 * interface. 946 * 947 * return 0 on success, positive on failure 948 **********************************************************************/ 949 950static int 951em_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 952{ 953 struct adapter *adapter = ifp->if_softc; 954 struct ifreq *ifr = (struct ifreq *)data; 955#ifdef INET 956 struct ifaddr *ifa = (struct ifaddr *)data; 957#endif 958 int error = 0; 959 960 if (adapter->in_detach) 961 return (error); 962 963 switch (command) { 964 case SIOCSIFADDR: 965#ifdef INET 966 if (ifa->ifa_addr->sa_family == AF_INET) { 967 /* 968 * XXX 969 * Since resetting hardware takes a very long time 970 * and results in link renegotiation we only 971 * initialize the hardware only when it is absolutely 972 * required. 973 */ 974 ifp->if_flags |= IFF_UP; 975 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 976 EM_CORE_LOCK(adapter); 977 em_init_locked(adapter); 978 EM_CORE_UNLOCK(adapter); 979 } 980 arp_ifinit(ifp, ifa); 981 } else 982#endif 983 error = ether_ioctl(ifp, command, data); 984 break; 985 case SIOCSIFMTU: 986 { 987 int max_frame_size; 988 989 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 990 991 EM_CORE_LOCK(adapter); 992 switch (adapter->hw.mac.type) { 993 case e1000_82571: 994 case e1000_82572: 995 case e1000_ich9lan: 996 case e1000_ich10lan: 997 case e1000_82574: 998 case e1000_80003es2lan: /* 9K Jumbo Frame size */ 999 max_frame_size = 9234; 1000 break; 1001 case e1000_pchlan: 1002 max_frame_size = 4096; 1003 break; 1004 /* Adapters that do not support jumbo frames */ 1005 case e1000_82583: 1006 case e1000_ich8lan: 1007 max_frame_size = ETHER_MAX_LEN; 1008 break; 1009 default: 1010 max_frame_size = MAX_JUMBO_FRAME_SIZE; 1011 } 1012 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1013 ETHER_CRC_LEN) { 1014 EM_CORE_UNLOCK(adapter); 1015 error = EINVAL; 1016 break; 1017 } 1018 1019 ifp->if_mtu = ifr->ifr_mtu; 1020 adapter->max_frame_size = 1021 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1022 em_init_locked(adapter); 1023 EM_CORE_UNLOCK(adapter); 1024 break; 1025 } 1026 case SIOCSIFFLAGS: 1027 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1028 SIOCSIFFLAGS (Set Interface Flags)"); 1029 EM_CORE_LOCK(adapter); 1030 if (ifp->if_flags & IFF_UP) { 1031 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1032 if ((ifp->if_flags ^ adapter->if_flags) & 1033 (IFF_PROMISC | IFF_ALLMULTI)) { 1034 em_disable_promisc(adapter); 1035 em_set_promisc(adapter); 1036 } 1037 } else 1038 em_init_locked(adapter); 1039 } else 1040 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1041 em_stop(adapter); 1042 adapter->if_flags = ifp->if_flags; 1043 EM_CORE_UNLOCK(adapter); 1044 break; 1045 case SIOCADDMULTI: 1046 case SIOCDELMULTI: 1047 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1048 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1049 EM_CORE_LOCK(adapter); 1050 em_disable_intr(adapter); 1051 em_set_multi(adapter); 1052#ifdef DEVICE_POLLING 1053 if (!(ifp->if_capenable & IFCAP_POLLING)) 1054#endif 1055 em_enable_intr(adapter); 1056 EM_CORE_UNLOCK(adapter); 1057 } 1058 break; 1059 case SIOCSIFMEDIA: 1060 /* Check SOL/IDER usage */ 1061 EM_CORE_LOCK(adapter); 1062 if (e1000_check_reset_block(&adapter->hw)) { 1063 EM_CORE_UNLOCK(adapter); 1064 device_printf(adapter->dev, "Media change is" 1065 " blocked due to SOL/IDER session.\n"); 1066 break; 1067 } 1068 EM_CORE_UNLOCK(adapter); 1069 case SIOCGIFMEDIA: 1070 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1071 SIOCxIFMEDIA (Get/Set Interface Media)"); 1072 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1073 break; 1074 case SIOCSIFCAP: 1075 { 1076 int mask, reinit; 1077 1078 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1079 reinit = 0; 1080 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1081#ifdef DEVICE_POLLING 1082 if (mask & IFCAP_POLLING) { 1083 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1084 error = ether_poll_register(em_poll, ifp); 1085 if (error) 1086 return (error); 1087 EM_CORE_LOCK(adapter); 1088 em_disable_intr(adapter); 1089 ifp->if_capenable |= IFCAP_POLLING; 1090 EM_CORE_UNLOCK(adapter); 1091 } else { 1092 error = ether_poll_deregister(ifp); 1093 /* Enable interrupt even in error case */ 1094 EM_CORE_LOCK(adapter); 1095 em_enable_intr(adapter); 1096 ifp->if_capenable &= ~IFCAP_POLLING; 1097 EM_CORE_UNLOCK(adapter); 1098 } 1099 } 1100#endif 1101 if (mask & IFCAP_HWCSUM) { 1102 ifp->if_capenable ^= IFCAP_HWCSUM; 1103 reinit = 1; 1104 } 1105 if (mask & IFCAP_TSO4) { 1106 ifp->if_capenable ^= IFCAP_TSO4; 1107 reinit = 1; 1108 } 1109 if (mask & IFCAP_VLAN_HWTAGGING) { 1110 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1111 reinit = 1; 1112 } 1113 if ((mask & IFCAP_WOL) && 1114 (ifp->if_capabilities & IFCAP_WOL) != 0) { 1115 if (mask & IFCAP_WOL_MCAST) 1116 ifp->if_capenable ^= IFCAP_WOL_MCAST; 1117 if (mask & IFCAP_WOL_MAGIC) 1118 ifp->if_capenable ^= IFCAP_WOL_MAGIC; 1119 } 1120 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1121 em_init(adapter); 1122 VLAN_CAPABILITIES(ifp); 1123 break; 1124 } 1125 1126 default: 1127 error = ether_ioctl(ifp, command, data); 1128 break; 1129 } 1130 1131 return (error); 1132} 1133 1134 1135/********************************************************************* 1136 * Init entry point 1137 * 1138 * This routine is used in two ways. It is used by the stack as 1139 * init entry point in network interface structure. It is also used 1140 * by the driver as a hw/sw initialization routine to get to a 1141 * consistent state. 1142 * 1143 * return 0 on success, positive on failure 1144 **********************************************************************/ 1145 1146static void 1147em_init_locked(struct adapter *adapter) 1148{ 1149 struct ifnet *ifp = adapter->ifp; 1150 device_t dev = adapter->dev; 1151 u32 pba; 1152 1153 INIT_DEBUGOUT("em_init: begin"); 1154 1155 EM_CORE_LOCK_ASSERT(adapter); 1156 1157 em_disable_intr(adapter); 1158 callout_stop(&adapter->timer); 1159 1160 /* 1161 * Packet Buffer Allocation (PBA) 1162 * Writing PBA sets the receive portion of the buffer 1163 * the remainder is used for the transmit buffer. 1164 */ 1165 switch (adapter->hw.mac.type) { 1166 /* Total Packet Buffer on these is 48K */ 1167 case e1000_82571: 1168 case e1000_82572: 1169 case e1000_80003es2lan: 1170 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1171 break; 1172 case e1000_82573: /* 82573: Total Packet Buffer is 32K */ 1173 pba = E1000_PBA_12K; /* 12K for Rx, 20K for Tx */ 1174 break; 1175 case e1000_82574: 1176 case e1000_82583: 1177 pba = E1000_PBA_20K; /* 20K for Rx, 20K for Tx */ 1178 break; 1179 case e1000_ich9lan: 1180 case e1000_ich10lan: 1181 case e1000_pchlan: 1182 pba = E1000_PBA_10K; 1183 break; 1184 case e1000_ich8lan: 1185 pba = E1000_PBA_8K; 1186 break; 1187 default: 1188 if (adapter->max_frame_size > 8192) 1189 pba = E1000_PBA_40K; /* 40K for Rx, 24K for Tx */ 1190 else 1191 pba = E1000_PBA_48K; /* 48K for Rx, 16K for Tx */ 1192 } 1193 1194 INIT_DEBUGOUT1("em_init: pba=%dK",pba); 1195 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1196 1197 /* Get the latest mac address, User can use a LAA */ 1198 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1199 ETHER_ADDR_LEN); 1200 1201 /* Put the address into the Receive Address Array */ 1202 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1203 1204 /* 1205 * With the 82571 adapter, RAR[0] may be overwritten 1206 * when the other port is reset, we make a duplicate 1207 * in RAR[14] for that eventuality, this assures 1208 * the interface continues to function. 1209 */ 1210 if (adapter->hw.mac.type == e1000_82571) { 1211 e1000_set_laa_state_82571(&adapter->hw, TRUE); 1212 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 1213 E1000_RAR_ENTRIES - 1); 1214 } 1215 1216 /* Initialize the hardware */ 1217 em_reset(adapter); 1218 em_update_link_status(adapter); 1219 1220 /* Setup VLAN support, basic and offload if available */ 1221 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1222 1223 /* Use real VLAN Filter support */ 1224 em_setup_vlan_hw_support(adapter); 1225 1226 /* Set hardware offload abilities */ 1227 ifp->if_hwassist = 0; 1228 if (ifp->if_capenable & IFCAP_TXCSUM) 1229 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1230 if (ifp->if_capenable & IFCAP_TSO4) 1231 ifp->if_hwassist |= CSUM_TSO; 1232 1233 /* Configure for OS presence */ 1234 em_init_manageability(adapter); 1235 1236 /* Prepare transmit descriptors and buffers */ 1237 em_setup_transmit_structures(adapter); 1238 em_initialize_transmit_unit(adapter); 1239 1240 /* Setup Multicast table */ 1241 em_set_multi(adapter); 1242 1243 /* Prepare receive descriptors and buffers */ 1244 if (em_setup_receive_structures(adapter)) { 1245 device_printf(dev, "Could not setup receive structures\n"); 1246 em_stop(adapter); 1247 return; 1248 } 1249 em_initialize_receive_unit(adapter); 1250 1251 /* Don't lose promiscuous settings */ 1252 em_set_promisc(adapter); 1253 1254 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1255 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1256 1257 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1258 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1259 1260 /* MSI/X configuration for 82574 */ 1261 if (adapter->hw.mac.type == e1000_82574) { 1262 int tmp; 1263 tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 1264 tmp |= E1000_CTRL_EXT_PBA_CLR; 1265 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, tmp); 1266 /* Set the IVAR - interrupt vector routing. */ 1267 E1000_WRITE_REG(&adapter->hw, E1000_IVAR, adapter->ivars); 1268 } 1269 1270#ifdef DEVICE_POLLING 1271 /* 1272 * Only enable interrupts if we are not polling, make sure 1273 * they are off otherwise. 1274 */ 1275 if (ifp->if_capenable & IFCAP_POLLING) 1276 em_disable_intr(adapter); 1277 else 1278#endif /* DEVICE_POLLING */ 1279 em_enable_intr(adapter); 1280 1281 /* AMT based hardware can now take control from firmware */ 1282 if (adapter->has_manage && adapter->has_amt) 1283 em_get_hw_control(adapter); 1284 1285 /* Don't reset the phy next time init gets called */ 1286 adapter->hw.phy.reset_disable = TRUE; 1287} 1288 1289static void 1290em_init(void *arg) 1291{ 1292 struct adapter *adapter = arg; 1293 1294 EM_CORE_LOCK(adapter); 1295 em_init_locked(adapter); 1296 EM_CORE_UNLOCK(adapter); 1297} 1298 1299 1300#ifdef DEVICE_POLLING 1301/********************************************************************* 1302 * 1303 * Legacy polling routine: note this only works with single queue 1304 * 1305 *********************************************************************/ 1306static int 1307em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1308{ 1309 struct adapter *adapter = ifp->if_softc; 1310 struct tx_ring *txr = adapter->tx_rings; 1311 struct rx_ring *rxr = adapter->rx_rings; 1312 u32 reg_icr, rx_done = 0; 1313 1314 EM_CORE_LOCK(adapter); 1315 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1316 EM_CORE_UNLOCK(adapter); 1317 return (rx_done); 1318 } 1319 1320 if (cmd == POLL_AND_CHECK_STATUS) { 1321 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1322 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1323 callout_stop(&adapter->timer); 1324 adapter->hw.mac.get_link_status = 1; 1325 em_update_link_status(adapter); 1326 callout_reset(&adapter->timer, hz, 1327 em_local_timer, adapter); 1328 } 1329 } 1330 EM_CORE_UNLOCK(adapter); 1331 1332 rx_done = em_rxeof(rxr, count); 1333 1334 EM_TX_LOCK(txr); 1335 em_txeof(txr); 1336#if __FreeBSD_version >= 800000 1337 if (!drbr_empty(ifp, txr->br)) 1338 em_mq_start_locked(ifp, txr, NULL); 1339#else 1340 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1341 em_start_locked(ifp, txr); 1342#endif 1343 EM_TX_UNLOCK(txr); 1344 1345 return (rx_done); 1346} 1347#endif /* DEVICE_POLLING */ 1348 1349 1350/********************************************************************* 1351 * 1352 * Fast Legacy/MSI Combined Interrupt Service routine 1353 * 1354 *********************************************************************/ 1355static int 1356em_irq_fast(void *arg) 1357{ 1358 struct adapter *adapter = arg; 1359 struct ifnet *ifp; 1360 u32 reg_icr; 1361 1362 ifp = adapter->ifp; 1363 1364 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1365 1366 /* Hot eject? */ 1367 if (reg_icr == 0xffffffff) 1368 return FILTER_STRAY; 1369 1370 /* Definitely not our interrupt. */ 1371 if (reg_icr == 0x0) 1372 return FILTER_STRAY; 1373 1374 /* 1375 * Starting with the 82571 chip, bit 31 should be used to 1376 * determine whether the interrupt belongs to us. 1377 */ 1378 if (adapter->hw.mac.type >= e1000_82571 && 1379 (reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1380 return FILTER_STRAY; 1381 1382 em_disable_intr(adapter); 1383 taskqueue_enqueue(adapter->tq, &adapter->que_task); 1384 1385 /* Link status change */ 1386 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1387 adapter->hw.mac.get_link_status = 1; 1388 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1389 } 1390 1391 if (reg_icr & E1000_ICR_RXO) 1392 adapter->rx_overruns++; 1393 return FILTER_HANDLED; 1394} 1395 1396/* Combined RX/TX handler, used by Legacy and MSI */ 1397static void 1398em_handle_que(void *context, int pending) 1399{ 1400 struct adapter *adapter = context; 1401 struct ifnet *ifp = adapter->ifp; 1402 struct tx_ring *txr = adapter->tx_rings; 1403 struct rx_ring *rxr = adapter->rx_rings; 1404 u32 loop = EM_MAX_LOOP; 1405 bool more_rx, more_tx; 1406 1407 1408 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1409 EM_TX_LOCK(txr); 1410 do { 1411 more_rx = em_rxeof(rxr, adapter->rx_process_limit); 1412 more_tx = em_txeof(txr); 1413 } while (loop-- && (more_rx || more_tx)); 1414 1415#if __FreeBSD_version >= 800000 1416 if (!drbr_empty(ifp, txr->br)) 1417 em_mq_start_locked(ifp, txr, NULL); 1418#else 1419 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1420 em_start_locked(ifp, txr); 1421#endif 1422 if (more_rx || more_tx) 1423 taskqueue_enqueue(adapter->tq, &adapter->que_task); 1424 1425 EM_TX_UNLOCK(txr); 1426 } 1427 1428 em_enable_intr(adapter); 1429 return; 1430} 1431 1432 1433/********************************************************************* 1434 * 1435 * MSIX Interrupt Service Routines 1436 * 1437 **********************************************************************/ 1438static void 1439em_msix_tx(void *arg) 1440{ 1441 struct tx_ring *txr = arg; 1442 struct adapter *adapter = txr->adapter; 1443 bool more; 1444 1445 ++txr->tx_irq; 1446 EM_TX_LOCK(txr); 1447 more = em_txeof(txr); 1448 EM_TX_UNLOCK(txr); 1449 if (more) 1450 taskqueue_enqueue(txr->tq, &txr->tx_task); 1451 else 1452 /* Reenable this interrupt */ 1453 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); 1454 return; 1455} 1456 1457/********************************************************************* 1458 * 1459 * MSIX RX Interrupt Service routine 1460 * 1461 **********************************************************************/ 1462 1463static void 1464em_msix_rx(void *arg) 1465{ 1466 struct rx_ring *rxr = arg; 1467 struct adapter *adapter = rxr->adapter; 1468 bool more; 1469 1470 ++rxr->rx_irq; 1471 more = em_rxeof(rxr, adapter->rx_process_limit); 1472 if (more) 1473 taskqueue_enqueue(rxr->tq, &rxr->rx_task); 1474 else 1475 /* Reenable this interrupt */ 1476 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); 1477 return; 1478} 1479 1480/********************************************************************* 1481 * 1482 * MSIX Link Fast Interrupt Service routine 1483 * 1484 **********************************************************************/ 1485static void 1486em_msix_link(void *arg) 1487{ 1488 struct adapter *adapter = arg; 1489 u32 reg_icr; 1490 1491 ++adapter->link_irq; 1492 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1493 1494 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1495 adapter->hw.mac.get_link_status = 1; 1496 taskqueue_enqueue(taskqueue_fast, &adapter->link_task); 1497 } else 1498 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1499 EM_MSIX_LINK | E1000_IMS_LSC); 1500 return; 1501} 1502 1503static void 1504em_handle_rx(void *context, int pending) 1505{ 1506 struct rx_ring *rxr = context; 1507 struct adapter *adapter = rxr->adapter; 1508 u32 loop = EM_MAX_LOOP; 1509 bool more; 1510 1511 do { 1512 more = em_rxeof(rxr, adapter->rx_process_limit); 1513 } while (loop-- && more); 1514 /* Reenable this interrupt */ 1515 E1000_WRITE_REG(&adapter->hw, E1000_IMS, rxr->ims); 1516} 1517 1518static void 1519em_handle_tx(void *context, int pending) 1520{ 1521 struct tx_ring *txr = context; 1522 struct adapter *adapter = txr->adapter; 1523 struct ifnet *ifp = adapter->ifp; 1524 u32 loop = EM_MAX_LOOP; 1525 bool more; 1526 1527 if (!EM_TX_TRYLOCK(txr)) 1528 return; 1529 do { 1530 more = em_txeof(txr); 1531 } while (loop-- && more); 1532 1533#if __FreeBSD_version >= 800000 1534 if (!drbr_empty(ifp, txr->br)) 1535 em_mq_start_locked(ifp, txr, NULL); 1536#else 1537 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1538 em_start_locked(ifp, txr); 1539#endif 1540 E1000_WRITE_REG(&adapter->hw, E1000_IMS, txr->ims); 1541 EM_TX_UNLOCK(txr); 1542} 1543 1544static void 1545em_handle_link(void *context, int pending) 1546{ 1547 struct adapter *adapter = context; 1548 struct ifnet *ifp = adapter->ifp; 1549 1550 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 1551 return; 1552 1553 EM_CORE_LOCK(adapter); 1554 callout_stop(&adapter->timer); 1555 em_update_link_status(adapter); 1556 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 1557 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 1558 EM_MSIX_LINK | E1000_IMS_LSC); 1559 EM_CORE_UNLOCK(adapter); 1560} 1561 1562 1563/********************************************************************* 1564 * 1565 * Media Ioctl callback 1566 * 1567 * This routine is called whenever the user queries the status of 1568 * the interface using ifconfig. 1569 * 1570 **********************************************************************/ 1571static void 1572em_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1573{ 1574 struct adapter *adapter = ifp->if_softc; 1575 u_char fiber_type = IFM_1000_SX; 1576 1577 INIT_DEBUGOUT("em_media_status: begin"); 1578 1579 EM_CORE_LOCK(adapter); 1580 em_update_link_status(adapter); 1581 1582 ifmr->ifm_status = IFM_AVALID; 1583 ifmr->ifm_active = IFM_ETHER; 1584 1585 if (!adapter->link_active) { 1586 EM_CORE_UNLOCK(adapter); 1587 return; 1588 } 1589 1590 ifmr->ifm_status |= IFM_ACTIVE; 1591 1592 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1593 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 1594 ifmr->ifm_active |= fiber_type | IFM_FDX; 1595 } else { 1596 switch (adapter->link_speed) { 1597 case 10: 1598 ifmr->ifm_active |= IFM_10_T; 1599 break; 1600 case 100: 1601 ifmr->ifm_active |= IFM_100_TX; 1602 break; 1603 case 1000: 1604 ifmr->ifm_active |= IFM_1000_T; 1605 break; 1606 } 1607 if (adapter->link_duplex == FULL_DUPLEX) 1608 ifmr->ifm_active |= IFM_FDX; 1609 else 1610 ifmr->ifm_active |= IFM_HDX; 1611 } 1612 EM_CORE_UNLOCK(adapter); 1613} 1614 1615/********************************************************************* 1616 * 1617 * Media Ioctl callback 1618 * 1619 * This routine is called when the user changes speed/duplex using 1620 * media/mediopt option with ifconfig. 1621 * 1622 **********************************************************************/ 1623static int 1624em_media_change(struct ifnet *ifp) 1625{ 1626 struct adapter *adapter = ifp->if_softc; 1627 struct ifmedia *ifm = &adapter->media; 1628 1629 INIT_DEBUGOUT("em_media_change: begin"); 1630 1631 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1632 return (EINVAL); 1633 1634 EM_CORE_LOCK(adapter); 1635 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1636 case IFM_AUTO: 1637 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1638 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1639 break; 1640 case IFM_1000_LX: 1641 case IFM_1000_SX: 1642 case IFM_1000_T: 1643 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1644 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1645 break; 1646 case IFM_100_TX: 1647 adapter->hw.mac.autoneg = FALSE; 1648 adapter->hw.phy.autoneg_advertised = 0; 1649 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1650 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1651 else 1652 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1653 break; 1654 case IFM_10_T: 1655 adapter->hw.mac.autoneg = FALSE; 1656 adapter->hw.phy.autoneg_advertised = 0; 1657 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1658 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1659 else 1660 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1661 break; 1662 default: 1663 device_printf(adapter->dev, "Unsupported media type\n"); 1664 } 1665 1666 /* As the speed/duplex settings my have changed we need to 1667 * reset the PHY. 1668 */ 1669 adapter->hw.phy.reset_disable = FALSE; 1670 1671 em_init_locked(adapter); 1672 EM_CORE_UNLOCK(adapter); 1673 1674 return (0); 1675} 1676 1677/********************************************************************* 1678 * 1679 * This routine maps the mbufs to tx descriptors. 1680 * 1681 * return 0 on success, positive on failure 1682 **********************************************************************/ 1683 1684static int 1685em_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1686{ 1687 struct adapter *adapter = txr->adapter; 1688 bus_dma_segment_t segs[EM_MAX_SCATTER]; 1689 bus_dmamap_t map; 1690 struct em_buffer *tx_buffer, *tx_buffer_mapped; 1691 struct e1000_tx_desc *ctxd = NULL; 1692 struct mbuf *m_head; 1693 u32 txd_upper, txd_lower, txd_used, txd_saved; 1694 int nsegs, i, j, first, last = 0; 1695 int error, do_tso, tso_desc = 0; 1696 1697 m_head = *m_headp; 1698 txd_upper = txd_lower = txd_used = txd_saved = 0; 1699 do_tso = ((m_head->m_pkthdr.csum_flags & CSUM_TSO) != 0); 1700 1701 /* 1702 * Force a cleanup if number of TX descriptors 1703 * available hits the threshold 1704 */ 1705 if (txr->tx_avail <= EM_TX_CLEANUP_THRESHOLD) 1706 em_txeof(txr); 1707 1708 /* 1709 * TSO workaround: 1710 * If an mbuf is only header we need 1711 * to pull 4 bytes of data into it. 1712 */ 1713 if (do_tso && (m_head->m_len <= M_TSO_LEN)) { 1714 m_head = m_pullup(m_head, M_TSO_LEN + 4); 1715 *m_headp = m_head; 1716 if (m_head == NULL) 1717 return (ENOBUFS); 1718 } 1719 1720 /* 1721 * Map the packet for DMA 1722 * 1723 * Capture the first descriptor index, 1724 * this descriptor will have the index 1725 * of the EOP which is the only one that 1726 * now gets a DONE bit writeback. 1727 */ 1728 first = txr->next_avail_desc; 1729 tx_buffer = &txr->tx_buffers[first]; 1730 tx_buffer_mapped = tx_buffer; 1731 map = tx_buffer->map; 1732 1733 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1734 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1735 1736 /* 1737 * There are two types of errors we can (try) to handle: 1738 * - EFBIG means the mbuf chain was too long and bus_dma ran 1739 * out of segments. Defragment the mbuf chain and try again. 1740 * - ENOMEM means bus_dma could not obtain enough bounce buffers 1741 * at this point in time. Defer sending and try again later. 1742 * All other errors, in particular EINVAL, are fatal and prevent the 1743 * mbuf chain from ever going through. Drop it and report error. 1744 */ 1745 if (error == EFBIG) { 1746 struct mbuf *m; 1747 1748 m = m_defrag(*m_headp, M_DONTWAIT); 1749 if (m == NULL) { 1750 adapter->mbuf_alloc_failed++; 1751 m_freem(*m_headp); 1752 *m_headp = NULL; 1753 return (ENOBUFS); 1754 } 1755 *m_headp = m; 1756 1757 /* Try it again */ 1758 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1759 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1760 1761 if (error) { 1762 adapter->no_tx_dma_setup++; 1763 m_freem(*m_headp); 1764 *m_headp = NULL; 1765 return (error); 1766 } 1767 } else if (error != 0) { 1768 adapter->no_tx_dma_setup++; 1769 return (error); 1770 } 1771 1772 /* 1773 * TSO Hardware workaround, if this packet is not 1774 * TSO, and is only a single descriptor long, and 1775 * it follows a TSO burst, then we need to add a 1776 * sentinel descriptor to prevent premature writeback. 1777 */ 1778 if ((do_tso == 0) && (txr->tx_tso == TRUE)) { 1779 if (nsegs == 1) 1780 tso_desc = TRUE; 1781 txr->tx_tso = FALSE; 1782 } 1783 1784 if (nsegs > (txr->tx_avail - 2)) { 1785 txr->no_desc_avail++; 1786 bus_dmamap_unload(txr->txtag, map); 1787 return (ENOBUFS); 1788 } 1789 m_head = *m_headp; 1790 1791 /* Do hardware assists */ 1792#if __FreeBSD_version >= 700000 1793 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1794 error = em_tso_setup(txr, m_head, &txd_upper, &txd_lower); 1795 if (error != TRUE) 1796 return (ENXIO); /* something foobar */ 1797 /* we need to make a final sentinel transmit desc */ 1798 tso_desc = TRUE; 1799 } else 1800#endif 1801 if (m_head->m_pkthdr.csum_flags & CSUM_OFFLOAD) 1802 em_transmit_checksum_setup(txr, m_head, 1803 &txd_upper, &txd_lower); 1804 1805 i = txr->next_avail_desc; 1806 1807 /* Set up our transmit descriptors */ 1808 for (j = 0; j < nsegs; j++) { 1809 bus_size_t seg_len; 1810 bus_addr_t seg_addr; 1811 1812 tx_buffer = &txr->tx_buffers[i]; 1813 ctxd = &txr->tx_base[i]; 1814 seg_addr = segs[j].ds_addr; 1815 seg_len = segs[j].ds_len; 1816 /* 1817 ** TSO Workaround: 1818 ** If this is the last descriptor, we want to 1819 ** split it so we have a small final sentinel 1820 */ 1821 if (tso_desc && (j == (nsegs -1)) && (seg_len > 8)) { 1822 seg_len -= 4; 1823 ctxd->buffer_addr = htole64(seg_addr); 1824 ctxd->lower.data = htole32( 1825 adapter->txd_cmd | txd_lower | seg_len); 1826 ctxd->upper.data = 1827 htole32(txd_upper); 1828 if (++i == adapter->num_tx_desc) 1829 i = 0; 1830 /* Now make the sentinel */ 1831 ++txd_used; /* using an extra txd */ 1832 ctxd = &txr->tx_base[i]; 1833 tx_buffer = &txr->tx_buffers[i]; 1834 ctxd->buffer_addr = 1835 htole64(seg_addr + seg_len); 1836 ctxd->lower.data = htole32( 1837 adapter->txd_cmd | txd_lower | 4); 1838 ctxd->upper.data = 1839 htole32(txd_upper); 1840 last = i; 1841 if (++i == adapter->num_tx_desc) 1842 i = 0; 1843 } else { 1844 ctxd->buffer_addr = htole64(seg_addr); 1845 ctxd->lower.data = htole32( 1846 adapter->txd_cmd | txd_lower | seg_len); 1847 ctxd->upper.data = 1848 htole32(txd_upper); 1849 last = i; 1850 if (++i == adapter->num_tx_desc) 1851 i = 0; 1852 } 1853 tx_buffer->m_head = NULL; 1854 tx_buffer->next_eop = -1; 1855 } 1856 1857 txr->next_avail_desc = i; 1858 txr->tx_avail -= nsegs; 1859 if (tso_desc) /* TSO used an extra for sentinel */ 1860 txr->tx_avail -= txd_used; 1861 1862 if (m_head->m_flags & M_VLANTAG) { 1863 /* Set the vlan id. */ 1864 ctxd->upper.fields.special = 1865 htole16(m_head->m_pkthdr.ether_vtag); 1866 /* Tell hardware to add tag */ 1867 ctxd->lower.data |= htole32(E1000_TXD_CMD_VLE); 1868 } 1869 1870 tx_buffer->m_head = m_head; 1871 tx_buffer_mapped->map = tx_buffer->map; 1872 tx_buffer->map = map; 1873 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1874 1875 /* 1876 * Last Descriptor of Packet 1877 * needs End Of Packet (EOP) 1878 * and Report Status (RS) 1879 */ 1880 ctxd->lower.data |= 1881 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 1882 /* 1883 * Keep track in the first buffer which 1884 * descriptor will be written back 1885 */ 1886 tx_buffer = &txr->tx_buffers[first]; 1887 tx_buffer->next_eop = last; 1888 1889 /* 1890 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 1891 * that this frame is available to transmit. 1892 */ 1893 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1894 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1895 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); 1896 1897 return (0); 1898} 1899 1900static void 1901em_set_promisc(struct adapter *adapter) 1902{ 1903 struct ifnet *ifp = adapter->ifp; 1904 u32 reg_rctl; 1905 1906 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1907 1908 if (ifp->if_flags & IFF_PROMISC) { 1909 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1910 /* Turn this on if you want to see bad packets */ 1911 if (em_debug_sbp) 1912 reg_rctl |= E1000_RCTL_SBP; 1913 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1914 } else if (ifp->if_flags & IFF_ALLMULTI) { 1915 reg_rctl |= E1000_RCTL_MPE; 1916 reg_rctl &= ~E1000_RCTL_UPE; 1917 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1918 } 1919} 1920 1921static void 1922em_disable_promisc(struct adapter *adapter) 1923{ 1924 u32 reg_rctl; 1925 1926 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1927 1928 reg_rctl &= (~E1000_RCTL_UPE); 1929 reg_rctl &= (~E1000_RCTL_MPE); 1930 reg_rctl &= (~E1000_RCTL_SBP); 1931 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1932} 1933 1934 1935/********************************************************************* 1936 * Multicast Update 1937 * 1938 * This routine is called whenever multicast address list is updated. 1939 * 1940 **********************************************************************/ 1941 1942static void 1943em_set_multi(struct adapter *adapter) 1944{ 1945 struct ifnet *ifp = adapter->ifp; 1946 struct ifmultiaddr *ifma; 1947 u32 reg_rctl = 0; 1948 u8 *mta; /* Multicast array memory */ 1949 int mcnt = 0; 1950 1951 IOCTL_DEBUGOUT("em_set_multi: begin"); 1952 1953 if (adapter->hw.mac.type == e1000_82542 && 1954 adapter->hw.revision_id == E1000_REVISION_2) { 1955 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1956 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 1957 e1000_pci_clear_mwi(&adapter->hw); 1958 reg_rctl |= E1000_RCTL_RST; 1959 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1960 msec_delay(5); 1961 } 1962 1963 /* Allocate temporary memory to setup array */ 1964 mta = malloc(sizeof(u8) * 1965 (ETH_ADDR_LEN * MAX_NUM_MULTICAST_ADDRESSES), 1966 M_DEVBUF, M_NOWAIT | M_ZERO); 1967 if (mta == NULL) 1968 panic("em_set_multi memory failure\n"); 1969 1970#if __FreeBSD_version < 800000 1971 IF_ADDR_LOCK(ifp); 1972#else 1973 if_maddr_rlock(ifp); 1974#endif 1975 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1976 if (ifma->ifma_addr->sa_family != AF_LINK) 1977 continue; 1978 1979 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 1980 break; 1981 1982 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1983 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 1984 mcnt++; 1985 } 1986#if __FreeBSD_version < 800000 1987 IF_ADDR_UNLOCK(ifp); 1988#else 1989 if_maddr_runlock(ifp); 1990#endif 1991 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 1992 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1993 reg_rctl |= E1000_RCTL_MPE; 1994 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1995 } else 1996 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 1997 1998 if (adapter->hw.mac.type == e1000_82542 && 1999 adapter->hw.revision_id == E1000_REVISION_2) { 2000 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 2001 reg_rctl &= ~E1000_RCTL_RST; 2002 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 2003 msec_delay(5); 2004 if (adapter->hw.bus.pci_cmd_word & CMD_MEM_WRT_INVALIDATE) 2005 e1000_pci_set_mwi(&adapter->hw); 2006 } 2007 free(mta, M_DEVBUF); 2008} 2009 2010 2011/********************************************************************* 2012 * Timer routine 2013 * 2014 * This routine checks for link status and updates statistics. 2015 * 2016 **********************************************************************/ 2017 2018static void 2019em_local_timer(void *arg) 2020{ 2021 struct adapter *adapter = arg; 2022 struct ifnet *ifp = adapter->ifp; 2023 struct tx_ring *txr = adapter->tx_rings; 2024 2025 EM_CORE_LOCK_ASSERT(adapter); 2026 2027 em_update_link_status(adapter); 2028 em_update_stats_counters(adapter); 2029 2030 /* Reset LAA into RAR[0] on 82571 */ 2031 if (e1000_get_laa_state_82571(&adapter->hw) == TRUE) 2032 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 2033 2034 if (em_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 2035 em_print_hw_stats(adapter); 2036 2037 /* 2038 ** Check for time since any descriptor was cleaned 2039 */ 2040 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2041 EM_TX_LOCK(txr); 2042 if (txr->watchdog_check == FALSE) { 2043 EM_TX_UNLOCK(txr); 2044 continue; 2045 } 2046 if ((ticks - txr->watchdog_time) > EM_WATCHDOG) 2047 goto hung; 2048 EM_TX_UNLOCK(txr); 2049 } 2050 2051 callout_reset(&adapter->timer, hz, em_local_timer, adapter); 2052 return; 2053hung: 2054 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 2055 ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 2056 adapter->watchdog_events++; 2057 EM_TX_UNLOCK(txr); 2058 em_init_locked(adapter); 2059} 2060 2061 2062static void 2063em_update_link_status(struct adapter *adapter) 2064{ 2065 struct e1000_hw *hw = &adapter->hw; 2066 struct ifnet *ifp = adapter->ifp; 2067 device_t dev = adapter->dev; 2068 u32 link_check = 0; 2069 2070 /* Get the cached link value or read phy for real */ 2071 switch (hw->phy.media_type) { 2072 case e1000_media_type_copper: 2073 if (hw->mac.get_link_status) { 2074 /* Do the work to read phy */ 2075 e1000_check_for_link(hw); 2076 link_check = !hw->mac.get_link_status; 2077 if (link_check) /* ESB2 fix */ 2078 e1000_cfg_on_link_up(hw); 2079 } else 2080 link_check = TRUE; 2081 break; 2082 case e1000_media_type_fiber: 2083 e1000_check_for_link(hw); 2084 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 2085 E1000_STATUS_LU); 2086 break; 2087 case e1000_media_type_internal_serdes: 2088 e1000_check_for_link(hw); 2089 link_check = adapter->hw.mac.serdes_has_link; 2090 break; 2091 default: 2092 case e1000_media_type_unknown: 2093 break; 2094 } 2095 2096 /* Now check for a transition */ 2097 if (link_check && (adapter->link_active == 0)) { 2098 e1000_get_speed_and_duplex(hw, &adapter->link_speed, 2099 &adapter->link_duplex); 2100 /* Check if we must disable SPEED_MODE bit on PCI-E */ 2101 if ((adapter->link_speed != SPEED_1000) && 2102 ((hw->mac.type == e1000_82571) || 2103 (hw->mac.type == e1000_82572))) { 2104 int tarc0; 2105 tarc0 = E1000_READ_REG(hw, E1000_TARC(0)); 2106 tarc0 &= ~SPEED_MODE_BIT; 2107 E1000_WRITE_REG(hw, E1000_TARC(0), tarc0); 2108 } 2109 if (bootverbose) 2110 device_printf(dev, "Link is up %d Mbps %s\n", 2111 adapter->link_speed, 2112 ((adapter->link_duplex == FULL_DUPLEX) ? 2113 "Full Duplex" : "Half Duplex")); 2114 adapter->link_active = 1; 2115 adapter->smartspeed = 0; 2116 ifp->if_baudrate = adapter->link_speed * 1000000; 2117 if_link_state_change(ifp, LINK_STATE_UP); 2118 } else if (!link_check && (adapter->link_active == 1)) { 2119 ifp->if_baudrate = adapter->link_speed = 0; 2120 adapter->link_duplex = 0; 2121 if (bootverbose) 2122 device_printf(dev, "Link is Down\n"); 2123 adapter->link_active = 0; 2124 /* Link down, disable watchdog */ 2125 // JFV change later 2126 //adapter->watchdog_check = FALSE; 2127 if_link_state_change(ifp, LINK_STATE_DOWN); 2128 } 2129} 2130 2131/********************************************************************* 2132 * 2133 * This routine disables all traffic on the adapter by issuing a 2134 * global reset on the MAC and deallocates TX/RX buffers. 2135 * 2136 * This routine should always be called with BOTH the CORE 2137 * and TX locks. 2138 **********************************************************************/ 2139 2140static void 2141em_stop(void *arg) 2142{ 2143 struct adapter *adapter = arg; 2144 struct ifnet *ifp = adapter->ifp; 2145 struct tx_ring *txr = adapter->tx_rings; 2146 2147 EM_CORE_LOCK_ASSERT(adapter); 2148 2149 INIT_DEBUGOUT("em_stop: begin"); 2150 2151 em_disable_intr(adapter); 2152 callout_stop(&adapter->timer); 2153 2154 /* Tell the stack that the interface is no longer active */ 2155 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2156 2157 /* Unarm watchdog timer. */ 2158 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2159 EM_TX_LOCK(txr); 2160 txr->watchdog_check = FALSE; 2161 EM_TX_UNLOCK(txr); 2162 } 2163 2164 e1000_reset_hw(&adapter->hw); 2165 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2166} 2167 2168 2169/********************************************************************* 2170 * 2171 * Determine hardware revision. 2172 * 2173 **********************************************************************/ 2174static void 2175em_identify_hardware(struct adapter *adapter) 2176{ 2177 device_t dev = adapter->dev; 2178 2179 /* Make sure our PCI config space has the necessary stuff set */ 2180 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2181 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2182 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2183 device_printf(dev, "Memory Access and/or Bus Master bits " 2184 "were not set!\n"); 2185 adapter->hw.bus.pci_cmd_word |= 2186 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2187 pci_write_config(dev, PCIR_COMMAND, 2188 adapter->hw.bus.pci_cmd_word, 2); 2189 } 2190 2191 /* Save off the information about this board */ 2192 adapter->hw.vendor_id = pci_get_vendor(dev); 2193 adapter->hw.device_id = pci_get_device(dev); 2194 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2195 adapter->hw.subsystem_vendor_id = 2196 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2197 adapter->hw.subsystem_device_id = 2198 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2199 2200 /* Do Shared Code Init and Setup */ 2201 if (e1000_set_mac_type(&adapter->hw)) { 2202 device_printf(dev, "Setup init failure\n"); 2203 return; 2204 } 2205} 2206 2207static int 2208em_allocate_pci_resources(struct adapter *adapter) 2209{ 2210 device_t dev = adapter->dev; 2211 int rid; 2212 2213 rid = PCIR_BAR(0); 2214 adapter->memory = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2215 &rid, RF_ACTIVE); 2216 if (adapter->memory == NULL) { 2217 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2218 return (ENXIO); 2219 } 2220 adapter->osdep.mem_bus_space_tag = 2221 rman_get_bustag(adapter->memory); 2222 adapter->osdep.mem_bus_space_handle = 2223 rman_get_bushandle(adapter->memory); 2224 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2225 2226 /* Default to a single queue */ 2227 adapter->num_queues = 1; 2228 2229 /* 2230 * Setup MSI/X or MSI if PCI Express 2231 */ 2232 adapter->msix = em_setup_msix(adapter); 2233 2234 adapter->hw.back = &adapter->osdep; 2235 2236 return (0); 2237} 2238 2239/********************************************************************* 2240 * 2241 * Setup the Legacy or MSI Interrupt handler 2242 * 2243 **********************************************************************/ 2244int 2245em_allocate_legacy(struct adapter *adapter) 2246{ 2247 device_t dev = adapter->dev; 2248 int error, rid = 0; 2249 2250 /* Manually turn off all interrupts */ 2251 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2252 2253 if (adapter->msix == 1) /* using MSI */ 2254 rid = 1; 2255 /* We allocate a single interrupt resource */ 2256 adapter->res = bus_alloc_resource_any(dev, 2257 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2258 if (adapter->res == NULL) { 2259 device_printf(dev, "Unable to allocate bus resource: " 2260 "interrupt\n"); 2261 return (ENXIO); 2262 } 2263 2264 /* 2265 * Allocate a fast interrupt and the associated 2266 * deferred processing contexts. 2267 */ 2268 TASK_INIT(&adapter->que_task, 0, em_handle_que, adapter); 2269 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2270 adapter->tq = taskqueue_create_fast("em_taskq", M_NOWAIT, 2271 taskqueue_thread_enqueue, &adapter->tq); 2272 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2273 device_get_nameunit(adapter->dev)); 2274 if ((error = bus_setup_intr(dev, adapter->res, INTR_TYPE_NET, 2275 em_irq_fast, NULL, adapter, &adapter->tag)) != 0) { 2276 device_printf(dev, "Failed to register fast interrupt " 2277 "handler: %d\n", error); 2278 taskqueue_free(adapter->tq); 2279 adapter->tq = NULL; 2280 return (error); 2281 } 2282 2283 return (0); 2284} 2285 2286/********************************************************************* 2287 * 2288 * Setup the MSIX Interrupt handlers 2289 * This is not really Multiqueue, rather 2290 * its just multiple interrupt vectors. 2291 * 2292 **********************************************************************/ 2293int 2294em_allocate_msix(struct adapter *adapter) 2295{ 2296 device_t dev = adapter->dev; 2297 struct tx_ring *txr = adapter->tx_rings; 2298 struct rx_ring *rxr = adapter->rx_rings; 2299 int error, rid, vector = 0; 2300 2301 2302 /* Make sure all interrupts are disabled */ 2303 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2304 2305 /* First set up ring resources */ 2306 for (int i = 0; i < adapter->num_queues; i++, txr++, rxr++) { 2307 2308 /* RX ring */ 2309 rid = vector + 1; 2310 2311 rxr->res = bus_alloc_resource_any(dev, 2312 SYS_RES_IRQ, &rid, RF_ACTIVE); 2313 if (rxr->res == NULL) { 2314 device_printf(dev, 2315 "Unable to allocate bus resource: " 2316 "RX MSIX Interrupt %d\n", i); 2317 return (ENXIO); 2318 } 2319 if ((error = bus_setup_intr(dev, rxr->res, 2320 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_rx, 2321 rxr, &rxr->tag)) != 0) { 2322 device_printf(dev, "Failed to register RX handler"); 2323 return (error); 2324 } 2325 rxr->msix = vector++; /* NOTE increment vector for TX */ 2326 TASK_INIT(&rxr->rx_task, 0, em_handle_rx, rxr); 2327 rxr->tq = taskqueue_create_fast("em_rxq", M_NOWAIT, 2328 taskqueue_thread_enqueue, &rxr->tq); 2329 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", 2330 device_get_nameunit(adapter->dev)); 2331 /* 2332 ** Set the bit to enable interrupt 2333 ** in E1000_IMS -- bits 20 and 21 2334 ** are for RX0 and RX1, note this has 2335 ** NOTHING to do with the MSIX vector 2336 */ 2337 rxr->ims = 1 << (20 + i); 2338 adapter->ivars |= (8 | rxr->msix) << (i * 4); 2339 2340 /* TX ring */ 2341 rid = vector + 1; 2342 txr->res = bus_alloc_resource_any(dev, 2343 SYS_RES_IRQ, &rid, RF_ACTIVE); 2344 if (txr->res == NULL) { 2345 device_printf(dev, 2346 "Unable to allocate bus resource: " 2347 "TX MSIX Interrupt %d\n", i); 2348 return (ENXIO); 2349 } 2350 if ((error = bus_setup_intr(dev, txr->res, 2351 INTR_TYPE_NET | INTR_MPSAFE, NULL, em_msix_tx, 2352 txr, &txr->tag)) != 0) { 2353 device_printf(dev, "Failed to register TX handler"); 2354 return (error); 2355 } 2356 txr->msix = vector++; /* Increment vector for next pass */ 2357 TASK_INIT(&txr->tx_task, 0, em_handle_tx, txr); 2358 txr->tq = taskqueue_create_fast("em_txq", M_NOWAIT, 2359 taskqueue_thread_enqueue, &txr->tq); 2360 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", 2361 device_get_nameunit(adapter->dev)); 2362 /* 2363 ** Set the bit to enable interrupt 2364 ** in E1000_IMS -- bits 22 and 23 2365 ** are for TX0 and TX1, note this has 2366 ** NOTHING to do with the MSIX vector 2367 */ 2368 txr->ims = 1 << (22 + i); 2369 adapter->ivars |= (8 | txr->msix) << (8 + (i * 4)); 2370 } 2371 2372 /* Link interrupt */ 2373 ++rid; 2374 adapter->res = bus_alloc_resource_any(dev, 2375 SYS_RES_IRQ, &rid, RF_ACTIVE); 2376 if (!adapter->res) { 2377 device_printf(dev,"Unable to allocate " 2378 "bus resource: Link interrupt [%d]\n", rid); 2379 return (ENXIO); 2380 } 2381 /* Set the link handler function */ 2382 error = bus_setup_intr(dev, adapter->res, 2383 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2384 em_msix_link, adapter, &adapter->tag); 2385 if (error) { 2386 adapter->res = NULL; 2387 device_printf(dev, "Failed to register LINK handler"); 2388 return (error); 2389 } 2390 adapter->linkvec = vector; 2391 adapter->ivars |= (8 | vector) << 16; 2392 adapter->ivars |= 0x80000000; 2393 TASK_INIT(&adapter->link_task, 0, em_handle_link, adapter); 2394 adapter->tq = taskqueue_create_fast("em_link", M_NOWAIT, 2395 taskqueue_thread_enqueue, &adapter->tq); 2396 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s linkq", 2397 device_get_nameunit(adapter->dev)); 2398 2399 return (0); 2400} 2401 2402 2403static void 2404em_free_pci_resources(struct adapter *adapter) 2405{ 2406 device_t dev = adapter->dev; 2407 struct tx_ring *txr; 2408 struct rx_ring *rxr; 2409 int rid; 2410 2411 2412 /* 2413 ** Release all the queue interrupt resources: 2414 */ 2415 for (int i = 0; i < adapter->num_queues; i++) { 2416 txr = &adapter->tx_rings[i]; 2417 rxr = &adapter->rx_rings[i]; 2418 rid = txr->msix +1; 2419 if (txr->tag != NULL) { 2420 bus_teardown_intr(dev, txr->res, txr->tag); 2421 txr->tag = NULL; 2422 } 2423 if (txr->res != NULL) 2424 bus_release_resource(dev, SYS_RES_IRQ, 2425 rid, txr->res); 2426 rid = rxr->msix +1; 2427 if (rxr->tag != NULL) { 2428 bus_teardown_intr(dev, rxr->res, rxr->tag); 2429 rxr->tag = NULL; 2430 } 2431 if (rxr->res != NULL) 2432 bus_release_resource(dev, SYS_RES_IRQ, 2433 rid, rxr->res); 2434 } 2435 2436 if (adapter->linkvec) /* we are doing MSIX */ 2437 rid = adapter->linkvec + 1; 2438 else 2439 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2440 2441 if (adapter->tag != NULL) { 2442 bus_teardown_intr(dev, adapter->res, adapter->tag); 2443 adapter->tag = NULL; 2444 } 2445 2446 if (adapter->res != NULL) 2447 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 2448 2449 2450 if (adapter->msix) 2451 pci_release_msi(dev); 2452 2453 if (adapter->msix_mem != NULL) 2454 bus_release_resource(dev, SYS_RES_MEMORY, 2455 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); 2456 2457 if (adapter->memory != NULL) 2458 bus_release_resource(dev, SYS_RES_MEMORY, 2459 PCIR_BAR(0), adapter->memory); 2460 2461 if (adapter->flash != NULL) 2462 bus_release_resource(dev, SYS_RES_MEMORY, 2463 EM_FLASH, adapter->flash); 2464} 2465 2466/* 2467 * Setup MSI or MSI/X 2468 */ 2469static int 2470em_setup_msix(struct adapter *adapter) 2471{ 2472 device_t dev = adapter->dev; 2473 int val = 0; 2474 2475 2476 /* Setup MSI/X for Hartwell */ 2477 if ((adapter->hw.mac.type == e1000_82574) && 2478 (em_enable_msix == TRUE)) { 2479 /* Map the MSIX BAR */ 2480 int rid = PCIR_BAR(EM_MSIX_BAR); 2481 adapter->msix_mem = bus_alloc_resource_any(dev, 2482 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2483 if (!adapter->msix_mem) { 2484 /* May not be enabled */ 2485 device_printf(adapter->dev, 2486 "Unable to map MSIX table \n"); 2487 goto msi; 2488 } 2489 val = pci_msix_count(dev); 2490 if (val != 5) { 2491 bus_release_resource(dev, SYS_RES_MEMORY, 2492 PCIR_BAR(EM_MSIX_BAR), adapter->msix_mem); 2493 adapter->msix_mem = NULL; 2494 device_printf(adapter->dev, 2495 "MSIX vectors wrong, using MSI \n"); 2496 goto msi; 2497 } 2498 if (em_msix_queues == 2) { 2499 val = 5; 2500 adapter->num_queues = 2; 2501 } else { 2502 val = 3; 2503 adapter->num_queues = 1; 2504 } 2505 if (pci_alloc_msix(dev, &val) == 0) { 2506 device_printf(adapter->dev, 2507 "Using MSIX interrupts " 2508 "with %d vectors\n", val); 2509 } 2510 2511 return (val); 2512 } 2513msi: 2514 val = pci_msi_count(dev); 2515 if (val == 1 && pci_alloc_msi(dev, &val) == 0) { 2516 adapter->msix = 1; 2517 device_printf(adapter->dev,"Using MSI interrupt\n"); 2518 return (val); 2519 } 2520 /* Should only happen due to manual invention */ 2521 device_printf(adapter->dev,"Setup MSIX failure\n"); 2522 return (0); 2523} 2524 2525 2526/********************************************************************* 2527 * 2528 * Initialize the hardware to a configuration 2529 * as specified by the adapter structure. 2530 * 2531 **********************************************************************/ 2532static void 2533em_reset(struct adapter *adapter) 2534{ 2535 device_t dev = adapter->dev; 2536 struct e1000_hw *hw = &adapter->hw; 2537 u16 rx_buffer_size; 2538 2539 INIT_DEBUGOUT("em_reset: begin"); 2540 2541 /* Set up smart power down as default off on newer adapters. */ 2542 if (!em_smart_pwr_down && (hw->mac.type == e1000_82571 || 2543 hw->mac.type == e1000_82572)) { 2544 u16 phy_tmp = 0; 2545 2546 /* Speed up time to link by disabling smart power down. */ 2547 e1000_read_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, &phy_tmp); 2548 phy_tmp &= ~IGP02E1000_PM_SPD; 2549 e1000_write_phy_reg(hw, IGP02E1000_PHY_POWER_MGMT, phy_tmp); 2550 } 2551 2552 /* 2553 * These parameters control the automatic generation (Tx) and 2554 * response (Rx) to Ethernet PAUSE frames. 2555 * - High water mark should allow for at least two frames to be 2556 * received after sending an XOFF. 2557 * - Low water mark works best when it is very near the high water mark. 2558 * This allows the receiver to restart by sending XON when it has 2559 * drained a bit. Here we use an arbitary value of 1500 which will 2560 * restart after one full frame is pulled from the buffer. There 2561 * could be several smaller frames in the buffer and if so they will 2562 * not trigger the XON until their total number reduces the buffer 2563 * by 1500. 2564 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 2565 */ 2566 rx_buffer_size = ((E1000_READ_REG(hw, E1000_PBA) & 0xffff) << 10 ); 2567 2568 hw->fc.high_water = rx_buffer_size - 2569 roundup2(adapter->max_frame_size, 1024); 2570 hw->fc.low_water = hw->fc.high_water - 1500; 2571 2572 if (hw->mac.type == e1000_80003es2lan) 2573 hw->fc.pause_time = 0xFFFF; 2574 else 2575 hw->fc.pause_time = EM_FC_PAUSE_TIME; 2576 2577 hw->fc.send_xon = TRUE; 2578 2579 /* Set Flow control, use the tunable location if sane */ 2580 if ((em_fc_setting >= 0) || (em_fc_setting < 4)) 2581 hw->fc.requested_mode = em_fc_setting; 2582 else 2583 hw->fc.requested_mode = e1000_fc_none; 2584 2585 /* Override - workaround for PCHLAN issue */ 2586 if (hw->mac.type == e1000_pchlan) 2587 hw->fc.requested_mode = e1000_fc_rx_pause; 2588 2589 /* Issue a global reset */ 2590 e1000_reset_hw(hw); 2591 E1000_WRITE_REG(hw, E1000_WUC, 0); 2592 2593 if (e1000_init_hw(hw) < 0) { 2594 device_printf(dev, "Hardware Initialization Failed\n"); 2595 return; 2596 } 2597 2598 E1000_WRITE_REG(hw, E1000_VET, ETHERTYPE_VLAN); 2599 e1000_get_phy_info(hw); 2600 e1000_check_for_link(hw); 2601 return; 2602} 2603 2604/********************************************************************* 2605 * 2606 * Setup networking device structure and register an interface. 2607 * 2608 **********************************************************************/ 2609static void 2610em_setup_interface(device_t dev, struct adapter *adapter) 2611{ 2612 struct ifnet *ifp; 2613 2614 INIT_DEBUGOUT("em_setup_interface: begin"); 2615 2616 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2617 if (ifp == NULL) 2618 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 2619 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2620 ifp->if_mtu = ETHERMTU; 2621 ifp->if_init = em_init; 2622 ifp->if_softc = adapter; 2623 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2624 ifp->if_ioctl = em_ioctl; 2625 ifp->if_start = em_start; 2626 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 2627 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 2628 IFQ_SET_READY(&ifp->if_snd); 2629 2630 ether_ifattach(ifp, adapter->hw.mac.addr); 2631 2632 ifp->if_capabilities = ifp->if_capenable = 0; 2633 2634#if __FreeBSD_version >= 800000 2635 /* Multiqueue tx functions */ 2636 ifp->if_transmit = em_mq_start; 2637 ifp->if_qflush = em_qflush; 2638#endif 2639 2640 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2641 ifp->if_capenable |= IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2642 2643 /* Enable TSO by default, can disable with ifconfig */ 2644 ifp->if_capabilities |= IFCAP_TSO4; 2645 ifp->if_capenable |= IFCAP_TSO4; 2646 2647 /* 2648 * Tell the upper layer(s) we support long frames. 2649 */ 2650 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2651 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2652 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2653 2654#ifdef DEVICE_POLLING 2655 ifp->if_capabilities |= IFCAP_POLLING; 2656#endif 2657 2658 /* Enable All WOL methods by default */ 2659 if (adapter->wol) { 2660 ifp->if_capabilities |= IFCAP_WOL; 2661 ifp->if_capenable |= IFCAP_WOL; 2662 } 2663 2664 /* 2665 * Specify the media types supported by this adapter and register 2666 * callbacks to update media and link information 2667 */ 2668 ifmedia_init(&adapter->media, IFM_IMASK, 2669 em_media_change, em_media_status); 2670 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2671 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2672 u_char fiber_type = IFM_1000_SX; /* default type */ 2673 2674 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type | IFM_FDX, 2675 0, NULL); 2676 ifmedia_add(&adapter->media, IFM_ETHER | fiber_type, 0, NULL); 2677 } else { 2678 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2679 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2680 0, NULL); 2681 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2682 0, NULL); 2683 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2684 0, NULL); 2685 if (adapter->hw.phy.type != e1000_phy_ife) { 2686 ifmedia_add(&adapter->media, 2687 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2688 ifmedia_add(&adapter->media, 2689 IFM_ETHER | IFM_1000_T, 0, NULL); 2690 } 2691 } 2692 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2693 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2694} 2695 2696 2697/* 2698 * Manage DMA'able memory. 2699 */ 2700static void 2701em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 2702{ 2703 if (error) 2704 return; 2705 *(bus_addr_t *) arg = segs[0].ds_addr; 2706} 2707 2708static int 2709em_dma_malloc(struct adapter *adapter, bus_size_t size, 2710 struct em_dma_alloc *dma, int mapflags) 2711{ 2712 int error; 2713 2714 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 2715 EM_DBA_ALIGN, 0, /* alignment, bounds */ 2716 BUS_SPACE_MAXADDR, /* lowaddr */ 2717 BUS_SPACE_MAXADDR, /* highaddr */ 2718 NULL, NULL, /* filter, filterarg */ 2719 size, /* maxsize */ 2720 1, /* nsegments */ 2721 size, /* maxsegsize */ 2722 0, /* flags */ 2723 NULL, /* lockfunc */ 2724 NULL, /* lockarg */ 2725 &dma->dma_tag); 2726 if (error) { 2727 device_printf(adapter->dev, 2728 "%s: bus_dma_tag_create failed: %d\n", 2729 __func__, error); 2730 goto fail_0; 2731 } 2732 2733 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 2734 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dma->dma_map); 2735 if (error) { 2736 device_printf(adapter->dev, 2737 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 2738 __func__, (uintmax_t)size, error); 2739 goto fail_2; 2740 } 2741 2742 dma->dma_paddr = 0; 2743 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2744 size, em_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 2745 if (error || dma->dma_paddr == 0) { 2746 device_printf(adapter->dev, 2747 "%s: bus_dmamap_load failed: %d\n", 2748 __func__, error); 2749 goto fail_3; 2750 } 2751 2752 return (0); 2753 2754fail_3: 2755 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2756fail_2: 2757 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2758 bus_dma_tag_destroy(dma->dma_tag); 2759fail_0: 2760 dma->dma_map = NULL; 2761 dma->dma_tag = NULL; 2762 2763 return (error); 2764} 2765 2766static void 2767em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) 2768{ 2769 if (dma->dma_tag == NULL) 2770 return; 2771 if (dma->dma_map != NULL) { 2772 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2773 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2774 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2775 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2776 dma->dma_map = NULL; 2777 } 2778 bus_dma_tag_destroy(dma->dma_tag); 2779 dma->dma_tag = NULL; 2780} 2781 2782 2783/********************************************************************* 2784 * 2785 * Allocate memory for the transmit and receive rings, and then 2786 * the descriptors associated with each, called only once at attach. 2787 * 2788 **********************************************************************/ 2789static int 2790em_allocate_queues(struct adapter *adapter) 2791{ 2792 device_t dev = adapter->dev; 2793 struct tx_ring *txr = NULL; 2794 struct rx_ring *rxr = NULL; 2795 int rsize, tsize, error = E1000_SUCCESS; 2796 int txconf = 0, rxconf = 0; 2797 2798 2799 /* Allocate the TX ring struct memory */ 2800 if (!(adapter->tx_rings = 2801 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2802 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2803 device_printf(dev, "Unable to allocate TX ring memory\n"); 2804 error = ENOMEM; 2805 goto fail; 2806 } 2807 2808 /* Now allocate the RX */ 2809 if (!(adapter->rx_rings = 2810 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2811 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2812 device_printf(dev, "Unable to allocate RX ring memory\n"); 2813 error = ENOMEM; 2814 goto rx_fail; 2815 } 2816 2817 tsize = roundup2(adapter->num_tx_desc * 2818 sizeof(struct e1000_tx_desc), EM_DBA_ALIGN); 2819 /* 2820 * Now set up the TX queues, txconf is needed to handle the 2821 * possibility that things fail midcourse and we need to 2822 * undo memory gracefully 2823 */ 2824 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2825 /* Set up some basics */ 2826 txr = &adapter->tx_rings[i]; 2827 txr->adapter = adapter; 2828 txr->me = i; 2829 2830 /* Initialize the TX lock */ 2831 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2832 device_get_nameunit(dev), txr->me); 2833 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2834 2835 if (em_dma_malloc(adapter, tsize, 2836 &txr->txdma, BUS_DMA_NOWAIT)) { 2837 device_printf(dev, 2838 "Unable to allocate TX Descriptor memory\n"); 2839 error = ENOMEM; 2840 goto err_tx_desc; 2841 } 2842 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; 2843 bzero((void *)txr->tx_base, tsize); 2844 2845 if (em_allocate_transmit_buffers(txr)) { 2846 device_printf(dev, 2847 "Critical Failure setting up transmit buffers\n"); 2848 error = ENOMEM; 2849 goto err_tx_desc; 2850 } 2851#if __FreeBSD_version >= 800000 2852 /* Allocate a buf ring */ 2853 txr->br = buf_ring_alloc(4096, M_DEVBUF, 2854 M_WAITOK, &txr->tx_mtx); 2855#endif 2856 } 2857 2858 /* 2859 * Next the RX queues... 2860 */ 2861 rsize = roundup2(adapter->num_rx_desc * 2862 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 2863 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2864 rxr = &adapter->rx_rings[i]; 2865 rxr->adapter = adapter; 2866 rxr->me = i; 2867 2868 /* Initialize the RX lock */ 2869 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2870 device_get_nameunit(dev), txr->me); 2871 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2872 2873 if (em_dma_malloc(adapter, rsize, 2874 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2875 device_printf(dev, 2876 "Unable to allocate RxDescriptor memory\n"); 2877 error = ENOMEM; 2878 goto err_rx_desc; 2879 } 2880 rxr->rx_base = (struct e1000_rx_desc *)rxr->rxdma.dma_vaddr; 2881 bzero((void *)rxr->rx_base, rsize); 2882 2883 /* Allocate receive buffers for the ring*/ 2884 if (em_allocate_receive_buffers(rxr)) { 2885 device_printf(dev, 2886 "Critical Failure setting up receive buffers\n"); 2887 error = ENOMEM; 2888 goto err_rx_desc; 2889 } 2890 } 2891 2892 return (0); 2893 2894err_rx_desc: 2895 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2896 em_dma_free(adapter, &rxr->rxdma); 2897err_tx_desc: 2898 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2899 em_dma_free(adapter, &txr->txdma); 2900 free(adapter->rx_rings, M_DEVBUF); 2901rx_fail: 2902 buf_ring_free(txr->br, M_DEVBUF); 2903 free(adapter->tx_rings, M_DEVBUF); 2904fail: 2905 return (error); 2906} 2907 2908 2909/********************************************************************* 2910 * 2911 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2912 * the information needed to transmit a packet on the wire. This is 2913 * called only once at attach, setup is done every reset. 2914 * 2915 **********************************************************************/ 2916static int 2917em_allocate_transmit_buffers(struct tx_ring *txr) 2918{ 2919 struct adapter *adapter = txr->adapter; 2920 device_t dev = adapter->dev; 2921 struct em_buffer *txbuf; 2922 int error, i; 2923 2924 /* 2925 * Setup DMA descriptor areas. 2926 */ 2927 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 2928 1, 0, /* alignment, bounds */ 2929 BUS_SPACE_MAXADDR, /* lowaddr */ 2930 BUS_SPACE_MAXADDR, /* highaddr */ 2931 NULL, NULL, /* filter, filterarg */ 2932 EM_TSO_SIZE, /* maxsize */ 2933 EM_MAX_SCATTER, /* nsegments */ 2934 PAGE_SIZE, /* maxsegsize */ 2935 0, /* flags */ 2936 NULL, /* lockfunc */ 2937 NULL, /* lockfuncarg */ 2938 &txr->txtag))) { 2939 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2940 goto fail; 2941 } 2942 2943 if (!(txr->tx_buffers = 2944 (struct em_buffer *) malloc(sizeof(struct em_buffer) * 2945 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2946 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2947 error = ENOMEM; 2948 goto fail; 2949 } 2950 2951 /* Create the descriptor buffer dma maps */ 2952 txbuf = txr->tx_buffers; 2953 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2954 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2955 if (error != 0) { 2956 device_printf(dev, "Unable to create TX DMA map\n"); 2957 goto fail; 2958 } 2959 } 2960 2961 return 0; 2962fail: 2963 /* We free all, it handles case where we are in the middle */ 2964 em_free_transmit_structures(adapter); 2965 return (error); 2966} 2967 2968/********************************************************************* 2969 * 2970 * Initialize a transmit ring. 2971 * 2972 **********************************************************************/ 2973static void 2974em_setup_transmit_ring(struct tx_ring *txr) 2975{ 2976 struct adapter *adapter = txr->adapter; 2977 struct em_buffer *txbuf; 2978 int i; 2979 2980 /* Clear the old descriptor contents */ 2981 EM_TX_LOCK(txr); 2982 bzero((void *)txr->tx_base, 2983 (sizeof(struct e1000_tx_desc)) * adapter->num_tx_desc); 2984 /* Reset indices */ 2985 txr->next_avail_desc = 0; 2986 txr->next_to_clean = 0; 2987 2988 /* Free any existing tx buffers. */ 2989 txbuf = txr->tx_buffers; 2990 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2991 if (txbuf->m_head != NULL) { 2992 bus_dmamap_sync(txr->txtag, txbuf->map, 2993 BUS_DMASYNC_POSTWRITE); 2994 bus_dmamap_unload(txr->txtag, txbuf->map); 2995 m_freem(txbuf->m_head); 2996 txbuf->m_head = NULL; 2997 } 2998 /* clear the watch index */ 2999 txbuf->next_eop = -1; 3000 } 3001 3002 /* Set number of descriptors available */ 3003 txr->tx_avail = adapter->num_tx_desc; 3004 3005 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3006 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3007 EM_TX_UNLOCK(txr); 3008} 3009 3010/********************************************************************* 3011 * 3012 * Initialize all transmit rings. 3013 * 3014 **********************************************************************/ 3015static void 3016em_setup_transmit_structures(struct adapter *adapter) 3017{ 3018 struct tx_ring *txr = adapter->tx_rings; 3019 3020 for (int i = 0; i < adapter->num_queues; i++, txr++) 3021 em_setup_transmit_ring(txr); 3022 3023 return; 3024} 3025 3026/********************************************************************* 3027 * 3028 * Enable transmit unit. 3029 * 3030 **********************************************************************/ 3031static void 3032em_initialize_transmit_unit(struct adapter *adapter) 3033{ 3034 struct tx_ring *txr = adapter->tx_rings; 3035 struct e1000_hw *hw = &adapter->hw; 3036 u32 tctl, tarc, tipg = 0; 3037 3038 INIT_DEBUGOUT("em_initialize_transmit_unit: begin"); 3039 3040 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3041 u64 bus_addr = txr->txdma.dma_paddr; 3042 /* Base and Len of TX Ring */ 3043 E1000_WRITE_REG(hw, E1000_TDLEN(i), 3044 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3045 E1000_WRITE_REG(hw, E1000_TDBAH(i), 3046 (u32)(bus_addr >> 32)); 3047 E1000_WRITE_REG(hw, E1000_TDBAL(i), 3048 (u32)bus_addr); 3049 /* Init the HEAD/TAIL indices */ 3050 E1000_WRITE_REG(hw, E1000_TDT(i), 0); 3051 E1000_WRITE_REG(hw, E1000_TDH(i), 0); 3052 3053 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3054 E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)), 3055 E1000_READ_REG(&adapter->hw, E1000_TDLEN(i))); 3056 3057 txr->watchdog_check = FALSE; 3058 } 3059 3060 /* Set the default values for the Tx Inter Packet Gap timer */ 3061 switch (adapter->hw.mac.type) { 3062 case e1000_82542: 3063 tipg = DEFAULT_82542_TIPG_IPGT; 3064 tipg |= DEFAULT_82542_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3065 tipg |= DEFAULT_82542_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3066 break; 3067 case e1000_80003es2lan: 3068 tipg = DEFAULT_82543_TIPG_IPGR1; 3069 tipg |= DEFAULT_80003ES2LAN_TIPG_IPGR2 << 3070 E1000_TIPG_IPGR2_SHIFT; 3071 break; 3072 default: 3073 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 3074 (adapter->hw.phy.media_type == 3075 e1000_media_type_internal_serdes)) 3076 tipg = DEFAULT_82543_TIPG_IPGT_FIBER; 3077 else 3078 tipg = DEFAULT_82543_TIPG_IPGT_COPPER; 3079 tipg |= DEFAULT_82543_TIPG_IPGR1 << E1000_TIPG_IPGR1_SHIFT; 3080 tipg |= DEFAULT_82543_TIPG_IPGR2 << E1000_TIPG_IPGR2_SHIFT; 3081 } 3082 3083 E1000_WRITE_REG(&adapter->hw, E1000_TIPG, tipg); 3084 E1000_WRITE_REG(&adapter->hw, E1000_TIDV, adapter->tx_int_delay.value); 3085 3086 if(adapter->hw.mac.type >= e1000_82540) 3087 E1000_WRITE_REG(&adapter->hw, E1000_TADV, 3088 adapter->tx_abs_int_delay.value); 3089 3090 if ((adapter->hw.mac.type == e1000_82571) || 3091 (adapter->hw.mac.type == e1000_82572)) { 3092 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3093 tarc |= SPEED_MODE_BIT; 3094 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3095 } else if (adapter->hw.mac.type == e1000_80003es2lan) { 3096 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(0)); 3097 tarc |= 1; 3098 E1000_WRITE_REG(&adapter->hw, E1000_TARC(0), tarc); 3099 tarc = E1000_READ_REG(&adapter->hw, E1000_TARC(1)); 3100 tarc |= 1; 3101 E1000_WRITE_REG(&adapter->hw, E1000_TARC(1), tarc); 3102 } 3103 3104 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 3105 if (adapter->tx_int_delay.value > 0) 3106 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 3107 3108 /* Program the Transmit Control Register */ 3109 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 3110 tctl &= ~E1000_TCTL_CT; 3111 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3112 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3113 3114 if (adapter->hw.mac.type >= e1000_82571) 3115 tctl |= E1000_TCTL_MULR; 3116 3117 /* This write will effectively turn on the transmit unit. */ 3118 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 3119 3120} 3121 3122 3123/********************************************************************* 3124 * 3125 * Free all transmit rings. 3126 * 3127 **********************************************************************/ 3128static void 3129em_free_transmit_structures(struct adapter *adapter) 3130{ 3131 struct tx_ring *txr = adapter->tx_rings; 3132 3133 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3134 EM_TX_LOCK(txr); 3135 em_free_transmit_buffers(txr); 3136 em_dma_free(adapter, &txr->txdma); 3137 EM_TX_UNLOCK(txr); 3138 EM_TX_LOCK_DESTROY(txr); 3139 } 3140 3141 free(adapter->tx_rings, M_DEVBUF); 3142} 3143 3144/********************************************************************* 3145 * 3146 * Free transmit ring related data structures. 3147 * 3148 **********************************************************************/ 3149static void 3150em_free_transmit_buffers(struct tx_ring *txr) 3151{ 3152 struct adapter *adapter = txr->adapter; 3153 struct em_buffer *txbuf; 3154 3155 INIT_DEBUGOUT("free_transmit_ring: begin"); 3156 3157 if (txr->tx_buffers == NULL) 3158 return; 3159 3160 for (int i = 0; i < adapter->num_tx_desc; i++) { 3161 txbuf = &txr->tx_buffers[i]; 3162 if (txbuf->m_head != NULL) { 3163 bus_dmamap_sync(txr->txtag, txbuf->map, 3164 BUS_DMASYNC_POSTWRITE); 3165 bus_dmamap_unload(txr->txtag, 3166 txbuf->map); 3167 m_freem(txbuf->m_head); 3168 txbuf->m_head = NULL; 3169 if (txbuf->map != NULL) { 3170 bus_dmamap_destroy(txr->txtag, 3171 txbuf->map); 3172 txbuf->map = NULL; 3173 } 3174 } else if (txbuf->map != NULL) { 3175 bus_dmamap_unload(txr->txtag, 3176 txbuf->map); 3177 bus_dmamap_destroy(txr->txtag, 3178 txbuf->map); 3179 txbuf->map = NULL; 3180 } 3181 } 3182#if __FreeBSD_version >= 800000 3183 if (txr->br != NULL) 3184 buf_ring_free(txr->br, M_DEVBUF); 3185#endif 3186 if (txr->tx_buffers != NULL) { 3187 free(txr->tx_buffers, M_DEVBUF); 3188 txr->tx_buffers = NULL; 3189 } 3190 if (txr->txtag != NULL) { 3191 bus_dma_tag_destroy(txr->txtag); 3192 txr->txtag = NULL; 3193 } 3194 return; 3195} 3196 3197 3198/********************************************************************* 3199 * 3200 * The offload context needs to be set when we transfer the first 3201 * packet of a particular protocol (TCP/UDP). This routine has been 3202 * enhanced to deal with inserted VLAN headers, and IPV6 (not complete) 3203 * 3204 * Added back the old method of keeping the current context type 3205 * and not setting if unnecessary, as this is reported to be a 3206 * big performance win. -jfv 3207 **********************************************************************/ 3208static void 3209em_transmit_checksum_setup(struct tx_ring *txr, struct mbuf *mp, 3210 u32 *txd_upper, u32 *txd_lower) 3211{ 3212 struct adapter *adapter = txr->adapter; 3213 struct e1000_context_desc *TXD = NULL; 3214 struct em_buffer *tx_buffer; 3215 struct ether_vlan_header *eh; 3216 struct ip *ip = NULL; 3217 struct ip6_hdr *ip6; 3218 int cur, ehdrlen; 3219 u32 cmd, hdr_len, ip_hlen; 3220 u16 etype; 3221 u8 ipproto; 3222 3223 3224 cmd = hdr_len = ipproto = 0; 3225 cur = txr->next_avail_desc; 3226 3227 /* 3228 * Determine where frame payload starts. 3229 * Jump over vlan headers if already present, 3230 * helpful for QinQ too. 3231 */ 3232 eh = mtod(mp, struct ether_vlan_header *); 3233 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3234 etype = ntohs(eh->evl_proto); 3235 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3236 } else { 3237 etype = ntohs(eh->evl_encap_proto); 3238 ehdrlen = ETHER_HDR_LEN; 3239 } 3240 3241 /* 3242 * We only support TCP/UDP for IPv4 and IPv6 for the moment. 3243 * TODO: Support SCTP too when it hits the tree. 3244 */ 3245 switch (etype) { 3246 case ETHERTYPE_IP: 3247 ip = (struct ip *)(mp->m_data + ehdrlen); 3248 ip_hlen = ip->ip_hl << 2; 3249 3250 /* Setup of IP header checksum. */ 3251 if (mp->m_pkthdr.csum_flags & CSUM_IP) { 3252 /* 3253 * Start offset for header checksum calculation. 3254 * End offset for header checksum calculation. 3255 * Offset of place to put the checksum. 3256 */ 3257 TXD = (struct e1000_context_desc *) 3258 &txr->tx_base[cur]; 3259 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3260 TXD->lower_setup.ip_fields.ipcse = 3261 htole16(ehdrlen + ip_hlen); 3262 TXD->lower_setup.ip_fields.ipcso = 3263 ehdrlen + offsetof(struct ip, ip_sum); 3264 cmd |= E1000_TXD_CMD_IP; 3265 *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 3266 } 3267 3268 if (mp->m_len < ehdrlen + ip_hlen) 3269 return; /* failure */ 3270 3271 hdr_len = ehdrlen + ip_hlen; 3272 ipproto = ip->ip_p; 3273 3274 break; 3275 case ETHERTYPE_IPV6: 3276 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3277 ip_hlen = sizeof(struct ip6_hdr); /* XXX: No header stacking. */ 3278 3279 if (mp->m_len < ehdrlen + ip_hlen) 3280 return; /* failure */ 3281 3282 /* IPv6 doesn't have a header checksum. */ 3283 3284 hdr_len = ehdrlen + ip_hlen; 3285 ipproto = ip6->ip6_nxt; 3286 3287 break; 3288 default: 3289 *txd_upper = 0; 3290 *txd_lower = 0; 3291 return; 3292 } 3293 3294 switch (ipproto) { 3295 case IPPROTO_TCP: 3296 if (mp->m_pkthdr.csum_flags & CSUM_TCP) { 3297 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 3298 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3299 /* no need for context if already set */ 3300 if (txr->last_hw_offload == CSUM_TCP) 3301 return; 3302 txr->last_hw_offload = CSUM_TCP; 3303 /* 3304 * Start offset for payload checksum calculation. 3305 * End offset for payload checksum calculation. 3306 * Offset of place to put the checksum. 3307 */ 3308 TXD = (struct e1000_context_desc *) 3309 &txr->tx_base[cur]; 3310 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3311 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3312 TXD->upper_setup.tcp_fields.tucso = 3313 hdr_len + offsetof(struct tcphdr, th_sum); 3314 cmd |= E1000_TXD_CMD_TCP; 3315 } 3316 break; 3317 case IPPROTO_UDP: 3318 { 3319 if (mp->m_pkthdr.csum_flags & CSUM_UDP) { 3320 *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 3321 *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 3322 /* no need for context if already set */ 3323 if (txr->last_hw_offload == CSUM_UDP) 3324 return; 3325 txr->last_hw_offload = CSUM_UDP; 3326 /* 3327 * Start offset for header checksum calculation. 3328 * End offset for header checksum calculation. 3329 * Offset of place to put the checksum. 3330 */ 3331 TXD = (struct e1000_context_desc *) 3332 &txr->tx_base[cur]; 3333 TXD->upper_setup.tcp_fields.tucss = hdr_len; 3334 TXD->upper_setup.tcp_fields.tucse = htole16(0); 3335 TXD->upper_setup.tcp_fields.tucso = 3336 hdr_len + offsetof(struct udphdr, uh_sum); 3337 } 3338 /* Fall Thru */ 3339 } 3340 default: 3341 break; 3342 } 3343 3344 TXD->tcp_seg_setup.data = htole32(0); 3345 TXD->cmd_and_length = 3346 htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT | cmd); 3347 tx_buffer = &txr->tx_buffers[cur]; 3348 tx_buffer->m_head = NULL; 3349 tx_buffer->next_eop = -1; 3350 3351 if (++cur == adapter->num_tx_desc) 3352 cur = 0; 3353 3354 txr->tx_avail--; 3355 txr->next_avail_desc = cur; 3356} 3357 3358 3359/********************************************************************** 3360 * 3361 * Setup work for hardware segmentation offload (TSO) 3362 * 3363 **********************************************************************/ 3364static bool 3365em_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *txd_upper, 3366 u32 *txd_lower) 3367{ 3368 struct adapter *adapter = txr->adapter; 3369 struct e1000_context_desc *TXD; 3370 struct em_buffer *tx_buffer; 3371 struct ether_vlan_header *eh; 3372 struct ip *ip; 3373 struct ip6_hdr *ip6; 3374 struct tcphdr *th; 3375 int cur, ehdrlen, hdr_len, ip_hlen, isip6; 3376 u16 etype; 3377 3378 /* 3379 * This function could/should be extended to support IP/IPv6 3380 * fragmentation as well. But as they say, one step at a time. 3381 */ 3382 3383 /* 3384 * Determine where frame payload starts. 3385 * Jump over vlan headers if already present, 3386 * helpful for QinQ too. 3387 */ 3388 eh = mtod(mp, struct ether_vlan_header *); 3389 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3390 etype = ntohs(eh->evl_proto); 3391 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3392 } else { 3393 etype = ntohs(eh->evl_encap_proto); 3394 ehdrlen = ETHER_HDR_LEN; 3395 } 3396 3397 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3398 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3399 return FALSE; /* -1 */ 3400 3401 /* 3402 * We only support TCP for IPv4 and IPv6 (notyet) for the moment. 3403 * TODO: Support SCTP too when it hits the tree. 3404 */ 3405 switch (etype) { 3406 case ETHERTYPE_IP: 3407 isip6 = 0; 3408 ip = (struct ip *)(mp->m_data + ehdrlen); 3409 if (ip->ip_p != IPPROTO_TCP) 3410 return FALSE; /* 0 */ 3411 ip->ip_len = 0; 3412 ip->ip_sum = 0; 3413 ip_hlen = ip->ip_hl << 2; 3414 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3415 return FALSE; /* -1 */ 3416 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3417#if 1 3418 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3419 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3420#else 3421 th->th_sum = mp->m_pkthdr.csum_data; 3422#endif 3423 break; 3424 case ETHERTYPE_IPV6: 3425 isip6 = 1; 3426 return FALSE; /* Not supported yet. */ 3427 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3428 if (ip6->ip6_nxt != IPPROTO_TCP) 3429 return FALSE; /* 0 */ 3430 ip6->ip6_plen = 0; 3431 ip_hlen = sizeof(struct ip6_hdr); /* XXX: no header stacking. */ 3432 if (mp->m_len < ehdrlen + ip_hlen + sizeof(struct tcphdr)) 3433 return FALSE; /* -1 */ 3434 th = (struct tcphdr *)((caddr_t)ip6 + ip_hlen); 3435#if 0 3436 th->th_sum = in6_pseudo(ip6->ip6_src, ip->ip6_dst, 3437 htons(IPPROTO_TCP)); /* XXX: function notyet. */ 3438#else 3439 th->th_sum = mp->m_pkthdr.csum_data; 3440#endif 3441 break; 3442 default: 3443 return FALSE; 3444 } 3445 hdr_len = ehdrlen + ip_hlen + (th->th_off << 2); 3446 3447 *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 3448 E1000_TXD_DTYP_D | /* Data descr type */ 3449 E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 3450 3451 /* IP and/or TCP header checksum calculation and insertion. */ 3452 *txd_upper = ((isip6 ? 0 : E1000_TXD_POPTS_IXSM) | 3453 E1000_TXD_POPTS_TXSM) << 8; 3454 3455 cur = txr->next_avail_desc; 3456 tx_buffer = &txr->tx_buffers[cur]; 3457 TXD = (struct e1000_context_desc *) &txr->tx_base[cur]; 3458 3459 /* IPv6 doesn't have a header checksum. */ 3460 if (!isip6) { 3461 /* 3462 * Start offset for header checksum calculation. 3463 * End offset for header checksum calculation. 3464 * Offset of place put the checksum. 3465 */ 3466 TXD->lower_setup.ip_fields.ipcss = ehdrlen; 3467 TXD->lower_setup.ip_fields.ipcse = 3468 htole16(ehdrlen + ip_hlen - 1); 3469 TXD->lower_setup.ip_fields.ipcso = 3470 ehdrlen + offsetof(struct ip, ip_sum); 3471 } 3472 /* 3473 * Start offset for payload checksum calculation. 3474 * End offset for payload checksum calculation. 3475 * Offset of place to put the checksum. 3476 */ 3477 TXD->upper_setup.tcp_fields.tucss = 3478 ehdrlen + ip_hlen; 3479 TXD->upper_setup.tcp_fields.tucse = 0; 3480 TXD->upper_setup.tcp_fields.tucso = 3481 ehdrlen + ip_hlen + offsetof(struct tcphdr, th_sum); 3482 /* 3483 * Payload size per packet w/o any headers. 3484 * Length of all headers up to payload. 3485 */ 3486 TXD->tcp_seg_setup.fields.mss = htole16(mp->m_pkthdr.tso_segsz); 3487 TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 3488 3489 TXD->cmd_and_length = htole32(adapter->txd_cmd | 3490 E1000_TXD_CMD_DEXT | /* Extended descr */ 3491 E1000_TXD_CMD_TSE | /* TSE context */ 3492 (isip6 ? 0 : E1000_TXD_CMD_IP) | 3493 E1000_TXD_CMD_TCP | /* Do TCP checksum */ 3494 (mp->m_pkthdr.len - (hdr_len))); /* Total len */ 3495 3496 tx_buffer->m_head = NULL; 3497 tx_buffer->next_eop = -1; 3498 3499 if (++cur == adapter->num_tx_desc) 3500 cur = 0; 3501 3502 txr->tx_avail--; 3503 txr->next_avail_desc = cur; 3504 txr->tx_tso = TRUE; 3505 3506 return TRUE; 3507} 3508 3509 3510/********************************************************************** 3511 * 3512 * Examine each tx_buffer in the used queue. If the hardware is done 3513 * processing the packet then free associated resources. The 3514 * tx_buffer is put back on the free queue. 3515 * 3516 **********************************************************************/ 3517static bool 3518em_txeof(struct tx_ring *txr) 3519{ 3520 struct adapter *adapter = txr->adapter; 3521 int first, last, done, num_avail; 3522 struct em_buffer *tx_buffer; 3523 struct e1000_tx_desc *tx_desc, *eop_desc; 3524 struct ifnet *ifp = adapter->ifp; 3525 3526 EM_TX_LOCK_ASSERT(txr); 3527 3528 if (txr->tx_avail == adapter->num_tx_desc) 3529 return (FALSE); 3530 3531 num_avail = txr->tx_avail; 3532 first = txr->next_to_clean; 3533 tx_desc = &txr->tx_base[first]; 3534 tx_buffer = &txr->tx_buffers[first]; 3535 last = tx_buffer->next_eop; 3536 eop_desc = &txr->tx_base[last]; 3537 3538 /* 3539 * What this does is get the index of the 3540 * first descriptor AFTER the EOP of the 3541 * first packet, that way we can do the 3542 * simple comparison on the inner while loop. 3543 */ 3544 if (++last == adapter->num_tx_desc) 3545 last = 0; 3546 done = last; 3547 3548 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3549 BUS_DMASYNC_POSTREAD); 3550 3551 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3552 /* We clean the range of the packet */ 3553 while (first != done) { 3554 tx_desc->upper.data = 0; 3555 tx_desc->lower.data = 0; 3556 tx_desc->buffer_addr = 0; 3557 ++num_avail; 3558 3559 if (tx_buffer->m_head) { 3560 ifp->if_opackets++; 3561 bus_dmamap_sync(txr->txtag, 3562 tx_buffer->map, 3563 BUS_DMASYNC_POSTWRITE); 3564 bus_dmamap_unload(txr->txtag, 3565 tx_buffer->map); 3566 3567 m_freem(tx_buffer->m_head); 3568 tx_buffer->m_head = NULL; 3569 } 3570 tx_buffer->next_eop = -1; 3571 txr->watchdog_time = ticks; 3572 3573 if (++first == adapter->num_tx_desc) 3574 first = 0; 3575 3576 tx_buffer = &txr->tx_buffers[first]; 3577 tx_desc = &txr->tx_base[first]; 3578 } 3579 /* See if we can continue to the next packet */ 3580 last = tx_buffer->next_eop; 3581 if (last != -1) { 3582 eop_desc = &txr->tx_base[last]; 3583 /* Get new done point */ 3584 if (++last == adapter->num_tx_desc) last = 0; 3585 done = last; 3586 } else 3587 break; 3588 } 3589 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3590 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3591 3592 txr->next_to_clean = first; 3593 3594 /* 3595 * If we have enough room, clear IFF_DRV_OACTIVE to 3596 * tell the stack that it is OK to send packets. 3597 * If there are no pending descriptors, clear the watchdog. 3598 */ 3599 if (num_avail > EM_TX_CLEANUP_THRESHOLD) { 3600 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3601 if (num_avail == adapter->num_tx_desc) { 3602 txr->watchdog_check = FALSE; 3603 txr->tx_avail = num_avail; 3604 return (FALSE); 3605 } 3606 } 3607 3608 txr->tx_avail = num_avail; 3609 return (TRUE); 3610} 3611 3612 3613/********************************************************************* 3614 * 3615 * Refresh RX descriptor mbufs from system mbuf buffer pool. 3616 * 3617 **********************************************************************/ 3618static void 3619em_refresh_mbufs(struct rx_ring *rxr, int limit) 3620{ 3621 struct adapter *adapter = rxr->adapter; 3622 struct mbuf *m; 3623 bus_dma_segment_t segs[1]; 3624 bus_dmamap_t map; 3625 struct em_buffer *rxbuf; 3626 int i, error, nsegs, cleaned; 3627 3628 i = rxr->next_to_refresh; 3629 cleaned = -1; 3630 while (i != limit) { 3631 m = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 3632 if (m == NULL) 3633 goto update; 3634 m->m_len = m->m_pkthdr.len = MCLBYTES; 3635 3636 if (adapter->max_frame_size <= (MCLBYTES - ETHER_ALIGN)) 3637 m_adj(m, ETHER_ALIGN); 3638 3639 /* 3640 * Using memory from the mbuf cluster pool, invoke the 3641 * bus_dma machinery to arrange the memory mapping. 3642 */ 3643 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, rxr->rx_sparemap, 3644 m, segs, &nsegs, BUS_DMA_NOWAIT); 3645 if (error != 0) { 3646 m_free(m); 3647 goto update; 3648 } 3649 3650 /* If nsegs is wrong then the stack is corrupt. */ 3651 KASSERT(nsegs == 1, ("Too many segments returned!")); 3652 3653 rxbuf = &rxr->rx_buffers[i]; 3654 if (rxbuf->m_head != NULL) 3655 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3656 3657 map = rxbuf->map; 3658 rxbuf->map = rxr->rx_sparemap; 3659 rxr->rx_sparemap = map; 3660 bus_dmamap_sync(rxr->rxtag, 3661 rxbuf->map, BUS_DMASYNC_PREREAD); 3662 rxbuf->m_head = m; 3663 rxr->rx_base[i].buffer_addr = htole64(segs[0].ds_addr); 3664 3665 cleaned = i; 3666 /* Calculate next index */ 3667 if (++i == adapter->num_rx_desc) 3668 i = 0; 3669 /* This is the work marker for refresh */ 3670 rxr->next_to_refresh = i; 3671 } 3672update: 3673 if (cleaned != -1) /* Update tail index */ 3674 E1000_WRITE_REG(&adapter->hw, 3675 E1000_RDT(rxr->me), cleaned); 3676 3677 return; 3678} 3679 3680 3681/********************************************************************* 3682 * 3683 * Allocate memory for rx_buffer structures. Since we use one 3684 * rx_buffer per received packet, the maximum number of rx_buffer's 3685 * that we'll need is equal to the number of receive descriptors 3686 * that we've allocated. 3687 * 3688 **********************************************************************/ 3689static int 3690em_allocate_receive_buffers(struct rx_ring *rxr) 3691{ 3692 struct adapter *adapter = rxr->adapter; 3693 device_t dev = adapter->dev; 3694 struct em_buffer *rxbuf; 3695 int error; 3696 3697 rxr->rx_buffers = malloc(sizeof(struct em_buffer) * 3698 adapter->num_rx_desc, M_DEVBUF, M_NOWAIT | M_ZERO); 3699 if (rxr->rx_buffers == NULL) { 3700 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3701 return (ENOMEM); 3702 } 3703 3704 error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 3705 1, 0, /* alignment, bounds */ 3706 BUS_SPACE_MAXADDR, /* lowaddr */ 3707 BUS_SPACE_MAXADDR, /* highaddr */ 3708 NULL, NULL, /* filter, filterarg */ 3709 MCLBYTES, /* maxsize */ 3710 1, /* nsegments */ 3711 MCLBYTES, /* maxsegsize */ 3712 0, /* flags */ 3713 NULL, /* lockfunc */ 3714 NULL, /* lockarg */ 3715 &rxr->rxtag); 3716 if (error) { 3717 device_printf(dev, "%s: bus_dma_tag_create failed %d\n", 3718 __func__, error); 3719 goto fail; 3720 } 3721 3722 /* Create the spare map (used by getbuf) */ 3723 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3724 &rxr->rx_sparemap); 3725 if (error) { 3726 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3727 __func__, error); 3728 goto fail; 3729 } 3730 3731 rxbuf = rxr->rx_buffers; 3732 for (int i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3733 rxbuf = &rxr->rx_buffers[i]; 3734 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3735 &rxbuf->map); 3736 if (error) { 3737 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3738 __func__, error); 3739 goto fail; 3740 } 3741 } 3742 3743 return (0); 3744 3745fail: 3746 em_free_receive_structures(adapter); 3747 return (error); 3748} 3749 3750 3751/********************************************************************* 3752 * 3753 * Initialize a receive ring and its buffers. 3754 * 3755 **********************************************************************/ 3756static int 3757em_setup_receive_ring(struct rx_ring *rxr) 3758{ 3759 struct adapter *adapter = rxr->adapter; 3760 struct em_buffer *rxbuf; 3761 bus_dma_segment_t seg[1]; 3762 int rsize, nsegs, error; 3763 3764 3765 /* Clear the ring contents */ 3766 EM_RX_LOCK(rxr); 3767 rsize = roundup2(adapter->num_rx_desc * 3768 sizeof(struct e1000_rx_desc), EM_DBA_ALIGN); 3769 bzero((void *)rxr->rx_base, rsize); 3770 3771 /* 3772 ** Free current RX buffer structs and their mbufs 3773 */ 3774 for (int i = 0; i < adapter->num_rx_desc; i++) { 3775 rxbuf = &rxr->rx_buffers[i]; 3776 if (rxbuf->m_head != NULL) { 3777 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3778 BUS_DMASYNC_POSTREAD); 3779 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3780 m_freem(rxbuf->m_head); 3781 } 3782 } 3783 3784 /* Now replenish the mbufs */ 3785 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3786 3787 rxbuf = &rxr->rx_buffers[j]; 3788 rxbuf->m_head = m_getcl(M_DONTWAIT, MT_DATA, M_PKTHDR); 3789 if (rxbuf->m_head == NULL) 3790 panic("RX ring hdr initialization failed!\n"); 3791 rxbuf->m_head->m_len = MCLBYTES; 3792 rxbuf->m_head->m_flags &= ~M_HASFCS; /* we strip it */ 3793 rxbuf->m_head->m_pkthdr.len = MCLBYTES; 3794 3795 /* Get the memory mapping */ 3796 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3797 rxbuf->map, rxbuf->m_head, seg, 3798 &nsegs, BUS_DMA_NOWAIT); 3799 if (error != 0) 3800 panic("RX ring dma initialization failed!\n"); 3801 bus_dmamap_sync(rxr->rxtag, 3802 rxbuf->map, BUS_DMASYNC_PREREAD); 3803 3804 /* Update descriptor */ 3805 rxr->rx_base[j].buffer_addr = htole64(seg[0].ds_addr); 3806 } 3807 3808 3809 /* Setup our descriptor indices */ 3810 rxr->next_to_check = 0; 3811 rxr->next_to_refresh = 0; 3812 3813 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3814 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3815 3816 EM_RX_UNLOCK(rxr); 3817 return (0); 3818} 3819 3820/********************************************************************* 3821 * 3822 * Initialize all receive rings. 3823 * 3824 **********************************************************************/ 3825static int 3826em_setup_receive_structures(struct adapter *adapter) 3827{ 3828 struct rx_ring *rxr = adapter->rx_rings; 3829 int j; 3830 3831 for (j = 0; j < adapter->num_queues; j++, rxr++) 3832 if (em_setup_receive_ring(rxr)) 3833 goto fail; 3834 3835 return (0); 3836fail: 3837 /* 3838 * Free RX buffers allocated so far, we will only handle 3839 * the rings that completed, the failing case will have 3840 * cleaned up for itself. 'j' failed, so its the terminus. 3841 */ 3842 for (int i = 0; i < j; ++i) { 3843 rxr = &adapter->rx_rings[i]; 3844 for (int n = 0; n < adapter->num_rx_desc; n++) { 3845 struct em_buffer *rxbuf; 3846 rxbuf = &rxr->rx_buffers[n]; 3847 if (rxbuf->m_head != NULL) { 3848 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3849 BUS_DMASYNC_POSTREAD); 3850 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3851 m_freem(rxbuf->m_head); 3852 rxbuf->m_head = NULL; 3853 } 3854 } 3855 } 3856 3857 return (ENOBUFS); 3858} 3859 3860/********************************************************************* 3861 * 3862 * Free all receive rings. 3863 * 3864 **********************************************************************/ 3865static void 3866em_free_receive_structures(struct adapter *adapter) 3867{ 3868 struct rx_ring *rxr = adapter->rx_rings; 3869 3870 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3871 em_free_receive_buffers(rxr); 3872 /* Free the ring memory as well */ 3873 em_dma_free(adapter, &rxr->rxdma); 3874 EM_RX_LOCK_DESTROY(rxr); 3875 } 3876 3877 free(adapter->rx_rings, M_DEVBUF); 3878} 3879 3880 3881/********************************************************************* 3882 * 3883 * Free receive ring data structures 3884 * 3885 **********************************************************************/ 3886static void 3887em_free_receive_buffers(struct rx_ring *rxr) 3888{ 3889 struct adapter *adapter = rxr->adapter; 3890 struct em_buffer *rxbuf = NULL; 3891 3892 INIT_DEBUGOUT("free_receive_buffers: begin"); 3893 3894 if (rxr->rx_sparemap) { 3895 bus_dmamap_destroy(rxr->rxtag, rxr->rx_sparemap); 3896 rxr->rx_sparemap = NULL; 3897 } 3898 3899 if (rxr->rx_buffers != NULL) { 3900 for (int i = 0; i < adapter->num_rx_desc; i++) { 3901 rxbuf = &rxr->rx_buffers[i]; 3902 if (rxbuf->map != NULL) { 3903 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3904 BUS_DMASYNC_POSTREAD); 3905 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3906 bus_dmamap_destroy(rxr->rxtag, rxbuf->map); 3907 } 3908 if (rxbuf->m_head != NULL) { 3909 m_freem(rxbuf->m_head); 3910 rxbuf->m_head = NULL; 3911 } 3912 } 3913 free(rxr->rx_buffers, M_DEVBUF); 3914 rxr->rx_buffers = NULL; 3915 } 3916 3917 if (rxr->rxtag != NULL) { 3918 bus_dma_tag_destroy(rxr->rxtag); 3919 rxr->rxtag = NULL; 3920 } 3921 3922 return; 3923} 3924 3925 3926/********************************************************************* 3927 * 3928 * Enable receive unit. 3929 * 3930 **********************************************************************/ 3931#define MAX_INTS_PER_SEC 8000 3932#define DEFAULT_ITR 1000000000/(MAX_INTS_PER_SEC * 256) 3933 3934static void 3935em_initialize_receive_unit(struct adapter *adapter) 3936{ 3937 struct rx_ring *rxr = adapter->rx_rings; 3938 struct ifnet *ifp = adapter->ifp; 3939 struct e1000_hw *hw = &adapter->hw; 3940 u64 bus_addr; 3941 u32 rctl, rxcsum; 3942 3943 INIT_DEBUGOUT("em_initialize_receive_units: begin"); 3944 3945 /* 3946 * Make sure receives are disabled while setting 3947 * up the descriptor ring 3948 */ 3949 rctl = E1000_READ_REG(hw, E1000_RCTL); 3950 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3951 3952 E1000_WRITE_REG(&adapter->hw, E1000_RADV, 3953 adapter->rx_abs_int_delay.value); 3954 /* 3955 * Set the interrupt throttling rate. Value is calculated 3956 * as DEFAULT_ITR = 1/(MAX_INTS_PER_SEC * 256ns) 3957 */ 3958 E1000_WRITE_REG(hw, E1000_ITR, DEFAULT_ITR); 3959 3960 /* 3961 ** When using MSIX interrupts we need to throttle 3962 ** using the EITR register (82574 only) 3963 */ 3964 if (adapter->msix) 3965 for (int i = 0; i < 4; i++) 3966 E1000_WRITE_REG(hw, E1000_EITR_82574(i), 3967 DEFAULT_ITR); 3968 3969 /* Disable accelerated ackknowledge */ 3970 if (adapter->hw.mac.type == e1000_82574) 3971 E1000_WRITE_REG(hw, E1000_RFCTL, E1000_RFCTL_ACK_DIS); 3972 3973 if (ifp->if_capenable & IFCAP_RXCSUM) { 3974 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 3975 rxcsum |= (E1000_RXCSUM_IPOFL | E1000_RXCSUM_TUOFL); 3976 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 3977 } 3978 3979 /* 3980 ** XXX TEMPORARY WORKAROUND: on some systems with 82573 3981 ** long latencies are observed, like Lenovo X60. This 3982 ** change eliminates the problem, but since having positive 3983 ** values in RDTR is a known source of problems on other 3984 ** platforms another solution is being sought. 3985 */ 3986 if (hw->mac.type == e1000_82573) 3987 E1000_WRITE_REG(hw, E1000_RDTR, 0x20); 3988 3989 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3990 /* Setup the Base and Length of the Rx Descriptor Ring */ 3991 bus_addr = rxr->rxdma.dma_paddr; 3992 E1000_WRITE_REG(hw, E1000_RDLEN(i), 3993 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 3994 E1000_WRITE_REG(hw, E1000_RDBAH(i), (u32)(bus_addr >> 32)); 3995 E1000_WRITE_REG(hw, E1000_RDBAL(i), (u32)bus_addr); 3996 /* Setup the Head and Tail Descriptor Pointers */ 3997 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 3998 E1000_WRITE_REG(hw, E1000_RDT(i), adapter->num_rx_desc - 1); 3999 } 4000 4001 /* Setup the Receive Control Register */ 4002 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4003 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | 4004 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF | 4005 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4006 4007 /* Make sure VLAN Filters are off */ 4008 rctl &= ~E1000_RCTL_VFE; 4009 rctl &= ~E1000_RCTL_SBP; 4010 rctl |= E1000_RCTL_SZ_2048; 4011 if (ifp->if_mtu > ETHERMTU) 4012 rctl |= E1000_RCTL_LPE; 4013 else 4014 rctl &= ~E1000_RCTL_LPE; 4015 4016 /* Write out the settings */ 4017 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4018 4019 return; 4020} 4021 4022 4023/********************************************************************* 4024 * 4025 * This routine executes in interrupt context. It replenishes 4026 * the mbufs in the descriptor and sends data which has been 4027 * dma'ed into host memory to upper layer. 4028 * 4029 * We loop at most count times if count is > 0, or until done if 4030 * count < 0. 4031 * 4032 * For polling we also now return the number of cleaned packets 4033 *********************************************************************/ 4034static int 4035em_rxeof(struct rx_ring *rxr, int count) 4036{ 4037 struct adapter *adapter = rxr->adapter; 4038 struct ifnet *ifp = adapter->ifp;; 4039 struct mbuf *mp, *sendmp; 4040 u8 status; 4041 u16 len; 4042 int i, processed, rxdone = 0; 4043 bool eop; 4044 struct e1000_rx_desc *cur; 4045 4046 EM_RX_LOCK(rxr); 4047 4048 for (i = rxr->next_to_check, processed = 0; count != 0;) { 4049 4050 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 4051 break; 4052 4053 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4054 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4055 4056 cur = &rxr->rx_base[i]; 4057 status = cur->status; 4058 mp = sendmp = NULL; 4059 4060 if ((status & E1000_RXD_STAT_DD) == 0) 4061 break; 4062 4063 len = le16toh(cur->length); 4064 eop = (status & E1000_RXD_STAT_EOP) != 0; 4065 count--; 4066 4067 if ((cur->errors & E1000_RXD_ERR_FRAME_ERR_MASK) == 0) { 4068 4069 /* Assign correct length to the current fragment */ 4070 mp = rxr->rx_buffers[i].m_head; 4071 mp->m_len = len; 4072 4073 if (rxr->fmp == NULL) { 4074 mp->m_pkthdr.len = len; 4075 rxr->fmp = mp; /* Store the first mbuf */ 4076 rxr->lmp = mp; 4077 } else { 4078 /* Chain mbuf's together */ 4079 mp->m_flags &= ~M_PKTHDR; 4080 rxr->lmp->m_next = mp; 4081 rxr->lmp = rxr->lmp->m_next; 4082 rxr->fmp->m_pkthdr.len += len; 4083 } 4084 4085 if (eop) { 4086 rxr->fmp->m_pkthdr.rcvif = ifp; 4087 ifp->if_ipackets++; 4088 em_receive_checksum(cur, rxr->fmp); 4089#ifndef __NO_STRICT_ALIGNMENT 4090 if (adapter->max_frame_size > 4091 (MCLBYTES - ETHER_ALIGN) && 4092 em_fixup_rx(rxr) != 0) 4093 goto skip; 4094#endif 4095 if (status & E1000_RXD_STAT_VP) { 4096 rxr->fmp->m_pkthdr.ether_vtag = 4097 (le16toh(cur->special) & 4098 E1000_RXD_SPC_VLAN_MASK); 4099 rxr->fmp->m_flags |= M_VLANTAG; 4100 } 4101#ifndef __NO_STRICT_ALIGNMENT 4102skip: 4103#endif 4104 sendmp = rxr->fmp; 4105 rxr->fmp = NULL; 4106 rxr->lmp = NULL; 4107 } 4108 } else { 4109 ifp->if_ierrors++; 4110 /* Reuse loaded DMA map and just update mbuf chain */ 4111 mp = rxr->rx_buffers[i].m_head; 4112 mp->m_len = mp->m_pkthdr.len = MCLBYTES; 4113 mp->m_data = mp->m_ext.ext_buf; 4114 mp->m_next = NULL; 4115 if (adapter->max_frame_size <= 4116 (MCLBYTES - ETHER_ALIGN)) 4117 m_adj(mp, ETHER_ALIGN); 4118 if (rxr->fmp != NULL) { 4119 m_freem(rxr->fmp); 4120 rxr->fmp = NULL; 4121 rxr->lmp = NULL; 4122 } 4123 sendmp = NULL; 4124 } 4125 4126 /* Zero out the receive descriptors status. */ 4127 cur->status = 0; 4128 ++rxdone; /* cumulative for POLL */ 4129 ++processed; 4130 4131 /* Advance our pointers to the next descriptor. */ 4132 if (++i == adapter->num_rx_desc) 4133 i = 0; 4134 4135 /* Send to the stack */ 4136 if (sendmp != NULL) 4137 (*ifp->if_input)(ifp, sendmp); 4138 4139 /* Only refresh mbufs every 8 descriptors */ 4140 if (processed == 8) { 4141 em_refresh_mbufs(rxr, i); 4142 processed = 0; 4143 } 4144 } 4145 4146 /* Catch any remaining refresh work */ 4147 if (processed != 0) { 4148 em_refresh_mbufs(rxr, i); 4149 processed = 0; 4150 } 4151 4152 rxr->next_to_check = i; 4153 4154 EM_RX_UNLOCK(rxr); 4155 return (rxdone); 4156} 4157 4158#ifndef __NO_STRICT_ALIGNMENT 4159/* 4160 * When jumbo frames are enabled we should realign entire payload on 4161 * architecures with strict alignment. This is serious design mistake of 8254x 4162 * as it nullifies DMA operations. 8254x just allows RX buffer size to be 4163 * 2048/4096/8192/16384. What we really want is 2048 - ETHER_ALIGN to align its 4164 * payload. On architecures without strict alignment restrictions 8254x still 4165 * performs unaligned memory access which would reduce the performance too. 4166 * To avoid copying over an entire frame to align, we allocate a new mbuf and 4167 * copy ethernet header to the new mbuf. The new mbuf is prepended into the 4168 * existing mbuf chain. 4169 * 4170 * Be aware, best performance of the 8254x is achived only when jumbo frame is 4171 * not used at all on architectures with strict alignment. 4172 */ 4173static int 4174em_fixup_rx(struct rx_ring *rxr) 4175{ 4176 struct adapter *adapter = rxr->adapter; 4177 struct mbuf *m, *n; 4178 int error; 4179 4180 error = 0; 4181 m = rxr->fmp; 4182 if (m->m_len <= (MCLBYTES - ETHER_HDR_LEN)) { 4183 bcopy(m->m_data, m->m_data + ETHER_HDR_LEN, m->m_len); 4184 m->m_data += ETHER_HDR_LEN; 4185 } else { 4186 MGETHDR(n, M_DONTWAIT, MT_DATA); 4187 if (n != NULL) { 4188 bcopy(m->m_data, n->m_data, ETHER_HDR_LEN); 4189 m->m_data += ETHER_HDR_LEN; 4190 m->m_len -= ETHER_HDR_LEN; 4191 n->m_len = ETHER_HDR_LEN; 4192 M_MOVE_PKTHDR(n, m); 4193 n->m_next = m; 4194 rxr->fmp = n; 4195 } else { 4196 adapter->dropped_pkts++; 4197 m_freem(rxr->fmp); 4198 rxr->fmp = NULL; 4199 error = ENOMEM; 4200 } 4201 } 4202 4203 return (error); 4204} 4205#endif 4206 4207/********************************************************************* 4208 * 4209 * Verify that the hardware indicated that the checksum is valid. 4210 * Inform the stack about the status of checksum so that stack 4211 * doesn't spend time verifying the checksum. 4212 * 4213 *********************************************************************/ 4214static void 4215em_receive_checksum(struct e1000_rx_desc *rx_desc, struct mbuf *mp) 4216{ 4217 /* Ignore Checksum bit is set */ 4218 if (rx_desc->status & E1000_RXD_STAT_IXSM) { 4219 mp->m_pkthdr.csum_flags = 0; 4220 return; 4221 } 4222 4223 if (rx_desc->status & E1000_RXD_STAT_IPCS) { 4224 /* Did it pass? */ 4225 if (!(rx_desc->errors & E1000_RXD_ERR_IPE)) { 4226 /* IP Checksum Good */ 4227 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4228 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4229 4230 } else { 4231 mp->m_pkthdr.csum_flags = 0; 4232 } 4233 } 4234 4235 if (rx_desc->status & E1000_RXD_STAT_TCPCS) { 4236 /* Did it pass? */ 4237 if (!(rx_desc->errors & E1000_RXD_ERR_TCPE)) { 4238 mp->m_pkthdr.csum_flags |= 4239 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4240 mp->m_pkthdr.csum_data = htons(0xffff); 4241 } 4242 } 4243} 4244 4245/* 4246 * This routine is run via an vlan 4247 * config EVENT 4248 */ 4249static void 4250em_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4251{ 4252 struct adapter *adapter = ifp->if_softc; 4253 u32 index, bit; 4254 4255 if (ifp->if_softc != arg) /* Not our event */ 4256 return; 4257 4258 if ((vtag == 0) || (vtag > 4095)) /* Invalid ID */ 4259 return; 4260 4261 index = (vtag >> 5) & 0x7F; 4262 bit = vtag & 0x1F; 4263 em_shadow_vfta[index] |= (1 << bit); 4264 ++adapter->num_vlans; 4265 /* Re-init to load the changes */ 4266 em_init(adapter); 4267} 4268 4269/* 4270 * This routine is run via an vlan 4271 * unconfig EVENT 4272 */ 4273static void 4274em_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4275{ 4276 struct adapter *adapter = ifp->if_softc; 4277 u32 index, bit; 4278 4279 if (ifp->if_softc != arg) 4280 return; 4281 4282 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4283 return; 4284 4285 index = (vtag >> 5) & 0x7F; 4286 bit = vtag & 0x1F; 4287 em_shadow_vfta[index] &= ~(1 << bit); 4288 --adapter->num_vlans; 4289 /* Re-init to load the changes */ 4290 em_init(adapter); 4291} 4292 4293static void 4294em_setup_vlan_hw_support(struct adapter *adapter) 4295{ 4296 struct e1000_hw *hw = &adapter->hw; 4297 u32 reg; 4298 4299 /* 4300 ** We get here thru init_locked, meaning 4301 ** a soft reset, this has already cleared 4302 ** the VFTA and other state, so if there 4303 ** have been no vlan's registered do nothing. 4304 */ 4305 if (adapter->num_vlans == 0) 4306 return; 4307 4308 /* 4309 ** A soft reset zero's out the VFTA, so 4310 ** we need to repopulate it now. 4311 */ 4312 for (int i = 0; i < EM_VFTA_SIZE; i++) 4313 if (em_shadow_vfta[i] != 0) 4314 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, 4315 i, em_shadow_vfta[i]); 4316 4317 reg = E1000_READ_REG(hw, E1000_CTRL); 4318 reg |= E1000_CTRL_VME; 4319 E1000_WRITE_REG(hw, E1000_CTRL, reg); 4320 4321 /* Enable the Filter Table */ 4322 reg = E1000_READ_REG(hw, E1000_RCTL); 4323 reg &= ~E1000_RCTL_CFIEN; 4324 reg |= E1000_RCTL_VFE; 4325 E1000_WRITE_REG(hw, E1000_RCTL, reg); 4326 4327 /* Update the frame size */ 4328 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4329 adapter->max_frame_size + VLAN_TAG_SIZE); 4330} 4331 4332static void 4333em_enable_intr(struct adapter *adapter) 4334{ 4335 struct e1000_hw *hw = &adapter->hw; 4336 u32 ims_mask = IMS_ENABLE_MASK; 4337 4338 if (adapter->msix) { 4339 E1000_WRITE_REG(hw, EM_EIAC, EM_MSIX_MASK); 4340 ims_mask |= EM_MSIX_MASK; 4341 } 4342 E1000_WRITE_REG(hw, E1000_IMS, ims_mask); 4343} 4344 4345static void 4346em_disable_intr(struct adapter *adapter) 4347{ 4348 struct e1000_hw *hw = &adapter->hw; 4349 4350 if (adapter->msix) 4351 E1000_WRITE_REG(hw, EM_EIAC, 0); 4352 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 4353} 4354 4355/* 4356 * Bit of a misnomer, what this really means is 4357 * to enable OS management of the system... aka 4358 * to disable special hardware management features 4359 */ 4360static void 4361em_init_manageability(struct adapter *adapter) 4362{ 4363 /* A shared code workaround */ 4364#define E1000_82542_MANC2H E1000_MANC2H 4365 if (adapter->has_manage) { 4366 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4367 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4368 4369 /* disable hardware interception of ARP */ 4370 manc &= ~(E1000_MANC_ARP_EN); 4371 4372 /* enable receiving management packets to the host */ 4373 manc |= E1000_MANC_EN_MNG2HOST; 4374#define E1000_MNG2HOST_PORT_623 (1 << 5) 4375#define E1000_MNG2HOST_PORT_664 (1 << 6) 4376 manc2h |= E1000_MNG2HOST_PORT_623; 4377 manc2h |= E1000_MNG2HOST_PORT_664; 4378 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4379 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4380 } 4381} 4382 4383/* 4384 * Give control back to hardware management 4385 * controller if there is one. 4386 */ 4387static void 4388em_release_manageability(struct adapter *adapter) 4389{ 4390 if (adapter->has_manage) { 4391 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4392 4393 /* re-enable hardware interception of ARP */ 4394 manc |= E1000_MANC_ARP_EN; 4395 manc &= ~E1000_MANC_EN_MNG2HOST; 4396 4397 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4398 } 4399} 4400 4401/* 4402 * em_get_hw_control sets the {CTRL_EXT|FWSM}:DRV_LOAD bit. 4403 * For ASF and Pass Through versions of f/w this means 4404 * that the driver is loaded. For AMT version type f/w 4405 * this means that the network i/f is open. 4406 */ 4407static void 4408em_get_hw_control(struct adapter *adapter) 4409{ 4410 u32 ctrl_ext, swsm; 4411 4412 if (adapter->hw.mac.type == e1000_82573) { 4413 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4414 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4415 swsm | E1000_SWSM_DRV_LOAD); 4416 return; 4417 } 4418 /* else */ 4419 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4420 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4421 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4422 return; 4423} 4424 4425/* 4426 * em_release_hw_control resets {CTRL_EXT|FWSM}:DRV_LOAD bit. 4427 * For ASF and Pass Through versions of f/w this means that 4428 * the driver is no longer loaded. For AMT versions of the 4429 * f/w this means that the network i/f is closed. 4430 */ 4431static void 4432em_release_hw_control(struct adapter *adapter) 4433{ 4434 u32 ctrl_ext, swsm; 4435 4436 if (!adapter->has_manage) 4437 return; 4438 4439 if (adapter->hw.mac.type == e1000_82573) { 4440 swsm = E1000_READ_REG(&adapter->hw, E1000_SWSM); 4441 E1000_WRITE_REG(&adapter->hw, E1000_SWSM, 4442 swsm & ~E1000_SWSM_DRV_LOAD); 4443 return; 4444 } 4445 /* else */ 4446 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4447 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4448 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4449 return; 4450} 4451 4452static int 4453em_is_valid_ether_addr(u8 *addr) 4454{ 4455 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4456 4457 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4458 return (FALSE); 4459 } 4460 4461 return (TRUE); 4462} 4463 4464/* 4465** Parse the interface capabilities with regard 4466** to both system management and wake-on-lan for 4467** later use. 4468*/ 4469static void 4470em_get_wakeup(device_t dev) 4471{ 4472 struct adapter *adapter = device_get_softc(dev); 4473 u16 eeprom_data = 0, device_id, apme_mask; 4474 4475 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 4476 apme_mask = EM_EEPROM_APME; 4477 4478 switch (adapter->hw.mac.type) { 4479 case e1000_82573: 4480 case e1000_82583: 4481 adapter->has_amt = TRUE; 4482 /* Falls thru */ 4483 case e1000_82571: 4484 case e1000_82572: 4485 case e1000_80003es2lan: 4486 if (adapter->hw.bus.func == 1) { 4487 e1000_read_nvm(&adapter->hw, 4488 NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data); 4489 break; 4490 } else 4491 e1000_read_nvm(&adapter->hw, 4492 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 4493 break; 4494 case e1000_ich8lan: 4495 case e1000_ich9lan: 4496 case e1000_ich10lan: 4497 case e1000_pchlan: 4498 apme_mask = E1000_WUC_APME; 4499 adapter->has_amt = TRUE; 4500 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC); 4501 break; 4502 default: 4503 e1000_read_nvm(&adapter->hw, 4504 NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data); 4505 break; 4506 } 4507 if (eeprom_data & apme_mask) 4508 adapter->wol = (E1000_WUFC_MAG | E1000_WUFC_MC); 4509 /* 4510 * We have the eeprom settings, now apply the special cases 4511 * where the eeprom may be wrong or the board won't support 4512 * wake on lan on a particular port 4513 */ 4514 device_id = pci_get_device(dev); 4515 switch (device_id) { 4516 case E1000_DEV_ID_82571EB_FIBER: 4517 /* Wake events only supported on port A for dual fiber 4518 * regardless of eeprom setting */ 4519 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 4520 E1000_STATUS_FUNC_1) 4521 adapter->wol = 0; 4522 break; 4523 case E1000_DEV_ID_82571EB_QUAD_COPPER: 4524 case E1000_DEV_ID_82571EB_QUAD_FIBER: 4525 case E1000_DEV_ID_82571EB_QUAD_COPPER_LP: 4526 /* if quad port adapter, disable WoL on all but port A */ 4527 if (global_quad_port_a != 0) 4528 adapter->wol = 0; 4529 /* Reset for multiple quad port adapters */ 4530 if (++global_quad_port_a == 4) 4531 global_quad_port_a = 0; 4532 break; 4533 } 4534 return; 4535} 4536 4537 4538/* 4539 * Enable PCI Wake On Lan capability 4540 */ 4541void 4542em_enable_wakeup(device_t dev) 4543{ 4544 struct adapter *adapter = device_get_softc(dev); 4545 struct ifnet *ifp = adapter->ifp; 4546 u32 pmc, ctrl, ctrl_ext, rctl; 4547 u16 status; 4548 4549 if ((pci_find_extcap(dev, PCIY_PMG, &pmc) != 0)) 4550 return; 4551 4552 /* Advertise the wakeup capability */ 4553 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4554 ctrl |= (E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN3); 4555 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4556 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 4557 4558 if ((adapter->hw.mac.type == e1000_ich8lan) || 4559 (adapter->hw.mac.type == e1000_pchlan) || 4560 (adapter->hw.mac.type == e1000_ich9lan) || 4561 (adapter->hw.mac.type == e1000_ich10lan)) { 4562 e1000_disable_gig_wol_ich8lan(&adapter->hw); 4563 e1000_hv_phy_powerdown_workaround_ich8lan(&adapter->hw); 4564 } 4565 4566 /* Keep the laser running on Fiber adapters */ 4567 if (adapter->hw.phy.media_type == e1000_media_type_fiber || 4568 adapter->hw.phy.media_type == e1000_media_type_internal_serdes) { 4569 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4570 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA; 4571 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, ctrl_ext); 4572 } 4573 4574 /* 4575 ** Determine type of Wakeup: note that wol 4576 ** is set with all bits on by default. 4577 */ 4578 if ((ifp->if_capenable & IFCAP_WOL_MAGIC) == 0) 4579 adapter->wol &= ~E1000_WUFC_MAG; 4580 4581 if ((ifp->if_capenable & IFCAP_WOL_MCAST) == 0) 4582 adapter->wol &= ~E1000_WUFC_MC; 4583 else { 4584 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4585 rctl |= E1000_RCTL_MPE; 4586 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4587 } 4588 4589 if (adapter->hw.mac.type == e1000_pchlan) { 4590 if (em_enable_phy_wakeup(adapter)) 4591 return; 4592 } else { 4593 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 4594 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 4595 } 4596 4597 if (adapter->hw.phy.type == e1000_phy_igp_3) 4598 e1000_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw); 4599 4600 /* Request PME */ 4601 status = pci_read_config(dev, pmc + PCIR_POWER_STATUS, 2); 4602 status &= ~(PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE); 4603 if (ifp->if_capenable & IFCAP_WOL) 4604 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4605 pci_write_config(dev, pmc + PCIR_POWER_STATUS, status, 2); 4606 4607 return; 4608} 4609 4610/* 4611** WOL in the newer chipset interfaces (pchlan) 4612** require thing to be copied into the phy 4613*/ 4614static int 4615em_enable_phy_wakeup(struct adapter *adapter) 4616{ 4617 struct e1000_hw *hw = &adapter->hw; 4618 u32 mreg, ret = 0; 4619 u16 preg; 4620 4621 /* copy MAC RARs to PHY RARs */ 4622 for (int i = 0; i < adapter->hw.mac.rar_entry_count; i++) { 4623 mreg = E1000_READ_REG(hw, E1000_RAL(i)); 4624 e1000_write_phy_reg(hw, BM_RAR_L(i), (u16)(mreg & 0xFFFF)); 4625 e1000_write_phy_reg(hw, BM_RAR_M(i), 4626 (u16)((mreg >> 16) & 0xFFFF)); 4627 mreg = E1000_READ_REG(hw, E1000_RAH(i)); 4628 e1000_write_phy_reg(hw, BM_RAR_H(i), (u16)(mreg & 0xFFFF)); 4629 e1000_write_phy_reg(hw, BM_RAR_CTRL(i), 4630 (u16)((mreg >> 16) & 0xFFFF)); 4631 } 4632 4633 /* copy MAC MTA to PHY MTA */ 4634 for (int i = 0; i < adapter->hw.mac.mta_reg_count; i++) { 4635 mreg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i); 4636 e1000_write_phy_reg(hw, BM_MTA(i), (u16)(mreg & 0xFFFF)); 4637 e1000_write_phy_reg(hw, BM_MTA(i) + 1, 4638 (u16)((mreg >> 16) & 0xFFFF)); 4639 } 4640 4641 /* configure PHY Rx Control register */ 4642 e1000_read_phy_reg(&adapter->hw, BM_RCTL, &preg); 4643 mreg = E1000_READ_REG(hw, E1000_RCTL); 4644 if (mreg & E1000_RCTL_UPE) 4645 preg |= BM_RCTL_UPE; 4646 if (mreg & E1000_RCTL_MPE) 4647 preg |= BM_RCTL_MPE; 4648 preg &= ~(BM_RCTL_MO_MASK); 4649 if (mreg & E1000_RCTL_MO_3) 4650 preg |= (((mreg & E1000_RCTL_MO_3) >> E1000_RCTL_MO_SHIFT) 4651 << BM_RCTL_MO_SHIFT); 4652 if (mreg & E1000_RCTL_BAM) 4653 preg |= BM_RCTL_BAM; 4654 if (mreg & E1000_RCTL_PMCF) 4655 preg |= BM_RCTL_PMCF; 4656 mreg = E1000_READ_REG(hw, E1000_CTRL); 4657 if (mreg & E1000_CTRL_RFCE) 4658 preg |= BM_RCTL_RFCE; 4659 e1000_write_phy_reg(&adapter->hw, BM_RCTL, preg); 4660 4661 /* enable PHY wakeup in MAC register */ 4662 E1000_WRITE_REG(hw, E1000_WUC, 4663 E1000_WUC_PHY_WAKE | E1000_WUC_PME_EN); 4664 E1000_WRITE_REG(hw, E1000_WUFC, adapter->wol); 4665 4666 /* configure and enable PHY wakeup in PHY registers */ 4667 e1000_write_phy_reg(&adapter->hw, BM_WUFC, adapter->wol); 4668 e1000_write_phy_reg(&adapter->hw, BM_WUC, E1000_WUC_PME_EN); 4669 4670 /* activate PHY wakeup */ 4671 ret = hw->phy.ops.acquire(hw); 4672 if (ret) { 4673 printf("Could not acquire PHY\n"); 4674 return ret; 4675 } 4676 e1000_write_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT, 4677 (BM_WUC_ENABLE_PAGE << IGP_PAGE_SHIFT)); 4678 ret = e1000_read_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, &preg); 4679 if (ret) { 4680 printf("Could not read PHY page 769\n"); 4681 goto out; 4682 } 4683 preg |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT; 4684 ret = e1000_write_phy_reg_mdic(hw, BM_WUC_ENABLE_REG, preg); 4685 if (ret) 4686 printf("Could not set PHY Host Wakeup bit\n"); 4687out: 4688 hw->phy.ops.release(hw); 4689 4690 return ret; 4691} 4692 4693/********************************************************************** 4694 * 4695 * Update the board statistics counters. 4696 * 4697 **********************************************************************/ 4698static void 4699em_update_stats_counters(struct adapter *adapter) 4700{ 4701 struct ifnet *ifp; 4702 4703 if(adapter->hw.phy.media_type == e1000_media_type_copper || 4704 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 4705 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 4706 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 4707 } 4708 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 4709 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 4710 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 4711 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 4712 4713 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 4714 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 4715 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 4716 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 4717 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 4718 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 4719 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 4720 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 4721 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 4722 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 4723 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 4724 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 4725 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 4726 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 4727 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 4728 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 4729 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 4730 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 4731 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 4732 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 4733 4734 /* For the 64-bit byte counters the low dword must be read first. */ 4735 /* Both registers clear on the read of the high dword */ 4736 4737 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 4738 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 4739 4740 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 4741 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 4742 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 4743 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 4744 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 4745 4746 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 4747 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 4748 4749 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 4750 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 4751 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 4752 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 4753 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 4754 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 4755 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 4756 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 4757 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 4758 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 4759 4760 if (adapter->hw.mac.type >= e1000_82543) { 4761 adapter->stats.algnerrc += 4762 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 4763 adapter->stats.rxerrc += 4764 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 4765 adapter->stats.tncrs += 4766 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 4767 adapter->stats.cexterr += 4768 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 4769 adapter->stats.tsctc += 4770 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 4771 adapter->stats.tsctfc += 4772 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 4773 } 4774 ifp = adapter->ifp; 4775 4776 ifp->if_collisions = adapter->stats.colc; 4777 4778 /* Rx Errors */ 4779 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 4780 adapter->stats.crcerrs + adapter->stats.algnerrc + 4781 adapter->stats.ruc + adapter->stats.roc + 4782 adapter->stats.mpc + adapter->stats.cexterr; 4783 4784 /* Tx Errors */ 4785 ifp->if_oerrors = adapter->stats.ecol + 4786 adapter->stats.latecol + adapter->watchdog_events; 4787} 4788 4789 4790/********************************************************************** 4791 * 4792 * This routine is called only when em_display_debug_stats is enabled. 4793 * This routine provides a way to take a look at important statistics 4794 * maintained by the driver and hardware. 4795 * 4796 **********************************************************************/ 4797static void 4798em_print_debug_info(struct adapter *adapter) 4799{ 4800 device_t dev = adapter->dev; 4801 u8 *hw_addr = adapter->hw.hw_addr; 4802 struct rx_ring *rxr = adapter->rx_rings; 4803 struct tx_ring *txr = adapter->tx_rings; 4804 4805 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 4806 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 4807 E1000_READ_REG(&adapter->hw, E1000_CTRL), 4808 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 4809 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 4810 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 4811 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 4812 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 4813 adapter->hw.fc.high_water, 4814 adapter->hw.fc.low_water); 4815 device_printf(dev, "tx_int_delay = %d, tx_abs_int_delay = %d\n", 4816 E1000_READ_REG(&adapter->hw, E1000_TIDV), 4817 E1000_READ_REG(&adapter->hw, E1000_TADV)); 4818 device_printf(dev, "rx_int_delay = %d, rx_abs_int_delay = %d\n", 4819 E1000_READ_REG(&adapter->hw, E1000_RDTR), 4820 E1000_READ_REG(&adapter->hw, E1000_RADV)); 4821 4822 for (int i = 0; i < adapter->num_queues; i++, txr++) { 4823 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d\n", i, 4824 E1000_READ_REG(&adapter->hw, E1000_TDH(i)), 4825 E1000_READ_REG(&adapter->hw, E1000_TDT(i))); 4826 device_printf(dev, "TX(%d) no descriptors avail event = %ld\n", 4827 txr->me, txr->no_desc_avail); 4828 device_printf(dev, "TX(%d) MSIX IRQ Handled = %ld\n", 4829 txr->me, txr->tx_irq); 4830 device_printf(dev, "Num Tx descriptors avail = %d\n", 4831 txr->tx_avail); 4832 device_printf(dev, "Tx Descriptors not avail1 = %ld\n", 4833 txr->no_desc_avail); 4834 } 4835 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4836 device_printf(dev, "RX(%d) MSIX IRQ Handled = %ld\n", 4837 rxr->me, rxr->rx_irq); 4838 device_printf(dev, "hw rdh = %d, hw rdt = %d\n", 4839 E1000_READ_REG(&adapter->hw, E1000_RDH(i)), 4840 E1000_READ_REG(&adapter->hw, E1000_RDT(i))); 4841 } 4842 device_printf(dev, "Std mbuf failed = %ld\n", 4843 adapter->mbuf_alloc_failed); 4844 device_printf(dev, "Std mbuf cluster failed = %ld\n", 4845 adapter->mbuf_cluster_failed); 4846 device_printf(dev, "Driver dropped packets = %ld\n", 4847 adapter->dropped_pkts); 4848} 4849 4850static void 4851em_print_hw_stats(struct adapter *adapter) 4852{ 4853 device_t dev = adapter->dev; 4854 4855 device_printf(dev, "Excessive collisions = %lld\n", 4856 (long long)adapter->stats.ecol); 4857#if (DEBUG_HW > 0) /* Dont output these errors normally */ 4858 device_printf(dev, "Symbol errors = %lld\n", 4859 (long long)adapter->stats.symerrs); 4860#endif 4861 device_printf(dev, "Sequence errors = %lld\n", 4862 (long long)adapter->stats.sec); 4863 device_printf(dev, "Defer count = %lld\n", 4864 (long long)adapter->stats.dc); 4865 device_printf(dev, "Missed Packets = %lld\n", 4866 (long long)adapter->stats.mpc); 4867 device_printf(dev, "Receive No Buffers = %lld\n", 4868 (long long)adapter->stats.rnbc); 4869 /* RLEC is inaccurate on some hardware, calculate our own. */ 4870 device_printf(dev, "Receive Length Errors = %lld\n", 4871 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 4872 device_printf(dev, "Receive errors = %lld\n", 4873 (long long)adapter->stats.rxerrc); 4874 device_printf(dev, "Crc errors = %lld\n", 4875 (long long)adapter->stats.crcerrs); 4876 device_printf(dev, "Alignment errors = %lld\n", 4877 (long long)adapter->stats.algnerrc); 4878 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 4879 (long long)adapter->stats.cexterr); 4880 device_printf(dev, "watchdog timeouts = %ld\n", 4881 adapter->watchdog_events); 4882 device_printf(dev, "XON Rcvd = %lld\n", 4883 (long long)adapter->stats.xonrxc); 4884 device_printf(dev, "XON Xmtd = %lld\n", 4885 (long long)adapter->stats.xontxc); 4886 device_printf(dev, "XOFF Rcvd = %lld\n", 4887 (long long)adapter->stats.xoffrxc); 4888 device_printf(dev, "XOFF Xmtd = %lld\n", 4889 (long long)adapter->stats.xofftxc); 4890 device_printf(dev, "Good Packets Rcvd = %lld\n", 4891 (long long)adapter->stats.gprc); 4892 device_printf(dev, "Good Packets Xmtd = %lld\n", 4893 (long long)adapter->stats.gptc); 4894 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 4895 (long long)adapter->stats.tsctc); 4896 device_printf(dev, "TSO Contexts Failed = %lld\n", 4897 (long long)adapter->stats.tsctfc); 4898} 4899 4900/********************************************************************** 4901 * 4902 * This routine provides a way to dump out the adapter eeprom, 4903 * often a useful debug/service tool. This only dumps the first 4904 * 32 words, stuff that matters is in that extent. 4905 * 4906 **********************************************************************/ 4907static void 4908em_print_nvm_info(struct adapter *adapter) 4909{ 4910 u16 eeprom_data; 4911 int i, j, row = 0; 4912 4913 /* Its a bit crude, but it gets the job done */ 4914 printf("\nInterface EEPROM Dump:\n"); 4915 printf("Offset\n0x0000 "); 4916 for (i = 0, j = 0; i < 32; i++, j++) { 4917 if (j == 8) { /* Make the offset block */ 4918 j = 0; ++row; 4919 printf("\n0x00%x0 ",row); 4920 } 4921 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 4922 printf("%04x ", eeprom_data); 4923 } 4924 printf("\n"); 4925} 4926 4927static int 4928em_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 4929{ 4930 struct adapter *adapter; 4931 int error; 4932 int result; 4933 4934 result = -1; 4935 error = sysctl_handle_int(oidp, &result, 0, req); 4936 4937 if (error || !req->newptr) 4938 return (error); 4939 4940 if (result == 1) { 4941 adapter = (struct adapter *)arg1; 4942 em_print_debug_info(adapter); 4943 } 4944 /* 4945 * This value will cause a hex dump of the 4946 * first 32 16-bit words of the EEPROM to 4947 * the screen. 4948 */ 4949 if (result == 2) { 4950 adapter = (struct adapter *)arg1; 4951 em_print_nvm_info(adapter); 4952 } 4953 4954 return (error); 4955} 4956 4957 4958static int 4959em_sysctl_stats(SYSCTL_HANDLER_ARGS) 4960{ 4961 struct adapter *adapter; 4962 int error; 4963 int result; 4964 4965 result = -1; 4966 error = sysctl_handle_int(oidp, &result, 0, req); 4967 4968 if (error || !req->newptr) 4969 return (error); 4970 4971 if (result == 1) { 4972 adapter = (struct adapter *)arg1; 4973 em_print_hw_stats(adapter); 4974 } 4975 4976 return (error); 4977} 4978 4979static int 4980em_sysctl_int_delay(SYSCTL_HANDLER_ARGS) 4981{ 4982 struct em_int_delay_info *info; 4983 struct adapter *adapter; 4984 u32 regval; 4985 int error, usecs, ticks; 4986 4987 info = (struct em_int_delay_info *)arg1; 4988 usecs = info->value; 4989 error = sysctl_handle_int(oidp, &usecs, 0, req); 4990 if (error != 0 || req->newptr == NULL) 4991 return (error); 4992 if (usecs < 0 || usecs > EM_TICKS_TO_USECS(65535)) 4993 return (EINVAL); 4994 info->value = usecs; 4995 ticks = EM_USECS_TO_TICKS(usecs); 4996 4997 adapter = info->adapter; 4998 4999 EM_CORE_LOCK(adapter); 5000 regval = E1000_READ_OFFSET(&adapter->hw, info->offset); 5001 regval = (regval & ~0xffff) | (ticks & 0xffff); 5002 /* Handle a few special cases. */ 5003 switch (info->offset) { 5004 case E1000_RDTR: 5005 break; 5006 case E1000_TIDV: 5007 if (ticks == 0) { 5008 adapter->txd_cmd &= ~E1000_TXD_CMD_IDE; 5009 /* Don't write 0 into the TIDV register. */ 5010 regval++; 5011 } else 5012 adapter->txd_cmd |= E1000_TXD_CMD_IDE; 5013 break; 5014 } 5015 E1000_WRITE_OFFSET(&adapter->hw, info->offset, regval); 5016 EM_CORE_UNLOCK(adapter); 5017 return (0); 5018} 5019 5020static void 5021em_add_int_delay_sysctl(struct adapter *adapter, const char *name, 5022 const char *description, struct em_int_delay_info *info, 5023 int offset, int value) 5024{ 5025 info->adapter = adapter; 5026 info->offset = offset; 5027 info->value = value; 5028 SYSCTL_ADD_PROC(device_get_sysctl_ctx(adapter->dev), 5029 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5030 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, 5031 info, 0, em_sysctl_int_delay, "I", description); 5032} 5033 5034static void 5035em_add_rx_process_limit(struct adapter *adapter, const char *name, 5036 const char *description, int *limit, int value) 5037{ 5038 *limit = value; 5039 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5040 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5041 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5042} 5043 5044 5045