if_igb.c revision 209068
151792Smarcel/****************************************************************************** 251792Smarcel 351792Smarcel Copyright (c) 2001-2010, Intel Corporation 451792Smarcel All rights reserved. 551792Smarcel 651792Smarcel Redistribution and use in source and binary forms, with or without 751792Smarcel modification, are permitted provided that the following conditions are met: 851792Smarcel 951792Smarcel 1. Redistributions of source code must retain the above copyright notice, 1051792Smarcel this list of conditions and the following disclaimer. 1151792Smarcel 1251792Smarcel 2. Redistributions in binary form must reproduce the above copyright 1351792Smarcel notice, this list of conditions and the following disclaimer in the 1451792Smarcel documentation and/or other materials provided with the distribution. 1551792Smarcel 1651792Smarcel 3. Neither the name of the Intel Corporation nor the names of its 1751792Smarcel contributors may be used to endorse or promote products derived from 1851792Smarcel this software without specific prior written permission. 1951792Smarcel 2051792Smarcel THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 2151792Smarcel AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2251792Smarcel IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2351792Smarcel ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 2451792Smarcel LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2551792Smarcel CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2651792Smarcel SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2751792Smarcel INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2851792Smarcel CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 2951792Smarcel ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 3051792Smarcel POSSIBILITY OF SUCH DAMAGE. 3151792Smarcel 3251907Smarcel******************************************************************************/ 3351792Smarcel/*$FreeBSD: head/sys/dev/e1000/if_igb.c 209068 2010-06-11 20:54:27Z jfv $*/ 3451907Smarcel 3551907Smarcel 3651907Smarcel#ifdef HAVE_KERNEL_OPTION_HEADERS 3751907Smarcel#include "opt_device_polling.h" 3851792Smarcel#include "opt_inet.h" 3951792Smarcel#include "opt_altq.h" 4051792Smarcel#endif 4151792Smarcel 4251792Smarcel#include <sys/param.h> 4351792Smarcel#include <sys/systm.h> 4451792Smarcel#if __FreeBSD_version >= 800000 4551792Smarcel#include <sys/buf_ring.h> 4651792Smarcel#endif 4751792Smarcel#include <sys/bus.h> 4851792Smarcel#include <sys/endian.h> 4951792Smarcel#include <sys/kernel.h> 5051792Smarcel#include <sys/kthread.h> 5151792Smarcel#include <sys/malloc.h> 5251792Smarcel#include <sys/mbuf.h> 5351792Smarcel#include <sys/module.h> 5451792Smarcel#include <sys/rman.h> 5551792Smarcel#include <sys/socket.h> 5651792Smarcel#include <sys/sockio.h> 5751792Smarcel#include <sys/sysctl.h> 5851792Smarcel#include <sys/taskqueue.h> 5951792Smarcel#include <sys/eventhandler.h> 6051792Smarcel#include <sys/pcpu.h> 6151792Smarcel#include <sys/smp.h> 6251792Smarcel#include <machine/smp.h> 6351792Smarcel#include <machine/bus.h> 6451792Smarcel#include <machine/resource.h> 6551792Smarcel 6651792Smarcel#include <net/bpf.h> 6751792Smarcel#include <net/ethernet.h> 6851792Smarcel#include <net/if.h> 6951792Smarcel#include <net/if_arp.h> 7051792Smarcel#include <net/if_dl.h> 7151792Smarcel#include <net/if_media.h> 7251792Smarcel 7351792Smarcel#include <net/if_types.h> 7451792Smarcel#include <net/if_vlan_var.h> 7551907Smarcel 7651907Smarcel#include <netinet/in_systm.h> 7751907Smarcel#include <netinet/in.h> 7851907Smarcel#include <netinet/if_ether.h> 7951792Smarcel#include <netinet/ip.h> 8051792Smarcel#include <netinet/ip6.h> 8151792Smarcel#include <netinet/tcp.h> 8251792Smarcel#include <netinet/tcp_lro.h> 8351907Smarcel#include <netinet/udp.h> 8451907Smarcel 8551792Smarcel#include <machine/in_cksum.h> 8651792Smarcel#include <dev/led/led.h> 8751792Smarcel#include <dev/pci/pcivar.h> 8851792Smarcel#include <dev/pci/pcireg.h> 8951792Smarcel 9051792Smarcel#include "e1000_api.h" 9151792Smarcel#include "e1000_82575.h" 9251792Smarcel#include "if_igb.h" 9351792Smarcel 9451907Smarcel/********************************************************************* 95 * Set this to one to display debug statistics 96 *********************************************************************/ 97int igb_display_debug_stats = 0; 98 99/********************************************************************* 100 * Driver version: 101 *********************************************************************/ 102char igb_driver_version[] = "version - 1.9.6"; 103 104 105/********************************************************************* 106 * PCI Device ID Table 107 * 108 * Used by probe to select devices to load on 109 * Last field stores an index into e1000_strings 110 * Last entry must be all 0s 111 * 112 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 113 *********************************************************************/ 114 115static igb_vendor_info_t igb_vendor_info_array[] = 116{ 117 { 0x8086, E1000_DEV_ID_82575EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 118 { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, 119 PCI_ANY_ID, PCI_ANY_ID, 0}, 120 { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, 121 PCI_ANY_ID, PCI_ANY_ID, 0}, 122 { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0}, 123 { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0}, 124 { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 125 { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 126 { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 127 { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD, 128 PCI_ANY_ID, PCI_ANY_ID, 0}, 129 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER, 130 PCI_ANY_ID, PCI_ANY_ID, 0}, 131 { 0x8086, E1000_DEV_ID_82580_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 132 { 0x8086, E1000_DEV_ID_82580_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 133 { 0x8086, E1000_DEV_ID_82580_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82580_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL, 136 PCI_ANY_ID, PCI_ANY_ID, 0}, 137 /* required last entry */ 138 { 0, 0, 0, 0, 0} 139}; 140 141/********************************************************************* 142 * Table of branding strings for all supported NICs. 143 *********************************************************************/ 144 145static char *igb_strings[] = { 146 "Intel(R) PRO/1000 Network Connection" 147}; 148 149/********************************************************************* 150 * Function prototypes 151 *********************************************************************/ 152static int igb_probe(device_t); 153static int igb_attach(device_t); 154static int igb_detach(device_t); 155static int igb_shutdown(device_t); 156static int igb_suspend(device_t); 157static int igb_resume(device_t); 158static void igb_start(struct ifnet *); 159static void igb_start_locked(struct tx_ring *, struct ifnet *ifp); 160#if __FreeBSD_version >= 800000 161static int igb_mq_start(struct ifnet *, struct mbuf *); 162static int igb_mq_start_locked(struct ifnet *, 163 struct tx_ring *, struct mbuf *); 164static void igb_qflush(struct ifnet *); 165#endif 166static int igb_ioctl(struct ifnet *, u_long, caddr_t); 167static void igb_init(void *); 168static void igb_init_locked(struct adapter *); 169static void igb_stop(void *); 170static void igb_media_status(struct ifnet *, struct ifmediareq *); 171static int igb_media_change(struct ifnet *); 172static void igb_identify_hardware(struct adapter *); 173static int igb_allocate_pci_resources(struct adapter *); 174static int igb_allocate_msix(struct adapter *); 175static int igb_allocate_legacy(struct adapter *); 176static int igb_setup_msix(struct adapter *); 177static void igb_free_pci_resources(struct adapter *); 178static void igb_local_timer(void *); 179static void igb_reset(struct adapter *); 180static void igb_setup_interface(device_t, struct adapter *); 181static int igb_allocate_queues(struct adapter *); 182static void igb_configure_queues(struct adapter *); 183 184static int igb_allocate_transmit_buffers(struct tx_ring *); 185static void igb_setup_transmit_structures(struct adapter *); 186static void igb_setup_transmit_ring(struct tx_ring *); 187static void igb_initialize_transmit_units(struct adapter *); 188static void igb_free_transmit_structures(struct adapter *); 189static void igb_free_transmit_buffers(struct tx_ring *); 190 191static int igb_allocate_receive_buffers(struct rx_ring *); 192static int igb_setup_receive_structures(struct adapter *); 193static int igb_setup_receive_ring(struct rx_ring *); 194static void igb_initialize_receive_units(struct adapter *); 195static void igb_free_receive_structures(struct adapter *); 196static void igb_free_receive_buffers(struct rx_ring *); 197static void igb_free_receive_ring(struct rx_ring *); 198 199static void igb_enable_intr(struct adapter *); 200static void igb_disable_intr(struct adapter *); 201static void igb_update_stats_counters(struct adapter *); 202static bool igb_txeof(struct tx_ring *); 203 204static __inline void igb_rx_discard(struct rx_ring *, int); 205static __inline void igb_rx_input(struct rx_ring *, 206 struct ifnet *, struct mbuf *, u32); 207 208static bool igb_rxeof(struct igb_queue *, int); 209static void igb_rx_checksum(u32, struct mbuf *, u32); 210static int igb_tx_ctx_setup(struct tx_ring *, struct mbuf *); 211static bool igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 212static void igb_set_promisc(struct adapter *); 213static void igb_disable_promisc(struct adapter *); 214static void igb_set_multi(struct adapter *); 215static void igb_print_hw_stats(struct adapter *); 216static void igb_update_link_status(struct adapter *); 217static void igb_refresh_mbufs(struct rx_ring *, int); 218 219static void igb_register_vlan(void *, struct ifnet *, u16); 220static void igb_unregister_vlan(void *, struct ifnet *, u16); 221static void igb_setup_vlan_hw_support(struct adapter *); 222 223static int igb_xmit(struct tx_ring *, struct mbuf **); 224static int igb_dma_malloc(struct adapter *, bus_size_t, 225 struct igb_dma_alloc *, int); 226static void igb_dma_free(struct adapter *, struct igb_dma_alloc *); 227static void igb_print_debug_info(struct adapter *); 228static void igb_print_nvm_info(struct adapter *); 229static int igb_is_valid_ether_addr(u8 *); 230static int igb_sysctl_stats(SYSCTL_HANDLER_ARGS); 231static int igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 232/* Management and WOL Support */ 233static void igb_init_manageability(struct adapter *); 234static void igb_release_manageability(struct adapter *); 235static void igb_get_hw_control(struct adapter *); 236static void igb_release_hw_control(struct adapter *); 237static void igb_enable_wakeup(device_t); 238static void igb_led_func(void *, int); 239 240static int igb_irq_fast(void *); 241static void igb_add_rx_process_limit(struct adapter *, const char *, 242 const char *, int *, int); 243static void igb_handle_que(void *context, int pending); 244static void igb_handle_link(void *context, int pending); 245 246/* These are MSIX only irq handlers */ 247static void igb_msix_que(void *); 248static void igb_msix_link(void *); 249 250#ifdef DEVICE_POLLING 251static poll_handler_t igb_poll; 252#endif /* POLLING */ 253 254/********************************************************************* 255 * FreeBSD Device Interface Entry Points 256 *********************************************************************/ 257 258static device_method_t igb_methods[] = { 259 /* Device interface */ 260 DEVMETHOD(device_probe, igb_probe), 261 DEVMETHOD(device_attach, igb_attach), 262 DEVMETHOD(device_detach, igb_detach), 263 DEVMETHOD(device_shutdown, igb_shutdown), 264 DEVMETHOD(device_suspend, igb_suspend), 265 DEVMETHOD(device_resume, igb_resume), 266 {0, 0} 267}; 268 269static driver_t igb_driver = { 270 "igb", igb_methods, sizeof(struct adapter), 271}; 272 273static devclass_t igb_devclass; 274DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); 275MODULE_DEPEND(igb, pci, 1, 1, 1); 276MODULE_DEPEND(igb, ether, 1, 1, 1); 277 278/********************************************************************* 279 * Tunable default values. 280 *********************************************************************/ 281 282/* Descriptor defaults */ 283static int igb_rxd = IGB_DEFAULT_RXD; 284static int igb_txd = IGB_DEFAULT_TXD; 285TUNABLE_INT("hw.igb.rxd", &igb_rxd); 286TUNABLE_INT("hw.igb.txd", &igb_txd); 287 288/* 289** AIM: Adaptive Interrupt Moderation 290** which means that the interrupt rate 291** is varied over time based on the 292** traffic for that interrupt vector 293*/ 294static int igb_enable_aim = TRUE; 295TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim); 296 297/* 298 * MSIX should be the default for best performance, 299 * but this allows it to be forced off for testing. 300 */ 301static int igb_enable_msix = 1; 302TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix); 303 304/* 305 * Header split has seemed to be beneficial in 306 * many circumstances tested, however there have 307 * been some stability issues, so the default is 308 * off. 309 */ 310static bool igb_header_split = FALSE; 311TUNABLE_INT("hw.igb.hdr_split", &igb_header_split); 312 313/* 314** This will autoconfigure based on 315** the number of CPUs if left at 0. 316*/ 317static int igb_num_queues = 0; 318TUNABLE_INT("hw.igb.num_queues", &igb_num_queues); 319 320/* How many packets rxeof tries to clean at a time */ 321static int igb_rx_process_limit = 100; 322TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit); 323 324/* Flow control setting - default to FULL */ 325static int igb_fc_setting = e1000_fc_full; 326TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting); 327 328/* 329** Shadow VFTA table, this is needed because 330** the real filter table gets cleared during 331** a soft reset and the driver needs to be able 332** to repopulate it. 333*/ 334static u32 igb_shadow_vfta[IGB_VFTA_SIZE]; 335 336 337/********************************************************************* 338 * Device identification routine 339 * 340 * igb_probe determines if the driver should be loaded on 341 * adapter based on PCI vendor/device id of the adapter. 342 * 343 * return BUS_PROBE_DEFAULT on success, positive on failure 344 *********************************************************************/ 345 346static int 347igb_probe(device_t dev) 348{ 349 char adapter_name[60]; 350 uint16_t pci_vendor_id = 0; 351 uint16_t pci_device_id = 0; 352 uint16_t pci_subvendor_id = 0; 353 uint16_t pci_subdevice_id = 0; 354 igb_vendor_info_t *ent; 355 356 INIT_DEBUGOUT("igb_probe: begin"); 357 358 pci_vendor_id = pci_get_vendor(dev); 359 if (pci_vendor_id != IGB_VENDOR_ID) 360 return (ENXIO); 361 362 pci_device_id = pci_get_device(dev); 363 pci_subvendor_id = pci_get_subvendor(dev); 364 pci_subdevice_id = pci_get_subdevice(dev); 365 366 ent = igb_vendor_info_array; 367 while (ent->vendor_id != 0) { 368 if ((pci_vendor_id == ent->vendor_id) && 369 (pci_device_id == ent->device_id) && 370 371 ((pci_subvendor_id == ent->subvendor_id) || 372 (ent->subvendor_id == PCI_ANY_ID)) && 373 374 ((pci_subdevice_id == ent->subdevice_id) || 375 (ent->subdevice_id == PCI_ANY_ID))) { 376 sprintf(adapter_name, "%s %s", 377 igb_strings[ent->index], 378 igb_driver_version); 379 device_set_desc_copy(dev, adapter_name); 380 return (BUS_PROBE_DEFAULT); 381 } 382 ent++; 383 } 384 385 return (ENXIO); 386} 387 388/********************************************************************* 389 * Device initialization routine 390 * 391 * The attach entry point is called when the driver is being loaded. 392 * This routine identifies the type of hardware, allocates all resources 393 * and initializes the hardware. 394 * 395 * return 0 on success, positive on failure 396 *********************************************************************/ 397 398static int 399igb_attach(device_t dev) 400{ 401 struct adapter *adapter; 402 int error = 0; 403 u16 eeprom_data; 404 405 INIT_DEBUGOUT("igb_attach: begin"); 406 407 adapter = device_get_softc(dev); 408 adapter->dev = adapter->osdep.dev = dev; 409 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 410 411 /* SYSCTL stuff */ 412 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 413 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 414 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 415 igb_sysctl_debug_info, "I", "Debug Information"); 416 417 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 418 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 419 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 420 igb_sysctl_stats, "I", "Statistics"); 421 422 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 423 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 424 OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW, 425 &igb_fc_setting, 0, "Flow Control"); 426 427 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 428 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 429 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 430 &igb_enable_aim, 1, "Interrupt Moderation"); 431 432 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 433 434 /* Determine hardware and mac info */ 435 igb_identify_hardware(adapter); 436 437 /* Setup PCI resources */ 438 if (igb_allocate_pci_resources(adapter)) { 439 device_printf(dev, "Allocation of PCI resources failed\n"); 440 error = ENXIO; 441 goto err_pci; 442 } 443 444 /* Do Shared Code initialization */ 445 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 446 device_printf(dev, "Setup of Shared code failed\n"); 447 error = ENXIO; 448 goto err_pci; 449 } 450 451 e1000_get_bus_info(&adapter->hw); 452 453 /* Sysctls for limiting the amount of work done in the taskqueue */ 454 igb_add_rx_process_limit(adapter, "rx_processing_limit", 455 "max number of rx packets to process", &adapter->rx_process_limit, 456 igb_rx_process_limit); 457 458 /* 459 * Validate number of transmit and receive descriptors. It 460 * must not exceed hardware maximum, and must be multiple 461 * of E1000_DBA_ALIGN. 462 */ 463 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 || 464 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) { 465 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 466 IGB_DEFAULT_TXD, igb_txd); 467 adapter->num_tx_desc = IGB_DEFAULT_TXD; 468 } else 469 adapter->num_tx_desc = igb_txd; 470 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 || 471 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) { 472 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 473 IGB_DEFAULT_RXD, igb_rxd); 474 adapter->num_rx_desc = IGB_DEFAULT_RXD; 475 } else 476 adapter->num_rx_desc = igb_rxd; 477 478 adapter->hw.mac.autoneg = DO_AUTO_NEG; 479 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 480 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 481 482 /* Copper options */ 483 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 484 adapter->hw.phy.mdix = AUTO_ALL_MODES; 485 adapter->hw.phy.disable_polarity_correction = FALSE; 486 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE; 487 } 488 489 /* 490 * Set the frame limits assuming 491 * standard ethernet sized frames. 492 */ 493 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 494 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 495 496 /* 497 ** Allocate and Setup Queues 498 */ 499 if (igb_allocate_queues(adapter)) { 500 error = ENOMEM; 501 goto err_pci; 502 } 503 504 /* 505 ** Start from a known state, this is 506 ** important in reading the nvm and 507 ** mac from that. 508 */ 509 e1000_reset_hw(&adapter->hw); 510 511 /* Make sure we have a good EEPROM before we read from it */ 512 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 513 /* 514 ** Some PCI-E parts fail the first check due to 515 ** the link being in sleep state, call it again, 516 ** if it fails a second time its a real issue. 517 */ 518 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 519 device_printf(dev, 520 "The EEPROM Checksum Is Not Valid\n"); 521 error = EIO; 522 goto err_late; 523 } 524 } 525 526 /* 527 ** Copy the permanent MAC address out of the EEPROM 528 */ 529 if (e1000_read_mac_addr(&adapter->hw) < 0) { 530 device_printf(dev, "EEPROM read error while reading MAC" 531 " address\n"); 532 error = EIO; 533 goto err_late; 534 } 535 /* Check its sanity */ 536 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) { 537 device_printf(dev, "Invalid MAC address\n"); 538 error = EIO; 539 goto err_late; 540 } 541 542 /* 543 ** Configure Interrupts 544 */ 545 if ((adapter->msix > 1) && (igb_enable_msix)) 546 error = igb_allocate_msix(adapter); 547 else /* MSI or Legacy */ 548 error = igb_allocate_legacy(adapter); 549 if (error) 550 goto err_late; 551 552 /* Setup OS specific network interface */ 553 igb_setup_interface(dev, adapter); 554 555 /* Now get a good starting state */ 556 igb_reset(adapter); 557 558 /* Initialize statistics */ 559 igb_update_stats_counters(adapter); 560 561 adapter->hw.mac.get_link_status = 1; 562 igb_update_link_status(adapter); 563 564 /* Indicate SOL/IDER usage */ 565 if (e1000_check_reset_block(&adapter->hw)) 566 device_printf(dev, 567 "PHY reset is blocked due to SOL/IDER session.\n"); 568 569 /* Determine if we have to control management hardware */ 570 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 571 572 /* 573 * Setup Wake-on-Lan 574 */ 575 /* APME bit in EEPROM is mapped to WUC.APME */ 576 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME; 577 if (eeprom_data) 578 adapter->wol = E1000_WUFC_MAG; 579 580 /* Register for VLAN events */ 581 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 582 igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 583 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 584 igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 585 586 /* Tell the stack that the interface is not active */ 587 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 588 589 adapter->led_dev = led_create(igb_led_func, adapter, 590 device_get_nameunit(dev)); 591 592 INIT_DEBUGOUT("igb_attach: end"); 593 594 return (0); 595 596err_late: 597 igb_free_transmit_structures(adapter); 598 igb_free_receive_structures(adapter); 599 igb_release_hw_control(adapter); 600err_pci: 601 igb_free_pci_resources(adapter); 602 IGB_CORE_LOCK_DESTROY(adapter); 603 604 return (error); 605} 606 607/********************************************************************* 608 * Device removal routine 609 * 610 * The detach entry point is called when the driver is being removed. 611 * This routine stops the adapter and deallocates all the resources 612 * that were allocated for driver operation. 613 * 614 * return 0 on success, positive on failure 615 *********************************************************************/ 616 617static int 618igb_detach(device_t dev) 619{ 620 struct adapter *adapter = device_get_softc(dev); 621 struct ifnet *ifp = adapter->ifp; 622 623 INIT_DEBUGOUT("igb_detach: begin"); 624 625 /* Make sure VLANS are not using driver */ 626 if (adapter->ifp->if_vlantrunk != NULL) { 627 device_printf(dev,"Vlan in use, detach first\n"); 628 return (EBUSY); 629 } 630 631 if (adapter->led_dev != NULL) 632 led_destroy(adapter->led_dev); 633 634#ifdef DEVICE_POLLING 635 if (ifp->if_capenable & IFCAP_POLLING) 636 ether_poll_deregister(ifp); 637#endif 638 639 IGB_CORE_LOCK(adapter); 640 adapter->in_detach = 1; 641 igb_stop(adapter); 642 IGB_CORE_UNLOCK(adapter); 643 644 e1000_phy_hw_reset(&adapter->hw); 645 646 /* Give control back to firmware */ 647 igb_release_manageability(adapter); 648 igb_release_hw_control(adapter); 649 650 if (adapter->wol) { 651 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 652 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 653 igb_enable_wakeup(dev); 654 } 655 656 /* Unregister VLAN events */ 657 if (adapter->vlan_attach != NULL) 658 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 659 if (adapter->vlan_detach != NULL) 660 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 661 662 ether_ifdetach(adapter->ifp); 663 664 callout_drain(&adapter->timer); 665 666 igb_free_pci_resources(adapter); 667 bus_generic_detach(dev); 668 if_free(ifp); 669 670 igb_free_transmit_structures(adapter); 671 igb_free_receive_structures(adapter); 672 673 IGB_CORE_LOCK_DESTROY(adapter); 674 675 return (0); 676} 677 678/********************************************************************* 679 * 680 * Shutdown entry point 681 * 682 **********************************************************************/ 683 684static int 685igb_shutdown(device_t dev) 686{ 687 return igb_suspend(dev); 688} 689 690/* 691 * Suspend/resume device methods. 692 */ 693static int 694igb_suspend(device_t dev) 695{ 696 struct adapter *adapter = device_get_softc(dev); 697 698 IGB_CORE_LOCK(adapter); 699 700 igb_stop(adapter); 701 702 igb_release_manageability(adapter); 703 igb_release_hw_control(adapter); 704 705 if (adapter->wol) { 706 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 707 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 708 igb_enable_wakeup(dev); 709 } 710 711 IGB_CORE_UNLOCK(adapter); 712 713 return bus_generic_suspend(dev); 714} 715 716static int 717igb_resume(device_t dev) 718{ 719 struct adapter *adapter = device_get_softc(dev); 720 struct ifnet *ifp = adapter->ifp; 721 722 IGB_CORE_LOCK(adapter); 723 igb_init_locked(adapter); 724 igb_init_manageability(adapter); 725 726 if ((ifp->if_flags & IFF_UP) && 727 (ifp->if_drv_flags & IFF_DRV_RUNNING)) 728 igb_start(ifp); 729 730 IGB_CORE_UNLOCK(adapter); 731 732 return bus_generic_resume(dev); 733} 734 735 736/********************************************************************* 737 * Transmit entry point 738 * 739 * igb_start is called by the stack to initiate a transmit. 740 * The driver will remain in this routine as long as there are 741 * packets to transmit and transmit resources are available. 742 * In case resources are not available stack is notified and 743 * the packet is requeued. 744 **********************************************************************/ 745 746static void 747igb_start_locked(struct tx_ring *txr, struct ifnet *ifp) 748{ 749 struct adapter *adapter = ifp->if_softc; 750 struct mbuf *m_head; 751 752 IGB_TX_LOCK_ASSERT(txr); 753 754 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 755 IFF_DRV_RUNNING) 756 return; 757 if (!adapter->link_active) 758 return; 759 760 /* Call cleanup if number of TX descriptors low */ 761 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) 762 igb_txeof(txr); 763 764 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 765 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 766 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 767 break; 768 } 769 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 770 if (m_head == NULL) 771 break; 772 /* 773 * Encapsulation can modify our pointer, and or make it 774 * NULL on failure. In that event, we can't requeue. 775 */ 776 if (igb_xmit(txr, &m_head)) { 777 if (m_head == NULL) 778 break; 779 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 780 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 781 break; 782 } 783 784 /* Send a copy of the frame to the BPF listener */ 785 ETHER_BPF_MTAP(ifp, m_head); 786 787 /* Set watchdog on */ 788 txr->watchdog_time = ticks; 789 txr->watchdog_check = TRUE; 790 } 791} 792 793/* 794 * Legacy TX driver routine, called from the 795 * stack, always uses tx[0], and spins for it. 796 * Should not be used with multiqueue tx 797 */ 798static void 799igb_start(struct ifnet *ifp) 800{ 801 struct adapter *adapter = ifp->if_softc; 802 struct tx_ring *txr = adapter->tx_rings; 803 804 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 805 IGB_TX_LOCK(txr); 806 igb_start_locked(txr, ifp); 807 IGB_TX_UNLOCK(txr); 808 } 809 return; 810} 811 812#if __FreeBSD_version >= 800000 813/* 814** Multiqueue Transmit driver 815** 816*/ 817static int 818igb_mq_start(struct ifnet *ifp, struct mbuf *m) 819{ 820 struct adapter *adapter = ifp->if_softc; 821 struct igb_queue *que; 822 struct tx_ring *txr; 823 int i = 0, err = 0; 824 825 /* Which queue to use */ 826 if ((m->m_flags & M_FLOWID) != 0) 827 i = m->m_pkthdr.flowid % adapter->num_queues; 828 829 txr = &adapter->tx_rings[i]; 830 que = &adapter->queues[i]; 831 832 if (IGB_TX_TRYLOCK(txr)) { 833 err = igb_mq_start_locked(ifp, txr, m); 834 IGB_TX_UNLOCK(txr); 835 } else { 836 err = drbr_enqueue(ifp, txr->br, m); 837 taskqueue_enqueue(que->tq, &que->que_task); 838 } 839 840 return (err); 841} 842 843static int 844igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 845{ 846 struct adapter *adapter = txr->adapter; 847 struct mbuf *next; 848 int err = 0, enq; 849 850 IGB_TX_LOCK_ASSERT(txr); 851 852 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 853 IFF_DRV_RUNNING || adapter->link_active == 0) { 854 if (m != NULL) 855 err = drbr_enqueue(ifp, txr->br, m); 856 return (err); 857 } 858 859 /* Call cleanup if number of TX descriptors low */ 860 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) 861 igb_txeof(txr); 862 863 enq = 0; 864 if (m == NULL) { 865 next = drbr_dequeue(ifp, txr->br); 866 } else if (drbr_needs_enqueue(ifp, txr->br)) { 867 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 868 return (err); 869 next = drbr_dequeue(ifp, txr->br); 870 } else 871 next = m; 872 873 /* Process the queue */ 874 while (next != NULL) { 875 if ((err = igb_xmit(txr, &next)) != 0) { 876 if (next != NULL) 877 err = drbr_enqueue(ifp, txr->br, next); 878 break; 879 } 880 enq++; 881 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 882 ETHER_BPF_MTAP(ifp, next); 883 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 884 break; 885 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 886 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 887 break; 888 } 889 next = drbr_dequeue(ifp, txr->br); 890 } 891 if (enq > 0) { 892 /* Set the watchdog */ 893 txr->watchdog_check = TRUE; 894 txr->watchdog_time = ticks; 895 } 896 return (err); 897} 898 899/* 900** Flush all ring buffers 901*/ 902static void 903igb_qflush(struct ifnet *ifp) 904{ 905 struct adapter *adapter = ifp->if_softc; 906 struct tx_ring *txr = adapter->tx_rings; 907 struct mbuf *m; 908 909 for (int i = 0; i < adapter->num_queues; i++, txr++) { 910 IGB_TX_LOCK(txr); 911 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 912 m_freem(m); 913 IGB_TX_UNLOCK(txr); 914 } 915 if_qflush(ifp); 916} 917#endif /* __FreeBSD_version >= 800000 */ 918 919/********************************************************************* 920 * Ioctl entry point 921 * 922 * igb_ioctl is called when the user wants to configure the 923 * interface. 924 * 925 * return 0 on success, positive on failure 926 **********************************************************************/ 927 928static int 929igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 930{ 931 struct adapter *adapter = ifp->if_softc; 932 struct ifreq *ifr = (struct ifreq *)data; 933#ifdef INET 934 struct ifaddr *ifa = (struct ifaddr *)data; 935#endif 936 int error = 0; 937 938 if (adapter->in_detach) 939 return (error); 940 941 switch (command) { 942 case SIOCSIFADDR: 943#ifdef INET 944 if (ifa->ifa_addr->sa_family == AF_INET) { 945 /* 946 * XXX 947 * Since resetting hardware takes a very long time 948 * and results in link renegotiation we only 949 * initialize the hardware only when it is absolutely 950 * required. 951 */ 952 ifp->if_flags |= IFF_UP; 953 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 954 IGB_CORE_LOCK(adapter); 955 igb_init_locked(adapter); 956 IGB_CORE_UNLOCK(adapter); 957 } 958 if (!(ifp->if_flags & IFF_NOARP)) 959 arp_ifinit(ifp, ifa); 960 } else 961#endif 962 error = ether_ioctl(ifp, command, data); 963 break; 964 case SIOCSIFMTU: 965 { 966 int max_frame_size; 967 968 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 969 970 IGB_CORE_LOCK(adapter); 971 max_frame_size = 9234; 972 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 973 ETHER_CRC_LEN) { 974 IGB_CORE_UNLOCK(adapter); 975 error = EINVAL; 976 break; 977 } 978 979 ifp->if_mtu = ifr->ifr_mtu; 980 adapter->max_frame_size = 981 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 982 igb_init_locked(adapter); 983 IGB_CORE_UNLOCK(adapter); 984 break; 985 } 986 case SIOCSIFFLAGS: 987 IOCTL_DEBUGOUT("ioctl rcv'd:\ 988 SIOCSIFFLAGS (Set Interface Flags)"); 989 IGB_CORE_LOCK(adapter); 990 if (ifp->if_flags & IFF_UP) { 991 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 992 if ((ifp->if_flags ^ adapter->if_flags) & 993 (IFF_PROMISC | IFF_ALLMULTI)) { 994 igb_disable_promisc(adapter); 995 igb_set_promisc(adapter); 996 } 997 } else 998 igb_init_locked(adapter); 999 } else 1000 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1001 igb_stop(adapter); 1002 adapter->if_flags = ifp->if_flags; 1003 IGB_CORE_UNLOCK(adapter); 1004 break; 1005 case SIOCADDMULTI: 1006 case SIOCDELMULTI: 1007 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1008 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1009 IGB_CORE_LOCK(adapter); 1010 igb_disable_intr(adapter); 1011 igb_set_multi(adapter); 1012#ifdef DEVICE_POLLING 1013 if (!(ifp->if_capenable & IFCAP_POLLING)) 1014#endif 1015 igb_enable_intr(adapter); 1016 IGB_CORE_UNLOCK(adapter); 1017 } 1018 break; 1019 case SIOCSIFMEDIA: 1020 /* Check SOL/IDER usage */ 1021 IGB_CORE_LOCK(adapter); 1022 if (e1000_check_reset_block(&adapter->hw)) { 1023 IGB_CORE_UNLOCK(adapter); 1024 device_printf(adapter->dev, "Media change is" 1025 " blocked due to SOL/IDER session.\n"); 1026 break; 1027 } 1028 IGB_CORE_UNLOCK(adapter); 1029 case SIOCGIFMEDIA: 1030 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1031 SIOCxIFMEDIA (Get/Set Interface Media)"); 1032 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1033 break; 1034 case SIOCSIFCAP: 1035 { 1036 int mask, reinit; 1037 1038 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1039 reinit = 0; 1040 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1041#ifdef DEVICE_POLLING 1042 if (mask & IFCAP_POLLING) { 1043 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1044 error = ether_poll_register(igb_poll, ifp); 1045 if (error) 1046 return (error); 1047 IGB_CORE_LOCK(adapter); 1048 igb_disable_intr(adapter); 1049 ifp->if_capenable |= IFCAP_POLLING; 1050 IGB_CORE_UNLOCK(adapter); 1051 } else { 1052 error = ether_poll_deregister(ifp); 1053 /* Enable interrupt even in error case */ 1054 IGB_CORE_LOCK(adapter); 1055 igb_enable_intr(adapter); 1056 ifp->if_capenable &= ~IFCAP_POLLING; 1057 IGB_CORE_UNLOCK(adapter); 1058 } 1059 } 1060#endif 1061 if (mask & IFCAP_HWCSUM) { 1062 ifp->if_capenable ^= IFCAP_HWCSUM; 1063 reinit = 1; 1064 } 1065 if (mask & IFCAP_TSO4) { 1066 ifp->if_capenable ^= IFCAP_TSO4; 1067 reinit = 1; 1068 } 1069 if (mask & IFCAP_VLAN_HWTAGGING) { 1070 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1071 reinit = 1; 1072 } 1073 if (mask & IFCAP_VLAN_HWFILTER) { 1074 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1075 reinit = 1; 1076 } 1077 if (mask & IFCAP_LRO) { 1078 ifp->if_capenable ^= IFCAP_LRO; 1079 reinit = 1; 1080 } 1081 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1082 igb_init(adapter); 1083 VLAN_CAPABILITIES(ifp); 1084 break; 1085 } 1086 1087 default: 1088 error = ether_ioctl(ifp, command, data); 1089 break; 1090 } 1091 1092 return (error); 1093} 1094 1095 1096/********************************************************************* 1097 * Init entry point 1098 * 1099 * This routine is used in two ways. It is used by the stack as 1100 * init entry point in network interface structure. It is also used 1101 * by the driver as a hw/sw initialization routine to get to a 1102 * consistent state. 1103 * 1104 * return 0 on success, positive on failure 1105 **********************************************************************/ 1106 1107static void 1108igb_init_locked(struct adapter *adapter) 1109{ 1110 struct ifnet *ifp = adapter->ifp; 1111 device_t dev = adapter->dev; 1112 1113 INIT_DEBUGOUT("igb_init: begin"); 1114 1115 IGB_CORE_LOCK_ASSERT(adapter); 1116 1117 igb_disable_intr(adapter); 1118 callout_stop(&adapter->timer); 1119 1120 /* Get the latest mac address, User can use a LAA */ 1121 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1122 ETHER_ADDR_LEN); 1123 1124 /* Put the address into the Receive Address Array */ 1125 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1126 1127 igb_reset(adapter); 1128 igb_update_link_status(adapter); 1129 1130 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1131 1132 /* Use real VLAN Filter support? */ 1133 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1134 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 1135 /* Use real VLAN Filter support */ 1136 igb_setup_vlan_hw_support(adapter); 1137 else { 1138 u32 ctrl; 1139 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1140 ctrl |= E1000_CTRL_VME; 1141 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1142 } 1143 } 1144 1145 /* Set hardware offload abilities */ 1146 ifp->if_hwassist = 0; 1147 if (ifp->if_capenable & IFCAP_TXCSUM) { 1148 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1149#if __FreeBSD_version >= 800000 1150 if (adapter->hw.mac.type == e1000_82576) 1151 ifp->if_hwassist |= CSUM_SCTP; 1152#endif 1153 } 1154 1155 if (ifp->if_capenable & IFCAP_TSO4) 1156 ifp->if_hwassist |= CSUM_TSO; 1157 1158 /* Configure for OS presence */ 1159 igb_init_manageability(adapter); 1160 1161 /* Prepare transmit descriptors and buffers */ 1162 igb_setup_transmit_structures(adapter); 1163 igb_initialize_transmit_units(adapter); 1164 1165 /* Setup Multicast table */ 1166 igb_set_multi(adapter); 1167 1168 /* 1169 ** Figure out the desired mbuf pool 1170 ** for doing jumbo/packetsplit 1171 */ 1172 if (ifp->if_mtu > ETHERMTU) 1173 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1174 else 1175 adapter->rx_mbuf_sz = MCLBYTES; 1176 1177 /* Prepare receive descriptors and buffers */ 1178 if (igb_setup_receive_structures(adapter)) { 1179 device_printf(dev, "Could not setup receive structures\n"); 1180 return; 1181 } 1182 igb_initialize_receive_units(adapter); 1183 1184 /* Don't lose promiscuous settings */ 1185 igb_set_promisc(adapter); 1186 1187 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1188 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1189 1190 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1191 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1192 1193 if (adapter->msix > 1) /* Set up queue routing */ 1194 igb_configure_queues(adapter); 1195 1196 /* Set up VLAN tag offload and filter */ 1197 igb_setup_vlan_hw_support(adapter); 1198 1199 /* this clears any pending interrupts */ 1200 E1000_READ_REG(&adapter->hw, E1000_ICR); 1201#ifdef DEVICE_POLLING 1202 /* 1203 * Only enable interrupts if we are not polling, make sure 1204 * they are off otherwise. 1205 */ 1206 if (ifp->if_capenable & IFCAP_POLLING) 1207 igb_disable_intr(adapter); 1208 else 1209#endif /* DEVICE_POLLING */ 1210 { 1211 igb_enable_intr(adapter); 1212 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); 1213 } 1214 1215 /* Don't reset the phy next time init gets called */ 1216 adapter->hw.phy.reset_disable = TRUE; 1217} 1218 1219static void 1220igb_init(void *arg) 1221{ 1222 struct adapter *adapter = arg; 1223 1224 IGB_CORE_LOCK(adapter); 1225 igb_init_locked(adapter); 1226 IGB_CORE_UNLOCK(adapter); 1227} 1228 1229 1230static void 1231igb_handle_que(void *context, int pending) 1232{ 1233 struct igb_queue *que = context; 1234 struct adapter *adapter = que->adapter; 1235 struct tx_ring *txr = que->txr; 1236 struct ifnet *ifp = adapter->ifp; 1237 bool more; 1238 1239 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1240 more = igb_rxeof(que, -1); 1241 1242 IGB_TX_LOCK(txr); 1243 igb_txeof(txr); 1244#if __FreeBSD_version >= 800000 1245 if (!drbr_empty(ifp, txr->br)) 1246 igb_mq_start_locked(ifp, txr, NULL); 1247#else 1248 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1249 igb_start_locked(txr, ifp); 1250#endif 1251 IGB_TX_UNLOCK(txr); 1252 if (more) { 1253 taskqueue_enqueue(que->tq, &que->que_task); 1254 return; 1255 } 1256 } 1257 1258#ifdef DEVICE_POLLING 1259 if (ifp->if_capenable & IFCAP_POLLING) 1260 return; 1261#endif 1262 /* Reenable this interrupt */ 1263 if (que->eims) 1264 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); 1265 else 1266 igb_enable_intr(adapter); 1267} 1268 1269/* Deal with link in a sleepable context */ 1270static void 1271igb_handle_link(void *context, int pending) 1272{ 1273 struct adapter *adapter = context; 1274 1275 adapter->hw.mac.get_link_status = 1; 1276 igb_update_link_status(adapter); 1277} 1278 1279/********************************************************************* 1280 * 1281 * MSI/Legacy Deferred 1282 * Interrupt Service routine 1283 * 1284 *********************************************************************/ 1285static int 1286igb_irq_fast(void *arg) 1287{ 1288 struct adapter *adapter = arg; 1289 uint32_t reg_icr; 1290 1291 1292 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1293 1294 /* Hot eject? */ 1295 if (reg_icr == 0xffffffff) 1296 return FILTER_STRAY; 1297 1298 /* Definitely not our interrupt. */ 1299 if (reg_icr == 0x0) 1300 return FILTER_STRAY; 1301 1302 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1303 return FILTER_STRAY; 1304 1305 /* 1306 * Mask interrupts until the taskqueue is finished running. This is 1307 * cheap, just assume that it is needed. This also works around the 1308 * MSI message reordering errata on certain systems. 1309 */ 1310 igb_disable_intr(adapter); 1311 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1312 1313 /* Link status change */ 1314 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 1315 taskqueue_enqueue(adapter->tq, &adapter->link_task); 1316 1317 if (reg_icr & E1000_ICR_RXO) 1318 adapter->rx_overruns++; 1319 return FILTER_HANDLED; 1320} 1321 1322#ifdef DEVICE_POLLING 1323/********************************************************************* 1324 * 1325 * Legacy polling routine : if using this code you MUST be sure that 1326 * multiqueue is not defined, ie, set igb_num_queues to 1. 1327 * 1328 *********************************************************************/ 1329#if __FreeBSD_version >= 800000 1330#define POLL_RETURN_COUNT(a) (a) 1331static int 1332#else 1333#define POLL_RETURN_COUNT(a) 1334static void 1335#endif 1336igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1337{ 1338 struct adapter *adapter = ifp->if_softc; 1339 struct igb_queue *que = adapter->queues; 1340 struct tx_ring *txr = adapter->tx_rings; 1341 u32 reg_icr, rx_done = 0; 1342 u32 loop = IGB_MAX_LOOP; 1343 bool more; 1344 1345 IGB_CORE_LOCK(adapter); 1346 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1347 IGB_CORE_UNLOCK(adapter); 1348 return POLL_RETURN_COUNT(rx_done); 1349 } 1350 1351 if (cmd == POLL_AND_CHECK_STATUS) { 1352 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1353 /* Link status change */ 1354 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 1355 taskqueue_enqueue(adapter->tq, &adapter->link_task); 1356 1357 if (reg_icr & E1000_ICR_RXO) 1358 adapter->rx_overruns++; 1359 } 1360 IGB_CORE_UNLOCK(adapter); 1361 1362 /* TODO: rx_count */ 1363 rx_done = igb_rxeof(que, count) ? 1 : 0; 1364 1365 IGB_TX_LOCK(txr); 1366 do { 1367 more = igb_txeof(txr); 1368 } while (loop-- && more); 1369#if __FreeBSD_version >= 800000 1370 if (!drbr_empty(ifp, txr->br)) 1371 igb_mq_start_locked(ifp, txr, NULL); 1372#else 1373 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1374 igb_start_locked(txr, ifp); 1375#endif 1376 IGB_TX_UNLOCK(txr); 1377 return POLL_RETURN_COUNT(rx_done); 1378} 1379#endif /* DEVICE_POLLING */ 1380 1381/********************************************************************* 1382 * 1383 * MSIX TX Interrupt Service routine 1384 * 1385 **********************************************************************/ 1386static void 1387igb_msix_que(void *arg) 1388{ 1389 struct igb_queue *que = arg; 1390 struct adapter *adapter = que->adapter; 1391 struct tx_ring *txr = que->txr; 1392 struct rx_ring *rxr = que->rxr; 1393 u32 newitr = 0; 1394 bool more_tx, more_rx; 1395 1396 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims); 1397 ++que->irqs; 1398 1399 IGB_TX_LOCK(txr); 1400 more_tx = igb_txeof(txr); 1401 IGB_TX_UNLOCK(txr); 1402 1403 more_rx = igb_rxeof(que, adapter->rx_process_limit); 1404 1405 if (igb_enable_aim == FALSE) 1406 goto no_calc; 1407 /* 1408 ** Do Adaptive Interrupt Moderation: 1409 ** - Write out last calculated setting 1410 ** - Calculate based on average size over 1411 ** the last interval. 1412 */ 1413 if (que->eitr_setting) 1414 E1000_WRITE_REG(&adapter->hw, 1415 E1000_EITR(que->msix), que->eitr_setting); 1416 1417 que->eitr_setting = 0; 1418 1419 /* Idle, do nothing */ 1420 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1421 goto no_calc; 1422 1423 /* Used half Default if sub-gig */ 1424 if (adapter->link_speed != 1000) 1425 newitr = IGB_DEFAULT_ITR / 2; 1426 else { 1427 if ((txr->bytes) && (txr->packets)) 1428 newitr = txr->bytes/txr->packets; 1429 if ((rxr->bytes) && (rxr->packets)) 1430 newitr = max(newitr, 1431 (rxr->bytes / rxr->packets)); 1432 newitr += 24; /* account for hardware frame, crc */ 1433 /* set an upper boundary */ 1434 newitr = min(newitr, 3000); 1435 /* Be nice to the mid range */ 1436 if ((newitr > 300) && (newitr < 1200)) 1437 newitr = (newitr / 3); 1438 else 1439 newitr = (newitr / 2); 1440 } 1441 newitr &= 0x7FFC; /* Mask invalid bits */ 1442 if (adapter->hw.mac.type == e1000_82575) 1443 newitr |= newitr << 16; 1444 else 1445 newitr |= E1000_EITR_CNT_IGNR; 1446 1447 /* save for next interrupt */ 1448 que->eitr_setting = newitr; 1449 1450 /* Reset state */ 1451 txr->bytes = 0; 1452 txr->packets = 0; 1453 rxr->bytes = 0; 1454 rxr->packets = 0; 1455 1456no_calc: 1457 /* Schedule a clean task if needed*/ 1458 if (more_tx || more_rx) 1459 taskqueue_enqueue(que->tq, &que->que_task); 1460 else 1461 /* Reenable this interrupt */ 1462 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); 1463 return; 1464} 1465 1466 1467/********************************************************************* 1468 * 1469 * MSIX Link Interrupt Service routine 1470 * 1471 **********************************************************************/ 1472 1473static void 1474igb_msix_link(void *arg) 1475{ 1476 struct adapter *adapter = arg; 1477 u32 icr; 1478 1479 ++adapter->link_irq; 1480 icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1481 if (!(icr & E1000_ICR_LSC)) 1482 goto spurious; 1483 taskqueue_enqueue(adapter->tq, &adapter->link_task); 1484 1485spurious: 1486 /* Rearm */ 1487 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); 1488 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); 1489 return; 1490} 1491 1492 1493/********************************************************************* 1494 * 1495 * Media Ioctl callback 1496 * 1497 * This routine is called whenever the user queries the status of 1498 * the interface using ifconfig. 1499 * 1500 **********************************************************************/ 1501static void 1502igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1503{ 1504 struct adapter *adapter = ifp->if_softc; 1505 u_char fiber_type = IFM_1000_SX; 1506 1507 INIT_DEBUGOUT("igb_media_status: begin"); 1508 1509 IGB_CORE_LOCK(adapter); 1510 igb_update_link_status(adapter); 1511 1512 ifmr->ifm_status = IFM_AVALID; 1513 ifmr->ifm_active = IFM_ETHER; 1514 1515 if (!adapter->link_active) { 1516 IGB_CORE_UNLOCK(adapter); 1517 return; 1518 } 1519 1520 ifmr->ifm_status |= IFM_ACTIVE; 1521 1522 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1523 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) 1524 ifmr->ifm_active |= fiber_type | IFM_FDX; 1525 else { 1526 switch (adapter->link_speed) { 1527 case 10: 1528 ifmr->ifm_active |= IFM_10_T; 1529 break; 1530 case 100: 1531 ifmr->ifm_active |= IFM_100_TX; 1532 break; 1533 case 1000: 1534 ifmr->ifm_active |= IFM_1000_T; 1535 break; 1536 } 1537 if (adapter->link_duplex == FULL_DUPLEX) 1538 ifmr->ifm_active |= IFM_FDX; 1539 else 1540 ifmr->ifm_active |= IFM_HDX; 1541 } 1542 IGB_CORE_UNLOCK(adapter); 1543} 1544 1545/********************************************************************* 1546 * 1547 * Media Ioctl callback 1548 * 1549 * This routine is called when the user changes speed/duplex using 1550 * media/mediopt option with ifconfig. 1551 * 1552 **********************************************************************/ 1553static int 1554igb_media_change(struct ifnet *ifp) 1555{ 1556 struct adapter *adapter = ifp->if_softc; 1557 struct ifmedia *ifm = &adapter->media; 1558 1559 INIT_DEBUGOUT("igb_media_change: begin"); 1560 1561 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1562 return (EINVAL); 1563 1564 IGB_CORE_LOCK(adapter); 1565 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1566 case IFM_AUTO: 1567 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1568 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1569 break; 1570 case IFM_1000_LX: 1571 case IFM_1000_SX: 1572 case IFM_1000_T: 1573 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1574 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1575 break; 1576 case IFM_100_TX: 1577 adapter->hw.mac.autoneg = FALSE; 1578 adapter->hw.phy.autoneg_advertised = 0; 1579 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1580 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1581 else 1582 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1583 break; 1584 case IFM_10_T: 1585 adapter->hw.mac.autoneg = FALSE; 1586 adapter->hw.phy.autoneg_advertised = 0; 1587 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1588 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1589 else 1590 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1591 break; 1592 default: 1593 device_printf(adapter->dev, "Unsupported media type\n"); 1594 } 1595 1596 /* As the speed/duplex settings my have changed we need to 1597 * reset the PHY. 1598 */ 1599 adapter->hw.phy.reset_disable = FALSE; 1600 1601 igb_init_locked(adapter); 1602 IGB_CORE_UNLOCK(adapter); 1603 1604 return (0); 1605} 1606 1607 1608/********************************************************************* 1609 * 1610 * This routine maps the mbufs to Advanced TX descriptors. 1611 * used by the 82575 adapter. 1612 * 1613 **********************************************************************/ 1614 1615static int 1616igb_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1617{ 1618 struct adapter *adapter = txr->adapter; 1619 bus_dma_segment_t segs[IGB_MAX_SCATTER]; 1620 bus_dmamap_t map; 1621 struct igb_tx_buffer *tx_buffer, *tx_buffer_mapped; 1622 union e1000_adv_tx_desc *txd = NULL; 1623 struct mbuf *m_head; 1624 u32 olinfo_status = 0, cmd_type_len = 0; 1625 int nsegs, i, j, error, first, last = 0; 1626 u32 hdrlen = 0; 1627 1628 m_head = *m_headp; 1629 1630 1631 /* Set basic descriptor constants */ 1632 cmd_type_len |= E1000_ADVTXD_DTYP_DATA; 1633 cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT; 1634 if (m_head->m_flags & M_VLANTAG) 1635 cmd_type_len |= E1000_ADVTXD_DCMD_VLE; 1636 1637 /* 1638 * Force a cleanup if number of TX descriptors 1639 * available hits the threshold 1640 */ 1641 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) { 1642 igb_txeof(txr); 1643 /* Now do we at least have a minimal? */ 1644 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 1645 txr->no_desc_avail++; 1646 return (ENOBUFS); 1647 } 1648 } 1649 1650 /* 1651 * Map the packet for DMA. 1652 * 1653 * Capture the first descriptor index, 1654 * this descriptor will have the index 1655 * of the EOP which is the only one that 1656 * now gets a DONE bit writeback. 1657 */ 1658 first = txr->next_avail_desc; 1659 tx_buffer = &txr->tx_buffers[first]; 1660 tx_buffer_mapped = tx_buffer; 1661 map = tx_buffer->map; 1662 1663 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1664 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1665 1666 if (error == EFBIG) { 1667 struct mbuf *m; 1668 1669 m = m_defrag(*m_headp, M_DONTWAIT); 1670 if (m == NULL) { 1671 adapter->mbuf_defrag_failed++; 1672 m_freem(*m_headp); 1673 *m_headp = NULL; 1674 return (ENOBUFS); 1675 } 1676 *m_headp = m; 1677 1678 /* Try it again */ 1679 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1680 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1681 1682 if (error == ENOMEM) { 1683 adapter->no_tx_dma_setup++; 1684 return (error); 1685 } else if (error != 0) { 1686 adapter->no_tx_dma_setup++; 1687 m_freem(*m_headp); 1688 *m_headp = NULL; 1689 return (error); 1690 } 1691 } else if (error == ENOMEM) { 1692 adapter->no_tx_dma_setup++; 1693 return (error); 1694 } else if (error != 0) { 1695 adapter->no_tx_dma_setup++; 1696 m_freem(*m_headp); 1697 *m_headp = NULL; 1698 return (error); 1699 } 1700 1701 /* Check again to be sure we have enough descriptors */ 1702 if (nsegs > (txr->tx_avail - 2)) { 1703 txr->no_desc_avail++; 1704 bus_dmamap_unload(txr->txtag, map); 1705 return (ENOBUFS); 1706 } 1707 m_head = *m_headp; 1708 1709 /* 1710 * Set up the context descriptor: 1711 * used when any hardware offload is done. 1712 * This includes CSUM, VLAN, and TSO. It 1713 * will use the first descriptor. 1714 */ 1715 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1716 if (igb_tso_setup(txr, m_head, &hdrlen)) { 1717 cmd_type_len |= E1000_ADVTXD_DCMD_TSE; 1718 olinfo_status |= E1000_TXD_POPTS_IXSM << 8; 1719 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 1720 } else 1721 return (ENXIO); 1722 } else if (igb_tx_ctx_setup(txr, m_head)) 1723 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 1724 1725 /* Calculate payload length */ 1726 olinfo_status |= ((m_head->m_pkthdr.len - hdrlen) 1727 << E1000_ADVTXD_PAYLEN_SHIFT); 1728 1729 /* 82575 needs the queue index added */ 1730 if (adapter->hw.mac.type == e1000_82575) 1731 olinfo_status |= txr->me << 4; 1732 1733 /* Set up our transmit descriptors */ 1734 i = txr->next_avail_desc; 1735 for (j = 0; j < nsegs; j++) { 1736 bus_size_t seg_len; 1737 bus_addr_t seg_addr; 1738 1739 tx_buffer = &txr->tx_buffers[i]; 1740 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i]; 1741 seg_addr = segs[j].ds_addr; 1742 seg_len = segs[j].ds_len; 1743 1744 txd->read.buffer_addr = htole64(seg_addr); 1745 txd->read.cmd_type_len = htole32(cmd_type_len | seg_len); 1746 txd->read.olinfo_status = htole32(olinfo_status); 1747 last = i; 1748 if (++i == adapter->num_tx_desc) 1749 i = 0; 1750 tx_buffer->m_head = NULL; 1751 tx_buffer->next_eop = -1; 1752 } 1753 1754 txr->next_avail_desc = i; 1755 txr->tx_avail -= nsegs; 1756 1757 tx_buffer->m_head = m_head; 1758 tx_buffer_mapped->map = tx_buffer->map; 1759 tx_buffer->map = map; 1760 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1761 1762 /* 1763 * Last Descriptor of Packet 1764 * needs End Of Packet (EOP) 1765 * and Report Status (RS) 1766 */ 1767 txd->read.cmd_type_len |= 1768 htole32(E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS); 1769 /* 1770 * Keep track in the first buffer which 1771 * descriptor will be written back 1772 */ 1773 tx_buffer = &txr->tx_buffers[first]; 1774 tx_buffer->next_eop = last; 1775 txr->watchdog_time = ticks; 1776 1777 /* 1778 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 1779 * that this frame is available to transmit. 1780 */ 1781 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1782 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1783 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); 1784 ++txr->tx_packets; 1785 1786 return (0); 1787 1788} 1789 1790static void 1791igb_set_promisc(struct adapter *adapter) 1792{ 1793 struct ifnet *ifp = adapter->ifp; 1794 uint32_t reg_rctl; 1795 1796 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1797 1798 if (ifp->if_flags & IFF_PROMISC) { 1799 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1800 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1801 } else if (ifp->if_flags & IFF_ALLMULTI) { 1802 reg_rctl |= E1000_RCTL_MPE; 1803 reg_rctl &= ~E1000_RCTL_UPE; 1804 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1805 } 1806} 1807 1808static void 1809igb_disable_promisc(struct adapter *adapter) 1810{ 1811 uint32_t reg_rctl; 1812 1813 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1814 1815 reg_rctl &= (~E1000_RCTL_UPE); 1816 reg_rctl &= (~E1000_RCTL_MPE); 1817 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1818} 1819 1820 1821/********************************************************************* 1822 * Multicast Update 1823 * 1824 * This routine is called whenever multicast address list is updated. 1825 * 1826 **********************************************************************/ 1827 1828static void 1829igb_set_multi(struct adapter *adapter) 1830{ 1831 struct ifnet *ifp = adapter->ifp; 1832 struct ifmultiaddr *ifma; 1833 u32 reg_rctl = 0; 1834 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_ADDR_LEN]; 1835 1836 int mcnt = 0; 1837 1838 IOCTL_DEBUGOUT("igb_set_multi: begin"); 1839 1840#if __FreeBSD_version < 800000 1841 IF_ADDR_LOCK(ifp); 1842#else 1843 if_maddr_rlock(ifp); 1844#endif 1845 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1846 if (ifma->ifma_addr->sa_family != AF_LINK) 1847 continue; 1848 1849 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 1850 break; 1851 1852 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1853 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 1854 mcnt++; 1855 } 1856#if __FreeBSD_version < 800000 1857 IF_ADDR_UNLOCK(ifp); 1858#else 1859 if_maddr_runlock(ifp); 1860#endif 1861 1862 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 1863 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1864 reg_rctl |= E1000_RCTL_MPE; 1865 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1866 } else 1867 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 1868} 1869 1870 1871/********************************************************************* 1872 * Timer routine: 1873 * This routine checks for link status, 1874 * updates statistics, and does the watchdog. 1875 * 1876 **********************************************************************/ 1877 1878static void 1879igb_local_timer(void *arg) 1880{ 1881 struct adapter *adapter = arg; 1882 struct ifnet *ifp = adapter->ifp; 1883 device_t dev = adapter->dev; 1884 struct tx_ring *txr = adapter->tx_rings; 1885 1886 1887 IGB_CORE_LOCK_ASSERT(adapter); 1888 1889 igb_update_link_status(adapter); 1890 igb_update_stats_counters(adapter); 1891 1892 if (igb_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 1893 igb_print_hw_stats(adapter); 1894 1895 /* 1896 ** Watchdog: check for time since any descriptor was cleaned 1897 */ 1898 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1899 if (txr->watchdog_check == FALSE) 1900 continue; 1901 if ((ticks - txr->watchdog_time) > IGB_WATCHDOG) 1902 goto timeout; 1903 } 1904 1905 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1906 return; 1907 1908timeout: 1909 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1910 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1911 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)), 1912 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me))); 1913 device_printf(dev,"TX(%d) desc avail = %d," 1914 "Next TX to Clean = %d\n", 1915 txr->me, txr->tx_avail, txr->next_to_clean); 1916 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1917 adapter->watchdog_events++; 1918 igb_init_locked(adapter); 1919} 1920 1921static void 1922igb_update_link_status(struct adapter *adapter) 1923{ 1924 struct e1000_hw *hw = &adapter->hw; 1925 struct ifnet *ifp = adapter->ifp; 1926 device_t dev = adapter->dev; 1927 struct tx_ring *txr = adapter->tx_rings; 1928 u32 link_check = 0; 1929 1930 /* Get the cached link value or read for real */ 1931 switch (hw->phy.media_type) { 1932 case e1000_media_type_copper: 1933 if (hw->mac.get_link_status) { 1934 /* Do the work to read phy */ 1935 e1000_check_for_link(hw); 1936 link_check = !hw->mac.get_link_status; 1937 } else 1938 link_check = TRUE; 1939 break; 1940 case e1000_media_type_fiber: 1941 e1000_check_for_link(hw); 1942 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 1943 E1000_STATUS_LU); 1944 break; 1945 case e1000_media_type_internal_serdes: 1946 e1000_check_for_link(hw); 1947 link_check = adapter->hw.mac.serdes_has_link; 1948 break; 1949 default: 1950 case e1000_media_type_unknown: 1951 break; 1952 } 1953 1954 /* Now we check if a transition has happened */ 1955 if (link_check && (adapter->link_active == 0)) { 1956 e1000_get_speed_and_duplex(&adapter->hw, 1957 &adapter->link_speed, &adapter->link_duplex); 1958 if (bootverbose) 1959 device_printf(dev, "Link is up %d Mbps %s\n", 1960 adapter->link_speed, 1961 ((adapter->link_duplex == FULL_DUPLEX) ? 1962 "Full Duplex" : "Half Duplex")); 1963 adapter->link_active = 1; 1964 ifp->if_baudrate = adapter->link_speed * 1000000; 1965 /* This can sleep */ 1966 if_link_state_change(ifp, LINK_STATE_UP); 1967 } else if (!link_check && (adapter->link_active == 1)) { 1968 ifp->if_baudrate = adapter->link_speed = 0; 1969 adapter->link_duplex = 0; 1970 if (bootverbose) 1971 device_printf(dev, "Link is Down\n"); 1972 adapter->link_active = 0; 1973 /* This can sleep */ 1974 if_link_state_change(ifp, LINK_STATE_DOWN); 1975 /* Turn off watchdogs */ 1976 for (int i = 0; i < adapter->num_queues; i++, txr++) 1977 txr->watchdog_check = FALSE; 1978 } 1979} 1980 1981/********************************************************************* 1982 * 1983 * This routine disables all traffic on the adapter by issuing a 1984 * global reset on the MAC and deallocates TX/RX buffers. 1985 * 1986 **********************************************************************/ 1987 1988static void 1989igb_stop(void *arg) 1990{ 1991 struct adapter *adapter = arg; 1992 struct ifnet *ifp = adapter->ifp; 1993 struct tx_ring *txr = adapter->tx_rings; 1994 1995 IGB_CORE_LOCK_ASSERT(adapter); 1996 1997 INIT_DEBUGOUT("igb_stop: begin"); 1998 1999 igb_disable_intr(adapter); 2000 2001 callout_stop(&adapter->timer); 2002 2003 /* Tell the stack that the interface is no longer active */ 2004 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2005 2006 /* Unarm watchdog timer. */ 2007 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2008 IGB_TX_LOCK(txr); 2009 txr->watchdog_check = FALSE; 2010 IGB_TX_UNLOCK(txr); 2011 } 2012 2013 e1000_reset_hw(&adapter->hw); 2014 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2015 2016 e1000_led_off(&adapter->hw); 2017 e1000_cleanup_led(&adapter->hw); 2018} 2019 2020 2021/********************************************************************* 2022 * 2023 * Determine hardware revision. 2024 * 2025 **********************************************************************/ 2026static void 2027igb_identify_hardware(struct adapter *adapter) 2028{ 2029 device_t dev = adapter->dev; 2030 2031 /* Make sure our PCI config space has the necessary stuff set */ 2032 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2033 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2034 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2035 device_printf(dev, "Memory Access and/or Bus Master bits " 2036 "were not set!\n"); 2037 adapter->hw.bus.pci_cmd_word |= 2038 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2039 pci_write_config(dev, PCIR_COMMAND, 2040 adapter->hw.bus.pci_cmd_word, 2); 2041 } 2042 2043 /* Save off the information about this board */ 2044 adapter->hw.vendor_id = pci_get_vendor(dev); 2045 adapter->hw.device_id = pci_get_device(dev); 2046 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2047 adapter->hw.subsystem_vendor_id = 2048 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2049 adapter->hw.subsystem_device_id = 2050 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2051 2052 /* Do Shared Code Init and Setup */ 2053 if (e1000_set_mac_type(&adapter->hw)) { 2054 device_printf(dev, "Setup init failure\n"); 2055 return; 2056 } 2057} 2058 2059static int 2060igb_allocate_pci_resources(struct adapter *adapter) 2061{ 2062 device_t dev = adapter->dev; 2063 int rid; 2064 2065 rid = PCIR_BAR(0); 2066 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2067 &rid, RF_ACTIVE); 2068 if (adapter->pci_mem == NULL) { 2069 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2070 return (ENXIO); 2071 } 2072 adapter->osdep.mem_bus_space_tag = 2073 rman_get_bustag(adapter->pci_mem); 2074 adapter->osdep.mem_bus_space_handle = 2075 rman_get_bushandle(adapter->pci_mem); 2076 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2077 2078 adapter->num_queues = 1; /* Defaults for Legacy or MSI */ 2079 2080 /* This will setup either MSI/X or MSI */ 2081 adapter->msix = igb_setup_msix(adapter); 2082 adapter->hw.back = &adapter->osdep; 2083 2084 return (0); 2085} 2086 2087/********************************************************************* 2088 * 2089 * Setup the Legacy or MSI Interrupt handler 2090 * 2091 **********************************************************************/ 2092static int 2093igb_allocate_legacy(struct adapter *adapter) 2094{ 2095 device_t dev = adapter->dev; 2096 struct igb_queue *que = adapter->queues; 2097 int error, rid = 0; 2098 2099 /* Turn off all interrupts */ 2100 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2101 2102 /* MSI RID is 1 */ 2103 if (adapter->msix == 1) 2104 rid = 1; 2105 2106 /* We allocate a single interrupt resource */ 2107 adapter->res = bus_alloc_resource_any(dev, 2108 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2109 if (adapter->res == NULL) { 2110 device_printf(dev, "Unable to allocate bus resource: " 2111 "interrupt\n"); 2112 return (ENXIO); 2113 } 2114 2115 /* 2116 * Try allocating a fast interrupt and the associated deferred 2117 * processing contexts. 2118 */ 2119 TASK_INIT(&adapter->rxtx_task, 0, igb_handle_que, que); 2120 /* Make tasklet for deferred link handling */ 2121 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); 2122 adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, 2123 taskqueue_thread_enqueue, &adapter->tq); 2124 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2125 device_get_nameunit(adapter->dev)); 2126 if ((error = bus_setup_intr(dev, adapter->res, 2127 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL, 2128 adapter, &adapter->tag)) != 0) { 2129 device_printf(dev, "Failed to register fast interrupt " 2130 "handler: %d\n", error); 2131 taskqueue_free(adapter->tq); 2132 adapter->tq = NULL; 2133 return (error); 2134 } 2135 2136 return (0); 2137} 2138 2139 2140/********************************************************************* 2141 * 2142 * Setup the MSIX Queue Interrupt handlers: 2143 * 2144 **********************************************************************/ 2145static int 2146igb_allocate_msix(struct adapter *adapter) 2147{ 2148 device_t dev = adapter->dev; 2149 struct igb_queue *que = adapter->queues; 2150 int error, rid, vector = 0; 2151 2152 2153 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2154 rid = vector +1; 2155 que->res = bus_alloc_resource_any(dev, 2156 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2157 if (que->res == NULL) { 2158 device_printf(dev, 2159 "Unable to allocate bus resource: " 2160 "MSIX Queue Interrupt\n"); 2161 return (ENXIO); 2162 } 2163 error = bus_setup_intr(dev, que->res, 2164 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2165 igb_msix_que, que, &que->tag); 2166 if (error) { 2167 que->res = NULL; 2168 device_printf(dev, "Failed to register Queue handler"); 2169 return (error); 2170 } 2171 que->msix = vector; 2172 if (adapter->hw.mac.type == e1000_82575) 2173 que->eims = E1000_EICR_TX_QUEUE0 << i; 2174 else 2175 que->eims = 1 << vector; 2176 /* 2177 ** Bind the msix vector, and thus the 2178 ** rings to the corresponding cpu. 2179 */ 2180 if (adapter->num_queues > 1) 2181 bus_bind_intr(dev, que->res, i); 2182 /* Make tasklet for deferred handling */ 2183 TASK_INIT(&que->que_task, 0, igb_handle_que, que); 2184 que->tq = taskqueue_create_fast("igb_que", M_NOWAIT, 2185 taskqueue_thread_enqueue, &que->tq); 2186 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 2187 device_get_nameunit(adapter->dev)); 2188 } 2189 2190 /* And Link */ 2191 rid = vector + 1; 2192 adapter->res = bus_alloc_resource_any(dev, 2193 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2194 if (adapter->res == NULL) { 2195 device_printf(dev, 2196 "Unable to allocate bus resource: " 2197 "MSIX Link Interrupt\n"); 2198 return (ENXIO); 2199 } 2200 if ((error = bus_setup_intr(dev, adapter->res, 2201 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2202 igb_msix_link, adapter, &adapter->tag)) != 0) { 2203 device_printf(dev, "Failed to register Link handler"); 2204 return (error); 2205 } 2206 adapter->linkvec = vector; 2207 2208 /* Make tasklet for deferred handling */ 2209 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); 2210 adapter->tq = taskqueue_create_fast("igb_link", M_NOWAIT, 2211 taskqueue_thread_enqueue, &adapter->tq); 2212 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s link", 2213 device_get_nameunit(adapter->dev)); 2214 2215 return (0); 2216} 2217 2218 2219static void 2220igb_configure_queues(struct adapter *adapter) 2221{ 2222 struct e1000_hw *hw = &adapter->hw; 2223 struct igb_queue *que; 2224 u32 tmp, ivar = 0; 2225 u32 newitr = IGB_DEFAULT_ITR; 2226 2227 /* First turn on RSS capability */ 2228 if (adapter->hw.mac.type > e1000_82575) 2229 E1000_WRITE_REG(hw, E1000_GPIE, 2230 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | 2231 E1000_GPIE_PBA | E1000_GPIE_NSICR); 2232 2233 /* Turn on MSIX */ 2234 switch (adapter->hw.mac.type) { 2235 case e1000_82580: 2236 /* RX entries */ 2237 for (int i = 0; i < adapter->num_queues; i++) { 2238 u32 index = i >> 1; 2239 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2240 que = &adapter->queues[i]; 2241 if (i & 1) { 2242 ivar &= 0xFF00FFFF; 2243 ivar |= (que->msix | E1000_IVAR_VALID) << 16; 2244 } else { 2245 ivar &= 0xFFFFFF00; 2246 ivar |= que->msix | E1000_IVAR_VALID; 2247 } 2248 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2249 } 2250 /* TX entries */ 2251 for (int i = 0; i < adapter->num_queues; i++) { 2252 u32 index = i >> 1; 2253 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2254 que = &adapter->queues[i]; 2255 if (i & 1) { 2256 ivar &= 0x00FFFFFF; 2257 ivar |= (que->msix | E1000_IVAR_VALID) << 24; 2258 } else { 2259 ivar &= 0xFFFF00FF; 2260 ivar |= (que->msix | E1000_IVAR_VALID) << 8; 2261 } 2262 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2263 adapter->eims_mask |= que->eims; 2264 } 2265 2266 /* And for the link interrupt */ 2267 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; 2268 adapter->link_mask = 1 << adapter->linkvec; 2269 adapter->eims_mask |= adapter->link_mask; 2270 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 2271 break; 2272 case e1000_82576: 2273 /* RX entries */ 2274 for (int i = 0; i < adapter->num_queues; i++) { 2275 u32 index = i & 0x7; /* Each IVAR has two entries */ 2276 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2277 que = &adapter->queues[i]; 2278 if (i < 8) { 2279 ivar &= 0xFFFFFF00; 2280 ivar |= que->msix | E1000_IVAR_VALID; 2281 } else { 2282 ivar &= 0xFF00FFFF; 2283 ivar |= (que->msix | E1000_IVAR_VALID) << 16; 2284 } 2285 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2286 adapter->eims_mask |= que->eims; 2287 } 2288 /* TX entries */ 2289 for (int i = 0; i < adapter->num_queues; i++) { 2290 u32 index = i & 0x7; /* Each IVAR has two entries */ 2291 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2292 que = &adapter->queues[i]; 2293 if (i < 8) { 2294 ivar &= 0xFFFF00FF; 2295 ivar |= (que->msix | E1000_IVAR_VALID) << 8; 2296 } else { 2297 ivar &= 0x00FFFFFF; 2298 ivar |= (que->msix | E1000_IVAR_VALID) << 24; 2299 } 2300 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2301 adapter->eims_mask |= que->eims; 2302 } 2303 2304 /* And for the link interrupt */ 2305 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; 2306 adapter->link_mask = 1 << adapter->linkvec; 2307 adapter->eims_mask |= adapter->link_mask; 2308 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 2309 break; 2310 2311 case e1000_82575: 2312 /* enable MSI-X support*/ 2313 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); 2314 tmp |= E1000_CTRL_EXT_PBA_CLR; 2315 /* Auto-Mask interrupts upon ICR read. */ 2316 tmp |= E1000_CTRL_EXT_EIAME; 2317 tmp |= E1000_CTRL_EXT_IRCA; 2318 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); 2319 2320 /* Queues */ 2321 for (int i = 0; i < adapter->num_queues; i++) { 2322 que = &adapter->queues[i]; 2323 tmp = E1000_EICR_RX_QUEUE0 << i; 2324 tmp |= E1000_EICR_TX_QUEUE0 << i; 2325 que->eims = tmp; 2326 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 2327 i, que->eims); 2328 adapter->eims_mask |= que->eims; 2329 } 2330 2331 /* Link */ 2332 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), 2333 E1000_EIMS_OTHER); 2334 adapter->link_mask |= E1000_EIMS_OTHER; 2335 adapter->eims_mask |= adapter->link_mask; 2336 default: 2337 break; 2338 } 2339 2340 /* Set the starting interrupt rate */ 2341 if (hw->mac.type == e1000_82575) 2342 newitr |= newitr << 16; 2343 else 2344 newitr |= E1000_EITR_CNT_IGNR; 2345 2346 for (int i = 0; i < adapter->num_queues; i++) { 2347 que = &adapter->queues[i]; 2348 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr); 2349 } 2350 2351 return; 2352} 2353 2354 2355static void 2356igb_free_pci_resources(struct adapter *adapter) 2357{ 2358 struct igb_queue *que = adapter->queues; 2359 device_t dev = adapter->dev; 2360 int rid; 2361 2362 /* 2363 ** There is a slight possibility of a failure mode 2364 ** in attach that will result in entering this function 2365 ** before interrupt resources have been initialized, and 2366 ** in that case we do not want to execute the loops below 2367 ** We can detect this reliably by the state of the adapter 2368 ** res pointer. 2369 */ 2370 if (adapter->res == NULL) 2371 goto mem; 2372 2373 /* 2374 * First release all the interrupt resources: 2375 */ 2376 for (int i = 0; i < adapter->num_queues; i++, que++) { 2377 rid = que->msix + 1; 2378 if (que->tag != NULL) { 2379 bus_teardown_intr(dev, que->res, que->tag); 2380 que->tag = NULL; 2381 } 2382 if (que->res != NULL) 2383 bus_release_resource(dev, 2384 SYS_RES_IRQ, rid, que->res); 2385 } 2386 2387 /* Clean the Legacy or Link interrupt last */ 2388 if (adapter->linkvec) /* we are doing MSIX */ 2389 rid = adapter->linkvec + 1; 2390 else 2391 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2392 2393 if (adapter->tag != NULL) { 2394 bus_teardown_intr(dev, adapter->res, adapter->tag); 2395 adapter->tag = NULL; 2396 } 2397 if (adapter->res != NULL) 2398 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 2399 2400mem: 2401 if (adapter->msix) 2402 pci_release_msi(dev); 2403 2404 if (adapter->msix_mem != NULL) 2405 bus_release_resource(dev, SYS_RES_MEMORY, 2406 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2407 2408 if (adapter->pci_mem != NULL) 2409 bus_release_resource(dev, SYS_RES_MEMORY, 2410 PCIR_BAR(0), adapter->pci_mem); 2411 2412} 2413 2414/* 2415 * Setup Either MSI/X or MSI 2416 */ 2417static int 2418igb_setup_msix(struct adapter *adapter) 2419{ 2420 device_t dev = adapter->dev; 2421 int rid, want, queues, msgs; 2422 2423 /* tuneable override */ 2424 if (igb_enable_msix == 0) 2425 goto msi; 2426 2427 /* First try MSI/X */ 2428 rid = PCIR_BAR(IGB_MSIX_BAR); 2429 adapter->msix_mem = bus_alloc_resource_any(dev, 2430 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2431 if (!adapter->msix_mem) { 2432 /* May not be enabled */ 2433 device_printf(adapter->dev, 2434 "Unable to map MSIX table \n"); 2435 goto msi; 2436 } 2437 2438 msgs = pci_msix_count(dev); 2439 if (msgs == 0) { /* system has msix disabled */ 2440 bus_release_resource(dev, SYS_RES_MEMORY, 2441 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2442 adapter->msix_mem = NULL; 2443 goto msi; 2444 } 2445 2446 /* Figure out a reasonable auto config value */ 2447 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 2448 2449 /* Manual override */ 2450 if (igb_num_queues != 0) 2451 queues = igb_num_queues; 2452 2453 /* Can have max of 4 queues on 82575 */ 2454 if ((adapter->hw.mac.type == e1000_82575) && (queues > 4)) 2455 queues = 4; 2456 2457 /* 2458 ** One vector (RX/TX pair) per queue 2459 ** plus an additional for Link interrupt 2460 */ 2461 want = queues + 1; 2462 if (msgs >= want) 2463 msgs = want; 2464 else { 2465 device_printf(adapter->dev, 2466 "MSIX Configuration Problem, " 2467 "%d vectors configured, but %d queues wanted!\n", 2468 msgs, want); 2469 return (ENXIO); 2470 } 2471 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2472 device_printf(adapter->dev, 2473 "Using MSIX interrupts with %d vectors\n", msgs); 2474 adapter->num_queues = queues; 2475 return (msgs); 2476 } 2477msi: 2478 msgs = pci_msi_count(dev); 2479 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2480 device_printf(adapter->dev,"Using MSI interrupt\n"); 2481 return (msgs); 2482} 2483 2484/********************************************************************* 2485 * 2486 * Set up an fresh starting state 2487 * 2488 **********************************************************************/ 2489static void 2490igb_reset(struct adapter *adapter) 2491{ 2492 device_t dev = adapter->dev; 2493 struct e1000_hw *hw = &adapter->hw; 2494 struct e1000_fc_info *fc = &hw->fc; 2495 struct ifnet *ifp = adapter->ifp; 2496 u32 pba = 0; 2497 u16 hwm; 2498 2499 INIT_DEBUGOUT("igb_reset: begin"); 2500 2501 /* Let the firmware know the OS is in control */ 2502 igb_get_hw_control(adapter); 2503 2504 /* 2505 * Packet Buffer Allocation (PBA) 2506 * Writing PBA sets the receive portion of the buffer 2507 * the remainder is used for the transmit buffer. 2508 */ 2509 switch (hw->mac.type) { 2510 case e1000_82575: 2511 pba = E1000_PBA_32K; 2512 break; 2513 case e1000_82576: 2514 pba = E1000_PBA_64K; 2515 break; 2516 case e1000_82580: 2517 pba = E1000_PBA_35K; 2518 default: 2519 break; 2520 } 2521 2522 /* Special needs in case of Jumbo frames */ 2523 if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) { 2524 u32 tx_space, min_tx, min_rx; 2525 pba = E1000_READ_REG(hw, E1000_PBA); 2526 tx_space = pba >> 16; 2527 pba &= 0xffff; 2528 min_tx = (adapter->max_frame_size + 2529 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; 2530 min_tx = roundup2(min_tx, 1024); 2531 min_tx >>= 10; 2532 min_rx = adapter->max_frame_size; 2533 min_rx = roundup2(min_rx, 1024); 2534 min_rx >>= 10; 2535 if (tx_space < min_tx && 2536 ((min_tx - tx_space) < pba)) { 2537 pba = pba - (min_tx - tx_space); 2538 /* 2539 * if short on rx space, rx wins 2540 * and must trump tx adjustment 2541 */ 2542 if (pba < min_rx) 2543 pba = min_rx; 2544 } 2545 E1000_WRITE_REG(hw, E1000_PBA, pba); 2546 } 2547 2548 INIT_DEBUGOUT1("igb_init: pba=%dK",pba); 2549 2550 /* 2551 * These parameters control the automatic generation (Tx) and 2552 * response (Rx) to Ethernet PAUSE frames. 2553 * - High water mark should allow for at least two frames to be 2554 * received after sending an XOFF. 2555 * - Low water mark works best when it is very near the high water mark. 2556 * This allows the receiver to restart by sending XON when it has 2557 * drained a bit. 2558 */ 2559 hwm = min(((pba << 10) * 9 / 10), 2560 ((pba << 10) - 2 * adapter->max_frame_size)); 2561 2562 if (hw->mac.type < e1000_82576) { 2563 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */ 2564 fc->low_water = fc->high_water - 8; 2565 } else { 2566 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */ 2567 fc->low_water = fc->high_water - 16; 2568 } 2569 2570 fc->pause_time = IGB_FC_PAUSE_TIME; 2571 fc->send_xon = TRUE; 2572 2573 /* Set Flow control, use the tunable location if sane */ 2574 if ((igb_fc_setting >= 0) || (igb_fc_setting < 4)) 2575 fc->requested_mode = igb_fc_setting; 2576 else 2577 fc->requested_mode = e1000_fc_none; 2578 2579 fc->current_mode = fc->requested_mode; 2580 2581 /* Issue a global reset */ 2582 e1000_reset_hw(hw); 2583 E1000_WRITE_REG(hw, E1000_WUC, 0); 2584 2585 if (e1000_init_hw(hw) < 0) 2586 device_printf(dev, "Hardware Initialization Failed\n"); 2587 2588 if (hw->mac.type == e1000_82580) { 2589 u32 reg; 2590 2591 hwm = (pba << 10) - (2 * adapter->max_frame_size); 2592 /* 2593 * 0x80000000 - enable DMA COAL 2594 * 0x10000000 - use L0s as low power 2595 * 0x20000000 - use L1 as low power 2596 * X << 16 - exit dma coal when rx data exceeds X kB 2597 * Y - upper limit to stay in dma coal in units of 32usecs 2598 */ 2599 E1000_WRITE_REG(hw, E1000_DMACR, 2600 0xA0000006 | ((hwm << 6) & 0x00FF0000)); 2601 2602 /* set hwm to PBA - 2 * max frame size */ 2603 E1000_WRITE_REG(hw, E1000_FCRTC, hwm); 2604 /* 2605 * This sets the time to wait before requesting transition to 2606 * low power state to number of usecs needed to receive 1 512 2607 * byte frame at gigabit line rate 2608 */ 2609 E1000_WRITE_REG(hw, E1000_DMCTLX, 4); 2610 2611 /* free space in tx packet buffer to wake from DMA coal */ 2612 E1000_WRITE_REG(hw, E1000_DMCTXTH, 2613 (20480 - (2 * adapter->max_frame_size)) >> 6); 2614 2615 /* make low power state decision controlled by DMA coal */ 2616 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 2617 E1000_WRITE_REG(hw, E1000_PCIEMISC, 2618 reg | E1000_PCIEMISC_LX_DECISION); 2619 } 2620 2621 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 2622 e1000_get_phy_info(hw); 2623 e1000_check_for_link(hw); 2624 return; 2625} 2626 2627/********************************************************************* 2628 * 2629 * Setup networking device structure and register an interface. 2630 * 2631 **********************************************************************/ 2632static void 2633igb_setup_interface(device_t dev, struct adapter *adapter) 2634{ 2635 struct ifnet *ifp; 2636 2637 INIT_DEBUGOUT("igb_setup_interface: begin"); 2638 2639 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2640 if (ifp == NULL) 2641 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 2642 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2643 ifp->if_mtu = ETHERMTU; 2644 ifp->if_init = igb_init; 2645 ifp->if_softc = adapter; 2646 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2647 ifp->if_ioctl = igb_ioctl; 2648 ifp->if_start = igb_start; 2649#if __FreeBSD_version >= 800000 2650 ifp->if_transmit = igb_mq_start; 2651 ifp->if_qflush = igb_qflush; 2652#endif 2653 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 2654 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 2655 IFQ_SET_READY(&ifp->if_snd); 2656 2657 ether_ifattach(ifp, adapter->hw.mac.addr); 2658 2659 ifp->if_capabilities = ifp->if_capenable = 0; 2660 2661 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_MTU; 2662 ifp->if_capabilities |= IFCAP_TSO4; 2663 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 2664 if (igb_header_split) 2665 ifp->if_capabilities |= IFCAP_LRO; 2666 2667 ifp->if_capenable = ifp->if_capabilities; 2668#ifdef DEVICE_POLLING 2669 ifp->if_capabilities |= IFCAP_POLLING; 2670#endif 2671 2672 /* 2673 * Tell the upper layer(s) we 2674 * support full VLAN capability. 2675 */ 2676 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2677 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2678 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2679 2680 /* 2681 ** Dont turn this on by default, if vlans are 2682 ** created on another pseudo device (eg. lagg) 2683 ** then vlan events are not passed thru, breaking 2684 ** operation, but with HW FILTER off it works. If 2685 ** using vlans directly on the em driver you can 2686 ** enable this and get full hardware tag filtering. 2687 */ 2688 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2689 2690 /* 2691 * Specify the media types supported by this adapter and register 2692 * callbacks to update media and link information 2693 */ 2694 ifmedia_init(&adapter->media, IFM_IMASK, 2695 igb_media_change, igb_media_status); 2696 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2697 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2698 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 2699 0, NULL); 2700 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 2701 } else { 2702 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2703 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2704 0, NULL); 2705 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2706 0, NULL); 2707 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2708 0, NULL); 2709 if (adapter->hw.phy.type != e1000_phy_ife) { 2710 ifmedia_add(&adapter->media, 2711 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2712 ifmedia_add(&adapter->media, 2713 IFM_ETHER | IFM_1000_T, 0, NULL); 2714 } 2715 } 2716 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2717 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2718} 2719 2720 2721/* 2722 * Manage DMA'able memory. 2723 */ 2724static void 2725igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 2726{ 2727 if (error) 2728 return; 2729 *(bus_addr_t *) arg = segs[0].ds_addr; 2730} 2731 2732static int 2733igb_dma_malloc(struct adapter *adapter, bus_size_t size, 2734 struct igb_dma_alloc *dma, int mapflags) 2735{ 2736 int error; 2737 2738 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 2739 IGB_DBA_ALIGN, 0, /* alignment, bounds */ 2740 BUS_SPACE_MAXADDR, /* lowaddr */ 2741 BUS_SPACE_MAXADDR, /* highaddr */ 2742 NULL, NULL, /* filter, filterarg */ 2743 size, /* maxsize */ 2744 1, /* nsegments */ 2745 size, /* maxsegsize */ 2746 0, /* flags */ 2747 NULL, /* lockfunc */ 2748 NULL, /* lockarg */ 2749 &dma->dma_tag); 2750 if (error) { 2751 device_printf(adapter->dev, 2752 "%s: bus_dma_tag_create failed: %d\n", 2753 __func__, error); 2754 goto fail_0; 2755 } 2756 2757 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 2758 BUS_DMA_NOWAIT, &dma->dma_map); 2759 if (error) { 2760 device_printf(adapter->dev, 2761 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 2762 __func__, (uintmax_t)size, error); 2763 goto fail_2; 2764 } 2765 2766 dma->dma_paddr = 0; 2767 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2768 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 2769 if (error || dma->dma_paddr == 0) { 2770 device_printf(adapter->dev, 2771 "%s: bus_dmamap_load failed: %d\n", 2772 __func__, error); 2773 goto fail_3; 2774 } 2775 2776 return (0); 2777 2778fail_3: 2779 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2780fail_2: 2781 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2782 bus_dma_tag_destroy(dma->dma_tag); 2783fail_0: 2784 dma->dma_map = NULL; 2785 dma->dma_tag = NULL; 2786 2787 return (error); 2788} 2789 2790static void 2791igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma) 2792{ 2793 if (dma->dma_tag == NULL) 2794 return; 2795 if (dma->dma_map != NULL) { 2796 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2797 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2798 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2799 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2800 dma->dma_map = NULL; 2801 } 2802 bus_dma_tag_destroy(dma->dma_tag); 2803 dma->dma_tag = NULL; 2804} 2805 2806 2807/********************************************************************* 2808 * 2809 * Allocate memory for the transmit and receive rings, and then 2810 * the descriptors associated with each, called only once at attach. 2811 * 2812 **********************************************************************/ 2813static int 2814igb_allocate_queues(struct adapter *adapter) 2815{ 2816 device_t dev = adapter->dev; 2817 struct igb_queue *que = NULL; 2818 struct tx_ring *txr = NULL; 2819 struct rx_ring *rxr = NULL; 2820 int rsize, tsize, error = E1000_SUCCESS; 2821 int txconf = 0, rxconf = 0; 2822 2823 /* First allocate the top level queue structs */ 2824 if (!(adapter->queues = 2825 (struct igb_queue *) malloc(sizeof(struct igb_queue) * 2826 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2827 device_printf(dev, "Unable to allocate queue memory\n"); 2828 error = ENOMEM; 2829 goto fail; 2830 } 2831 2832 /* Next allocate the TX ring struct memory */ 2833 if (!(adapter->tx_rings = 2834 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2835 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2836 device_printf(dev, "Unable to allocate TX ring memory\n"); 2837 error = ENOMEM; 2838 goto tx_fail; 2839 } 2840 2841 /* Now allocate the RX */ 2842 if (!(adapter->rx_rings = 2843 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2844 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2845 device_printf(dev, "Unable to allocate RX ring memory\n"); 2846 error = ENOMEM; 2847 goto rx_fail; 2848 } 2849 2850 tsize = roundup2(adapter->num_tx_desc * 2851 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN); 2852 /* 2853 * Now set up the TX queues, txconf is needed to handle the 2854 * possibility that things fail midcourse and we need to 2855 * undo memory gracefully 2856 */ 2857 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2858 /* Set up some basics */ 2859 txr = &adapter->tx_rings[i]; 2860 txr->adapter = adapter; 2861 txr->me = i; 2862 2863 /* Initialize the TX lock */ 2864 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2865 device_get_nameunit(dev), txr->me); 2866 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2867 2868 if (igb_dma_malloc(adapter, tsize, 2869 &txr->txdma, BUS_DMA_NOWAIT)) { 2870 device_printf(dev, 2871 "Unable to allocate TX Descriptor memory\n"); 2872 error = ENOMEM; 2873 goto err_tx_desc; 2874 } 2875 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; 2876 bzero((void *)txr->tx_base, tsize); 2877 2878 /* Now allocate transmit buffers for the ring */ 2879 if (igb_allocate_transmit_buffers(txr)) { 2880 device_printf(dev, 2881 "Critical Failure setting up transmit buffers\n"); 2882 error = ENOMEM; 2883 goto err_tx_desc; 2884 } 2885#if __FreeBSD_version >= 800000 2886 /* Allocate a buf ring */ 2887 txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF, 2888 M_WAITOK, &txr->tx_mtx); 2889#endif 2890 } 2891 2892 /* 2893 * Next the RX queues... 2894 */ 2895 rsize = roundup2(adapter->num_rx_desc * 2896 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 2897 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2898 rxr = &adapter->rx_rings[i]; 2899 rxr->adapter = adapter; 2900 rxr->me = i; 2901 2902 /* Initialize the RX lock */ 2903 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2904 device_get_nameunit(dev), txr->me); 2905 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2906 2907 if (igb_dma_malloc(adapter, rsize, 2908 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2909 device_printf(dev, 2910 "Unable to allocate RxDescriptor memory\n"); 2911 error = ENOMEM; 2912 goto err_rx_desc; 2913 } 2914 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2915 bzero((void *)rxr->rx_base, rsize); 2916 2917 /* Allocate receive buffers for the ring*/ 2918 if (igb_allocate_receive_buffers(rxr)) { 2919 device_printf(dev, 2920 "Critical Failure setting up receive buffers\n"); 2921 error = ENOMEM; 2922 goto err_rx_desc; 2923 } 2924 } 2925 2926 /* 2927 ** Finally set up the queue holding structs 2928 */ 2929 for (int i = 0; i < adapter->num_queues; i++) { 2930 que = &adapter->queues[i]; 2931 que->adapter = adapter; 2932 que->txr = &adapter->tx_rings[i]; 2933 que->rxr = &adapter->rx_rings[i]; 2934 } 2935 2936 return (0); 2937 2938err_rx_desc: 2939 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2940 igb_dma_free(adapter, &rxr->rxdma); 2941err_tx_desc: 2942 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2943 igb_dma_free(adapter, &txr->txdma); 2944 free(adapter->rx_rings, M_DEVBUF); 2945rx_fail: 2946#if __FreeBSD_version >= 800000 2947 buf_ring_free(txr->br, M_DEVBUF); 2948#endif 2949 free(adapter->tx_rings, M_DEVBUF); 2950tx_fail: 2951 free(adapter->queues, M_DEVBUF); 2952fail: 2953 return (error); 2954} 2955 2956/********************************************************************* 2957 * 2958 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2959 * the information needed to transmit a packet on the wire. This is 2960 * called only once at attach, setup is done every reset. 2961 * 2962 **********************************************************************/ 2963static int 2964igb_allocate_transmit_buffers(struct tx_ring *txr) 2965{ 2966 struct adapter *adapter = txr->adapter; 2967 device_t dev = adapter->dev; 2968 struct igb_tx_buffer *txbuf; 2969 int error, i; 2970 2971 /* 2972 * Setup DMA descriptor areas. 2973 */ 2974 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 2975 1, 0, /* alignment, bounds */ 2976 BUS_SPACE_MAXADDR, /* lowaddr */ 2977 BUS_SPACE_MAXADDR, /* highaddr */ 2978 NULL, NULL, /* filter, filterarg */ 2979 IGB_TSO_SIZE, /* maxsize */ 2980 IGB_MAX_SCATTER, /* nsegments */ 2981 PAGE_SIZE, /* maxsegsize */ 2982 0, /* flags */ 2983 NULL, /* lockfunc */ 2984 NULL, /* lockfuncarg */ 2985 &txr->txtag))) { 2986 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2987 goto fail; 2988 } 2989 2990 if (!(txr->tx_buffers = 2991 (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) * 2992 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2993 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2994 error = ENOMEM; 2995 goto fail; 2996 } 2997 2998 /* Create the descriptor buffer dma maps */ 2999 txbuf = txr->tx_buffers; 3000 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3001 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 3002 if (error != 0) { 3003 device_printf(dev, "Unable to create TX DMA map\n"); 3004 goto fail; 3005 } 3006 } 3007 3008 return 0; 3009fail: 3010 /* We free all, it handles case where we are in the middle */ 3011 igb_free_transmit_structures(adapter); 3012 return (error); 3013} 3014 3015/********************************************************************* 3016 * 3017 * Initialize a transmit ring. 3018 * 3019 **********************************************************************/ 3020static void 3021igb_setup_transmit_ring(struct tx_ring *txr) 3022{ 3023 struct adapter *adapter = txr->adapter; 3024 struct igb_tx_buffer *txbuf; 3025 int i; 3026 3027 /* Clear the old descriptor contents */ 3028 IGB_TX_LOCK(txr); 3029 bzero((void *)txr->tx_base, 3030 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc); 3031 /* Reset indices */ 3032 txr->next_avail_desc = 0; 3033 txr->next_to_clean = 0; 3034 3035 /* Free any existing tx buffers. */ 3036 txbuf = txr->tx_buffers; 3037 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3038 if (txbuf->m_head != NULL) { 3039 bus_dmamap_sync(txr->txtag, txbuf->map, 3040 BUS_DMASYNC_POSTWRITE); 3041 bus_dmamap_unload(txr->txtag, txbuf->map); 3042 m_freem(txbuf->m_head); 3043 txbuf->m_head = NULL; 3044 } 3045 /* clear the watch index */ 3046 txbuf->next_eop = -1; 3047 } 3048 3049 /* Set number of descriptors available */ 3050 txr->tx_avail = adapter->num_tx_desc; 3051 3052 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3053 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3054 IGB_TX_UNLOCK(txr); 3055} 3056 3057/********************************************************************* 3058 * 3059 * Initialize all transmit rings. 3060 * 3061 **********************************************************************/ 3062static void 3063igb_setup_transmit_structures(struct adapter *adapter) 3064{ 3065 struct tx_ring *txr = adapter->tx_rings; 3066 3067 for (int i = 0; i < adapter->num_queues; i++, txr++) 3068 igb_setup_transmit_ring(txr); 3069 3070 return; 3071} 3072 3073/********************************************************************* 3074 * 3075 * Enable transmit unit. 3076 * 3077 **********************************************************************/ 3078static void 3079igb_initialize_transmit_units(struct adapter *adapter) 3080{ 3081 struct tx_ring *txr = adapter->tx_rings; 3082 struct e1000_hw *hw = &adapter->hw; 3083 u32 tctl, txdctl; 3084 3085 INIT_DEBUGOUT("igb_initialize_transmit_units: begin"); 3086 3087 /* Setup the Tx Descriptor Rings */ 3088 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3089 u64 bus_addr = txr->txdma.dma_paddr; 3090 3091 E1000_WRITE_REG(hw, E1000_TDLEN(i), 3092 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3093 E1000_WRITE_REG(hw, E1000_TDBAH(i), 3094 (uint32_t)(bus_addr >> 32)); 3095 E1000_WRITE_REG(hw, E1000_TDBAL(i), 3096 (uint32_t)bus_addr); 3097 3098 /* Setup the HW Tx Head and Tail descriptor pointers */ 3099 E1000_WRITE_REG(hw, E1000_TDT(i), 0); 3100 E1000_WRITE_REG(hw, E1000_TDH(i), 0); 3101 3102 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3103 E1000_READ_REG(hw, E1000_TDBAL(i)), 3104 E1000_READ_REG(hw, E1000_TDLEN(i))); 3105 3106 txr->watchdog_check = FALSE; 3107 3108 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i)); 3109 txdctl |= IGB_TX_PTHRESH; 3110 txdctl |= IGB_TX_HTHRESH << 8; 3111 txdctl |= IGB_TX_WTHRESH << 16; 3112 txdctl |= E1000_TXDCTL_QUEUE_ENABLE; 3113 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); 3114 } 3115 3116 /* Program the Transmit Control Register */ 3117 tctl = E1000_READ_REG(hw, E1000_TCTL); 3118 tctl &= ~E1000_TCTL_CT; 3119 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3120 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3121 3122 e1000_config_collision_dist(hw); 3123 3124 /* This write will effectively turn on the transmit unit. */ 3125 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 3126} 3127 3128/********************************************************************* 3129 * 3130 * Free all transmit rings. 3131 * 3132 **********************************************************************/ 3133static void 3134igb_free_transmit_structures(struct adapter *adapter) 3135{ 3136 struct tx_ring *txr = adapter->tx_rings; 3137 3138 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3139 IGB_TX_LOCK(txr); 3140 igb_free_transmit_buffers(txr); 3141 igb_dma_free(adapter, &txr->txdma); 3142 IGB_TX_UNLOCK(txr); 3143 IGB_TX_LOCK_DESTROY(txr); 3144 } 3145 free(adapter->tx_rings, M_DEVBUF); 3146} 3147 3148/********************************************************************* 3149 * 3150 * Free transmit ring related data structures. 3151 * 3152 **********************************************************************/ 3153static void 3154igb_free_transmit_buffers(struct tx_ring *txr) 3155{ 3156 struct adapter *adapter = txr->adapter; 3157 struct igb_tx_buffer *tx_buffer; 3158 int i; 3159 3160 INIT_DEBUGOUT("free_transmit_ring: begin"); 3161 3162 if (txr->tx_buffers == NULL) 3163 return; 3164 3165 tx_buffer = txr->tx_buffers; 3166 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3167 if (tx_buffer->m_head != NULL) { 3168 bus_dmamap_sync(txr->txtag, tx_buffer->map, 3169 BUS_DMASYNC_POSTWRITE); 3170 bus_dmamap_unload(txr->txtag, 3171 tx_buffer->map); 3172 m_freem(tx_buffer->m_head); 3173 tx_buffer->m_head = NULL; 3174 if (tx_buffer->map != NULL) { 3175 bus_dmamap_destroy(txr->txtag, 3176 tx_buffer->map); 3177 tx_buffer->map = NULL; 3178 } 3179 } else if (tx_buffer->map != NULL) { 3180 bus_dmamap_unload(txr->txtag, 3181 tx_buffer->map); 3182 bus_dmamap_destroy(txr->txtag, 3183 tx_buffer->map); 3184 tx_buffer->map = NULL; 3185 } 3186 } 3187#if __FreeBSD_version >= 800000 3188 if (txr->br != NULL) 3189 buf_ring_free(txr->br, M_DEVBUF); 3190#endif 3191 if (txr->tx_buffers != NULL) { 3192 free(txr->tx_buffers, M_DEVBUF); 3193 txr->tx_buffers = NULL; 3194 } 3195 if (txr->txtag != NULL) { 3196 bus_dma_tag_destroy(txr->txtag); 3197 txr->txtag = NULL; 3198 } 3199 return; 3200} 3201 3202/********************************************************************** 3203 * 3204 * Setup work for hardware segmentation offload (TSO) 3205 * 3206 **********************************************************************/ 3207static boolean_t 3208igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen) 3209{ 3210 struct adapter *adapter = txr->adapter; 3211 struct e1000_adv_tx_context_desc *TXD; 3212 struct igb_tx_buffer *tx_buffer; 3213 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3214 u32 mss_l4len_idx = 0; 3215 u16 vtag = 0; 3216 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 3217 struct ether_vlan_header *eh; 3218 struct ip *ip; 3219 struct tcphdr *th; 3220 3221 3222 /* 3223 * Determine where frame payload starts. 3224 * Jump over vlan headers if already present 3225 */ 3226 eh = mtod(mp, struct ether_vlan_header *); 3227 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 3228 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3229 else 3230 ehdrlen = ETHER_HDR_LEN; 3231 3232 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3233 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3234 return FALSE; 3235 3236 /* Only supports IPV4 for now */ 3237 ctxd = txr->next_avail_desc; 3238 tx_buffer = &txr->tx_buffers[ctxd]; 3239 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3240 3241 ip = (struct ip *)(mp->m_data + ehdrlen); 3242 if (ip->ip_p != IPPROTO_TCP) 3243 return FALSE; /* 0 */ 3244 ip->ip_sum = 0; 3245 ip_hlen = ip->ip_hl << 2; 3246 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3247 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3248 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3249 tcp_hlen = th->th_off << 2; 3250 /* 3251 * Calculate header length, this is used 3252 * in the transmit desc in igb_xmit 3253 */ 3254 *hdrlen = ehdrlen + ip_hlen + tcp_hlen; 3255 3256 /* VLAN MACLEN IPLEN */ 3257 if (mp->m_flags & M_VLANTAG) { 3258 vtag = htole16(mp->m_pkthdr.ether_vtag); 3259 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3260 } 3261 3262 vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT); 3263 vlan_macip_lens |= ip_hlen; 3264 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3265 3266 /* ADV DTYPE TUCMD */ 3267 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3268 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3269 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3270 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3271 3272 /* MSS L4LEN IDX */ 3273 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT); 3274 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); 3275 /* 82575 needs the queue index added */ 3276 if (adapter->hw.mac.type == e1000_82575) 3277 mss_l4len_idx |= txr->me << 4; 3278 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3279 3280 TXD->seqnum_seed = htole32(0); 3281 tx_buffer->m_head = NULL; 3282 tx_buffer->next_eop = -1; 3283 3284 if (++ctxd == adapter->num_tx_desc) 3285 ctxd = 0; 3286 3287 txr->tx_avail--; 3288 txr->next_avail_desc = ctxd; 3289 return TRUE; 3290} 3291 3292 3293/********************************************************************* 3294 * 3295 * Context Descriptor setup for VLAN or CSUM 3296 * 3297 **********************************************************************/ 3298 3299static bool 3300igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3301{ 3302 struct adapter *adapter = txr->adapter; 3303 struct e1000_adv_tx_context_desc *TXD; 3304 struct igb_tx_buffer *tx_buffer; 3305 u32 vlan_macip_lens, type_tucmd_mlhl, mss_l4len_idx; 3306 struct ether_vlan_header *eh; 3307 struct ip *ip = NULL; 3308 struct ip6_hdr *ip6; 3309 int ehdrlen, ctxd, ip_hlen = 0; 3310 u16 etype, vtag = 0; 3311 u8 ipproto = 0; 3312 bool offload = TRUE; 3313 3314 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 3315 offload = FALSE; 3316 3317 vlan_macip_lens = type_tucmd_mlhl = mss_l4len_idx = 0; 3318 ctxd = txr->next_avail_desc; 3319 tx_buffer = &txr->tx_buffers[ctxd]; 3320 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3321 3322 /* 3323 ** In advanced descriptors the vlan tag must 3324 ** be placed into the context descriptor, thus 3325 ** we need to be here just for that setup. 3326 */ 3327 if (mp->m_flags & M_VLANTAG) { 3328 vtag = htole16(mp->m_pkthdr.ether_vtag); 3329 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3330 } else if (offload == FALSE) 3331 return FALSE; 3332 3333 /* 3334 * Determine where frame payload starts. 3335 * Jump over vlan headers if already present, 3336 * helpful for QinQ too. 3337 */ 3338 eh = mtod(mp, struct ether_vlan_header *); 3339 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3340 etype = ntohs(eh->evl_proto); 3341 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3342 } else { 3343 etype = ntohs(eh->evl_encap_proto); 3344 ehdrlen = ETHER_HDR_LEN; 3345 } 3346 3347 /* Set the ether header length */ 3348 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; 3349 3350 switch (etype) { 3351 case ETHERTYPE_IP: 3352 ip = (struct ip *)(mp->m_data + ehdrlen); 3353 ip_hlen = ip->ip_hl << 2; 3354 if (mp->m_len < ehdrlen + ip_hlen) { 3355 offload = FALSE; 3356 break; 3357 } 3358 ipproto = ip->ip_p; 3359 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3360 break; 3361 case ETHERTYPE_IPV6: 3362 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3363 ip_hlen = sizeof(struct ip6_hdr); 3364 if (mp->m_len < ehdrlen + ip_hlen) 3365 return (FALSE); 3366 ipproto = ip6->ip6_nxt; 3367 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; 3368 break; 3369 default: 3370 offload = FALSE; 3371 break; 3372 } 3373 3374 vlan_macip_lens |= ip_hlen; 3375 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3376 3377 switch (ipproto) { 3378 case IPPROTO_TCP: 3379 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 3380 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3381 break; 3382 case IPPROTO_UDP: 3383 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 3384 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; 3385 break; 3386#if __FreeBSD_version >= 800000 3387 case IPPROTO_SCTP: 3388 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 3389 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; 3390 break; 3391#endif 3392 default: 3393 offload = FALSE; 3394 break; 3395 } 3396 3397 /* 82575 needs the queue index added */ 3398 if (adapter->hw.mac.type == e1000_82575) 3399 mss_l4len_idx = txr->me << 4; 3400 3401 /* Now copy bits into descriptor */ 3402 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3403 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3404 TXD->seqnum_seed = htole32(0); 3405 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3406 3407 tx_buffer->m_head = NULL; 3408 tx_buffer->next_eop = -1; 3409 3410 /* We've consumed the first desc, adjust counters */ 3411 if (++ctxd == adapter->num_tx_desc) 3412 ctxd = 0; 3413 txr->next_avail_desc = ctxd; 3414 --txr->tx_avail; 3415 3416 return (offload); 3417} 3418 3419 3420/********************************************************************** 3421 * 3422 * Examine each tx_buffer in the used queue. If the hardware is done 3423 * processing the packet then free associated resources. The 3424 * tx_buffer is put back on the free queue. 3425 * 3426 * TRUE return means there's work in the ring to clean, FALSE its empty. 3427 **********************************************************************/ 3428static bool 3429igb_txeof(struct tx_ring *txr) 3430{ 3431 struct adapter *adapter = txr->adapter; 3432 int first, last, done; 3433 struct igb_tx_buffer *tx_buffer; 3434 struct e1000_tx_desc *tx_desc, *eop_desc; 3435 struct ifnet *ifp = adapter->ifp; 3436 3437 IGB_TX_LOCK_ASSERT(txr); 3438 3439 if (txr->tx_avail == adapter->num_tx_desc) 3440 return FALSE; 3441 3442 first = txr->next_to_clean; 3443 tx_desc = &txr->tx_base[first]; 3444 tx_buffer = &txr->tx_buffers[first]; 3445 last = tx_buffer->next_eop; 3446 eop_desc = &txr->tx_base[last]; 3447 3448 /* 3449 * What this does is get the index of the 3450 * first descriptor AFTER the EOP of the 3451 * first packet, that way we can do the 3452 * simple comparison on the inner while loop. 3453 */ 3454 if (++last == adapter->num_tx_desc) 3455 last = 0; 3456 done = last; 3457 3458 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3459 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3460 3461 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3462 /* We clean the range of the packet */ 3463 while (first != done) { 3464 tx_desc->upper.data = 0; 3465 tx_desc->lower.data = 0; 3466 tx_desc->buffer_addr = 0; 3467 ++txr->tx_avail; 3468 3469 if (tx_buffer->m_head) { 3470 txr->bytes += 3471 tx_buffer->m_head->m_pkthdr.len; 3472 bus_dmamap_sync(txr->txtag, 3473 tx_buffer->map, 3474 BUS_DMASYNC_POSTWRITE); 3475 bus_dmamap_unload(txr->txtag, 3476 tx_buffer->map); 3477 3478 m_freem(tx_buffer->m_head); 3479 tx_buffer->m_head = NULL; 3480 } 3481 tx_buffer->next_eop = -1; 3482 txr->watchdog_time = ticks; 3483 3484 if (++first == adapter->num_tx_desc) 3485 first = 0; 3486 3487 tx_buffer = &txr->tx_buffers[first]; 3488 tx_desc = &txr->tx_base[first]; 3489 } 3490 ++txr->packets; 3491 ++ifp->if_opackets; 3492 /* See if we can continue to the next packet */ 3493 last = tx_buffer->next_eop; 3494 if (last != -1) { 3495 eop_desc = &txr->tx_base[last]; 3496 /* Get new done point */ 3497 if (++last == adapter->num_tx_desc) last = 0; 3498 done = last; 3499 } else 3500 break; 3501 } 3502 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3503 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3504 3505 txr->next_to_clean = first; 3506 3507 /* 3508 * If we have enough room, clear IFF_DRV_OACTIVE 3509 * to tell the stack that it is OK to send packets. 3510 */ 3511 if (txr->tx_avail > IGB_TX_CLEANUP_THRESHOLD) { 3512 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3513 /* All clean, turn off the watchdog */ 3514 if (txr->tx_avail == adapter->num_tx_desc) { 3515 txr->watchdog_check = FALSE; 3516 return FALSE; 3517 } 3518 } 3519 3520 return (TRUE); 3521} 3522 3523 3524/********************************************************************* 3525 * 3526 * Refresh mbuf buffers for RX descriptor rings 3527 * - now keeps its own state so discards due to resource 3528 * exhaustion are unnecessary, if an mbuf cannot be obtained 3529 * it just returns, keeping its placeholder, thus it can simply 3530 * be recalled to try again. 3531 * 3532 **********************************************************************/ 3533static void 3534igb_refresh_mbufs(struct rx_ring *rxr, int limit) 3535{ 3536 struct adapter *adapter = rxr->adapter; 3537 bus_dma_segment_t hseg[1]; 3538 bus_dma_segment_t pseg[1]; 3539 struct igb_rx_buf *rxbuf; 3540 struct mbuf *mh, *mp; 3541 int i, nsegs, error, cleaned; 3542 3543 i = rxr->next_to_refresh; 3544 cleaned = -1; /* Signify no completions */ 3545 while (i != limit) { 3546 rxbuf = &rxr->rx_buffers[i]; 3547 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 3548 mh = m_gethdr(M_DONTWAIT, MT_DATA); 3549 if (mh == NULL) 3550 goto update; 3551 mh->m_pkthdr.len = mh->m_len = MHLEN; 3552 mh->m_len = MHLEN; 3553 mh->m_flags |= M_PKTHDR; 3554 m_adj(mh, ETHER_ALIGN); 3555 /* Get the memory mapping */ 3556 error = bus_dmamap_load_mbuf_sg(rxr->htag, 3557 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); 3558 if (error != 0) { 3559 printf("GET BUF: dmamap load" 3560 " failure - %d\n", error); 3561 m_free(mh); 3562 goto update; 3563 } 3564 rxbuf->m_head = mh; 3565 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3566 BUS_DMASYNC_PREREAD); 3567 rxr->rx_base[i].read.hdr_addr = 3568 htole64(hseg[0].ds_addr); 3569 } 3570 3571 if (rxbuf->m_pack == NULL) { 3572 mp = m_getjcl(M_DONTWAIT, MT_DATA, 3573 M_PKTHDR, adapter->rx_mbuf_sz); 3574 if (mp == NULL) 3575 goto update; 3576 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3577 /* Get the memory mapping */ 3578 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 3579 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); 3580 if (error != 0) { 3581 printf("GET BUF: dmamap load" 3582 " failure - %d\n", error); 3583 m_free(mp); 3584 goto update; 3585 } 3586 rxbuf->m_pack = mp; 3587 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3588 BUS_DMASYNC_PREREAD); 3589 rxr->rx_base[i].read.pkt_addr = 3590 htole64(pseg[0].ds_addr); 3591 } 3592 3593 cleaned = i; 3594 /* Calculate next index */ 3595 if (++i == adapter->num_rx_desc) 3596 i = 0; 3597 /* This is the work marker for refresh */ 3598 rxr->next_to_refresh = i; 3599 } 3600update: 3601 if (cleaned != -1) /* If we refreshed some, bump tail */ 3602 E1000_WRITE_REG(&adapter->hw, 3603 E1000_RDT(rxr->me), cleaned); 3604 return; 3605} 3606 3607 3608/********************************************************************* 3609 * 3610 * Allocate memory for rx_buffer structures. Since we use one 3611 * rx_buffer per received packet, the maximum number of rx_buffer's 3612 * that we'll need is equal to the number of receive descriptors 3613 * that we've allocated. 3614 * 3615 **********************************************************************/ 3616static int 3617igb_allocate_receive_buffers(struct rx_ring *rxr) 3618{ 3619 struct adapter *adapter = rxr->adapter; 3620 device_t dev = adapter->dev; 3621 struct igb_rx_buf *rxbuf; 3622 int i, bsize, error; 3623 3624 bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc; 3625 if (!(rxr->rx_buffers = 3626 (struct igb_rx_buf *) malloc(bsize, 3627 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3628 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3629 error = ENOMEM; 3630 goto fail; 3631 } 3632 3633 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 3634 1, 0, /* alignment, bounds */ 3635 BUS_SPACE_MAXADDR, /* lowaddr */ 3636 BUS_SPACE_MAXADDR, /* highaddr */ 3637 NULL, NULL, /* filter, filterarg */ 3638 MSIZE, /* maxsize */ 3639 1, /* nsegments */ 3640 MSIZE, /* maxsegsize */ 3641 0, /* flags */ 3642 NULL, /* lockfunc */ 3643 NULL, /* lockfuncarg */ 3644 &rxr->htag))) { 3645 device_printf(dev, "Unable to create RX DMA tag\n"); 3646 goto fail; 3647 } 3648 3649 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 3650 1, 0, /* alignment, bounds */ 3651 BUS_SPACE_MAXADDR, /* lowaddr */ 3652 BUS_SPACE_MAXADDR, /* highaddr */ 3653 NULL, NULL, /* filter, filterarg */ 3654 MJUMPAGESIZE, /* maxsize */ 3655 1, /* nsegments */ 3656 MJUMPAGESIZE, /* maxsegsize */ 3657 0, /* flags */ 3658 NULL, /* lockfunc */ 3659 NULL, /* lockfuncarg */ 3660 &rxr->ptag))) { 3661 device_printf(dev, "Unable to create RX payload DMA tag\n"); 3662 goto fail; 3663 } 3664 3665 for (i = 0; i < adapter->num_rx_desc; i++) { 3666 rxbuf = &rxr->rx_buffers[i]; 3667 error = bus_dmamap_create(rxr->htag, 3668 BUS_DMA_NOWAIT, &rxbuf->hmap); 3669 if (error) { 3670 device_printf(dev, 3671 "Unable to create RX head DMA maps\n"); 3672 goto fail; 3673 } 3674 error = bus_dmamap_create(rxr->ptag, 3675 BUS_DMA_NOWAIT, &rxbuf->pmap); 3676 if (error) { 3677 device_printf(dev, 3678 "Unable to create RX packet DMA maps\n"); 3679 goto fail; 3680 } 3681 } 3682 3683 return (0); 3684 3685fail: 3686 /* Frees all, but can handle partial completion */ 3687 igb_free_receive_structures(adapter); 3688 return (error); 3689} 3690 3691 3692static void 3693igb_free_receive_ring(struct rx_ring *rxr) 3694{ 3695 struct adapter *adapter; 3696 struct igb_rx_buf *rxbuf; 3697 int i; 3698 3699 adapter = rxr->adapter; 3700 for (i = 0; i < adapter->num_rx_desc; i++) { 3701 rxbuf = &rxr->rx_buffers[i]; 3702 if (rxbuf->m_head != NULL) { 3703 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3704 BUS_DMASYNC_POSTREAD); 3705 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 3706 rxbuf->m_head->m_flags |= M_PKTHDR; 3707 m_freem(rxbuf->m_head); 3708 } 3709 if (rxbuf->m_pack != NULL) { 3710 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3711 BUS_DMASYNC_POSTREAD); 3712 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 3713 rxbuf->m_pack->m_flags |= M_PKTHDR; 3714 m_freem(rxbuf->m_pack); 3715 } 3716 rxbuf->m_head = NULL; 3717 rxbuf->m_pack = NULL; 3718 } 3719} 3720 3721 3722/********************************************************************* 3723 * 3724 * Initialize a receive ring and its buffers. 3725 * 3726 **********************************************************************/ 3727static int 3728igb_setup_receive_ring(struct rx_ring *rxr) 3729{ 3730 struct adapter *adapter; 3731 struct ifnet *ifp; 3732 device_t dev; 3733 struct igb_rx_buf *rxbuf; 3734 bus_dma_segment_t pseg[1], hseg[1]; 3735 struct lro_ctrl *lro = &rxr->lro; 3736 int rsize, nsegs, error = 0; 3737 3738 adapter = rxr->adapter; 3739 dev = adapter->dev; 3740 ifp = adapter->ifp; 3741 3742 /* Clear the ring contents */ 3743 IGB_RX_LOCK(rxr); 3744 rsize = roundup2(adapter->num_rx_desc * 3745 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 3746 bzero((void *)rxr->rx_base, rsize); 3747 3748 /* 3749 ** Free current RX buffer structures and their mbufs 3750 */ 3751 igb_free_receive_ring(rxr); 3752 3753 /* Configure for header split? */ 3754 if (igb_header_split) 3755 rxr->hdr_split = TRUE; 3756 3757 /* Now replenish the ring mbufs */ 3758 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3759 struct mbuf *mh, *mp; 3760 3761 rxbuf = &rxr->rx_buffers[j]; 3762 if (rxr->hdr_split == FALSE) 3763 goto skip_head; 3764 3765 /* First the header */ 3766 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 3767 if (rxbuf->m_head == NULL) { 3768 error = ENOBUFS; 3769 goto fail; 3770 } 3771 m_adj(rxbuf->m_head, ETHER_ALIGN); 3772 mh = rxbuf->m_head; 3773 mh->m_len = mh->m_pkthdr.len = MHLEN; 3774 mh->m_flags |= M_PKTHDR; 3775 /* Get the memory mapping */ 3776 error = bus_dmamap_load_mbuf_sg(rxr->htag, 3777 rxbuf->hmap, rxbuf->m_head, hseg, 3778 &nsegs, BUS_DMA_NOWAIT); 3779 if (error != 0) /* Nothing elegant to do here */ 3780 goto fail; 3781 bus_dmamap_sync(rxr->htag, 3782 rxbuf->hmap, BUS_DMASYNC_PREREAD); 3783 /* Update descriptor */ 3784 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 3785 3786skip_head: 3787 /* Now the payload cluster */ 3788 rxbuf->m_pack = m_getjcl(M_DONTWAIT, MT_DATA, 3789 M_PKTHDR, adapter->rx_mbuf_sz); 3790 if (rxbuf->m_pack == NULL) { 3791 error = ENOBUFS; 3792 goto fail; 3793 } 3794 mp = rxbuf->m_pack; 3795 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3796 /* Get the memory mapping */ 3797 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 3798 rxbuf->pmap, mp, pseg, 3799 &nsegs, BUS_DMA_NOWAIT); 3800 if (error != 0) 3801 goto fail; 3802 bus_dmamap_sync(rxr->ptag, 3803 rxbuf->pmap, BUS_DMASYNC_PREREAD); 3804 /* Update descriptor */ 3805 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 3806 } 3807 3808 /* Setup our descriptor indices */ 3809 rxr->next_to_check = 0; 3810 rxr->next_to_refresh = 0; 3811 rxr->lro_enabled = FALSE; 3812 rxr->rx_split_packets = 0; 3813 rxr->rx_bytes = 0; 3814 3815 rxr->fmp = NULL; 3816 rxr->lmp = NULL; 3817 rxr->discard = FALSE; 3818 3819 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3820 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3821 3822 /* 3823 ** Now set up the LRO interface, we 3824 ** also only do head split when LRO 3825 ** is enabled, since so often they 3826 ** are undesireable in similar setups. 3827 */ 3828 if (ifp->if_capenable & IFCAP_LRO) { 3829 int err = tcp_lro_init(lro); 3830 if (err) { 3831 device_printf(dev, "LRO Initialization failed!\n"); 3832 goto fail; 3833 } 3834 INIT_DEBUGOUT("RX LRO Initialized\n"); 3835 rxr->lro_enabled = TRUE; 3836 lro->ifp = adapter->ifp; 3837 } 3838 3839 IGB_RX_UNLOCK(rxr); 3840 return (0); 3841 3842fail: 3843 igb_free_receive_ring(rxr); 3844 IGB_RX_UNLOCK(rxr); 3845 return (error); 3846} 3847 3848/********************************************************************* 3849 * 3850 * Initialize all receive rings. 3851 * 3852 **********************************************************************/ 3853static int 3854igb_setup_receive_structures(struct adapter *adapter) 3855{ 3856 struct rx_ring *rxr = adapter->rx_rings; 3857 int i; 3858 3859 for (i = 0; i < adapter->num_queues; i++, rxr++) 3860 if (igb_setup_receive_ring(rxr)) 3861 goto fail; 3862 3863 return (0); 3864fail: 3865 /* 3866 * Free RX buffers allocated so far, we will only handle 3867 * the rings that completed, the failing case will have 3868 * cleaned up for itself. 'i' is the endpoint. 3869 */ 3870 for (int j = 0; j > i; ++j) { 3871 rxr = &adapter->rx_rings[i]; 3872 igb_free_receive_ring(rxr); 3873 } 3874 3875 return (ENOBUFS); 3876} 3877 3878/********************************************************************* 3879 * 3880 * Enable receive unit. 3881 * 3882 **********************************************************************/ 3883static void 3884igb_initialize_receive_units(struct adapter *adapter) 3885{ 3886 struct rx_ring *rxr = adapter->rx_rings; 3887 struct ifnet *ifp = adapter->ifp; 3888 struct e1000_hw *hw = &adapter->hw; 3889 u32 rctl, rxcsum, psize, srrctl = 0; 3890 3891 INIT_DEBUGOUT("igb_initialize_receive_unit: begin"); 3892 3893 /* 3894 * Make sure receives are disabled while setting 3895 * up the descriptor ring 3896 */ 3897 rctl = E1000_READ_REG(hw, E1000_RCTL); 3898 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3899 3900 /* 3901 ** Set up for header split 3902 */ 3903 if (rxr->hdr_split) { 3904 /* Use a standard mbuf for the header */ 3905 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; 3906 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3907 } else 3908 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 3909 3910 /* 3911 ** Set up for jumbo frames 3912 */ 3913 if (ifp->if_mtu > ETHERMTU) { 3914 rctl |= E1000_RCTL_LPE; 3915 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 3916 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 3917 3918 /* Set maximum packet len */ 3919 psize = adapter->max_frame_size; 3920 /* are we on a vlan? */ 3921 if (adapter->ifp->if_vlantrunk != NULL) 3922 psize += VLAN_TAG_SIZE; 3923 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); 3924 } else { 3925 rctl &= ~E1000_RCTL_LPE; 3926 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 3927 rctl |= E1000_RCTL_SZ_2048; 3928 } 3929 3930 /* Setup the Base and Length of the Rx Descriptor Rings */ 3931 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3932 u64 bus_addr = rxr->rxdma.dma_paddr; 3933 u32 rxdctl; 3934 3935 E1000_WRITE_REG(hw, E1000_RDLEN(i), 3936 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 3937 E1000_WRITE_REG(hw, E1000_RDBAH(i), 3938 (uint32_t)(bus_addr >> 32)); 3939 E1000_WRITE_REG(hw, E1000_RDBAL(i), 3940 (uint32_t)bus_addr); 3941 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); 3942 /* Enable this Queue */ 3943 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3944 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 3945 rxdctl &= 0xFFF00000; 3946 rxdctl |= IGB_RX_PTHRESH; 3947 rxdctl |= IGB_RX_HTHRESH << 8; 3948 rxdctl |= IGB_RX_WTHRESH << 16; 3949 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); 3950 } 3951 3952 /* 3953 ** Setup for RX MultiQueue 3954 */ 3955 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 3956 if (adapter->num_queues >1) { 3957 u32 random[10], mrqc, shift = 0; 3958 union igb_reta { 3959 u32 dword; 3960 u8 bytes[4]; 3961 } reta; 3962 3963 arc4rand(&random, sizeof(random), 0); 3964 if (adapter->hw.mac.type == e1000_82575) 3965 shift = 6; 3966 /* Warning FM follows */ 3967 for (int i = 0; i < 128; i++) { 3968 reta.bytes[i & 3] = 3969 (i % adapter->num_queues) << shift; 3970 if ((i & 3) == 3) 3971 E1000_WRITE_REG(hw, 3972 E1000_RETA(i >> 2), reta.dword); 3973 } 3974 /* Now fill in hash table */ 3975 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 3976 for (int i = 0; i < 10; i++) 3977 E1000_WRITE_REG_ARRAY(hw, 3978 E1000_RSSRK(0), i, random[i]); 3979 3980 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 3981 E1000_MRQC_RSS_FIELD_IPV4_TCP); 3982 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | 3983 E1000_MRQC_RSS_FIELD_IPV6_TCP); 3984 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | 3985 E1000_MRQC_RSS_FIELD_IPV6_UDP); 3986 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 3987 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 3988 3989 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 3990 3991 /* 3992 ** NOTE: Receive Full-Packet Checksum Offload 3993 ** is mutually exclusive with Multiqueue. However 3994 ** this is not the same as TCP/IP checksums which 3995 ** still work. 3996 */ 3997 rxcsum |= E1000_RXCSUM_PCSD; 3998#if __FreeBSD_version >= 800000 3999 /* For SCTP Offload */ 4000 if ((hw->mac.type == e1000_82576) 4001 && (ifp->if_capenable & IFCAP_RXCSUM)) 4002 rxcsum |= E1000_RXCSUM_CRCOFL; 4003#endif 4004 } else { 4005 /* Non RSS setup */ 4006 if (ifp->if_capenable & IFCAP_RXCSUM) { 4007 rxcsum |= E1000_RXCSUM_IPPCSE; 4008#if __FreeBSD_version >= 800000 4009 if (adapter->hw.mac.type == e1000_82576) 4010 rxcsum |= E1000_RXCSUM_CRCOFL; 4011#endif 4012 } else 4013 rxcsum &= ~E1000_RXCSUM_TUOFL; 4014 } 4015 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 4016 4017 /* Setup the Receive Control Register */ 4018 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4019 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 4020 E1000_RCTL_RDMTS_HALF | 4021 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4022 /* Strip CRC bytes. */ 4023 rctl |= E1000_RCTL_SECRC; 4024 /* Make sure VLAN Filters are off */ 4025 rctl &= ~E1000_RCTL_VFE; 4026 /* Don't store bad packets */ 4027 rctl &= ~E1000_RCTL_SBP; 4028 4029 /* Enable Receives */ 4030 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4031 4032 /* 4033 * Setup the HW Rx Head and Tail Descriptor Pointers 4034 * - needs to be after enable 4035 */ 4036 for (int i = 0; i < adapter->num_queues; i++) { 4037 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 4038 E1000_WRITE_REG(hw, E1000_RDT(i), 4039 adapter->num_rx_desc - 1); 4040 } 4041 return; 4042} 4043 4044/********************************************************************* 4045 * 4046 * Free receive rings. 4047 * 4048 **********************************************************************/ 4049static void 4050igb_free_receive_structures(struct adapter *adapter) 4051{ 4052 struct rx_ring *rxr = adapter->rx_rings; 4053 4054 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4055 struct lro_ctrl *lro = &rxr->lro; 4056 igb_free_receive_buffers(rxr); 4057 tcp_lro_free(lro); 4058 igb_dma_free(adapter, &rxr->rxdma); 4059 } 4060 4061 free(adapter->rx_rings, M_DEVBUF); 4062} 4063 4064/********************************************************************* 4065 * 4066 * Free receive ring data structures. 4067 * 4068 **********************************************************************/ 4069static void 4070igb_free_receive_buffers(struct rx_ring *rxr) 4071{ 4072 struct adapter *adapter = rxr->adapter; 4073 struct igb_rx_buf *rxbuf; 4074 int i; 4075 4076 INIT_DEBUGOUT("free_receive_structures: begin"); 4077 4078 /* Cleanup any existing buffers */ 4079 if (rxr->rx_buffers != NULL) { 4080 for (i = 0; i < adapter->num_rx_desc; i++) { 4081 rxbuf = &rxr->rx_buffers[i]; 4082 if (rxbuf->m_head != NULL) { 4083 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 4084 BUS_DMASYNC_POSTREAD); 4085 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 4086 rxbuf->m_head->m_flags |= M_PKTHDR; 4087 m_freem(rxbuf->m_head); 4088 } 4089 if (rxbuf->m_pack != NULL) { 4090 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 4091 BUS_DMASYNC_POSTREAD); 4092 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 4093 rxbuf->m_pack->m_flags |= M_PKTHDR; 4094 m_freem(rxbuf->m_pack); 4095 } 4096 rxbuf->m_head = NULL; 4097 rxbuf->m_pack = NULL; 4098 if (rxbuf->hmap != NULL) { 4099 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 4100 rxbuf->hmap = NULL; 4101 } 4102 if (rxbuf->pmap != NULL) { 4103 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4104 rxbuf->pmap = NULL; 4105 } 4106 } 4107 if (rxr->rx_buffers != NULL) { 4108 free(rxr->rx_buffers, M_DEVBUF); 4109 rxr->rx_buffers = NULL; 4110 } 4111 } 4112 4113 if (rxr->htag != NULL) { 4114 bus_dma_tag_destroy(rxr->htag); 4115 rxr->htag = NULL; 4116 } 4117 if (rxr->ptag != NULL) { 4118 bus_dma_tag_destroy(rxr->ptag); 4119 rxr->ptag = NULL; 4120 } 4121} 4122 4123static __inline void 4124igb_rx_discard(struct rx_ring *rxr, int i) 4125{ 4126 struct adapter *adapter = rxr->adapter; 4127 struct igb_rx_buf *rbuf; 4128 struct mbuf *mh, *mp; 4129 4130 rbuf = &rxr->rx_buffers[i]; 4131 if (rxr->fmp != NULL) { 4132 rxr->fmp->m_flags |= M_PKTHDR; 4133 m_freem(rxr->fmp); 4134 rxr->fmp = NULL; 4135 rxr->lmp = NULL; 4136 } 4137 4138 mh = rbuf->m_head; 4139 mp = rbuf->m_pack; 4140 4141 /* Reuse loaded DMA map and just update mbuf chain */ 4142 mh->m_len = MHLEN; 4143 mh->m_flags |= M_PKTHDR; 4144 mh->m_next = NULL; 4145 4146 mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz; 4147 mp->m_data = mp->m_ext.ext_buf; 4148 mp->m_next = NULL; 4149 return; 4150} 4151 4152static __inline void 4153igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4154{ 4155 4156 /* 4157 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 4158 * should be computed by hardware. Also it should not have VLAN tag in 4159 * ethernet header. 4160 */ 4161 if (rxr->lro_enabled && 4162 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 4163 (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && 4164 (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) == 4165 (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) && 4166 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4167 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4168 /* 4169 * Send to the stack if: 4170 ** - LRO not enabled, or 4171 ** - no LRO resources, or 4172 ** - lro enqueue fails 4173 */ 4174 if (rxr->lro.lro_cnt != 0) 4175 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4176 return; 4177 } 4178 IGB_RX_UNLOCK(rxr); 4179 (*ifp->if_input)(ifp, m); 4180 IGB_RX_LOCK(rxr); 4181} 4182 4183/********************************************************************* 4184 * 4185 * This routine executes in interrupt context. It replenishes 4186 * the mbufs in the descriptor and sends data which has been 4187 * dma'ed into host memory to upper layer. 4188 * 4189 * We loop at most count times if count is > 0, or until done if 4190 * count < 0. 4191 * 4192 * Return TRUE if more to clean, FALSE otherwise 4193 *********************************************************************/ 4194static bool 4195igb_rxeof(struct igb_queue *que, int count) 4196{ 4197 struct adapter *adapter = que->adapter; 4198 struct rx_ring *rxr = que->rxr; 4199 struct ifnet *ifp = adapter->ifp; 4200 struct lro_ctrl *lro = &rxr->lro; 4201 struct lro_entry *queued; 4202 int i, processed = 0; 4203 u32 ptype, staterr = 0; 4204 union e1000_adv_rx_desc *cur; 4205 4206 IGB_RX_LOCK(rxr); 4207 /* Sync the ring. */ 4208 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4209 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4210 4211 /* Main clean loop */ 4212 for (i = rxr->next_to_check; count != 0;) { 4213 struct mbuf *sendmp, *mh, *mp; 4214 struct igb_rx_buf *rxbuf; 4215 u16 hlen, plen, hdr, vtag; 4216 bool eop = FALSE; 4217 4218 cur = &rxr->rx_base[i]; 4219 staterr = le32toh(cur->wb.upper.status_error); 4220 if ((staterr & E1000_RXD_STAT_DD) == 0) 4221 break; 4222 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 4223 break; 4224 count--; 4225 sendmp = mh = mp = NULL; 4226 cur->wb.upper.status_error = 0; 4227 rxbuf = &rxr->rx_buffers[i]; 4228 plen = le16toh(cur->wb.upper.length); 4229 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; 4230 vtag = le16toh(cur->wb.upper.vlan); 4231 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4232 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); 4233 4234 /* Make sure all segments of a bad packet are discarded */ 4235 if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) || 4236 (rxr->discard)) { 4237 ifp->if_ierrors++; 4238 ++rxr->rx_discarded; 4239 if (!eop) /* Catch subsequent segs */ 4240 rxr->discard = TRUE; 4241 else 4242 rxr->discard = FALSE; 4243 igb_rx_discard(rxr, i); 4244 goto next_desc; 4245 } 4246 4247 /* 4248 ** The way the hardware is configured to 4249 ** split, it will ONLY use the header buffer 4250 ** when header split is enabled, otherwise we 4251 ** get normal behavior, ie, both header and 4252 ** payload are DMA'd into the payload buffer. 4253 ** 4254 ** The fmp test is to catch the case where a 4255 ** packet spans multiple descriptors, in that 4256 ** case only the first header is valid. 4257 */ 4258 if (rxr->hdr_split && rxr->fmp == NULL) { 4259 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >> 4260 E1000_RXDADV_HDRBUFLEN_SHIFT; 4261 if (hlen > IGB_HDR_BUF) 4262 hlen = IGB_HDR_BUF; 4263 /* Handle the header mbuf */ 4264 mh = rxr->rx_buffers[i].m_head; 4265 mh->m_len = hlen; 4266 /* clear buf info for refresh */ 4267 rxbuf->m_head = NULL; 4268 /* 4269 ** Get the payload length, this 4270 ** could be zero if its a small 4271 ** packet. 4272 */ 4273 if (plen > 0) { 4274 mp = rxr->rx_buffers[i].m_pack; 4275 mp->m_len = plen; 4276 mh->m_next = mp; 4277 /* clear buf info for refresh */ 4278 rxbuf->m_pack = NULL; 4279 rxr->rx_split_packets++; 4280 } 4281 } else { 4282 /* 4283 ** Either no header split, or a 4284 ** secondary piece of a fragmented 4285 ** split packet. 4286 */ 4287 mh = rxr->rx_buffers[i].m_pack; 4288 mh->m_len = plen; 4289 /* clear buf info for refresh */ 4290 rxbuf->m_pack = NULL; 4291 } 4292 4293 ++processed; /* So we know when to refresh */ 4294 4295 /* Initial frame - setup */ 4296 if (rxr->fmp == NULL) { 4297 mh->m_pkthdr.len = mh->m_len; 4298 /* Store the first mbuf */ 4299 rxr->fmp = mh; 4300 rxr->lmp = mh; 4301 if (mp != NULL) { 4302 /* Add payload if split */ 4303 mh->m_pkthdr.len += mp->m_len; 4304 rxr->lmp = mh->m_next; 4305 } 4306 } else { 4307 /* Chain mbuf's together */ 4308 rxr->lmp->m_next = mh; 4309 rxr->lmp = rxr->lmp->m_next; 4310 rxr->fmp->m_pkthdr.len += mh->m_len; 4311 } 4312 4313 if (eop) { 4314 rxr->fmp->m_pkthdr.rcvif = ifp; 4315 ifp->if_ipackets++; 4316 rxr->rx_packets++; 4317 /* capture data for AIM */ 4318 rxr->packets++; 4319 rxr->bytes += rxr->fmp->m_pkthdr.len; 4320 rxr->rx_bytes += rxr->fmp->m_pkthdr.len; 4321 4322 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 4323 igb_rx_checksum(staterr, rxr->fmp, ptype); 4324 4325 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 4326 (staterr & E1000_RXD_STAT_VP) != 0) { 4327 rxr->fmp->m_pkthdr.ether_vtag = vtag; 4328 rxr->fmp->m_flags |= M_VLANTAG; 4329 } 4330#if __FreeBSD_version >= 800000 4331 rxr->fmp->m_pkthdr.flowid = que->msix; 4332 rxr->fmp->m_flags |= M_FLOWID; 4333#endif 4334 sendmp = rxr->fmp; 4335 /* Make sure to set M_PKTHDR. */ 4336 sendmp->m_flags |= M_PKTHDR; 4337 rxr->fmp = NULL; 4338 rxr->lmp = NULL; 4339 } 4340 4341next_desc: 4342 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4343 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4344 4345 /* Advance our pointers to the next descriptor. */ 4346 if (++i == adapter->num_rx_desc) 4347 i = 0; 4348 /* 4349 ** Send to the stack or LRO 4350 */ 4351 if (sendmp != NULL) { 4352 rxr->next_to_check = i; 4353 igb_rx_input(rxr, ifp, sendmp, ptype); 4354 i = rxr->next_to_check; 4355 } 4356 4357 /* Every 8 descriptors we go to refresh mbufs */ 4358 if (processed == 8) { 4359 igb_refresh_mbufs(rxr, i); 4360 processed = 0; 4361 } 4362 } 4363 4364 /* Catch any remainders */ 4365 if (processed != 0) { 4366 igb_refresh_mbufs(rxr, i); 4367 processed = 0; 4368 } 4369 4370 rxr->next_to_check = i; 4371 4372 /* 4373 * Flush any outstanding LRO work 4374 */ 4375 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4376 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4377 tcp_lro_flush(lro, queued); 4378 } 4379 4380 IGB_RX_UNLOCK(rxr); 4381 4382 /* 4383 ** We still have cleaning to do? 4384 ** Schedule another interrupt if so. 4385 */ 4386 if ((staterr & E1000_RXD_STAT_DD) != 0) 4387 return (TRUE); 4388 4389 return (FALSE); 4390} 4391 4392/********************************************************************* 4393 * 4394 * Verify that the hardware indicated that the checksum is valid. 4395 * Inform the stack about the status of checksum so that stack 4396 * doesn't spend time verifying the checksum. 4397 * 4398 *********************************************************************/ 4399static void 4400igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype) 4401{ 4402 u16 status = (u16)staterr; 4403 u8 errors = (u8) (staterr >> 24); 4404 int sctp; 4405 4406 /* Ignore Checksum bit is set */ 4407 if (status & E1000_RXD_STAT_IXSM) { 4408 mp->m_pkthdr.csum_flags = 0; 4409 return; 4410 } 4411 4412 if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && 4413 (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) 4414 sctp = 1; 4415 else 4416 sctp = 0; 4417 if (status & E1000_RXD_STAT_IPCS) { 4418 /* Did it pass? */ 4419 if (!(errors & E1000_RXD_ERR_IPE)) { 4420 /* IP Checksum Good */ 4421 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4422 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4423 } else 4424 mp->m_pkthdr.csum_flags = 0; 4425 } 4426 4427 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { 4428 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4429#if __FreeBSD_version >= 800000 4430 if (sctp) /* reassign */ 4431 type = CSUM_SCTP_VALID; 4432#endif 4433 /* Did it pass? */ 4434 if (!(errors & E1000_RXD_ERR_TCPE)) { 4435 mp->m_pkthdr.csum_flags |= type; 4436 if (sctp == 0) 4437 mp->m_pkthdr.csum_data = htons(0xffff); 4438 } 4439 } 4440 return; 4441} 4442 4443/* 4444 * This routine is run via an vlan 4445 * config EVENT 4446 */ 4447static void 4448igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4449{ 4450 struct adapter *adapter = ifp->if_softc; 4451 u32 index, bit; 4452 4453 if (ifp->if_softc != arg) /* Not our event */ 4454 return; 4455 4456 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4457 return; 4458 4459 index = (vtag >> 5) & 0x7F; 4460 bit = vtag & 0x1F; 4461 igb_shadow_vfta[index] |= (1 << bit); 4462 ++adapter->num_vlans; 4463 /* Re-init to load the changes */ 4464 igb_init(adapter); 4465} 4466 4467/* 4468 * This routine is run via an vlan 4469 * unconfig EVENT 4470 */ 4471static void 4472igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4473{ 4474 struct adapter *adapter = ifp->if_softc; 4475 u32 index, bit; 4476 4477 if (ifp->if_softc != arg) 4478 return; 4479 4480 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4481 return; 4482 4483 index = (vtag >> 5) & 0x7F; 4484 bit = vtag & 0x1F; 4485 igb_shadow_vfta[index] &= ~(1 << bit); 4486 --adapter->num_vlans; 4487 /* Re-init to load the changes */ 4488 igb_init(adapter); 4489} 4490 4491static void 4492igb_setup_vlan_hw_support(struct adapter *adapter) 4493{ 4494 struct e1000_hw *hw = &adapter->hw; 4495 u32 reg; 4496 4497 /* 4498 ** We get here thru init_locked, meaning 4499 ** a soft reset, this has already cleared 4500 ** the VFTA and other state, so if there 4501 ** have been no vlan's registered do nothing. 4502 */ 4503 if (adapter->num_vlans == 0) 4504 return; 4505 4506 /* 4507 ** A soft reset zero's out the VFTA, so 4508 ** we need to repopulate it now. 4509 */ 4510 for (int i = 0; i < IGB_VFTA_SIZE; i++) 4511 if (igb_shadow_vfta[i] != 0) 4512 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, 4513 i, igb_shadow_vfta[i]); 4514 4515 reg = E1000_READ_REG(hw, E1000_CTRL); 4516 reg |= E1000_CTRL_VME; 4517 E1000_WRITE_REG(hw, E1000_CTRL, reg); 4518 4519 /* Enable the Filter Table */ 4520 reg = E1000_READ_REG(hw, E1000_RCTL); 4521 reg &= ~E1000_RCTL_CFIEN; 4522 reg |= E1000_RCTL_VFE; 4523 E1000_WRITE_REG(hw, E1000_RCTL, reg); 4524 4525 /* Update the frame size */ 4526 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4527 adapter->max_frame_size + VLAN_TAG_SIZE); 4528} 4529 4530static void 4531igb_enable_intr(struct adapter *adapter) 4532{ 4533 /* With RSS set up what to auto clear */ 4534 if (adapter->msix_mem) { 4535 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 4536 adapter->eims_mask); 4537 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, 4538 adapter->eims_mask); 4539 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, 4540 adapter->eims_mask); 4541 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 4542 E1000_IMS_LSC); 4543 } else { 4544 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 4545 IMS_ENABLE_MASK); 4546 } 4547 E1000_WRITE_FLUSH(&adapter->hw); 4548 4549 return; 4550} 4551 4552static void 4553igb_disable_intr(struct adapter *adapter) 4554{ 4555 if (adapter->msix_mem) { 4556 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); 4557 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); 4558 } 4559 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); 4560 E1000_WRITE_FLUSH(&adapter->hw); 4561 return; 4562} 4563 4564/* 4565 * Bit of a misnomer, what this really means is 4566 * to enable OS management of the system... aka 4567 * to disable special hardware management features 4568 */ 4569static void 4570igb_init_manageability(struct adapter *adapter) 4571{ 4572 if (adapter->has_manage) { 4573 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4574 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4575 4576 /* disable hardware interception of ARP */ 4577 manc &= ~(E1000_MANC_ARP_EN); 4578 4579 /* enable receiving management packets to the host */ 4580 manc |= E1000_MANC_EN_MNG2HOST; 4581 manc2h |= 1 << 5; /* Mng Port 623 */ 4582 manc2h |= 1 << 6; /* Mng Port 664 */ 4583 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4584 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4585 } 4586} 4587 4588/* 4589 * Give control back to hardware management 4590 * controller if there is one. 4591 */ 4592static void 4593igb_release_manageability(struct adapter *adapter) 4594{ 4595 if (adapter->has_manage) { 4596 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4597 4598 /* re-enable hardware interception of ARP */ 4599 manc |= E1000_MANC_ARP_EN; 4600 manc &= ~E1000_MANC_EN_MNG2HOST; 4601 4602 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4603 } 4604} 4605 4606/* 4607 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 4608 * For ASF and Pass Through versions of f/w this means that 4609 * the driver is loaded. 4610 * 4611 */ 4612static void 4613igb_get_hw_control(struct adapter *adapter) 4614{ 4615 u32 ctrl_ext; 4616 4617 /* Let firmware know the driver has taken over */ 4618 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4619 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4620 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4621} 4622 4623/* 4624 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 4625 * For ASF and Pass Through versions of f/w this means that the 4626 * driver is no longer loaded. 4627 * 4628 */ 4629static void 4630igb_release_hw_control(struct adapter *adapter) 4631{ 4632 u32 ctrl_ext; 4633 4634 /* Let firmware taken over control of h/w */ 4635 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4636 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4637 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4638} 4639 4640static int 4641igb_is_valid_ether_addr(uint8_t *addr) 4642{ 4643 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4644 4645 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4646 return (FALSE); 4647 } 4648 4649 return (TRUE); 4650} 4651 4652 4653/* 4654 * Enable PCI Wake On Lan capability 4655 */ 4656static void 4657igb_enable_wakeup(device_t dev) 4658{ 4659 u16 cap, status; 4660 u8 id; 4661 4662 /* First find the capabilities pointer*/ 4663 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 4664 /* Read the PM Capabilities */ 4665 id = pci_read_config(dev, cap, 1); 4666 if (id != PCIY_PMG) /* Something wrong */ 4667 return; 4668 /* OK, we have the power capabilities, so 4669 now get the status register */ 4670 cap += PCIR_POWER_STATUS; 4671 status = pci_read_config(dev, cap, 2); 4672 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4673 pci_write_config(dev, cap, status, 2); 4674 return; 4675} 4676 4677static void 4678igb_led_func(void *arg, int onoff) 4679{ 4680 struct adapter *adapter = arg; 4681 4682 IGB_CORE_LOCK(adapter); 4683 if (onoff) { 4684 e1000_setup_led(&adapter->hw); 4685 e1000_led_on(&adapter->hw); 4686 } else { 4687 e1000_led_off(&adapter->hw); 4688 e1000_cleanup_led(&adapter->hw); 4689 } 4690 IGB_CORE_UNLOCK(adapter); 4691} 4692 4693/********************************************************************** 4694 * 4695 * Update the board statistics counters. 4696 * 4697 **********************************************************************/ 4698static void 4699igb_update_stats_counters(struct adapter *adapter) 4700{ 4701 struct ifnet *ifp; 4702 4703 if (adapter->hw.phy.media_type == e1000_media_type_copper || 4704 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 4705 adapter->stats.symerrs += 4706 E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 4707 adapter->stats.sec += 4708 E1000_READ_REG(&adapter->hw, E1000_SEC); 4709 } 4710 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 4711 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 4712 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 4713 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 4714 4715 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 4716 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 4717 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 4718 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 4719 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 4720 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 4721 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 4722 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 4723 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 4724 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 4725 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 4726 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 4727 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 4728 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 4729 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 4730 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 4731 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 4732 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 4733 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 4734 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 4735 4736 /* For the 64-bit byte counters the low dword must be read first. */ 4737 /* Both registers clear on the read of the high dword */ 4738 4739 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 4740 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 4741 4742 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 4743 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 4744 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 4745 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 4746 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 4747 4748 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 4749 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 4750 4751 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 4752 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 4753 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 4754 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 4755 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 4756 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 4757 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 4758 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 4759 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 4760 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 4761 4762 adapter->stats.algnerrc += 4763 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 4764 adapter->stats.rxerrc += 4765 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 4766 adapter->stats.tncrs += 4767 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 4768 adapter->stats.cexterr += 4769 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 4770 adapter->stats.tsctc += 4771 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 4772 adapter->stats.tsctfc += 4773 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 4774 ifp = adapter->ifp; 4775 4776 ifp->if_collisions = adapter->stats.colc; 4777 4778 /* Rx Errors */ 4779 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 4780 adapter->stats.crcerrs + adapter->stats.algnerrc + 4781 adapter->stats.ruc + adapter->stats.roc + 4782 adapter->stats.mpc + adapter->stats.cexterr; 4783 4784 /* Tx Errors */ 4785 ifp->if_oerrors = adapter->stats.ecol + 4786 adapter->stats.latecol + adapter->watchdog_events; 4787} 4788 4789 4790/********************************************************************** 4791 * 4792 * This routine is called only when igb_display_debug_stats is enabled. 4793 * This routine provides a way to take a look at important statistics 4794 * maintained by the driver and hardware. 4795 * 4796 **********************************************************************/ 4797static void 4798igb_print_debug_info(struct adapter *adapter) 4799{ 4800 device_t dev = adapter->dev; 4801 struct igb_queue *que = adapter->queues; 4802 struct rx_ring *rxr = adapter->rx_rings; 4803 struct tx_ring *txr = adapter->tx_rings; 4804 uint8_t *hw_addr = adapter->hw.hw_addr; 4805 4806 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 4807 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 4808 E1000_READ_REG(&adapter->hw, E1000_CTRL), 4809 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 4810 4811#if (DEBUG_HW > 0) /* Dont output these errors normally */ 4812 device_printf(dev, "IMS = 0x%x EIMS = 0x%x \n", 4813 E1000_READ_REG(&adapter->hw, E1000_IMS), 4814 E1000_READ_REG(&adapter->hw, E1000_EIMS)); 4815#endif 4816 4817 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 4818 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 4819 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 4820 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 4821 adapter->hw.fc.high_water, 4822 adapter->hw.fc.low_water); 4823 4824 for (int i = 0; i < adapter->num_queues; i++, txr++) { 4825 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d ", i, 4826 E1000_READ_REG(&adapter->hw, E1000_TDH(i)), 4827 E1000_READ_REG(&adapter->hw, E1000_TDT(i))); 4828 device_printf(dev, "TX(%d) no descriptors = %lld\n", 4829 txr->me, (long long)txr->no_desc_avail); 4830 device_printf(dev, "TX(%d) Packets sent = %lld\n", 4831 txr->me, (long long)txr->tx_packets); 4832 } 4833 4834 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4835 struct lro_ctrl *lro = &rxr->lro; 4836 device_printf(dev, "Queue(%d) rdh = %d, rdt = %d\n", i, 4837 E1000_READ_REG(&adapter->hw, E1000_RDH(i)), 4838 E1000_READ_REG(&adapter->hw, E1000_RDT(i))); 4839 device_printf(dev, "RX(%d) Packets received = %lld\n", rxr->me, 4840 (long long)rxr->rx_packets); 4841 device_printf(dev, " Split Packets = %lld ", 4842 (long long)rxr->rx_split_packets); 4843 device_printf(dev, " Byte count = %lld\n", 4844 (long long)rxr->rx_bytes); 4845 device_printf(dev,"RX(%d) LRO Queued= %d ", 4846 i, lro->lro_queued); 4847 device_printf(dev,"LRO Flushed= %d\n",lro->lro_flushed); 4848 } 4849 4850 for (int i = 0; i < adapter->num_queues; i++, que++) 4851 device_printf(dev,"QUE(%d) IRQs = %lld\n", 4852 i, (long long)que->irqs); 4853 4854 device_printf(dev, "LINK MSIX IRQ Handled = %u\n", adapter->link_irq); 4855 device_printf(dev, "Mbuf defrag failed = %ld\n", 4856 adapter->mbuf_defrag_failed); 4857 device_printf(dev, "Std mbuf header failed = %ld\n", 4858 adapter->mbuf_header_failed); 4859 device_printf(dev, "Std mbuf packet failed = %ld\n", 4860 adapter->mbuf_packet_failed); 4861 device_printf(dev, "Driver dropped packets = %ld\n", 4862 adapter->dropped_pkts); 4863 device_printf(dev, "Driver tx dma failure in xmit = %ld\n", 4864 adapter->no_tx_dma_setup); 4865} 4866 4867static void 4868igb_print_hw_stats(struct adapter *adapter) 4869{ 4870 device_t dev = adapter->dev; 4871 4872 device_printf(dev, "Excessive collisions = %lld\n", 4873 (long long)adapter->stats.ecol); 4874#if (DEBUG_HW > 0) /* Dont output these errors normally */ 4875 device_printf(dev, "Symbol errors = %lld\n", 4876 (long long)adapter->stats.symerrs); 4877#endif 4878 device_printf(dev, "Sequence errors = %lld\n", 4879 (long long)adapter->stats.sec); 4880 device_printf(dev, "Defer count = %lld\n", 4881 (long long)adapter->stats.dc); 4882 device_printf(dev, "Missed Packets = %lld\n", 4883 (long long)adapter->stats.mpc); 4884 device_printf(dev, "Receive No Buffers = %lld\n", 4885 (long long)adapter->stats.rnbc); 4886 /* RLEC is inaccurate on some hardware, calculate our own. */ 4887 device_printf(dev, "Receive Length Errors = %lld\n", 4888 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 4889 device_printf(dev, "Receive errors = %lld\n", 4890 (long long)adapter->stats.rxerrc); 4891 device_printf(dev, "Crc errors = %lld\n", 4892 (long long)adapter->stats.crcerrs); 4893 device_printf(dev, "Alignment errors = %lld\n", 4894 (long long)adapter->stats.algnerrc); 4895 /* On 82575 these are collision counts */ 4896 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 4897 (long long)adapter->stats.cexterr); 4898 device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); 4899 device_printf(dev, "watchdog timeouts = %ld\n", 4900 adapter->watchdog_events); 4901 device_printf(dev, "XON Rcvd = %lld\n", 4902 (long long)adapter->stats.xonrxc); 4903 device_printf(dev, "XON Xmtd = %lld\n", 4904 (long long)adapter->stats.xontxc); 4905 device_printf(dev, "XOFF Rcvd = %lld\n", 4906 (long long)adapter->stats.xoffrxc); 4907 device_printf(dev, "XOFF Xmtd = %lld\n", 4908 (long long)adapter->stats.xofftxc); 4909 device_printf(dev, "Good Packets Rcvd = %lld\n", 4910 (long long)adapter->stats.gprc); 4911 device_printf(dev, "Good Packets Xmtd = %lld\n", 4912 (long long)adapter->stats.gptc); 4913 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 4914 (long long)adapter->stats.tsctc); 4915 device_printf(dev, "TSO Contexts Failed = %lld\n", 4916 (long long)adapter->stats.tsctfc); 4917} 4918 4919/********************************************************************** 4920 * 4921 * This routine provides a way to dump out the adapter eeprom, 4922 * often a useful debug/service tool. This only dumps the first 4923 * 32 words, stuff that matters is in that extent. 4924 * 4925 **********************************************************************/ 4926static void 4927igb_print_nvm_info(struct adapter *adapter) 4928{ 4929 u16 eeprom_data; 4930 int i, j, row = 0; 4931 4932 /* Its a bit crude, but it gets the job done */ 4933 printf("\nInterface EEPROM Dump:\n"); 4934 printf("Offset\n0x0000 "); 4935 for (i = 0, j = 0; i < 32; i++, j++) { 4936 if (j == 8) { /* Make the offset block */ 4937 j = 0; ++row; 4938 printf("\n0x00%x0 ",row); 4939 } 4940 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 4941 printf("%04x ", eeprom_data); 4942 } 4943 printf("\n"); 4944} 4945 4946static int 4947igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 4948{ 4949 struct adapter *adapter; 4950 int error; 4951 int result; 4952 4953 result = -1; 4954 error = sysctl_handle_int(oidp, &result, 0, req); 4955 4956 if (error || !req->newptr) 4957 return (error); 4958 4959 if (result == 1) { 4960 adapter = (struct adapter *)arg1; 4961 igb_print_debug_info(adapter); 4962 } 4963 /* 4964 * This value will cause a hex dump of the 4965 * first 32 16-bit words of the EEPROM to 4966 * the screen. 4967 */ 4968 if (result == 2) { 4969 adapter = (struct adapter *)arg1; 4970 igb_print_nvm_info(adapter); 4971 } 4972 4973 return (error); 4974} 4975 4976 4977static int 4978igb_sysctl_stats(SYSCTL_HANDLER_ARGS) 4979{ 4980 struct adapter *adapter; 4981 int error; 4982 int result; 4983 4984 result = -1; 4985 error = sysctl_handle_int(oidp, &result, 0, req); 4986 4987 if (error || !req->newptr) 4988 return (error); 4989 4990 if (result == 1) { 4991 adapter = (struct adapter *)arg1; 4992 igb_print_hw_stats(adapter); 4993 } 4994 4995 return (error); 4996} 4997 4998static void 4999igb_add_rx_process_limit(struct adapter *adapter, const char *name, 5000 const char *description, int *limit, int value) 5001{ 5002 *limit = value; 5003 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5004 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5005 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5006} 5007