if_igb.c revision 211913
1/****************************************************************************** 2 3 Copyright (c) 2001-2010, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ 33/*$FreeBSD: head/sys/dev/e1000/if_igb.c 211913 2010-08-28 00:34:22Z yongari $*/ 34 35 36#ifdef HAVE_KERNEL_OPTION_HEADERS 37#include "opt_device_polling.h" 38#include "opt_inet.h" 39#include "opt_altq.h" 40#endif 41 42#include <sys/param.h> 43#include <sys/systm.h> 44#if __FreeBSD_version >= 800000 45#include <sys/buf_ring.h> 46#endif 47#include <sys/bus.h> 48#include <sys/endian.h> 49#include <sys/kernel.h> 50#include <sys/kthread.h> 51#include <sys/malloc.h> 52#include <sys/mbuf.h> 53#include <sys/module.h> 54#include <sys/rman.h> 55#include <sys/socket.h> 56#include <sys/sockio.h> 57#include <sys/sysctl.h> 58#include <sys/taskqueue.h> 59#include <sys/eventhandler.h> 60#include <sys/pcpu.h> 61#include <sys/smp.h> 62#include <machine/smp.h> 63#include <machine/bus.h> 64#include <machine/resource.h> 65 66#include <net/bpf.h> 67#include <net/ethernet.h> 68#include <net/if.h> 69#include <net/if_arp.h> 70#include <net/if_dl.h> 71#include <net/if_media.h> 72 73#include <net/if_types.h> 74#include <net/if_vlan_var.h> 75 76#include <netinet/in_systm.h> 77#include <netinet/in.h> 78#include <netinet/if_ether.h> 79#include <netinet/ip.h> 80#include <netinet/ip6.h> 81#include <netinet/tcp.h> 82#include <netinet/tcp_lro.h> 83#include <netinet/udp.h> 84 85#include <machine/in_cksum.h> 86#include <dev/led/led.h> 87#include <dev/pci/pcivar.h> 88#include <dev/pci/pcireg.h> 89 90#include "e1000_api.h" 91#include "e1000_82575.h" 92#include "if_igb.h" 93 94/********************************************************************* 95 * Set this to one to display debug statistics 96 *********************************************************************/ 97int igb_display_debug_stats = 0; 98 99/********************************************************************* 100 * Driver version: 101 *********************************************************************/ 102char igb_driver_version[] = "version - 2.0.1"; 103 104 105/********************************************************************* 106 * PCI Device ID Table 107 * 108 * Used by probe to select devices to load on 109 * Last field stores an index into e1000_strings 110 * Last entry must be all 0s 111 * 112 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 113 *********************************************************************/ 114 115static igb_vendor_info_t igb_vendor_info_array[] = 116{ 117 { 0x8086, E1000_DEV_ID_82575EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 118 { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, 119 PCI_ANY_ID, PCI_ANY_ID, 0}, 120 { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, 121 PCI_ANY_ID, PCI_ANY_ID, 0}, 122 { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0}, 123 { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0}, 124 { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 125 { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 126 { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 127 { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD, 128 PCI_ANY_ID, PCI_ANY_ID, 0}, 129 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER, 130 PCI_ANY_ID, PCI_ANY_ID, 0}, 131 { 0x8086, E1000_DEV_ID_82576_VF, PCI_ANY_ID, PCI_ANY_ID, 0}, 132 { 0x8086, E1000_DEV_ID_82580_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 133 { 0x8086, E1000_DEV_ID_82580_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82580_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_82580_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0}, 136 { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL, 137 PCI_ANY_ID, PCI_ANY_ID, 0}, 138 /* required last entry */ 139 { 0, 0, 0, 0, 0} 140}; 141 142/********************************************************************* 143 * Table of branding strings for all supported NICs. 144 *********************************************************************/ 145 146static char *igb_strings[] = { 147 "Intel(R) PRO/1000 Network Connection" 148}; 149 150/********************************************************************* 151 * Function prototypes 152 *********************************************************************/ 153static int igb_probe(device_t); 154static int igb_attach(device_t); 155static int igb_detach(device_t); 156static int igb_shutdown(device_t); 157static int igb_suspend(device_t); 158static int igb_resume(device_t); 159static void igb_start(struct ifnet *); 160static void igb_start_locked(struct tx_ring *, struct ifnet *ifp); 161#if __FreeBSD_version >= 800000 162static int igb_mq_start(struct ifnet *, struct mbuf *); 163static int igb_mq_start_locked(struct ifnet *, 164 struct tx_ring *, struct mbuf *); 165static void igb_qflush(struct ifnet *); 166#endif 167static int igb_ioctl(struct ifnet *, u_long, caddr_t); 168static void igb_init(void *); 169static void igb_init_locked(struct adapter *); 170static void igb_stop(void *); 171static void igb_media_status(struct ifnet *, struct ifmediareq *); 172static int igb_media_change(struct ifnet *); 173static void igb_identify_hardware(struct adapter *); 174static int igb_allocate_pci_resources(struct adapter *); 175static int igb_allocate_msix(struct adapter *); 176static int igb_allocate_legacy(struct adapter *); 177static int igb_setup_msix(struct adapter *); 178static void igb_free_pci_resources(struct adapter *); 179static void igb_local_timer(void *); 180static void igb_reset(struct adapter *); 181static int igb_setup_interface(device_t, struct adapter *); 182static int igb_allocate_queues(struct adapter *); 183static void igb_configure_queues(struct adapter *); 184 185static int igb_allocate_transmit_buffers(struct tx_ring *); 186static void igb_setup_transmit_structures(struct adapter *); 187static void igb_setup_transmit_ring(struct tx_ring *); 188static void igb_initialize_transmit_units(struct adapter *); 189static void igb_free_transmit_structures(struct adapter *); 190static void igb_free_transmit_buffers(struct tx_ring *); 191 192static int igb_allocate_receive_buffers(struct rx_ring *); 193static int igb_setup_receive_structures(struct adapter *); 194static int igb_setup_receive_ring(struct rx_ring *); 195static void igb_initialize_receive_units(struct adapter *); 196static void igb_free_receive_structures(struct adapter *); 197static void igb_free_receive_buffers(struct rx_ring *); 198static void igb_free_receive_ring(struct rx_ring *); 199 200static void igb_enable_intr(struct adapter *); 201static void igb_disable_intr(struct adapter *); 202static void igb_update_stats_counters(struct adapter *); 203static bool igb_txeof(struct tx_ring *); 204 205static __inline void igb_rx_discard(struct rx_ring *, int); 206static __inline void igb_rx_input(struct rx_ring *, 207 struct ifnet *, struct mbuf *, u32); 208 209static bool igb_rxeof(struct igb_queue *, int, int *); 210static void igb_rx_checksum(u32, struct mbuf *, u32); 211static int igb_tx_ctx_setup(struct tx_ring *, struct mbuf *); 212static bool igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 213static void igb_set_promisc(struct adapter *); 214static void igb_disable_promisc(struct adapter *); 215static void igb_set_multi(struct adapter *); 216static void igb_update_link_status(struct adapter *); 217static void igb_refresh_mbufs(struct rx_ring *, int); 218 219static void igb_register_vlan(void *, struct ifnet *, u16); 220static void igb_unregister_vlan(void *, struct ifnet *, u16); 221static void igb_setup_vlan_hw_support(struct adapter *); 222 223static int igb_xmit(struct tx_ring *, struct mbuf **); 224static int igb_dma_malloc(struct adapter *, bus_size_t, 225 struct igb_dma_alloc *, int); 226static void igb_dma_free(struct adapter *, struct igb_dma_alloc *); 227static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); 228static void igb_print_nvm_info(struct adapter *); 229static int igb_is_valid_ether_addr(u8 *); 230static void igb_add_hw_stats(struct adapter *); 231 232static void igb_vf_init_stats(struct adapter *); 233static void igb_update_vf_stats_counters(struct adapter *); 234 235/* Management and WOL Support */ 236static void igb_init_manageability(struct adapter *); 237static void igb_release_manageability(struct adapter *); 238static void igb_get_hw_control(struct adapter *); 239static void igb_release_hw_control(struct adapter *); 240static void igb_enable_wakeup(device_t); 241static void igb_led_func(void *, int); 242 243static int igb_irq_fast(void *); 244static void igb_add_rx_process_limit(struct adapter *, const char *, 245 const char *, int *, int); 246static void igb_handle_que(void *context, int pending); 247static void igb_handle_link(void *context, int pending); 248 249/* These are MSIX only irq handlers */ 250static void igb_msix_que(void *); 251static void igb_msix_link(void *); 252 253#ifdef DEVICE_POLLING 254static poll_handler_t igb_poll; 255#endif /* POLLING */ 256 257/********************************************************************* 258 * FreeBSD Device Interface Entry Points 259 *********************************************************************/ 260 261static device_method_t igb_methods[] = { 262 /* Device interface */ 263 DEVMETHOD(device_probe, igb_probe), 264 DEVMETHOD(device_attach, igb_attach), 265 DEVMETHOD(device_detach, igb_detach), 266 DEVMETHOD(device_shutdown, igb_shutdown), 267 DEVMETHOD(device_suspend, igb_suspend), 268 DEVMETHOD(device_resume, igb_resume), 269 {0, 0} 270}; 271 272static driver_t igb_driver = { 273 "igb", igb_methods, sizeof(struct adapter), 274}; 275 276static devclass_t igb_devclass; 277DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); 278MODULE_DEPEND(igb, pci, 1, 1, 1); 279MODULE_DEPEND(igb, ether, 1, 1, 1); 280 281/********************************************************************* 282 * Tunable default values. 283 *********************************************************************/ 284 285/* Descriptor defaults */ 286static int igb_rxd = IGB_DEFAULT_RXD; 287static int igb_txd = IGB_DEFAULT_TXD; 288TUNABLE_INT("hw.igb.rxd", &igb_rxd); 289TUNABLE_INT("hw.igb.txd", &igb_txd); 290 291/* 292** AIM: Adaptive Interrupt Moderation 293** which means that the interrupt rate 294** is varied over time based on the 295** traffic for that interrupt vector 296*/ 297static int igb_enable_aim = TRUE; 298TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim); 299 300/* 301 * MSIX should be the default for best performance, 302 * but this allows it to be forced off for testing. 303 */ 304static int igb_enable_msix = 1; 305TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix); 306 307/* 308 * Header split has seemed to be beneficial in 309 * many circumstances tested, however there have 310 * been some stability issues, so the default is 311 * off. 312 */ 313static bool igb_header_split = FALSE; 314TUNABLE_INT("hw.igb.hdr_split", &igb_header_split); 315 316/* 317** This will autoconfigure based on 318** the number of CPUs if left at 0. 319*/ 320static int igb_num_queues = 0; 321TUNABLE_INT("hw.igb.num_queues", &igb_num_queues); 322 323/* How many packets rxeof tries to clean at a time */ 324static int igb_rx_process_limit = 100; 325TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit); 326 327/* Flow control setting - default to FULL */ 328static int igb_fc_setting = e1000_fc_full; 329TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting); 330 331/* 332** Shadow VFTA table, this is needed because 333** the real filter table gets cleared during 334** a soft reset and the driver needs to be able 335** to repopulate it. 336*/ 337static u32 igb_shadow_vfta[IGB_VFTA_SIZE]; 338 339 340/********************************************************************* 341 * Device identification routine 342 * 343 * igb_probe determines if the driver should be loaded on 344 * adapter based on PCI vendor/device id of the adapter. 345 * 346 * return BUS_PROBE_DEFAULT on success, positive on failure 347 *********************************************************************/ 348 349static int 350igb_probe(device_t dev) 351{ 352 char adapter_name[60]; 353 uint16_t pci_vendor_id = 0; 354 uint16_t pci_device_id = 0; 355 uint16_t pci_subvendor_id = 0; 356 uint16_t pci_subdevice_id = 0; 357 igb_vendor_info_t *ent; 358 359 INIT_DEBUGOUT("igb_probe: begin"); 360 361 pci_vendor_id = pci_get_vendor(dev); 362 if (pci_vendor_id != IGB_VENDOR_ID) 363 return (ENXIO); 364 365 pci_device_id = pci_get_device(dev); 366 pci_subvendor_id = pci_get_subvendor(dev); 367 pci_subdevice_id = pci_get_subdevice(dev); 368 369 ent = igb_vendor_info_array; 370 while (ent->vendor_id != 0) { 371 if ((pci_vendor_id == ent->vendor_id) && 372 (pci_device_id == ent->device_id) && 373 374 ((pci_subvendor_id == ent->subvendor_id) || 375 (ent->subvendor_id == PCI_ANY_ID)) && 376 377 ((pci_subdevice_id == ent->subdevice_id) || 378 (ent->subdevice_id == PCI_ANY_ID))) { 379 sprintf(adapter_name, "%s %s", 380 igb_strings[ent->index], 381 igb_driver_version); 382 device_set_desc_copy(dev, adapter_name); 383 return (BUS_PROBE_DEFAULT); 384 } 385 ent++; 386 } 387 388 return (ENXIO); 389} 390 391/********************************************************************* 392 * Device initialization routine 393 * 394 * The attach entry point is called when the driver is being loaded. 395 * This routine identifies the type of hardware, allocates all resources 396 * and initializes the hardware. 397 * 398 * return 0 on success, positive on failure 399 *********************************************************************/ 400 401static int 402igb_attach(device_t dev) 403{ 404 struct adapter *adapter; 405 int error = 0; 406 u16 eeprom_data; 407 408 INIT_DEBUGOUT("igb_attach: begin"); 409 410 adapter = device_get_softc(dev); 411 adapter->dev = adapter->osdep.dev = dev; 412 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 413 414 /* SYSCTL stuff */ 415 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 416 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 417 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 418 igb_sysctl_nvm_info, "I", "NVM Information"); 419 420 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 421 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 422 OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW, 423 &igb_fc_setting, 0, "Flow Control"); 424 425 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 426 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 427 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 428 &igb_enable_aim, 1, "Interrupt Moderation"); 429 430 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 431 432 /* Determine hardware and mac info */ 433 igb_identify_hardware(adapter); 434 435 /* Setup PCI resources */ 436 if (igb_allocate_pci_resources(adapter)) { 437 device_printf(dev, "Allocation of PCI resources failed\n"); 438 error = ENXIO; 439 goto err_pci; 440 } 441 442 /* Do Shared Code initialization */ 443 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 444 device_printf(dev, "Setup of Shared code failed\n"); 445 error = ENXIO; 446 goto err_pci; 447 } 448 449 e1000_get_bus_info(&adapter->hw); 450 451 /* Sysctls for limiting the amount of work done in the taskqueue */ 452 igb_add_rx_process_limit(adapter, "rx_processing_limit", 453 "max number of rx packets to process", &adapter->rx_process_limit, 454 igb_rx_process_limit); 455 456 /* 457 * Validate number of transmit and receive descriptors. It 458 * must not exceed hardware maximum, and must be multiple 459 * of E1000_DBA_ALIGN. 460 */ 461 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 || 462 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) { 463 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 464 IGB_DEFAULT_TXD, igb_txd); 465 adapter->num_tx_desc = IGB_DEFAULT_TXD; 466 } else 467 adapter->num_tx_desc = igb_txd; 468 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 || 469 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) { 470 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 471 IGB_DEFAULT_RXD, igb_rxd); 472 adapter->num_rx_desc = IGB_DEFAULT_RXD; 473 } else 474 adapter->num_rx_desc = igb_rxd; 475 476 adapter->hw.mac.autoneg = DO_AUTO_NEG; 477 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 478 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 479 480 /* Copper options */ 481 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 482 adapter->hw.phy.mdix = AUTO_ALL_MODES; 483 adapter->hw.phy.disable_polarity_correction = FALSE; 484 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE; 485 } 486 487 /* 488 * Set the frame limits assuming 489 * standard ethernet sized frames. 490 */ 491 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 492 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 493 494 /* 495 ** Allocate and Setup Queues 496 */ 497 if (igb_allocate_queues(adapter)) { 498 error = ENOMEM; 499 goto err_pci; 500 } 501 502 /* Allocate the appropriate stats memory */ 503 if (adapter->hw.mac.type == e1000_vfadapt) { 504 adapter->stats = 505 (struct e1000_vf_stats *)malloc(sizeof \ 506 (struct e1000_vf_stats), M_DEVBUF, M_NOWAIT | M_ZERO); 507 igb_vf_init_stats(adapter); 508 } else 509 adapter->stats = 510 (struct e1000_hw_stats *)malloc(sizeof \ 511 (struct e1000_hw_stats), M_DEVBUF, M_NOWAIT | M_ZERO); 512 if (adapter->stats == NULL) { 513 device_printf(dev, "Can not allocate stats memory\n"); 514 error = ENOMEM; 515 goto err_late; 516 } 517 518 /* Allocate multicast array memory. */ 519 adapter->mta = malloc(sizeof(u8) * ETH_ADDR_LEN * 520 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 521 if (adapter->mta == NULL) { 522 device_printf(dev, "Can not allocate multicast setup array\n"); 523 error = ENOMEM; 524 goto err_late; 525 } 526 527 /* 528 ** Start from a known state, this is 529 ** important in reading the nvm and 530 ** mac from that. 531 */ 532 e1000_reset_hw(&adapter->hw); 533 534 /* Make sure we have a good EEPROM before we read from it */ 535 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 536 /* 537 ** Some PCI-E parts fail the first check due to 538 ** the link being in sleep state, call it again, 539 ** if it fails a second time its a real issue. 540 */ 541 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 542 device_printf(dev, 543 "The EEPROM Checksum Is Not Valid\n"); 544 error = EIO; 545 goto err_late; 546 } 547 } 548 549 /* 550 ** Copy the permanent MAC address out of the EEPROM 551 */ 552 if (e1000_read_mac_addr(&adapter->hw) < 0) { 553 device_printf(dev, "EEPROM read error while reading MAC" 554 " address\n"); 555 error = EIO; 556 goto err_late; 557 } 558 /* Check its sanity */ 559 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) { 560 device_printf(dev, "Invalid MAC address\n"); 561 error = EIO; 562 goto err_late; 563 } 564 565 /* 566 ** Configure Interrupts 567 */ 568 if ((adapter->msix > 1) && (igb_enable_msix)) 569 error = igb_allocate_msix(adapter); 570 else /* MSI or Legacy */ 571 error = igb_allocate_legacy(adapter); 572 if (error) 573 goto err_late; 574 575 /* Setup OS specific network interface */ 576 if (igb_setup_interface(dev, adapter) != 0) 577 goto err_late; 578 579 /* Now get a good starting state */ 580 igb_reset(adapter); 581 582 /* Initialize statistics */ 583 igb_update_stats_counters(adapter); 584 585 adapter->hw.mac.get_link_status = 1; 586 igb_update_link_status(adapter); 587 588 /* Indicate SOL/IDER usage */ 589 if (e1000_check_reset_block(&adapter->hw)) 590 device_printf(dev, 591 "PHY reset is blocked due to SOL/IDER session.\n"); 592 593 /* Determine if we have to control management hardware */ 594 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 595 596 /* 597 * Setup Wake-on-Lan 598 */ 599 /* APME bit in EEPROM is mapped to WUC.APME */ 600 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME; 601 if (eeprom_data) 602 adapter->wol = E1000_WUFC_MAG; 603 604 /* Register for VLAN events */ 605 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 606 igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 607 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 608 igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 609 610 igb_add_hw_stats(adapter); 611 612 /* Tell the stack that the interface is not active */ 613 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 614 615 adapter->led_dev = led_create(igb_led_func, adapter, 616 device_get_nameunit(dev)); 617 618 INIT_DEBUGOUT("igb_attach: end"); 619 620 return (0); 621 622err_late: 623 igb_free_transmit_structures(adapter); 624 igb_free_receive_structures(adapter); 625 igb_release_hw_control(adapter); 626 if (adapter->ifp != NULL) 627 if_free(adapter->ifp); 628err_pci: 629 igb_free_pci_resources(adapter); 630 free(adapter->mta, M_DEVBUF); 631 IGB_CORE_LOCK_DESTROY(adapter); 632 633 return (error); 634} 635 636/********************************************************************* 637 * Device removal routine 638 * 639 * The detach entry point is called when the driver is being removed. 640 * This routine stops the adapter and deallocates all the resources 641 * that were allocated for driver operation. 642 * 643 * return 0 on success, positive on failure 644 *********************************************************************/ 645 646static int 647igb_detach(device_t dev) 648{ 649 struct adapter *adapter = device_get_softc(dev); 650 struct ifnet *ifp = adapter->ifp; 651 652 INIT_DEBUGOUT("igb_detach: begin"); 653 654 /* Make sure VLANS are not using driver */ 655 if (adapter->ifp->if_vlantrunk != NULL) { 656 device_printf(dev,"Vlan in use, detach first\n"); 657 return (EBUSY); 658 } 659 660 if (adapter->led_dev != NULL) 661 led_destroy(adapter->led_dev); 662 663#ifdef DEVICE_POLLING 664 if (ifp->if_capenable & IFCAP_POLLING) 665 ether_poll_deregister(ifp); 666#endif 667 668 IGB_CORE_LOCK(adapter); 669 adapter->in_detach = 1; 670 igb_stop(adapter); 671 IGB_CORE_UNLOCK(adapter); 672 673 e1000_phy_hw_reset(&adapter->hw); 674 675 /* Give control back to firmware */ 676 igb_release_manageability(adapter); 677 igb_release_hw_control(adapter); 678 679 if (adapter->wol) { 680 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 681 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 682 igb_enable_wakeup(dev); 683 } 684 685 /* Unregister VLAN events */ 686 if (adapter->vlan_attach != NULL) 687 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 688 if (adapter->vlan_detach != NULL) 689 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 690 691 ether_ifdetach(adapter->ifp); 692 693 callout_drain(&adapter->timer); 694 695 igb_free_pci_resources(adapter); 696 bus_generic_detach(dev); 697 if_free(ifp); 698 699 igb_free_transmit_structures(adapter); 700 igb_free_receive_structures(adapter); 701 free(adapter->mta, M_DEVBUF); 702 703 IGB_CORE_LOCK_DESTROY(adapter); 704 705 return (0); 706} 707 708/********************************************************************* 709 * 710 * Shutdown entry point 711 * 712 **********************************************************************/ 713 714static int 715igb_shutdown(device_t dev) 716{ 717 return igb_suspend(dev); 718} 719 720/* 721 * Suspend/resume device methods. 722 */ 723static int 724igb_suspend(device_t dev) 725{ 726 struct adapter *adapter = device_get_softc(dev); 727 728 IGB_CORE_LOCK(adapter); 729 730 igb_stop(adapter); 731 732 igb_release_manageability(adapter); 733 igb_release_hw_control(adapter); 734 735 if (adapter->wol) { 736 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 737 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 738 igb_enable_wakeup(dev); 739 } 740 741 IGB_CORE_UNLOCK(adapter); 742 743 return bus_generic_suspend(dev); 744} 745 746static int 747igb_resume(device_t dev) 748{ 749 struct adapter *adapter = device_get_softc(dev); 750 struct ifnet *ifp = adapter->ifp; 751 752 IGB_CORE_LOCK(adapter); 753 igb_init_locked(adapter); 754 igb_init_manageability(adapter); 755 756 if ((ifp->if_flags & IFF_UP) && 757 (ifp->if_drv_flags & IFF_DRV_RUNNING)) 758 igb_start(ifp); 759 760 IGB_CORE_UNLOCK(adapter); 761 762 return bus_generic_resume(dev); 763} 764 765 766/********************************************************************* 767 * Transmit entry point 768 * 769 * igb_start is called by the stack to initiate a transmit. 770 * The driver will remain in this routine as long as there are 771 * packets to transmit and transmit resources are available. 772 * In case resources are not available stack is notified and 773 * the packet is requeued. 774 **********************************************************************/ 775 776static void 777igb_start_locked(struct tx_ring *txr, struct ifnet *ifp) 778{ 779 struct adapter *adapter = ifp->if_softc; 780 struct mbuf *m_head; 781 782 IGB_TX_LOCK_ASSERT(txr); 783 784 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 785 IFF_DRV_RUNNING) 786 return; 787 if (!adapter->link_active) 788 return; 789 790 /* Call cleanup if number of TX descriptors low */ 791 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) 792 igb_txeof(txr); 793 794 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 795 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 796 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 797 break; 798 } 799 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 800 if (m_head == NULL) 801 break; 802 /* 803 * Encapsulation can modify our pointer, and or make it 804 * NULL on failure. In that event, we can't requeue. 805 */ 806 if (igb_xmit(txr, &m_head)) { 807 if (m_head == NULL) 808 break; 809 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 810 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 811 break; 812 } 813 814 /* Send a copy of the frame to the BPF listener */ 815 ETHER_BPF_MTAP(ifp, m_head); 816 817 /* Set watchdog on */ 818 txr->watchdog_time = ticks; 819 txr->watchdog_check = TRUE; 820 } 821} 822 823/* 824 * Legacy TX driver routine, called from the 825 * stack, always uses tx[0], and spins for it. 826 * Should not be used with multiqueue tx 827 */ 828static void 829igb_start(struct ifnet *ifp) 830{ 831 struct adapter *adapter = ifp->if_softc; 832 struct tx_ring *txr = adapter->tx_rings; 833 834 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 835 IGB_TX_LOCK(txr); 836 igb_start_locked(txr, ifp); 837 IGB_TX_UNLOCK(txr); 838 } 839 return; 840} 841 842#if __FreeBSD_version >= 800000 843/* 844** Multiqueue Transmit driver 845** 846*/ 847static int 848igb_mq_start(struct ifnet *ifp, struct mbuf *m) 849{ 850 struct adapter *adapter = ifp->if_softc; 851 struct igb_queue *que; 852 struct tx_ring *txr; 853 int i = 0, err = 0; 854 855 /* Which queue to use */ 856 if ((m->m_flags & M_FLOWID) != 0) 857 i = m->m_pkthdr.flowid % adapter->num_queues; 858 859 txr = &adapter->tx_rings[i]; 860 que = &adapter->queues[i]; 861 862 if (IGB_TX_TRYLOCK(txr)) { 863 err = igb_mq_start_locked(ifp, txr, m); 864 IGB_TX_UNLOCK(txr); 865 } else { 866 err = drbr_enqueue(ifp, txr->br, m); 867 taskqueue_enqueue(que->tq, &que->que_task); 868 } 869 870 return (err); 871} 872 873static int 874igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 875{ 876 struct adapter *adapter = txr->adapter; 877 struct mbuf *next; 878 int err = 0, enq; 879 880 IGB_TX_LOCK_ASSERT(txr); 881 882 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 883 IFF_DRV_RUNNING || adapter->link_active == 0) { 884 if (m != NULL) 885 err = drbr_enqueue(ifp, txr->br, m); 886 return (err); 887 } 888 889 /* Call cleanup if number of TX descriptors low */ 890 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) 891 igb_txeof(txr); 892 893 enq = 0; 894 if (m == NULL) { 895 next = drbr_dequeue(ifp, txr->br); 896 } else if (drbr_needs_enqueue(ifp, txr->br)) { 897 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 898 return (err); 899 next = drbr_dequeue(ifp, txr->br); 900 } else 901 next = m; 902 903 /* Process the queue */ 904 while (next != NULL) { 905 if ((err = igb_xmit(txr, &next)) != 0) { 906 if (next != NULL) 907 err = drbr_enqueue(ifp, txr->br, next); 908 break; 909 } 910 enq++; 911 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 912 ETHER_BPF_MTAP(ifp, next); 913 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 914 break; 915 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 916 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 917 break; 918 } 919 next = drbr_dequeue(ifp, txr->br); 920 } 921 if (enq > 0) { 922 /* Set the watchdog */ 923 txr->watchdog_check = TRUE; 924 txr->watchdog_time = ticks; 925 } 926 return (err); 927} 928 929/* 930** Flush all ring buffers 931*/ 932static void 933igb_qflush(struct ifnet *ifp) 934{ 935 struct adapter *adapter = ifp->if_softc; 936 struct tx_ring *txr = adapter->tx_rings; 937 struct mbuf *m; 938 939 for (int i = 0; i < adapter->num_queues; i++, txr++) { 940 IGB_TX_LOCK(txr); 941 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 942 m_freem(m); 943 IGB_TX_UNLOCK(txr); 944 } 945 if_qflush(ifp); 946} 947#endif /* __FreeBSD_version >= 800000 */ 948 949/********************************************************************* 950 * Ioctl entry point 951 * 952 * igb_ioctl is called when the user wants to configure the 953 * interface. 954 * 955 * return 0 on success, positive on failure 956 **********************************************************************/ 957 958static int 959igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 960{ 961 struct adapter *adapter = ifp->if_softc; 962 struct ifreq *ifr = (struct ifreq *)data; 963#ifdef INET 964 struct ifaddr *ifa = (struct ifaddr *)data; 965#endif 966 int error = 0; 967 968 if (adapter->in_detach) 969 return (error); 970 971 switch (command) { 972 case SIOCSIFADDR: 973#ifdef INET 974 if (ifa->ifa_addr->sa_family == AF_INET) { 975 /* 976 * XXX 977 * Since resetting hardware takes a very long time 978 * and results in link renegotiation we only 979 * initialize the hardware only when it is absolutely 980 * required. 981 */ 982 ifp->if_flags |= IFF_UP; 983 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 984 IGB_CORE_LOCK(adapter); 985 igb_init_locked(adapter); 986 IGB_CORE_UNLOCK(adapter); 987 } 988 if (!(ifp->if_flags & IFF_NOARP)) 989 arp_ifinit(ifp, ifa); 990 } else 991#endif 992 error = ether_ioctl(ifp, command, data); 993 break; 994 case SIOCSIFMTU: 995 { 996 int max_frame_size; 997 998 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 999 1000 IGB_CORE_LOCK(adapter); 1001 max_frame_size = 9234; 1002 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 1003 ETHER_CRC_LEN) { 1004 IGB_CORE_UNLOCK(adapter); 1005 error = EINVAL; 1006 break; 1007 } 1008 1009 ifp->if_mtu = ifr->ifr_mtu; 1010 adapter->max_frame_size = 1011 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1012 igb_init_locked(adapter); 1013 IGB_CORE_UNLOCK(adapter); 1014 break; 1015 } 1016 case SIOCSIFFLAGS: 1017 IOCTL_DEBUGOUT("ioctl rcv'd:\ 1018 SIOCSIFFLAGS (Set Interface Flags)"); 1019 IGB_CORE_LOCK(adapter); 1020 if (ifp->if_flags & IFF_UP) { 1021 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 1022 if ((ifp->if_flags ^ adapter->if_flags) & 1023 (IFF_PROMISC | IFF_ALLMULTI)) { 1024 igb_disable_promisc(adapter); 1025 igb_set_promisc(adapter); 1026 } 1027 } else 1028 igb_init_locked(adapter); 1029 } else 1030 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1031 igb_stop(adapter); 1032 adapter->if_flags = ifp->if_flags; 1033 IGB_CORE_UNLOCK(adapter); 1034 break; 1035 case SIOCADDMULTI: 1036 case SIOCDELMULTI: 1037 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1038 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1039 IGB_CORE_LOCK(adapter); 1040 igb_disable_intr(adapter); 1041 igb_set_multi(adapter); 1042#ifdef DEVICE_POLLING 1043 if (!(ifp->if_capenable & IFCAP_POLLING)) 1044#endif 1045 igb_enable_intr(adapter); 1046 IGB_CORE_UNLOCK(adapter); 1047 } 1048 break; 1049 case SIOCSIFMEDIA: 1050 /* Check SOL/IDER usage */ 1051 IGB_CORE_LOCK(adapter); 1052 if (e1000_check_reset_block(&adapter->hw)) { 1053 IGB_CORE_UNLOCK(adapter); 1054 device_printf(adapter->dev, "Media change is" 1055 " blocked due to SOL/IDER session.\n"); 1056 break; 1057 } 1058 IGB_CORE_UNLOCK(adapter); 1059 case SIOCGIFMEDIA: 1060 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1061 SIOCxIFMEDIA (Get/Set Interface Media)"); 1062 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1063 break; 1064 case SIOCSIFCAP: 1065 { 1066 int mask, reinit; 1067 1068 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1069 reinit = 0; 1070 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1071#ifdef DEVICE_POLLING 1072 if (mask & IFCAP_POLLING) { 1073 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1074 error = ether_poll_register(igb_poll, ifp); 1075 if (error) 1076 return (error); 1077 IGB_CORE_LOCK(adapter); 1078 igb_disable_intr(adapter); 1079 ifp->if_capenable |= IFCAP_POLLING; 1080 IGB_CORE_UNLOCK(adapter); 1081 } else { 1082 error = ether_poll_deregister(ifp); 1083 /* Enable interrupt even in error case */ 1084 IGB_CORE_LOCK(adapter); 1085 igb_enable_intr(adapter); 1086 ifp->if_capenable &= ~IFCAP_POLLING; 1087 IGB_CORE_UNLOCK(adapter); 1088 } 1089 } 1090#endif 1091 if (mask & IFCAP_HWCSUM) { 1092 ifp->if_capenable ^= IFCAP_HWCSUM; 1093 reinit = 1; 1094 } 1095 if (mask & IFCAP_TSO4) { 1096 ifp->if_capenable ^= IFCAP_TSO4; 1097 reinit = 1; 1098 } 1099 if (mask & IFCAP_VLAN_HWTAGGING) { 1100 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1101 reinit = 1; 1102 } 1103 if (mask & IFCAP_VLAN_HWFILTER) { 1104 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1105 reinit = 1; 1106 } 1107 if (mask & IFCAP_LRO) { 1108 ifp->if_capenable ^= IFCAP_LRO; 1109 reinit = 1; 1110 } 1111 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1112 igb_init(adapter); 1113 VLAN_CAPABILITIES(ifp); 1114 break; 1115 } 1116 1117 default: 1118 error = ether_ioctl(ifp, command, data); 1119 break; 1120 } 1121 1122 return (error); 1123} 1124 1125 1126/********************************************************************* 1127 * Init entry point 1128 * 1129 * This routine is used in two ways. It is used by the stack as 1130 * init entry point in network interface structure. It is also used 1131 * by the driver as a hw/sw initialization routine to get to a 1132 * consistent state. 1133 * 1134 * return 0 on success, positive on failure 1135 **********************************************************************/ 1136 1137static void 1138igb_init_locked(struct adapter *adapter) 1139{ 1140 struct ifnet *ifp = adapter->ifp; 1141 device_t dev = adapter->dev; 1142 1143 INIT_DEBUGOUT("igb_init: begin"); 1144 1145 IGB_CORE_LOCK_ASSERT(adapter); 1146 1147 igb_disable_intr(adapter); 1148 callout_stop(&adapter->timer); 1149 1150 /* Get the latest mac address, User can use a LAA */ 1151 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1152 ETHER_ADDR_LEN); 1153 1154 /* Put the address into the Receive Address Array */ 1155 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1156 1157 igb_reset(adapter); 1158 igb_update_link_status(adapter); 1159 1160 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1161 1162 /* Use real VLAN Filter support? */ 1163 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1164 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 1165 /* Use real VLAN Filter support */ 1166 igb_setup_vlan_hw_support(adapter); 1167 else { 1168 u32 ctrl; 1169 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1170 ctrl |= E1000_CTRL_VME; 1171 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1172 } 1173 } 1174 1175 /* Set hardware offload abilities */ 1176 ifp->if_hwassist = 0; 1177 if (ifp->if_capenable & IFCAP_TXCSUM) { 1178 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1179#if __FreeBSD_version >= 800000 1180 if (adapter->hw.mac.type == e1000_82576) 1181 ifp->if_hwassist |= CSUM_SCTP; 1182#endif 1183 } 1184 1185 if (ifp->if_capenable & IFCAP_TSO4) 1186 ifp->if_hwassist |= CSUM_TSO; 1187 1188 /* Configure for OS presence */ 1189 igb_init_manageability(adapter); 1190 1191 /* Prepare transmit descriptors and buffers */ 1192 igb_setup_transmit_structures(adapter); 1193 igb_initialize_transmit_units(adapter); 1194 1195 /* Setup Multicast table */ 1196 igb_set_multi(adapter); 1197 1198 /* 1199 ** Figure out the desired mbuf pool 1200 ** for doing jumbo/packetsplit 1201 */ 1202 if (ifp->if_mtu > ETHERMTU) 1203 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1204 else 1205 adapter->rx_mbuf_sz = MCLBYTES; 1206 1207 /* Prepare receive descriptors and buffers */ 1208 if (igb_setup_receive_structures(adapter)) { 1209 device_printf(dev, "Could not setup receive structures\n"); 1210 return; 1211 } 1212 igb_initialize_receive_units(adapter); 1213 1214 /* Don't lose promiscuous settings */ 1215 igb_set_promisc(adapter); 1216 1217 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1218 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1219 1220 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1221 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1222 1223 if (adapter->msix > 1) /* Set up queue routing */ 1224 igb_configure_queues(adapter); 1225 1226 /* Set up VLAN tag offload and filter */ 1227 igb_setup_vlan_hw_support(adapter); 1228 1229 /* this clears any pending interrupts */ 1230 E1000_READ_REG(&adapter->hw, E1000_ICR); 1231#ifdef DEVICE_POLLING 1232 /* 1233 * Only enable interrupts if we are not polling, make sure 1234 * they are off otherwise. 1235 */ 1236 if (ifp->if_capenable & IFCAP_POLLING) 1237 igb_disable_intr(adapter); 1238 else 1239#endif /* DEVICE_POLLING */ 1240 { 1241 igb_enable_intr(adapter); 1242 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); 1243 } 1244 1245 /* Don't reset the phy next time init gets called */ 1246 adapter->hw.phy.reset_disable = TRUE; 1247} 1248 1249static void 1250igb_init(void *arg) 1251{ 1252 struct adapter *adapter = arg; 1253 1254 IGB_CORE_LOCK(adapter); 1255 igb_init_locked(adapter); 1256 IGB_CORE_UNLOCK(adapter); 1257} 1258 1259 1260static void 1261igb_handle_que(void *context, int pending) 1262{ 1263 struct igb_queue *que = context; 1264 struct adapter *adapter = que->adapter; 1265 struct tx_ring *txr = que->txr; 1266 struct ifnet *ifp = adapter->ifp; 1267 1268 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1269 bool more; 1270 1271 more = igb_rxeof(que, -1, NULL); 1272 1273 IGB_TX_LOCK(txr); 1274 if (igb_txeof(txr)) 1275 more = TRUE; 1276#if __FreeBSD_version >= 800000 1277 if (!drbr_empty(ifp, txr->br)) 1278 igb_mq_start_locked(ifp, txr, NULL); 1279#else 1280 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1281 igb_start_locked(txr, ifp); 1282#endif 1283 IGB_TX_UNLOCK(txr); 1284 if (more) { 1285 taskqueue_enqueue(que->tq, &que->que_task); 1286 return; 1287 } 1288 } 1289 1290#ifdef DEVICE_POLLING 1291 if (ifp->if_capenable & IFCAP_POLLING) 1292 return; 1293#endif 1294 /* Reenable this interrupt */ 1295 if (que->eims) 1296 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); 1297 else 1298 igb_enable_intr(adapter); 1299} 1300 1301/* Deal with link in a sleepable context */ 1302static void 1303igb_handle_link(void *context, int pending) 1304{ 1305 struct adapter *adapter = context; 1306 1307 adapter->hw.mac.get_link_status = 1; 1308 igb_update_link_status(adapter); 1309} 1310 1311/********************************************************************* 1312 * 1313 * MSI/Legacy Deferred 1314 * Interrupt Service routine 1315 * 1316 *********************************************************************/ 1317static int 1318igb_irq_fast(void *arg) 1319{ 1320 struct adapter *adapter = arg; 1321 struct igb_queue *que = adapter->queues; 1322 u32 reg_icr; 1323 1324 1325 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1326 1327 /* Hot eject? */ 1328 if (reg_icr == 0xffffffff) 1329 return FILTER_STRAY; 1330 1331 /* Definitely not our interrupt. */ 1332 if (reg_icr == 0x0) 1333 return FILTER_STRAY; 1334 1335 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1336 return FILTER_STRAY; 1337 1338 /* 1339 * Mask interrupts until the taskqueue is finished running. This is 1340 * cheap, just assume that it is needed. This also works around the 1341 * MSI message reordering errata on certain systems. 1342 */ 1343 igb_disable_intr(adapter); 1344 taskqueue_enqueue(que->tq, &que->que_task); 1345 1346 /* Link status change */ 1347 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 1348 taskqueue_enqueue(que->tq, &adapter->link_task); 1349 1350 if (reg_icr & E1000_ICR_RXO) 1351 adapter->rx_overruns++; 1352 return FILTER_HANDLED; 1353} 1354 1355#ifdef DEVICE_POLLING 1356/********************************************************************* 1357 * 1358 * Legacy polling routine : if using this code you MUST be sure that 1359 * multiqueue is not defined, ie, set igb_num_queues to 1. 1360 * 1361 *********************************************************************/ 1362#if __FreeBSD_version >= 800000 1363#define POLL_RETURN_COUNT(a) (a) 1364static int 1365#else 1366#define POLL_RETURN_COUNT(a) 1367static void 1368#endif 1369igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1370{ 1371 struct adapter *adapter = ifp->if_softc; 1372 struct igb_queue *que = adapter->queues; 1373 struct tx_ring *txr = adapter->tx_rings; 1374 u32 reg_icr, rx_done = 0; 1375 u32 loop = IGB_MAX_LOOP; 1376 bool more; 1377 1378 IGB_CORE_LOCK(adapter); 1379 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1380 IGB_CORE_UNLOCK(adapter); 1381 return POLL_RETURN_COUNT(rx_done); 1382 } 1383 1384 if (cmd == POLL_AND_CHECK_STATUS) { 1385 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1386 /* Link status change */ 1387 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 1388 igb_handle_link(adapter, 0); 1389 1390 if (reg_icr & E1000_ICR_RXO) 1391 adapter->rx_overruns++; 1392 } 1393 IGB_CORE_UNLOCK(adapter); 1394 1395 igb_rxeof(que, count, &rx_done); 1396 1397 IGB_TX_LOCK(txr); 1398 do { 1399 more = igb_txeof(txr); 1400 } while (loop-- && more); 1401#if __FreeBSD_version >= 800000 1402 if (!drbr_empty(ifp, txr->br)) 1403 igb_mq_start_locked(ifp, txr, NULL); 1404#else 1405 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1406 igb_start_locked(txr, ifp); 1407#endif 1408 IGB_TX_UNLOCK(txr); 1409 return POLL_RETURN_COUNT(rx_done); 1410} 1411#endif /* DEVICE_POLLING */ 1412 1413/********************************************************************* 1414 * 1415 * MSIX TX Interrupt Service routine 1416 * 1417 **********************************************************************/ 1418static void 1419igb_msix_que(void *arg) 1420{ 1421 struct igb_queue *que = arg; 1422 struct adapter *adapter = que->adapter; 1423 struct tx_ring *txr = que->txr; 1424 struct rx_ring *rxr = que->rxr; 1425 u32 newitr = 0; 1426 bool more_tx, more_rx; 1427 1428 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims); 1429 ++que->irqs; 1430 1431 IGB_TX_LOCK(txr); 1432 more_tx = igb_txeof(txr); 1433 IGB_TX_UNLOCK(txr); 1434 1435 more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL); 1436 1437 if (igb_enable_aim == FALSE) 1438 goto no_calc; 1439 /* 1440 ** Do Adaptive Interrupt Moderation: 1441 ** - Write out last calculated setting 1442 ** - Calculate based on average size over 1443 ** the last interval. 1444 */ 1445 if (que->eitr_setting) 1446 E1000_WRITE_REG(&adapter->hw, 1447 E1000_EITR(que->msix), que->eitr_setting); 1448 1449 que->eitr_setting = 0; 1450 1451 /* Idle, do nothing */ 1452 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1453 goto no_calc; 1454 1455 /* Used half Default if sub-gig */ 1456 if (adapter->link_speed != 1000) 1457 newitr = IGB_DEFAULT_ITR / 2; 1458 else { 1459 if ((txr->bytes) && (txr->packets)) 1460 newitr = txr->bytes/txr->packets; 1461 if ((rxr->bytes) && (rxr->packets)) 1462 newitr = max(newitr, 1463 (rxr->bytes / rxr->packets)); 1464 newitr += 24; /* account for hardware frame, crc */ 1465 /* set an upper boundary */ 1466 newitr = min(newitr, 3000); 1467 /* Be nice to the mid range */ 1468 if ((newitr > 300) && (newitr < 1200)) 1469 newitr = (newitr / 3); 1470 else 1471 newitr = (newitr / 2); 1472 } 1473 newitr &= 0x7FFC; /* Mask invalid bits */ 1474 if (adapter->hw.mac.type == e1000_82575) 1475 newitr |= newitr << 16; 1476 else 1477 newitr |= E1000_EITR_CNT_IGNR; 1478 1479 /* save for next interrupt */ 1480 que->eitr_setting = newitr; 1481 1482 /* Reset state */ 1483 txr->bytes = 0; 1484 txr->packets = 0; 1485 rxr->bytes = 0; 1486 rxr->packets = 0; 1487 1488no_calc: 1489 /* Schedule a clean task if needed*/ 1490 if (more_tx || more_rx) 1491 taskqueue_enqueue(que->tq, &que->que_task); 1492 else 1493 /* Reenable this interrupt */ 1494 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); 1495 return; 1496} 1497 1498 1499/********************************************************************* 1500 * 1501 * MSIX Link Interrupt Service routine 1502 * 1503 **********************************************************************/ 1504 1505static void 1506igb_msix_link(void *arg) 1507{ 1508 struct adapter *adapter = arg; 1509 u32 icr; 1510 1511 ++adapter->link_irq; 1512 icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1513 if (!(icr & E1000_ICR_LSC)) 1514 goto spurious; 1515 igb_handle_link(adapter, 0); 1516 1517spurious: 1518 /* Rearm */ 1519 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); 1520 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); 1521 return; 1522} 1523 1524 1525/********************************************************************* 1526 * 1527 * Media Ioctl callback 1528 * 1529 * This routine is called whenever the user queries the status of 1530 * the interface using ifconfig. 1531 * 1532 **********************************************************************/ 1533static void 1534igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1535{ 1536 struct adapter *adapter = ifp->if_softc; 1537 u_char fiber_type = IFM_1000_SX; 1538 1539 INIT_DEBUGOUT("igb_media_status: begin"); 1540 1541 IGB_CORE_LOCK(adapter); 1542 igb_update_link_status(adapter); 1543 1544 ifmr->ifm_status = IFM_AVALID; 1545 ifmr->ifm_active = IFM_ETHER; 1546 1547 if (!adapter->link_active) { 1548 IGB_CORE_UNLOCK(adapter); 1549 return; 1550 } 1551 1552 ifmr->ifm_status |= IFM_ACTIVE; 1553 1554 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1555 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) 1556 ifmr->ifm_active |= fiber_type | IFM_FDX; 1557 else { 1558 switch (adapter->link_speed) { 1559 case 10: 1560 ifmr->ifm_active |= IFM_10_T; 1561 break; 1562 case 100: 1563 ifmr->ifm_active |= IFM_100_TX; 1564 break; 1565 case 1000: 1566 ifmr->ifm_active |= IFM_1000_T; 1567 break; 1568 } 1569 if (adapter->link_duplex == FULL_DUPLEX) 1570 ifmr->ifm_active |= IFM_FDX; 1571 else 1572 ifmr->ifm_active |= IFM_HDX; 1573 } 1574 IGB_CORE_UNLOCK(adapter); 1575} 1576 1577/********************************************************************* 1578 * 1579 * Media Ioctl callback 1580 * 1581 * This routine is called when the user changes speed/duplex using 1582 * media/mediopt option with ifconfig. 1583 * 1584 **********************************************************************/ 1585static int 1586igb_media_change(struct ifnet *ifp) 1587{ 1588 struct adapter *adapter = ifp->if_softc; 1589 struct ifmedia *ifm = &adapter->media; 1590 1591 INIT_DEBUGOUT("igb_media_change: begin"); 1592 1593 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1594 return (EINVAL); 1595 1596 IGB_CORE_LOCK(adapter); 1597 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1598 case IFM_AUTO: 1599 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1600 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1601 break; 1602 case IFM_1000_LX: 1603 case IFM_1000_SX: 1604 case IFM_1000_T: 1605 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1606 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1607 break; 1608 case IFM_100_TX: 1609 adapter->hw.mac.autoneg = FALSE; 1610 adapter->hw.phy.autoneg_advertised = 0; 1611 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1612 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1613 else 1614 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1615 break; 1616 case IFM_10_T: 1617 adapter->hw.mac.autoneg = FALSE; 1618 adapter->hw.phy.autoneg_advertised = 0; 1619 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1620 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1621 else 1622 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1623 break; 1624 default: 1625 device_printf(adapter->dev, "Unsupported media type\n"); 1626 } 1627 1628 /* As the speed/duplex settings my have changed we need to 1629 * reset the PHY. 1630 */ 1631 adapter->hw.phy.reset_disable = FALSE; 1632 1633 igb_init_locked(adapter); 1634 IGB_CORE_UNLOCK(adapter); 1635 1636 return (0); 1637} 1638 1639 1640/********************************************************************* 1641 * 1642 * This routine maps the mbufs to Advanced TX descriptors. 1643 * used by the 82575 adapter. 1644 * 1645 **********************************************************************/ 1646 1647static int 1648igb_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1649{ 1650 struct adapter *adapter = txr->adapter; 1651 bus_dma_segment_t segs[IGB_MAX_SCATTER]; 1652 bus_dmamap_t map; 1653 struct igb_tx_buffer *tx_buffer, *tx_buffer_mapped; 1654 union e1000_adv_tx_desc *txd = NULL; 1655 struct mbuf *m_head; 1656 u32 olinfo_status = 0, cmd_type_len = 0; 1657 int nsegs, i, j, error, first, last = 0; 1658 u32 hdrlen = 0; 1659 1660 m_head = *m_headp; 1661 1662 1663 /* Set basic descriptor constants */ 1664 cmd_type_len |= E1000_ADVTXD_DTYP_DATA; 1665 cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT; 1666 if (m_head->m_flags & M_VLANTAG) 1667 cmd_type_len |= E1000_ADVTXD_DCMD_VLE; 1668 1669 /* 1670 * Force a cleanup if number of TX descriptors 1671 * available hits the threshold 1672 */ 1673 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) { 1674 igb_txeof(txr); 1675 /* Now do we at least have a minimal? */ 1676 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 1677 txr->no_desc_avail++; 1678 return (ENOBUFS); 1679 } 1680 } 1681 1682 /* 1683 * Map the packet for DMA. 1684 * 1685 * Capture the first descriptor index, 1686 * this descriptor will have the index 1687 * of the EOP which is the only one that 1688 * now gets a DONE bit writeback. 1689 */ 1690 first = txr->next_avail_desc; 1691 tx_buffer = &txr->tx_buffers[first]; 1692 tx_buffer_mapped = tx_buffer; 1693 map = tx_buffer->map; 1694 1695 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1696 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1697 1698 if (error == EFBIG) { 1699 struct mbuf *m; 1700 1701 m = m_defrag(*m_headp, M_DONTWAIT); 1702 if (m == NULL) { 1703 adapter->mbuf_defrag_failed++; 1704 m_freem(*m_headp); 1705 *m_headp = NULL; 1706 return (ENOBUFS); 1707 } 1708 *m_headp = m; 1709 1710 /* Try it again */ 1711 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1712 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1713 1714 if (error == ENOMEM) { 1715 adapter->no_tx_dma_setup++; 1716 return (error); 1717 } else if (error != 0) { 1718 adapter->no_tx_dma_setup++; 1719 m_freem(*m_headp); 1720 *m_headp = NULL; 1721 return (error); 1722 } 1723 } else if (error == ENOMEM) { 1724 adapter->no_tx_dma_setup++; 1725 return (error); 1726 } else if (error != 0) { 1727 adapter->no_tx_dma_setup++; 1728 m_freem(*m_headp); 1729 *m_headp = NULL; 1730 return (error); 1731 } 1732 1733 /* Check again to be sure we have enough descriptors */ 1734 if (nsegs > (txr->tx_avail - 2)) { 1735 txr->no_desc_avail++; 1736 bus_dmamap_unload(txr->txtag, map); 1737 return (ENOBUFS); 1738 } 1739 m_head = *m_headp; 1740 1741 /* 1742 * Set up the context descriptor: 1743 * used when any hardware offload is done. 1744 * This includes CSUM, VLAN, and TSO. It 1745 * will use the first descriptor. 1746 */ 1747 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1748 if (igb_tso_setup(txr, m_head, &hdrlen)) { 1749 cmd_type_len |= E1000_ADVTXD_DCMD_TSE; 1750 olinfo_status |= E1000_TXD_POPTS_IXSM << 8; 1751 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 1752 } else 1753 return (ENXIO); 1754 } else if (igb_tx_ctx_setup(txr, m_head)) 1755 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 1756 1757 /* Calculate payload length */ 1758 olinfo_status |= ((m_head->m_pkthdr.len - hdrlen) 1759 << E1000_ADVTXD_PAYLEN_SHIFT); 1760 1761 /* 82575 needs the queue index added */ 1762 if (adapter->hw.mac.type == e1000_82575) 1763 olinfo_status |= txr->me << 4; 1764 1765 /* Set up our transmit descriptors */ 1766 i = txr->next_avail_desc; 1767 for (j = 0; j < nsegs; j++) { 1768 bus_size_t seg_len; 1769 bus_addr_t seg_addr; 1770 1771 tx_buffer = &txr->tx_buffers[i]; 1772 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i]; 1773 seg_addr = segs[j].ds_addr; 1774 seg_len = segs[j].ds_len; 1775 1776 txd->read.buffer_addr = htole64(seg_addr); 1777 txd->read.cmd_type_len = htole32(cmd_type_len | seg_len); 1778 txd->read.olinfo_status = htole32(olinfo_status); 1779 last = i; 1780 if (++i == adapter->num_tx_desc) 1781 i = 0; 1782 tx_buffer->m_head = NULL; 1783 tx_buffer->next_eop = -1; 1784 } 1785 1786 txr->next_avail_desc = i; 1787 txr->tx_avail -= nsegs; 1788 1789 tx_buffer->m_head = m_head; 1790 tx_buffer_mapped->map = tx_buffer->map; 1791 tx_buffer->map = map; 1792 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1793 1794 /* 1795 * Last Descriptor of Packet 1796 * needs End Of Packet (EOP) 1797 * and Report Status (RS) 1798 */ 1799 txd->read.cmd_type_len |= 1800 htole32(E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS); 1801 /* 1802 * Keep track in the first buffer which 1803 * descriptor will be written back 1804 */ 1805 tx_buffer = &txr->tx_buffers[first]; 1806 tx_buffer->next_eop = last; 1807 txr->watchdog_time = ticks; 1808 1809 /* 1810 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 1811 * that this frame is available to transmit. 1812 */ 1813 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1814 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1815 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); 1816 ++txr->tx_packets; 1817 1818 return (0); 1819 1820} 1821 1822static void 1823igb_set_promisc(struct adapter *adapter) 1824{ 1825 struct ifnet *ifp = adapter->ifp; 1826 struct e1000_hw *hw = &adapter->hw; 1827 u32 reg; 1828 1829 if (hw->mac.type == e1000_vfadapt) { 1830 e1000_promisc_set_vf(hw, e1000_promisc_enabled); 1831 return; 1832 } 1833 1834 reg = E1000_READ_REG(hw, E1000_RCTL); 1835 if (ifp->if_flags & IFF_PROMISC) { 1836 reg |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1837 E1000_WRITE_REG(hw, E1000_RCTL, reg); 1838 } else if (ifp->if_flags & IFF_ALLMULTI) { 1839 reg |= E1000_RCTL_MPE; 1840 reg &= ~E1000_RCTL_UPE; 1841 E1000_WRITE_REG(hw, E1000_RCTL, reg); 1842 } 1843} 1844 1845static void 1846igb_disable_promisc(struct adapter *adapter) 1847{ 1848 struct e1000_hw *hw = &adapter->hw; 1849 u32 reg; 1850 1851 if (hw->mac.type == e1000_vfadapt) { 1852 e1000_promisc_set_vf(hw, e1000_promisc_disabled); 1853 return; 1854 } 1855 reg = E1000_READ_REG(hw, E1000_RCTL); 1856 reg &= (~E1000_RCTL_UPE); 1857 reg &= (~E1000_RCTL_MPE); 1858 E1000_WRITE_REG(hw, E1000_RCTL, reg); 1859} 1860 1861 1862/********************************************************************* 1863 * Multicast Update 1864 * 1865 * This routine is called whenever multicast address list is updated. 1866 * 1867 **********************************************************************/ 1868 1869static void 1870igb_set_multi(struct adapter *adapter) 1871{ 1872 struct ifnet *ifp = adapter->ifp; 1873 struct ifmultiaddr *ifma; 1874 u32 reg_rctl = 0; 1875 u8 *mta; 1876 1877 int mcnt = 0; 1878 1879 IOCTL_DEBUGOUT("igb_set_multi: begin"); 1880 1881 mta = adapter->mta; 1882 bzero(mta, sizeof(uint8_t) * ETH_ADDR_LEN * 1883 MAX_NUM_MULTICAST_ADDRESSES); 1884 1885#if __FreeBSD_version < 800000 1886 IF_ADDR_LOCK(ifp); 1887#else 1888 if_maddr_rlock(ifp); 1889#endif 1890 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1891 if (ifma->ifma_addr->sa_family != AF_LINK) 1892 continue; 1893 1894 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 1895 break; 1896 1897 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1898 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 1899 mcnt++; 1900 } 1901#if __FreeBSD_version < 800000 1902 IF_ADDR_UNLOCK(ifp); 1903#else 1904 if_maddr_runlock(ifp); 1905#endif 1906 1907 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 1908 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1909 reg_rctl |= E1000_RCTL_MPE; 1910 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1911 } else 1912 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 1913} 1914 1915 1916/********************************************************************* 1917 * Timer routine: 1918 * This routine checks for link status, 1919 * updates statistics, and does the watchdog. 1920 * 1921 **********************************************************************/ 1922 1923static void 1924igb_local_timer(void *arg) 1925{ 1926 struct adapter *adapter = arg; 1927 device_t dev = adapter->dev; 1928 struct tx_ring *txr = adapter->tx_rings; 1929 1930 1931 IGB_CORE_LOCK_ASSERT(adapter); 1932 1933 igb_update_link_status(adapter); 1934 igb_update_stats_counters(adapter); 1935 1936 /* 1937 ** Watchdog: check for time since any descriptor was cleaned 1938 */ 1939 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1940 if (txr->watchdog_check == FALSE) 1941 continue; 1942 if ((ticks - txr->watchdog_time) > IGB_WATCHDOG) 1943 goto timeout; 1944 } 1945 1946 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1947 return; 1948 1949timeout: 1950 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1951 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1952 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)), 1953 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me))); 1954 device_printf(dev,"TX(%d) desc avail = %d," 1955 "Next TX to Clean = %d\n", 1956 txr->me, txr->tx_avail, txr->next_to_clean); 1957 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1958 adapter->watchdog_events++; 1959 igb_init_locked(adapter); 1960} 1961 1962static void 1963igb_update_link_status(struct adapter *adapter) 1964{ 1965 struct e1000_hw *hw = &adapter->hw; 1966 struct ifnet *ifp = adapter->ifp; 1967 device_t dev = adapter->dev; 1968 struct tx_ring *txr = adapter->tx_rings; 1969 u32 link_check = 0; 1970 1971 /* Get the cached link value or read for real */ 1972 switch (hw->phy.media_type) { 1973 case e1000_media_type_copper: 1974 if (hw->mac.get_link_status) { 1975 /* Do the work to read phy */ 1976 e1000_check_for_link(hw); 1977 link_check = !hw->mac.get_link_status; 1978 } else 1979 link_check = TRUE; 1980 break; 1981 case e1000_media_type_fiber: 1982 e1000_check_for_link(hw); 1983 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 1984 E1000_STATUS_LU); 1985 break; 1986 case e1000_media_type_internal_serdes: 1987 e1000_check_for_link(hw); 1988 link_check = adapter->hw.mac.serdes_has_link; 1989 break; 1990 /* VF device is type_unknown */ 1991 case e1000_media_type_unknown: 1992 e1000_check_for_link(hw); 1993 link_check = !hw->mac.get_link_status; 1994 /* Fall thru */ 1995 default: 1996 break; 1997 } 1998 1999 /* Now we check if a transition has happened */ 2000 if (link_check && (adapter->link_active == 0)) { 2001 e1000_get_speed_and_duplex(&adapter->hw, 2002 &adapter->link_speed, &adapter->link_duplex); 2003 if (bootverbose) 2004 device_printf(dev, "Link is up %d Mbps %s\n", 2005 adapter->link_speed, 2006 ((adapter->link_duplex == FULL_DUPLEX) ? 2007 "Full Duplex" : "Half Duplex")); 2008 adapter->link_active = 1; 2009 ifp->if_baudrate = adapter->link_speed * 1000000; 2010 /* This can sleep */ 2011 if_link_state_change(ifp, LINK_STATE_UP); 2012 } else if (!link_check && (adapter->link_active == 1)) { 2013 ifp->if_baudrate = adapter->link_speed = 0; 2014 adapter->link_duplex = 0; 2015 if (bootverbose) 2016 device_printf(dev, "Link is Down\n"); 2017 adapter->link_active = 0; 2018 /* This can sleep */ 2019 if_link_state_change(ifp, LINK_STATE_DOWN); 2020 /* Turn off watchdogs */ 2021 for (int i = 0; i < adapter->num_queues; i++, txr++) 2022 txr->watchdog_check = FALSE; 2023 } 2024} 2025 2026/********************************************************************* 2027 * 2028 * This routine disables all traffic on the adapter by issuing a 2029 * global reset on the MAC and deallocates TX/RX buffers. 2030 * 2031 **********************************************************************/ 2032 2033static void 2034igb_stop(void *arg) 2035{ 2036 struct adapter *adapter = arg; 2037 struct ifnet *ifp = adapter->ifp; 2038 struct tx_ring *txr = adapter->tx_rings; 2039 2040 IGB_CORE_LOCK_ASSERT(adapter); 2041 2042 INIT_DEBUGOUT("igb_stop: begin"); 2043 2044 igb_disable_intr(adapter); 2045 2046 callout_stop(&adapter->timer); 2047 2048 /* Tell the stack that the interface is no longer active */ 2049 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2050 2051 /* Unarm watchdog timer. */ 2052 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2053 IGB_TX_LOCK(txr); 2054 txr->watchdog_check = FALSE; 2055 IGB_TX_UNLOCK(txr); 2056 } 2057 2058 e1000_reset_hw(&adapter->hw); 2059 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2060 2061 e1000_led_off(&adapter->hw); 2062 e1000_cleanup_led(&adapter->hw); 2063} 2064 2065 2066/********************************************************************* 2067 * 2068 * Determine hardware revision. 2069 * 2070 **********************************************************************/ 2071static void 2072igb_identify_hardware(struct adapter *adapter) 2073{ 2074 device_t dev = adapter->dev; 2075 2076 /* Make sure our PCI config space has the necessary stuff set */ 2077 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2078 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2079 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2080 INIT_DEBUGOUT("Memory Access and/or Bus Master " 2081 "bits were not set!\n"); 2082 adapter->hw.bus.pci_cmd_word |= 2083 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2084 pci_write_config(dev, PCIR_COMMAND, 2085 adapter->hw.bus.pci_cmd_word, 2); 2086 } 2087 2088 /* Save off the information about this board */ 2089 adapter->hw.vendor_id = pci_get_vendor(dev); 2090 adapter->hw.device_id = pci_get_device(dev); 2091 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2092 adapter->hw.subsystem_vendor_id = 2093 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2094 adapter->hw.subsystem_device_id = 2095 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2096 2097 /* Set MAC type early for PCI setup */ 2098 e1000_set_mac_type(&adapter->hw); 2099} 2100 2101static int 2102igb_allocate_pci_resources(struct adapter *adapter) 2103{ 2104 device_t dev = adapter->dev; 2105 int rid; 2106 2107 rid = PCIR_BAR(0); 2108 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2109 &rid, RF_ACTIVE); 2110 if (adapter->pci_mem == NULL) { 2111 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2112 return (ENXIO); 2113 } 2114 adapter->osdep.mem_bus_space_tag = 2115 rman_get_bustag(adapter->pci_mem); 2116 adapter->osdep.mem_bus_space_handle = 2117 rman_get_bushandle(adapter->pci_mem); 2118 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2119 2120 adapter->num_queues = 1; /* Defaults for Legacy or MSI */ 2121 2122 /* This will setup either MSI/X or MSI */ 2123 adapter->msix = igb_setup_msix(adapter); 2124 adapter->hw.back = &adapter->osdep; 2125 2126 return (0); 2127} 2128 2129/********************************************************************* 2130 * 2131 * Setup the Legacy or MSI Interrupt handler 2132 * 2133 **********************************************************************/ 2134static int 2135igb_allocate_legacy(struct adapter *adapter) 2136{ 2137 device_t dev = adapter->dev; 2138 struct igb_queue *que = adapter->queues; 2139 int error, rid = 0; 2140 2141 /* Turn off all interrupts */ 2142 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2143 2144 /* MSI RID is 1 */ 2145 if (adapter->msix == 1) 2146 rid = 1; 2147 2148 /* We allocate a single interrupt resource */ 2149 adapter->res = bus_alloc_resource_any(dev, 2150 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2151 if (adapter->res == NULL) { 2152 device_printf(dev, "Unable to allocate bus resource: " 2153 "interrupt\n"); 2154 return (ENXIO); 2155 } 2156 2157 /* 2158 * Try allocating a fast interrupt and the associated deferred 2159 * processing contexts. 2160 */ 2161 TASK_INIT(&que->que_task, 0, igb_handle_que, que); 2162 /* Make tasklet for deferred link handling */ 2163 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); 2164 que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, 2165 taskqueue_thread_enqueue, &que->tq); 2166 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq", 2167 device_get_nameunit(adapter->dev)); 2168 if ((error = bus_setup_intr(dev, adapter->res, 2169 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL, 2170 adapter, &adapter->tag)) != 0) { 2171 device_printf(dev, "Failed to register fast interrupt " 2172 "handler: %d\n", error); 2173 taskqueue_free(que->tq); 2174 que->tq = NULL; 2175 return (error); 2176 } 2177 2178 return (0); 2179} 2180 2181 2182/********************************************************************* 2183 * 2184 * Setup the MSIX Queue Interrupt handlers: 2185 * 2186 **********************************************************************/ 2187static int 2188igb_allocate_msix(struct adapter *adapter) 2189{ 2190 device_t dev = adapter->dev; 2191 struct igb_queue *que = adapter->queues; 2192 int error, rid, vector = 0; 2193 2194 2195 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2196 rid = vector +1; 2197 que->res = bus_alloc_resource_any(dev, 2198 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2199 if (que->res == NULL) { 2200 device_printf(dev, 2201 "Unable to allocate bus resource: " 2202 "MSIX Queue Interrupt\n"); 2203 return (ENXIO); 2204 } 2205 error = bus_setup_intr(dev, que->res, 2206 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2207 igb_msix_que, que, &que->tag); 2208 if (error) { 2209 que->res = NULL; 2210 device_printf(dev, "Failed to register Queue handler"); 2211 return (error); 2212 } 2213#if __FreeBSD_version >= 800504 2214 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 2215#endif 2216 que->msix = vector; 2217 if (adapter->hw.mac.type == e1000_82575) 2218 que->eims = E1000_EICR_TX_QUEUE0 << i; 2219 else 2220 que->eims = 1 << vector; 2221 /* 2222 ** Bind the msix vector, and thus the 2223 ** rings to the corresponding cpu. 2224 */ 2225 if (adapter->num_queues > 1) 2226 bus_bind_intr(dev, que->res, i); 2227 /* Make tasklet for deferred handling */ 2228 TASK_INIT(&que->que_task, 0, igb_handle_que, que); 2229 que->tq = taskqueue_create_fast("igb_que", M_NOWAIT, 2230 taskqueue_thread_enqueue, &que->tq); 2231 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 2232 device_get_nameunit(adapter->dev)); 2233 } 2234 2235 /* And Link */ 2236 rid = vector + 1; 2237 adapter->res = bus_alloc_resource_any(dev, 2238 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2239 if (adapter->res == NULL) { 2240 device_printf(dev, 2241 "Unable to allocate bus resource: " 2242 "MSIX Link Interrupt\n"); 2243 return (ENXIO); 2244 } 2245 if ((error = bus_setup_intr(dev, adapter->res, 2246 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2247 igb_msix_link, adapter, &adapter->tag)) != 0) { 2248 device_printf(dev, "Failed to register Link handler"); 2249 return (error); 2250 } 2251#if __FreeBSD_version >= 800504 2252 bus_describe_intr(dev, adapter->res, adapter->tag, "link"); 2253#endif 2254 adapter->linkvec = vector; 2255 2256 return (0); 2257} 2258 2259 2260static void 2261igb_configure_queues(struct adapter *adapter) 2262{ 2263 struct e1000_hw *hw = &adapter->hw; 2264 struct igb_queue *que; 2265 u32 tmp, ivar = 0; 2266 u32 newitr = IGB_DEFAULT_ITR; 2267 2268 /* First turn on RSS capability */ 2269 if (adapter->hw.mac.type > e1000_82575) 2270 E1000_WRITE_REG(hw, E1000_GPIE, 2271 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | 2272 E1000_GPIE_PBA | E1000_GPIE_NSICR); 2273 2274 /* Turn on MSIX */ 2275 switch (adapter->hw.mac.type) { 2276 case e1000_82580: 2277 case e1000_vfadapt: 2278 /* RX entries */ 2279 for (int i = 0; i < adapter->num_queues; i++) { 2280 u32 index = i >> 1; 2281 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2282 que = &adapter->queues[i]; 2283 if (i & 1) { 2284 ivar &= 0xFF00FFFF; 2285 ivar |= (que->msix | E1000_IVAR_VALID) << 16; 2286 } else { 2287 ivar &= 0xFFFFFF00; 2288 ivar |= que->msix | E1000_IVAR_VALID; 2289 } 2290 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2291 } 2292 /* TX entries */ 2293 for (int i = 0; i < adapter->num_queues; i++) { 2294 u32 index = i >> 1; 2295 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2296 que = &adapter->queues[i]; 2297 if (i & 1) { 2298 ivar &= 0x00FFFFFF; 2299 ivar |= (que->msix | E1000_IVAR_VALID) << 24; 2300 } else { 2301 ivar &= 0xFFFF00FF; 2302 ivar |= (que->msix | E1000_IVAR_VALID) << 8; 2303 } 2304 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2305 adapter->eims_mask |= que->eims; 2306 } 2307 2308 /* And for the link interrupt */ 2309 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; 2310 adapter->link_mask = 1 << adapter->linkvec; 2311 adapter->eims_mask |= adapter->link_mask; 2312 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 2313 break; 2314 case e1000_82576: 2315 /* RX entries */ 2316 for (int i = 0; i < adapter->num_queues; i++) { 2317 u32 index = i & 0x7; /* Each IVAR has two entries */ 2318 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2319 que = &adapter->queues[i]; 2320 if (i < 8) { 2321 ivar &= 0xFFFFFF00; 2322 ivar |= que->msix | E1000_IVAR_VALID; 2323 } else { 2324 ivar &= 0xFF00FFFF; 2325 ivar |= (que->msix | E1000_IVAR_VALID) << 16; 2326 } 2327 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2328 adapter->eims_mask |= que->eims; 2329 } 2330 /* TX entries */ 2331 for (int i = 0; i < adapter->num_queues; i++) { 2332 u32 index = i & 0x7; /* Each IVAR has two entries */ 2333 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2334 que = &adapter->queues[i]; 2335 if (i < 8) { 2336 ivar &= 0xFFFF00FF; 2337 ivar |= (que->msix | E1000_IVAR_VALID) << 8; 2338 } else { 2339 ivar &= 0x00FFFFFF; 2340 ivar |= (que->msix | E1000_IVAR_VALID) << 24; 2341 } 2342 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2343 adapter->eims_mask |= que->eims; 2344 } 2345 2346 /* And for the link interrupt */ 2347 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; 2348 adapter->link_mask = 1 << adapter->linkvec; 2349 adapter->eims_mask |= adapter->link_mask; 2350 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 2351 break; 2352 2353 case e1000_82575: 2354 /* enable MSI-X support*/ 2355 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); 2356 tmp |= E1000_CTRL_EXT_PBA_CLR; 2357 /* Auto-Mask interrupts upon ICR read. */ 2358 tmp |= E1000_CTRL_EXT_EIAME; 2359 tmp |= E1000_CTRL_EXT_IRCA; 2360 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); 2361 2362 /* Queues */ 2363 for (int i = 0; i < adapter->num_queues; i++) { 2364 que = &adapter->queues[i]; 2365 tmp = E1000_EICR_RX_QUEUE0 << i; 2366 tmp |= E1000_EICR_TX_QUEUE0 << i; 2367 que->eims = tmp; 2368 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 2369 i, que->eims); 2370 adapter->eims_mask |= que->eims; 2371 } 2372 2373 /* Link */ 2374 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), 2375 E1000_EIMS_OTHER); 2376 adapter->link_mask |= E1000_EIMS_OTHER; 2377 adapter->eims_mask |= adapter->link_mask; 2378 default: 2379 break; 2380 } 2381 2382 /* Set the starting interrupt rate */ 2383 if (hw->mac.type == e1000_82575) 2384 newitr |= newitr << 16; 2385 else 2386 newitr |= E1000_EITR_CNT_IGNR; 2387 2388 for (int i = 0; i < adapter->num_queues; i++) { 2389 que = &adapter->queues[i]; 2390 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr); 2391 } 2392 2393 return; 2394} 2395 2396 2397static void 2398igb_free_pci_resources(struct adapter *adapter) 2399{ 2400 struct igb_queue *que = adapter->queues; 2401 device_t dev = adapter->dev; 2402 int rid; 2403 2404 /* 2405 ** There is a slight possibility of a failure mode 2406 ** in attach that will result in entering this function 2407 ** before interrupt resources have been initialized, and 2408 ** in that case we do not want to execute the loops below 2409 ** We can detect this reliably by the state of the adapter 2410 ** res pointer. 2411 */ 2412 if (adapter->res == NULL) 2413 goto mem; 2414 2415 /* 2416 * First release all the interrupt resources: 2417 */ 2418 for (int i = 0; i < adapter->num_queues; i++, que++) { 2419 rid = que->msix + 1; 2420 if (que->tag != NULL) { 2421 bus_teardown_intr(dev, que->res, que->tag); 2422 que->tag = NULL; 2423 } 2424 if (que->res != NULL) 2425 bus_release_resource(dev, 2426 SYS_RES_IRQ, rid, que->res); 2427 } 2428 2429 /* Clean the Legacy or Link interrupt last */ 2430 if (adapter->linkvec) /* we are doing MSIX */ 2431 rid = adapter->linkvec + 1; 2432 else 2433 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2434 2435 if (adapter->tag != NULL) { 2436 bus_teardown_intr(dev, adapter->res, adapter->tag); 2437 adapter->tag = NULL; 2438 } 2439 if (adapter->res != NULL) 2440 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 2441 2442mem: 2443 if (adapter->msix) 2444 pci_release_msi(dev); 2445 2446 if (adapter->msix_mem != NULL) 2447 bus_release_resource(dev, SYS_RES_MEMORY, 2448 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2449 2450 if (adapter->pci_mem != NULL) 2451 bus_release_resource(dev, SYS_RES_MEMORY, 2452 PCIR_BAR(0), adapter->pci_mem); 2453 2454} 2455 2456/* 2457 * Setup Either MSI/X or MSI 2458 */ 2459static int 2460igb_setup_msix(struct adapter *adapter) 2461{ 2462 device_t dev = adapter->dev; 2463 int rid, want, queues, msgs; 2464 2465 /* tuneable override */ 2466 if (igb_enable_msix == 0) 2467 goto msi; 2468 2469 /* First try MSI/X */ 2470 rid = PCIR_BAR(IGB_MSIX_BAR); 2471 adapter->msix_mem = bus_alloc_resource_any(dev, 2472 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2473 if (!adapter->msix_mem) { 2474 /* May not be enabled */ 2475 device_printf(adapter->dev, 2476 "Unable to map MSIX table \n"); 2477 goto msi; 2478 } 2479 2480 msgs = pci_msix_count(dev); 2481 if (msgs == 0) { /* system has msix disabled */ 2482 bus_release_resource(dev, SYS_RES_MEMORY, 2483 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2484 adapter->msix_mem = NULL; 2485 goto msi; 2486 } 2487 2488 /* Figure out a reasonable auto config value */ 2489 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 2490 2491 /* Manual override */ 2492 if (igb_num_queues != 0) 2493 queues = igb_num_queues; 2494 2495 /* Can have max of 4 queues on 82575 */ 2496 if ((adapter->hw.mac.type == e1000_82575) && (queues > 4)) 2497 queues = 4; 2498 2499 /* Limit the VF adapter to one queue */ 2500 if (adapter->hw.mac.type == e1000_vfadapt) 2501 queues = 1; 2502 2503 /* 2504 ** One vector (RX/TX pair) per queue 2505 ** plus an additional for Link interrupt 2506 */ 2507 want = queues + 1; 2508 if (msgs >= want) 2509 msgs = want; 2510 else { 2511 device_printf(adapter->dev, 2512 "MSIX Configuration Problem, " 2513 "%d vectors configured, but %d queues wanted!\n", 2514 msgs, want); 2515 return (ENXIO); 2516 } 2517 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2518 device_printf(adapter->dev, 2519 "Using MSIX interrupts with %d vectors\n", msgs); 2520 adapter->num_queues = queues; 2521 return (msgs); 2522 } 2523msi: 2524 msgs = pci_msi_count(dev); 2525 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2526 device_printf(adapter->dev,"Using MSI interrupt\n"); 2527 return (msgs); 2528} 2529 2530/********************************************************************* 2531 * 2532 * Set up an fresh starting state 2533 * 2534 **********************************************************************/ 2535static void 2536igb_reset(struct adapter *adapter) 2537{ 2538 device_t dev = adapter->dev; 2539 struct e1000_hw *hw = &adapter->hw; 2540 struct e1000_fc_info *fc = &hw->fc; 2541 struct ifnet *ifp = adapter->ifp; 2542 u32 pba = 0; 2543 u16 hwm; 2544 2545 INIT_DEBUGOUT("igb_reset: begin"); 2546 2547 /* Let the firmware know the OS is in control */ 2548 igb_get_hw_control(adapter); 2549 2550 /* 2551 * Packet Buffer Allocation (PBA) 2552 * Writing PBA sets the receive portion of the buffer 2553 * the remainder is used for the transmit buffer. 2554 */ 2555 switch (hw->mac.type) { 2556 case e1000_82575: 2557 pba = E1000_PBA_32K; 2558 break; 2559 case e1000_82576: 2560 case e1000_vfadapt: 2561 pba = E1000_PBA_64K; 2562 break; 2563 case e1000_82580: 2564 pba = E1000_PBA_35K; 2565 default: 2566 break; 2567 } 2568 2569 /* Special needs in case of Jumbo frames */ 2570 if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) { 2571 u32 tx_space, min_tx, min_rx; 2572 pba = E1000_READ_REG(hw, E1000_PBA); 2573 tx_space = pba >> 16; 2574 pba &= 0xffff; 2575 min_tx = (adapter->max_frame_size + 2576 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; 2577 min_tx = roundup2(min_tx, 1024); 2578 min_tx >>= 10; 2579 min_rx = adapter->max_frame_size; 2580 min_rx = roundup2(min_rx, 1024); 2581 min_rx >>= 10; 2582 if (tx_space < min_tx && 2583 ((min_tx - tx_space) < pba)) { 2584 pba = pba - (min_tx - tx_space); 2585 /* 2586 * if short on rx space, rx wins 2587 * and must trump tx adjustment 2588 */ 2589 if (pba < min_rx) 2590 pba = min_rx; 2591 } 2592 E1000_WRITE_REG(hw, E1000_PBA, pba); 2593 } 2594 2595 INIT_DEBUGOUT1("igb_init: pba=%dK",pba); 2596 2597 /* 2598 * These parameters control the automatic generation (Tx) and 2599 * response (Rx) to Ethernet PAUSE frames. 2600 * - High water mark should allow for at least two frames to be 2601 * received after sending an XOFF. 2602 * - Low water mark works best when it is very near the high water mark. 2603 * This allows the receiver to restart by sending XON when it has 2604 * drained a bit. 2605 */ 2606 hwm = min(((pba << 10) * 9 / 10), 2607 ((pba << 10) - 2 * adapter->max_frame_size)); 2608 2609 if (hw->mac.type < e1000_82576) { 2610 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */ 2611 fc->low_water = fc->high_water - 8; 2612 } else { 2613 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */ 2614 fc->low_water = fc->high_water - 16; 2615 } 2616 2617 fc->pause_time = IGB_FC_PAUSE_TIME; 2618 fc->send_xon = TRUE; 2619 2620 /* Set Flow control, use the tunable location if sane */ 2621 if ((igb_fc_setting >= 0) || (igb_fc_setting < 4)) 2622 fc->requested_mode = igb_fc_setting; 2623 else 2624 fc->requested_mode = e1000_fc_none; 2625 2626 fc->current_mode = fc->requested_mode; 2627 2628 /* Issue a global reset */ 2629 e1000_reset_hw(hw); 2630 E1000_WRITE_REG(hw, E1000_WUC, 0); 2631 2632 if (e1000_init_hw(hw) < 0) 2633 device_printf(dev, "Hardware Initialization Failed\n"); 2634 2635 if (hw->mac.type == e1000_82580) { 2636 u32 reg; 2637 2638 hwm = (pba << 10) - (2 * adapter->max_frame_size); 2639 /* 2640 * 0x80000000 - enable DMA COAL 2641 * 0x10000000 - use L0s as low power 2642 * 0x20000000 - use L1 as low power 2643 * X << 16 - exit dma coal when rx data exceeds X kB 2644 * Y - upper limit to stay in dma coal in units of 32usecs 2645 */ 2646 E1000_WRITE_REG(hw, E1000_DMACR, 2647 0xA0000006 | ((hwm << 6) & 0x00FF0000)); 2648 2649 /* set hwm to PBA - 2 * max frame size */ 2650 E1000_WRITE_REG(hw, E1000_FCRTC, hwm); 2651 /* 2652 * This sets the time to wait before requesting transition to 2653 * low power state to number of usecs needed to receive 1 512 2654 * byte frame at gigabit line rate 2655 */ 2656 E1000_WRITE_REG(hw, E1000_DMCTLX, 4); 2657 2658 /* free space in tx packet buffer to wake from DMA coal */ 2659 E1000_WRITE_REG(hw, E1000_DMCTXTH, 2660 (20480 - (2 * adapter->max_frame_size)) >> 6); 2661 2662 /* make low power state decision controlled by DMA coal */ 2663 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 2664 E1000_WRITE_REG(hw, E1000_PCIEMISC, 2665 reg | E1000_PCIEMISC_LX_DECISION); 2666 } 2667 2668 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 2669 e1000_get_phy_info(hw); 2670 e1000_check_for_link(hw); 2671 return; 2672} 2673 2674/********************************************************************* 2675 * 2676 * Setup networking device structure and register an interface. 2677 * 2678 **********************************************************************/ 2679static int 2680igb_setup_interface(device_t dev, struct adapter *adapter) 2681{ 2682 struct ifnet *ifp; 2683 2684 INIT_DEBUGOUT("igb_setup_interface: begin"); 2685 2686 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2687 if (ifp == NULL) { 2688 device_printf(dev, "can not allocate ifnet structure\n"); 2689 return (-1); 2690 } 2691 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2692 ifp->if_mtu = ETHERMTU; 2693 ifp->if_init = igb_init; 2694 ifp->if_softc = adapter; 2695 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2696 ifp->if_ioctl = igb_ioctl; 2697 ifp->if_start = igb_start; 2698#if __FreeBSD_version >= 800000 2699 ifp->if_transmit = igb_mq_start; 2700 ifp->if_qflush = igb_qflush; 2701#endif 2702 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 2703 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 2704 IFQ_SET_READY(&ifp->if_snd); 2705 2706 ether_ifattach(ifp, adapter->hw.mac.addr); 2707 2708 ifp->if_capabilities = ifp->if_capenable = 0; 2709 2710 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2711 ifp->if_capabilities |= IFCAP_TSO4; 2712 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 2713 if (igb_header_split) 2714 ifp->if_capabilities |= IFCAP_LRO; 2715 2716 ifp->if_capenable = ifp->if_capabilities; 2717#ifdef DEVICE_POLLING 2718 ifp->if_capabilities |= IFCAP_POLLING; 2719#endif 2720 2721 /* 2722 * Tell the upper layer(s) we 2723 * support full VLAN capability. 2724 */ 2725 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2726 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2727 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2728 2729 /* 2730 ** Dont turn this on by default, if vlans are 2731 ** created on another pseudo device (eg. lagg) 2732 ** then vlan events are not passed thru, breaking 2733 ** operation, but with HW FILTER off it works. If 2734 ** using vlans directly on the em driver you can 2735 ** enable this and get full hardware tag filtering. 2736 */ 2737 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2738 2739 /* 2740 * Specify the media types supported by this adapter and register 2741 * callbacks to update media and link information 2742 */ 2743 ifmedia_init(&adapter->media, IFM_IMASK, 2744 igb_media_change, igb_media_status); 2745 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2746 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2747 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 2748 0, NULL); 2749 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 2750 } else { 2751 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2752 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2753 0, NULL); 2754 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2755 0, NULL); 2756 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2757 0, NULL); 2758 if (adapter->hw.phy.type != e1000_phy_ife) { 2759 ifmedia_add(&adapter->media, 2760 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2761 ifmedia_add(&adapter->media, 2762 IFM_ETHER | IFM_1000_T, 0, NULL); 2763 } 2764 } 2765 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2766 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2767 return (0); 2768} 2769 2770 2771/* 2772 * Manage DMA'able memory. 2773 */ 2774static void 2775igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 2776{ 2777 if (error) 2778 return; 2779 *(bus_addr_t *) arg = segs[0].ds_addr; 2780} 2781 2782static int 2783igb_dma_malloc(struct adapter *adapter, bus_size_t size, 2784 struct igb_dma_alloc *dma, int mapflags) 2785{ 2786 int error; 2787 2788 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 2789 IGB_DBA_ALIGN, 0, /* alignment, bounds */ 2790 BUS_SPACE_MAXADDR, /* lowaddr */ 2791 BUS_SPACE_MAXADDR, /* highaddr */ 2792 NULL, NULL, /* filter, filterarg */ 2793 size, /* maxsize */ 2794 1, /* nsegments */ 2795 size, /* maxsegsize */ 2796 0, /* flags */ 2797 NULL, /* lockfunc */ 2798 NULL, /* lockarg */ 2799 &dma->dma_tag); 2800 if (error) { 2801 device_printf(adapter->dev, 2802 "%s: bus_dma_tag_create failed: %d\n", 2803 __func__, error); 2804 goto fail_0; 2805 } 2806 2807 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 2808 BUS_DMA_NOWAIT, &dma->dma_map); 2809 if (error) { 2810 device_printf(adapter->dev, 2811 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 2812 __func__, (uintmax_t)size, error); 2813 goto fail_2; 2814 } 2815 2816 dma->dma_paddr = 0; 2817 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2818 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 2819 if (error || dma->dma_paddr == 0) { 2820 device_printf(adapter->dev, 2821 "%s: bus_dmamap_load failed: %d\n", 2822 __func__, error); 2823 goto fail_3; 2824 } 2825 2826 return (0); 2827 2828fail_3: 2829 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2830fail_2: 2831 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2832 bus_dma_tag_destroy(dma->dma_tag); 2833fail_0: 2834 dma->dma_map = NULL; 2835 dma->dma_tag = NULL; 2836 2837 return (error); 2838} 2839 2840static void 2841igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma) 2842{ 2843 if (dma->dma_tag == NULL) 2844 return; 2845 if (dma->dma_map != NULL) { 2846 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2847 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2848 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2849 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2850 dma->dma_map = NULL; 2851 } 2852 bus_dma_tag_destroy(dma->dma_tag); 2853 dma->dma_tag = NULL; 2854} 2855 2856 2857/********************************************************************* 2858 * 2859 * Allocate memory for the transmit and receive rings, and then 2860 * the descriptors associated with each, called only once at attach. 2861 * 2862 **********************************************************************/ 2863static int 2864igb_allocate_queues(struct adapter *adapter) 2865{ 2866 device_t dev = adapter->dev; 2867 struct igb_queue *que = NULL; 2868 struct tx_ring *txr = NULL; 2869 struct rx_ring *rxr = NULL; 2870 int rsize, tsize, error = E1000_SUCCESS; 2871 int txconf = 0, rxconf = 0; 2872 2873 /* First allocate the top level queue structs */ 2874 if (!(adapter->queues = 2875 (struct igb_queue *) malloc(sizeof(struct igb_queue) * 2876 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2877 device_printf(dev, "Unable to allocate queue memory\n"); 2878 error = ENOMEM; 2879 goto fail; 2880 } 2881 2882 /* Next allocate the TX ring struct memory */ 2883 if (!(adapter->tx_rings = 2884 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2885 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2886 device_printf(dev, "Unable to allocate TX ring memory\n"); 2887 error = ENOMEM; 2888 goto tx_fail; 2889 } 2890 2891 /* Now allocate the RX */ 2892 if (!(adapter->rx_rings = 2893 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2894 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2895 device_printf(dev, "Unable to allocate RX ring memory\n"); 2896 error = ENOMEM; 2897 goto rx_fail; 2898 } 2899 2900 tsize = roundup2(adapter->num_tx_desc * 2901 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN); 2902 /* 2903 * Now set up the TX queues, txconf is needed to handle the 2904 * possibility that things fail midcourse and we need to 2905 * undo memory gracefully 2906 */ 2907 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2908 /* Set up some basics */ 2909 txr = &adapter->tx_rings[i]; 2910 txr->adapter = adapter; 2911 txr->me = i; 2912 2913 /* Initialize the TX lock */ 2914 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2915 device_get_nameunit(dev), txr->me); 2916 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2917 2918 if (igb_dma_malloc(adapter, tsize, 2919 &txr->txdma, BUS_DMA_NOWAIT)) { 2920 device_printf(dev, 2921 "Unable to allocate TX Descriptor memory\n"); 2922 error = ENOMEM; 2923 goto err_tx_desc; 2924 } 2925 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; 2926 bzero((void *)txr->tx_base, tsize); 2927 2928 /* Now allocate transmit buffers for the ring */ 2929 if (igb_allocate_transmit_buffers(txr)) { 2930 device_printf(dev, 2931 "Critical Failure setting up transmit buffers\n"); 2932 error = ENOMEM; 2933 goto err_tx_desc; 2934 } 2935#if __FreeBSD_version >= 800000 2936 /* Allocate a buf ring */ 2937 txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF, 2938 M_WAITOK, &txr->tx_mtx); 2939#endif 2940 } 2941 2942 /* 2943 * Next the RX queues... 2944 */ 2945 rsize = roundup2(adapter->num_rx_desc * 2946 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 2947 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2948 rxr = &adapter->rx_rings[i]; 2949 rxr->adapter = adapter; 2950 rxr->me = i; 2951 2952 /* Initialize the RX lock */ 2953 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2954 device_get_nameunit(dev), txr->me); 2955 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2956 2957 if (igb_dma_malloc(adapter, rsize, 2958 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2959 device_printf(dev, 2960 "Unable to allocate RxDescriptor memory\n"); 2961 error = ENOMEM; 2962 goto err_rx_desc; 2963 } 2964 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2965 bzero((void *)rxr->rx_base, rsize); 2966 2967 /* Allocate receive buffers for the ring*/ 2968 if (igb_allocate_receive_buffers(rxr)) { 2969 device_printf(dev, 2970 "Critical Failure setting up receive buffers\n"); 2971 error = ENOMEM; 2972 goto err_rx_desc; 2973 } 2974 } 2975 2976 /* 2977 ** Finally set up the queue holding structs 2978 */ 2979 for (int i = 0; i < adapter->num_queues; i++) { 2980 que = &adapter->queues[i]; 2981 que->adapter = adapter; 2982 que->txr = &adapter->tx_rings[i]; 2983 que->rxr = &adapter->rx_rings[i]; 2984 } 2985 2986 return (0); 2987 2988err_rx_desc: 2989 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2990 igb_dma_free(adapter, &rxr->rxdma); 2991err_tx_desc: 2992 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2993 igb_dma_free(adapter, &txr->txdma); 2994 free(adapter->rx_rings, M_DEVBUF); 2995rx_fail: 2996#if __FreeBSD_version >= 800000 2997 buf_ring_free(txr->br, M_DEVBUF); 2998#endif 2999 free(adapter->tx_rings, M_DEVBUF); 3000tx_fail: 3001 free(adapter->queues, M_DEVBUF); 3002fail: 3003 return (error); 3004} 3005 3006/********************************************************************* 3007 * 3008 * Allocate memory for tx_buffer structures. The tx_buffer stores all 3009 * the information needed to transmit a packet on the wire. This is 3010 * called only once at attach, setup is done every reset. 3011 * 3012 **********************************************************************/ 3013static int 3014igb_allocate_transmit_buffers(struct tx_ring *txr) 3015{ 3016 struct adapter *adapter = txr->adapter; 3017 device_t dev = adapter->dev; 3018 struct igb_tx_buffer *txbuf; 3019 int error, i; 3020 3021 /* 3022 * Setup DMA descriptor areas. 3023 */ 3024 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 3025 1, 0, /* alignment, bounds */ 3026 BUS_SPACE_MAXADDR, /* lowaddr */ 3027 BUS_SPACE_MAXADDR, /* highaddr */ 3028 NULL, NULL, /* filter, filterarg */ 3029 IGB_TSO_SIZE, /* maxsize */ 3030 IGB_MAX_SCATTER, /* nsegments */ 3031 PAGE_SIZE, /* maxsegsize */ 3032 0, /* flags */ 3033 NULL, /* lockfunc */ 3034 NULL, /* lockfuncarg */ 3035 &txr->txtag))) { 3036 device_printf(dev,"Unable to allocate TX DMA tag\n"); 3037 goto fail; 3038 } 3039 3040 if (!(txr->tx_buffers = 3041 (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) * 3042 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 3043 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 3044 error = ENOMEM; 3045 goto fail; 3046 } 3047 3048 /* Create the descriptor buffer dma maps */ 3049 txbuf = txr->tx_buffers; 3050 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3051 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 3052 if (error != 0) { 3053 device_printf(dev, "Unable to create TX DMA map\n"); 3054 goto fail; 3055 } 3056 } 3057 3058 return 0; 3059fail: 3060 /* We free all, it handles case where we are in the middle */ 3061 igb_free_transmit_structures(adapter); 3062 return (error); 3063} 3064 3065/********************************************************************* 3066 * 3067 * Initialize a transmit ring. 3068 * 3069 **********************************************************************/ 3070static void 3071igb_setup_transmit_ring(struct tx_ring *txr) 3072{ 3073 struct adapter *adapter = txr->adapter; 3074 struct igb_tx_buffer *txbuf; 3075 int i; 3076 3077 /* Clear the old descriptor contents */ 3078 IGB_TX_LOCK(txr); 3079 bzero((void *)txr->tx_base, 3080 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc); 3081 /* Reset indices */ 3082 txr->next_avail_desc = 0; 3083 txr->next_to_clean = 0; 3084 3085 /* Free any existing tx buffers. */ 3086 txbuf = txr->tx_buffers; 3087 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3088 if (txbuf->m_head != NULL) { 3089 bus_dmamap_sync(txr->txtag, txbuf->map, 3090 BUS_DMASYNC_POSTWRITE); 3091 bus_dmamap_unload(txr->txtag, txbuf->map); 3092 m_freem(txbuf->m_head); 3093 txbuf->m_head = NULL; 3094 } 3095 /* clear the watch index */ 3096 txbuf->next_eop = -1; 3097 } 3098 3099 /* Set number of descriptors available */ 3100 txr->tx_avail = adapter->num_tx_desc; 3101 3102 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3103 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3104 IGB_TX_UNLOCK(txr); 3105} 3106 3107/********************************************************************* 3108 * 3109 * Initialize all transmit rings. 3110 * 3111 **********************************************************************/ 3112static void 3113igb_setup_transmit_structures(struct adapter *adapter) 3114{ 3115 struct tx_ring *txr = adapter->tx_rings; 3116 3117 for (int i = 0; i < adapter->num_queues; i++, txr++) 3118 igb_setup_transmit_ring(txr); 3119 3120 return; 3121} 3122 3123/********************************************************************* 3124 * 3125 * Enable transmit unit. 3126 * 3127 **********************************************************************/ 3128static void 3129igb_initialize_transmit_units(struct adapter *adapter) 3130{ 3131 struct tx_ring *txr = adapter->tx_rings; 3132 struct e1000_hw *hw = &adapter->hw; 3133 u32 tctl, txdctl; 3134 3135 INIT_DEBUGOUT("igb_initialize_transmit_units: begin"); 3136 tctl = txdctl = 0; 3137 3138 /* Setup the Tx Descriptor Rings */ 3139 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3140 u64 bus_addr = txr->txdma.dma_paddr; 3141 3142 E1000_WRITE_REG(hw, E1000_TDLEN(i), 3143 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3144 E1000_WRITE_REG(hw, E1000_TDBAH(i), 3145 (uint32_t)(bus_addr >> 32)); 3146 E1000_WRITE_REG(hw, E1000_TDBAL(i), 3147 (uint32_t)bus_addr); 3148 3149 /* Setup the HW Tx Head and Tail descriptor pointers */ 3150 E1000_WRITE_REG(hw, E1000_TDT(i), 0); 3151 E1000_WRITE_REG(hw, E1000_TDH(i), 0); 3152 3153 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3154 E1000_READ_REG(hw, E1000_TDBAL(i)), 3155 E1000_READ_REG(hw, E1000_TDLEN(i))); 3156 3157 txr->watchdog_check = FALSE; 3158 3159 txdctl |= IGB_TX_PTHRESH; 3160 txdctl |= IGB_TX_HTHRESH << 8; 3161 txdctl |= IGB_TX_WTHRESH << 16; 3162 txdctl |= E1000_TXDCTL_QUEUE_ENABLE; 3163 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); 3164 } 3165 3166 if (adapter->hw.mac.type == e1000_vfadapt) 3167 return; 3168 3169 /* Program the Transmit Control Register */ 3170 tctl = E1000_READ_REG(hw, E1000_TCTL); 3171 tctl &= ~E1000_TCTL_CT; 3172 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3173 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3174 3175 e1000_config_collision_dist(hw); 3176 3177 /* This write will effectively turn on the transmit unit. */ 3178 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 3179} 3180 3181/********************************************************************* 3182 * 3183 * Free all transmit rings. 3184 * 3185 **********************************************************************/ 3186static void 3187igb_free_transmit_structures(struct adapter *adapter) 3188{ 3189 struct tx_ring *txr = adapter->tx_rings; 3190 3191 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3192 IGB_TX_LOCK(txr); 3193 igb_free_transmit_buffers(txr); 3194 igb_dma_free(adapter, &txr->txdma); 3195 IGB_TX_UNLOCK(txr); 3196 IGB_TX_LOCK_DESTROY(txr); 3197 } 3198 free(adapter->tx_rings, M_DEVBUF); 3199} 3200 3201/********************************************************************* 3202 * 3203 * Free transmit ring related data structures. 3204 * 3205 **********************************************************************/ 3206static void 3207igb_free_transmit_buffers(struct tx_ring *txr) 3208{ 3209 struct adapter *adapter = txr->adapter; 3210 struct igb_tx_buffer *tx_buffer; 3211 int i; 3212 3213 INIT_DEBUGOUT("free_transmit_ring: begin"); 3214 3215 if (txr->tx_buffers == NULL) 3216 return; 3217 3218 tx_buffer = txr->tx_buffers; 3219 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3220 if (tx_buffer->m_head != NULL) { 3221 bus_dmamap_sync(txr->txtag, tx_buffer->map, 3222 BUS_DMASYNC_POSTWRITE); 3223 bus_dmamap_unload(txr->txtag, 3224 tx_buffer->map); 3225 m_freem(tx_buffer->m_head); 3226 tx_buffer->m_head = NULL; 3227 if (tx_buffer->map != NULL) { 3228 bus_dmamap_destroy(txr->txtag, 3229 tx_buffer->map); 3230 tx_buffer->map = NULL; 3231 } 3232 } else if (tx_buffer->map != NULL) { 3233 bus_dmamap_unload(txr->txtag, 3234 tx_buffer->map); 3235 bus_dmamap_destroy(txr->txtag, 3236 tx_buffer->map); 3237 tx_buffer->map = NULL; 3238 } 3239 } 3240#if __FreeBSD_version >= 800000 3241 if (txr->br != NULL) 3242 buf_ring_free(txr->br, M_DEVBUF); 3243#endif 3244 if (txr->tx_buffers != NULL) { 3245 free(txr->tx_buffers, M_DEVBUF); 3246 txr->tx_buffers = NULL; 3247 } 3248 if (txr->txtag != NULL) { 3249 bus_dma_tag_destroy(txr->txtag); 3250 txr->txtag = NULL; 3251 } 3252 return; 3253} 3254 3255/********************************************************************** 3256 * 3257 * Setup work for hardware segmentation offload (TSO) 3258 * 3259 **********************************************************************/ 3260static boolean_t 3261igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen) 3262{ 3263 struct adapter *adapter = txr->adapter; 3264 struct e1000_adv_tx_context_desc *TXD; 3265 struct igb_tx_buffer *tx_buffer; 3266 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3267 u32 mss_l4len_idx = 0; 3268 u16 vtag = 0; 3269 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 3270 struct ether_vlan_header *eh; 3271 struct ip *ip; 3272 struct tcphdr *th; 3273 3274 3275 /* 3276 * Determine where frame payload starts. 3277 * Jump over vlan headers if already present 3278 */ 3279 eh = mtod(mp, struct ether_vlan_header *); 3280 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 3281 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3282 else 3283 ehdrlen = ETHER_HDR_LEN; 3284 3285 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3286 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3287 return FALSE; 3288 3289 /* Only supports IPV4 for now */ 3290 ctxd = txr->next_avail_desc; 3291 tx_buffer = &txr->tx_buffers[ctxd]; 3292 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3293 3294 ip = (struct ip *)(mp->m_data + ehdrlen); 3295 if (ip->ip_p != IPPROTO_TCP) 3296 return FALSE; /* 0 */ 3297 ip->ip_sum = 0; 3298 ip_hlen = ip->ip_hl << 2; 3299 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3300 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3301 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3302 tcp_hlen = th->th_off << 2; 3303 /* 3304 * Calculate header length, this is used 3305 * in the transmit desc in igb_xmit 3306 */ 3307 *hdrlen = ehdrlen + ip_hlen + tcp_hlen; 3308 3309 /* VLAN MACLEN IPLEN */ 3310 if (mp->m_flags & M_VLANTAG) { 3311 vtag = htole16(mp->m_pkthdr.ether_vtag); 3312 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3313 } 3314 3315 vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT); 3316 vlan_macip_lens |= ip_hlen; 3317 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3318 3319 /* ADV DTYPE TUCMD */ 3320 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3321 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3322 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3323 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3324 3325 /* MSS L4LEN IDX */ 3326 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT); 3327 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); 3328 /* 82575 needs the queue index added */ 3329 if (adapter->hw.mac.type == e1000_82575) 3330 mss_l4len_idx |= txr->me << 4; 3331 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3332 3333 TXD->seqnum_seed = htole32(0); 3334 tx_buffer->m_head = NULL; 3335 tx_buffer->next_eop = -1; 3336 3337 if (++ctxd == adapter->num_tx_desc) 3338 ctxd = 0; 3339 3340 txr->tx_avail--; 3341 txr->next_avail_desc = ctxd; 3342 return TRUE; 3343} 3344 3345 3346/********************************************************************* 3347 * 3348 * Context Descriptor setup for VLAN or CSUM 3349 * 3350 **********************************************************************/ 3351 3352static bool 3353igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3354{ 3355 struct adapter *adapter = txr->adapter; 3356 struct e1000_adv_tx_context_desc *TXD; 3357 struct igb_tx_buffer *tx_buffer; 3358 u32 vlan_macip_lens, type_tucmd_mlhl, mss_l4len_idx; 3359 struct ether_vlan_header *eh; 3360 struct ip *ip = NULL; 3361 struct ip6_hdr *ip6; 3362 int ehdrlen, ctxd, ip_hlen = 0; 3363 u16 etype, vtag = 0; 3364 u8 ipproto = 0; 3365 bool offload = TRUE; 3366 3367 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 3368 offload = FALSE; 3369 3370 vlan_macip_lens = type_tucmd_mlhl = mss_l4len_idx = 0; 3371 ctxd = txr->next_avail_desc; 3372 tx_buffer = &txr->tx_buffers[ctxd]; 3373 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3374 3375 /* 3376 ** In advanced descriptors the vlan tag must 3377 ** be placed into the context descriptor, thus 3378 ** we need to be here just for that setup. 3379 */ 3380 if (mp->m_flags & M_VLANTAG) { 3381 vtag = htole16(mp->m_pkthdr.ether_vtag); 3382 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3383 } else if (offload == FALSE) 3384 return FALSE; 3385 3386 /* 3387 * Determine where frame payload starts. 3388 * Jump over vlan headers if already present, 3389 * helpful for QinQ too. 3390 */ 3391 eh = mtod(mp, struct ether_vlan_header *); 3392 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3393 etype = ntohs(eh->evl_proto); 3394 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3395 } else { 3396 etype = ntohs(eh->evl_encap_proto); 3397 ehdrlen = ETHER_HDR_LEN; 3398 } 3399 3400 /* Set the ether header length */ 3401 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; 3402 3403 switch (etype) { 3404 case ETHERTYPE_IP: 3405 ip = (struct ip *)(mp->m_data + ehdrlen); 3406 ip_hlen = ip->ip_hl << 2; 3407 if (mp->m_len < ehdrlen + ip_hlen) { 3408 offload = FALSE; 3409 break; 3410 } 3411 ipproto = ip->ip_p; 3412 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3413 break; 3414 case ETHERTYPE_IPV6: 3415 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3416 ip_hlen = sizeof(struct ip6_hdr); 3417 if (mp->m_len < ehdrlen + ip_hlen) 3418 return (FALSE); 3419 ipproto = ip6->ip6_nxt; 3420 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; 3421 break; 3422 default: 3423 offload = FALSE; 3424 break; 3425 } 3426 3427 vlan_macip_lens |= ip_hlen; 3428 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3429 3430 switch (ipproto) { 3431 case IPPROTO_TCP: 3432 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 3433 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3434 break; 3435 case IPPROTO_UDP: 3436 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 3437 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; 3438 break; 3439#if __FreeBSD_version >= 800000 3440 case IPPROTO_SCTP: 3441 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 3442 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; 3443 break; 3444#endif 3445 default: 3446 offload = FALSE; 3447 break; 3448 } 3449 3450 /* 82575 needs the queue index added */ 3451 if (adapter->hw.mac.type == e1000_82575) 3452 mss_l4len_idx = txr->me << 4; 3453 3454 /* Now copy bits into descriptor */ 3455 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3456 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3457 TXD->seqnum_seed = htole32(0); 3458 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3459 3460 tx_buffer->m_head = NULL; 3461 tx_buffer->next_eop = -1; 3462 3463 /* We've consumed the first desc, adjust counters */ 3464 if (++ctxd == adapter->num_tx_desc) 3465 ctxd = 0; 3466 txr->next_avail_desc = ctxd; 3467 --txr->tx_avail; 3468 3469 return (offload); 3470} 3471 3472 3473/********************************************************************** 3474 * 3475 * Examine each tx_buffer in the used queue. If the hardware is done 3476 * processing the packet then free associated resources. The 3477 * tx_buffer is put back on the free queue. 3478 * 3479 * TRUE return means there's work in the ring to clean, FALSE its empty. 3480 **********************************************************************/ 3481static bool 3482igb_txeof(struct tx_ring *txr) 3483{ 3484 struct adapter *adapter = txr->adapter; 3485 int first, last, done; 3486 struct igb_tx_buffer *tx_buffer; 3487 struct e1000_tx_desc *tx_desc, *eop_desc; 3488 struct ifnet *ifp = adapter->ifp; 3489 3490 IGB_TX_LOCK_ASSERT(txr); 3491 3492 if (txr->tx_avail == adapter->num_tx_desc) 3493 return FALSE; 3494 3495 first = txr->next_to_clean; 3496 tx_desc = &txr->tx_base[first]; 3497 tx_buffer = &txr->tx_buffers[first]; 3498 last = tx_buffer->next_eop; 3499 eop_desc = &txr->tx_base[last]; 3500 3501 /* 3502 * What this does is get the index of the 3503 * first descriptor AFTER the EOP of the 3504 * first packet, that way we can do the 3505 * simple comparison on the inner while loop. 3506 */ 3507 if (++last == adapter->num_tx_desc) 3508 last = 0; 3509 done = last; 3510 3511 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3512 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3513 3514 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3515 /* We clean the range of the packet */ 3516 while (first != done) { 3517 tx_desc->upper.data = 0; 3518 tx_desc->lower.data = 0; 3519 tx_desc->buffer_addr = 0; 3520 ++txr->tx_avail; 3521 3522 if (tx_buffer->m_head) { 3523 txr->bytes += 3524 tx_buffer->m_head->m_pkthdr.len; 3525 bus_dmamap_sync(txr->txtag, 3526 tx_buffer->map, 3527 BUS_DMASYNC_POSTWRITE); 3528 bus_dmamap_unload(txr->txtag, 3529 tx_buffer->map); 3530 3531 m_freem(tx_buffer->m_head); 3532 tx_buffer->m_head = NULL; 3533 } 3534 tx_buffer->next_eop = -1; 3535 txr->watchdog_time = ticks; 3536 3537 if (++first == adapter->num_tx_desc) 3538 first = 0; 3539 3540 tx_buffer = &txr->tx_buffers[first]; 3541 tx_desc = &txr->tx_base[first]; 3542 } 3543 ++txr->packets; 3544 ++ifp->if_opackets; 3545 /* See if we can continue to the next packet */ 3546 last = tx_buffer->next_eop; 3547 if (last != -1) { 3548 eop_desc = &txr->tx_base[last]; 3549 /* Get new done point */ 3550 if (++last == adapter->num_tx_desc) last = 0; 3551 done = last; 3552 } else 3553 break; 3554 } 3555 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3556 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3557 3558 txr->next_to_clean = first; 3559 3560 /* 3561 * If we have enough room, clear IFF_DRV_OACTIVE 3562 * to tell the stack that it is OK to send packets. 3563 */ 3564 if (txr->tx_avail > IGB_TX_CLEANUP_THRESHOLD) { 3565 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3566 /* All clean, turn off the watchdog */ 3567 if (txr->tx_avail == adapter->num_tx_desc) { 3568 txr->watchdog_check = FALSE; 3569 return (FALSE); 3570 } 3571 } 3572 3573 return (TRUE); 3574} 3575 3576 3577/********************************************************************* 3578 * 3579 * Refresh mbuf buffers for RX descriptor rings 3580 * - now keeps its own state so discards due to resource 3581 * exhaustion are unnecessary, if an mbuf cannot be obtained 3582 * it just returns, keeping its placeholder, thus it can simply 3583 * be recalled to try again. 3584 * 3585 **********************************************************************/ 3586static void 3587igb_refresh_mbufs(struct rx_ring *rxr, int limit) 3588{ 3589 struct adapter *adapter = rxr->adapter; 3590 bus_dma_segment_t hseg[1]; 3591 bus_dma_segment_t pseg[1]; 3592 struct igb_rx_buf *rxbuf; 3593 struct mbuf *mh, *mp; 3594 int i, nsegs, error, cleaned; 3595 3596 i = rxr->next_to_refresh; 3597 cleaned = -1; /* Signify no completions */ 3598 while (i != limit) { 3599 rxbuf = &rxr->rx_buffers[i]; 3600 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 3601 mh = m_gethdr(M_DONTWAIT, MT_DATA); 3602 if (mh == NULL) 3603 goto update; 3604 mh->m_pkthdr.len = mh->m_len = MHLEN; 3605 mh->m_len = MHLEN; 3606 mh->m_flags |= M_PKTHDR; 3607 m_adj(mh, ETHER_ALIGN); 3608 /* Get the memory mapping */ 3609 error = bus_dmamap_load_mbuf_sg(rxr->htag, 3610 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); 3611 if (error != 0) { 3612 printf("GET BUF: dmamap load" 3613 " failure - %d\n", error); 3614 m_free(mh); 3615 goto update; 3616 } 3617 rxbuf->m_head = mh; 3618 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3619 BUS_DMASYNC_PREREAD); 3620 rxr->rx_base[i].read.hdr_addr = 3621 htole64(hseg[0].ds_addr); 3622 } 3623 3624 if (rxbuf->m_pack == NULL) { 3625 mp = m_getjcl(M_DONTWAIT, MT_DATA, 3626 M_PKTHDR, adapter->rx_mbuf_sz); 3627 if (mp == NULL) 3628 goto update; 3629 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3630 /* Get the memory mapping */ 3631 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 3632 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); 3633 if (error != 0) { 3634 printf("GET BUF: dmamap load" 3635 " failure - %d\n", error); 3636 m_free(mp); 3637 goto update; 3638 } 3639 rxbuf->m_pack = mp; 3640 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3641 BUS_DMASYNC_PREREAD); 3642 rxr->rx_base[i].read.pkt_addr = 3643 htole64(pseg[0].ds_addr); 3644 } 3645 3646 cleaned = i; 3647 /* Calculate next index */ 3648 if (++i == adapter->num_rx_desc) 3649 i = 0; 3650 /* This is the work marker for refresh */ 3651 rxr->next_to_refresh = i; 3652 } 3653update: 3654 if (cleaned != -1) /* If we refreshed some, bump tail */ 3655 E1000_WRITE_REG(&adapter->hw, 3656 E1000_RDT(rxr->me), cleaned); 3657 return; 3658} 3659 3660 3661/********************************************************************* 3662 * 3663 * Allocate memory for rx_buffer structures. Since we use one 3664 * rx_buffer per received packet, the maximum number of rx_buffer's 3665 * that we'll need is equal to the number of receive descriptors 3666 * that we've allocated. 3667 * 3668 **********************************************************************/ 3669static int 3670igb_allocate_receive_buffers(struct rx_ring *rxr) 3671{ 3672 struct adapter *adapter = rxr->adapter; 3673 device_t dev = adapter->dev; 3674 struct igb_rx_buf *rxbuf; 3675 int i, bsize, error; 3676 3677 bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc; 3678 if (!(rxr->rx_buffers = 3679 (struct igb_rx_buf *) malloc(bsize, 3680 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3681 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3682 error = ENOMEM; 3683 goto fail; 3684 } 3685 3686 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 3687 1, 0, /* alignment, bounds */ 3688 BUS_SPACE_MAXADDR, /* lowaddr */ 3689 BUS_SPACE_MAXADDR, /* highaddr */ 3690 NULL, NULL, /* filter, filterarg */ 3691 MSIZE, /* maxsize */ 3692 1, /* nsegments */ 3693 MSIZE, /* maxsegsize */ 3694 0, /* flags */ 3695 NULL, /* lockfunc */ 3696 NULL, /* lockfuncarg */ 3697 &rxr->htag))) { 3698 device_printf(dev, "Unable to create RX DMA tag\n"); 3699 goto fail; 3700 } 3701 3702 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 3703 1, 0, /* alignment, bounds */ 3704 BUS_SPACE_MAXADDR, /* lowaddr */ 3705 BUS_SPACE_MAXADDR, /* highaddr */ 3706 NULL, NULL, /* filter, filterarg */ 3707 MJUMPAGESIZE, /* maxsize */ 3708 1, /* nsegments */ 3709 MJUMPAGESIZE, /* maxsegsize */ 3710 0, /* flags */ 3711 NULL, /* lockfunc */ 3712 NULL, /* lockfuncarg */ 3713 &rxr->ptag))) { 3714 device_printf(dev, "Unable to create RX payload DMA tag\n"); 3715 goto fail; 3716 } 3717 3718 for (i = 0; i < adapter->num_rx_desc; i++) { 3719 rxbuf = &rxr->rx_buffers[i]; 3720 error = bus_dmamap_create(rxr->htag, 3721 BUS_DMA_NOWAIT, &rxbuf->hmap); 3722 if (error) { 3723 device_printf(dev, 3724 "Unable to create RX head DMA maps\n"); 3725 goto fail; 3726 } 3727 error = bus_dmamap_create(rxr->ptag, 3728 BUS_DMA_NOWAIT, &rxbuf->pmap); 3729 if (error) { 3730 device_printf(dev, 3731 "Unable to create RX packet DMA maps\n"); 3732 goto fail; 3733 } 3734 } 3735 3736 return (0); 3737 3738fail: 3739 /* Frees all, but can handle partial completion */ 3740 igb_free_receive_structures(adapter); 3741 return (error); 3742} 3743 3744 3745static void 3746igb_free_receive_ring(struct rx_ring *rxr) 3747{ 3748 struct adapter *adapter; 3749 struct igb_rx_buf *rxbuf; 3750 int i; 3751 3752 adapter = rxr->adapter; 3753 for (i = 0; i < adapter->num_rx_desc; i++) { 3754 rxbuf = &rxr->rx_buffers[i]; 3755 if (rxbuf->m_head != NULL) { 3756 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3757 BUS_DMASYNC_POSTREAD); 3758 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 3759 rxbuf->m_head->m_flags |= M_PKTHDR; 3760 m_freem(rxbuf->m_head); 3761 } 3762 if (rxbuf->m_pack != NULL) { 3763 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3764 BUS_DMASYNC_POSTREAD); 3765 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 3766 rxbuf->m_pack->m_flags |= M_PKTHDR; 3767 m_freem(rxbuf->m_pack); 3768 } 3769 rxbuf->m_head = NULL; 3770 rxbuf->m_pack = NULL; 3771 } 3772} 3773 3774 3775/********************************************************************* 3776 * 3777 * Initialize a receive ring and its buffers. 3778 * 3779 **********************************************************************/ 3780static int 3781igb_setup_receive_ring(struct rx_ring *rxr) 3782{ 3783 struct adapter *adapter; 3784 struct ifnet *ifp; 3785 device_t dev; 3786 struct igb_rx_buf *rxbuf; 3787 bus_dma_segment_t pseg[1], hseg[1]; 3788 struct lro_ctrl *lro = &rxr->lro; 3789 int rsize, nsegs, error = 0; 3790 3791 adapter = rxr->adapter; 3792 dev = adapter->dev; 3793 ifp = adapter->ifp; 3794 3795 /* Clear the ring contents */ 3796 IGB_RX_LOCK(rxr); 3797 rsize = roundup2(adapter->num_rx_desc * 3798 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 3799 bzero((void *)rxr->rx_base, rsize); 3800 3801 /* 3802 ** Free current RX buffer structures and their mbufs 3803 */ 3804 igb_free_receive_ring(rxr); 3805 3806 /* Configure for header split? */ 3807 if (igb_header_split) 3808 rxr->hdr_split = TRUE; 3809 3810 /* Now replenish the ring mbufs */ 3811 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3812 struct mbuf *mh, *mp; 3813 3814 rxbuf = &rxr->rx_buffers[j]; 3815 if (rxr->hdr_split == FALSE) 3816 goto skip_head; 3817 3818 /* First the header */ 3819 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 3820 if (rxbuf->m_head == NULL) { 3821 error = ENOBUFS; 3822 goto fail; 3823 } 3824 m_adj(rxbuf->m_head, ETHER_ALIGN); 3825 mh = rxbuf->m_head; 3826 mh->m_len = mh->m_pkthdr.len = MHLEN; 3827 mh->m_flags |= M_PKTHDR; 3828 /* Get the memory mapping */ 3829 error = bus_dmamap_load_mbuf_sg(rxr->htag, 3830 rxbuf->hmap, rxbuf->m_head, hseg, 3831 &nsegs, BUS_DMA_NOWAIT); 3832 if (error != 0) /* Nothing elegant to do here */ 3833 goto fail; 3834 bus_dmamap_sync(rxr->htag, 3835 rxbuf->hmap, BUS_DMASYNC_PREREAD); 3836 /* Update descriptor */ 3837 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 3838 3839skip_head: 3840 /* Now the payload cluster */ 3841 rxbuf->m_pack = m_getjcl(M_DONTWAIT, MT_DATA, 3842 M_PKTHDR, adapter->rx_mbuf_sz); 3843 if (rxbuf->m_pack == NULL) { 3844 error = ENOBUFS; 3845 goto fail; 3846 } 3847 mp = rxbuf->m_pack; 3848 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3849 /* Get the memory mapping */ 3850 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 3851 rxbuf->pmap, mp, pseg, 3852 &nsegs, BUS_DMA_NOWAIT); 3853 if (error != 0) 3854 goto fail; 3855 bus_dmamap_sync(rxr->ptag, 3856 rxbuf->pmap, BUS_DMASYNC_PREREAD); 3857 /* Update descriptor */ 3858 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 3859 } 3860 3861 /* Setup our descriptor indices */ 3862 rxr->next_to_check = 0; 3863 rxr->next_to_refresh = 0; 3864 rxr->lro_enabled = FALSE; 3865 rxr->rx_split_packets = 0; 3866 rxr->rx_bytes = 0; 3867 3868 rxr->fmp = NULL; 3869 rxr->lmp = NULL; 3870 rxr->discard = FALSE; 3871 3872 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3873 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3874 3875 /* 3876 ** Now set up the LRO interface, we 3877 ** also only do head split when LRO 3878 ** is enabled, since so often they 3879 ** are undesireable in similar setups. 3880 */ 3881 if (ifp->if_capenable & IFCAP_LRO) { 3882 int err = tcp_lro_init(lro); 3883 if (err) { 3884 device_printf(dev, "LRO Initialization failed!\n"); 3885 goto fail; 3886 } 3887 INIT_DEBUGOUT("RX LRO Initialized\n"); 3888 rxr->lro_enabled = TRUE; 3889 lro->ifp = adapter->ifp; 3890 } 3891 3892 IGB_RX_UNLOCK(rxr); 3893 return (0); 3894 3895fail: 3896 igb_free_receive_ring(rxr); 3897 IGB_RX_UNLOCK(rxr); 3898 return (error); 3899} 3900 3901/********************************************************************* 3902 * 3903 * Initialize all receive rings. 3904 * 3905 **********************************************************************/ 3906static int 3907igb_setup_receive_structures(struct adapter *adapter) 3908{ 3909 struct rx_ring *rxr = adapter->rx_rings; 3910 int i; 3911 3912 for (i = 0; i < adapter->num_queues; i++, rxr++) 3913 if (igb_setup_receive_ring(rxr)) 3914 goto fail; 3915 3916 return (0); 3917fail: 3918 /* 3919 * Free RX buffers allocated so far, we will only handle 3920 * the rings that completed, the failing case will have 3921 * cleaned up for itself. 'i' is the endpoint. 3922 */ 3923 for (int j = 0; j > i; ++j) { 3924 rxr = &adapter->rx_rings[i]; 3925 igb_free_receive_ring(rxr); 3926 } 3927 3928 return (ENOBUFS); 3929} 3930 3931/********************************************************************* 3932 * 3933 * Enable receive unit. 3934 * 3935 **********************************************************************/ 3936static void 3937igb_initialize_receive_units(struct adapter *adapter) 3938{ 3939 struct rx_ring *rxr = adapter->rx_rings; 3940 struct ifnet *ifp = adapter->ifp; 3941 struct e1000_hw *hw = &adapter->hw; 3942 u32 rctl, rxcsum, psize, srrctl = 0; 3943 3944 INIT_DEBUGOUT("igb_initialize_receive_unit: begin"); 3945 3946 /* 3947 * Make sure receives are disabled while setting 3948 * up the descriptor ring 3949 */ 3950 rctl = E1000_READ_REG(hw, E1000_RCTL); 3951 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3952 3953 /* 3954 ** Set up for header split 3955 */ 3956 if (rxr->hdr_split) { 3957 /* Use a standard mbuf for the header */ 3958 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; 3959 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3960 } else 3961 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 3962 3963 /* 3964 ** Set up for jumbo frames 3965 */ 3966 if (ifp->if_mtu > ETHERMTU) { 3967 rctl |= E1000_RCTL_LPE; 3968 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 3969 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 3970 3971 /* Set maximum packet len */ 3972 psize = adapter->max_frame_size; 3973 /* are we on a vlan? */ 3974 if (adapter->ifp->if_vlantrunk != NULL) 3975 psize += VLAN_TAG_SIZE; 3976 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); 3977 } else { 3978 rctl &= ~E1000_RCTL_LPE; 3979 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 3980 rctl |= E1000_RCTL_SZ_2048; 3981 } 3982 3983 /* Setup the Base and Length of the Rx Descriptor Rings */ 3984 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3985 u64 bus_addr = rxr->rxdma.dma_paddr; 3986 u32 rxdctl; 3987 3988 E1000_WRITE_REG(hw, E1000_RDLEN(i), 3989 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 3990 E1000_WRITE_REG(hw, E1000_RDBAH(i), 3991 (uint32_t)(bus_addr >> 32)); 3992 E1000_WRITE_REG(hw, E1000_RDBAL(i), 3993 (uint32_t)bus_addr); 3994 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); 3995 /* Enable this Queue */ 3996 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3997 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 3998 rxdctl &= 0xFFF00000; 3999 rxdctl |= IGB_RX_PTHRESH; 4000 rxdctl |= IGB_RX_HTHRESH << 8; 4001 rxdctl |= IGB_RX_WTHRESH << 16; 4002 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); 4003 } 4004 4005 /* 4006 ** Setup for RX MultiQueue 4007 */ 4008 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 4009 if (adapter->num_queues >1) { 4010 u32 random[10], mrqc, shift = 0; 4011 union igb_reta { 4012 u32 dword; 4013 u8 bytes[4]; 4014 } reta; 4015 4016 arc4rand(&random, sizeof(random), 0); 4017 if (adapter->hw.mac.type == e1000_82575) 4018 shift = 6; 4019 /* Warning FM follows */ 4020 for (int i = 0; i < 128; i++) { 4021 reta.bytes[i & 3] = 4022 (i % adapter->num_queues) << shift; 4023 if ((i & 3) == 3) 4024 E1000_WRITE_REG(hw, 4025 E1000_RETA(i >> 2), reta.dword); 4026 } 4027 /* Now fill in hash table */ 4028 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 4029 for (int i = 0; i < 10; i++) 4030 E1000_WRITE_REG_ARRAY(hw, 4031 E1000_RSSRK(0), i, random[i]); 4032 4033 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 4034 E1000_MRQC_RSS_FIELD_IPV4_TCP); 4035 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | 4036 E1000_MRQC_RSS_FIELD_IPV6_TCP); 4037 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | 4038 E1000_MRQC_RSS_FIELD_IPV6_UDP); 4039 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 4040 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 4041 4042 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 4043 4044 /* 4045 ** NOTE: Receive Full-Packet Checksum Offload 4046 ** is mutually exclusive with Multiqueue. However 4047 ** this is not the same as TCP/IP checksums which 4048 ** still work. 4049 */ 4050 rxcsum |= E1000_RXCSUM_PCSD; 4051#if __FreeBSD_version >= 800000 4052 /* For SCTP Offload */ 4053 if ((hw->mac.type == e1000_82576) 4054 && (ifp->if_capenable & IFCAP_RXCSUM)) 4055 rxcsum |= E1000_RXCSUM_CRCOFL; 4056#endif 4057 } else { 4058 /* Non RSS setup */ 4059 if (ifp->if_capenable & IFCAP_RXCSUM) { 4060 rxcsum |= E1000_RXCSUM_IPPCSE; 4061#if __FreeBSD_version >= 800000 4062 if (adapter->hw.mac.type == e1000_82576) 4063 rxcsum |= E1000_RXCSUM_CRCOFL; 4064#endif 4065 } else 4066 rxcsum &= ~E1000_RXCSUM_TUOFL; 4067 } 4068 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 4069 4070 /* Setup the Receive Control Register */ 4071 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4072 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 4073 E1000_RCTL_RDMTS_HALF | 4074 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4075 /* Strip CRC bytes. */ 4076 rctl |= E1000_RCTL_SECRC; 4077 /* Make sure VLAN Filters are off */ 4078 rctl &= ~E1000_RCTL_VFE; 4079 /* Don't store bad packets */ 4080 rctl &= ~E1000_RCTL_SBP; 4081 4082 /* Enable Receives */ 4083 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4084 4085 /* 4086 * Setup the HW Rx Head and Tail Descriptor Pointers 4087 * - needs to be after enable 4088 */ 4089 for (int i = 0; i < adapter->num_queues; i++) { 4090 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 4091 E1000_WRITE_REG(hw, E1000_RDT(i), 4092 adapter->num_rx_desc - 1); 4093 } 4094 return; 4095} 4096 4097/********************************************************************* 4098 * 4099 * Free receive rings. 4100 * 4101 **********************************************************************/ 4102static void 4103igb_free_receive_structures(struct adapter *adapter) 4104{ 4105 struct rx_ring *rxr = adapter->rx_rings; 4106 4107 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4108 struct lro_ctrl *lro = &rxr->lro; 4109 igb_free_receive_buffers(rxr); 4110 tcp_lro_free(lro); 4111 igb_dma_free(adapter, &rxr->rxdma); 4112 } 4113 4114 free(adapter->rx_rings, M_DEVBUF); 4115} 4116 4117/********************************************************************* 4118 * 4119 * Free receive ring data structures. 4120 * 4121 **********************************************************************/ 4122static void 4123igb_free_receive_buffers(struct rx_ring *rxr) 4124{ 4125 struct adapter *adapter = rxr->adapter; 4126 struct igb_rx_buf *rxbuf; 4127 int i; 4128 4129 INIT_DEBUGOUT("free_receive_structures: begin"); 4130 4131 /* Cleanup any existing buffers */ 4132 if (rxr->rx_buffers != NULL) { 4133 for (i = 0; i < adapter->num_rx_desc; i++) { 4134 rxbuf = &rxr->rx_buffers[i]; 4135 if (rxbuf->m_head != NULL) { 4136 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 4137 BUS_DMASYNC_POSTREAD); 4138 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 4139 rxbuf->m_head->m_flags |= M_PKTHDR; 4140 m_freem(rxbuf->m_head); 4141 } 4142 if (rxbuf->m_pack != NULL) { 4143 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 4144 BUS_DMASYNC_POSTREAD); 4145 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 4146 rxbuf->m_pack->m_flags |= M_PKTHDR; 4147 m_freem(rxbuf->m_pack); 4148 } 4149 rxbuf->m_head = NULL; 4150 rxbuf->m_pack = NULL; 4151 if (rxbuf->hmap != NULL) { 4152 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 4153 rxbuf->hmap = NULL; 4154 } 4155 if (rxbuf->pmap != NULL) { 4156 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4157 rxbuf->pmap = NULL; 4158 } 4159 } 4160 if (rxr->rx_buffers != NULL) { 4161 free(rxr->rx_buffers, M_DEVBUF); 4162 rxr->rx_buffers = NULL; 4163 } 4164 } 4165 4166 if (rxr->htag != NULL) { 4167 bus_dma_tag_destroy(rxr->htag); 4168 rxr->htag = NULL; 4169 } 4170 if (rxr->ptag != NULL) { 4171 bus_dma_tag_destroy(rxr->ptag); 4172 rxr->ptag = NULL; 4173 } 4174} 4175 4176static __inline void 4177igb_rx_discard(struct rx_ring *rxr, int i) 4178{ 4179 struct adapter *adapter = rxr->adapter; 4180 struct igb_rx_buf *rbuf; 4181 struct mbuf *mh, *mp; 4182 4183 rbuf = &rxr->rx_buffers[i]; 4184 if (rxr->fmp != NULL) { 4185 rxr->fmp->m_flags |= M_PKTHDR; 4186 m_freem(rxr->fmp); 4187 rxr->fmp = NULL; 4188 rxr->lmp = NULL; 4189 } 4190 4191 mh = rbuf->m_head; 4192 mp = rbuf->m_pack; 4193 4194 /* Reuse loaded DMA map and just update mbuf chain */ 4195 mh->m_len = MHLEN; 4196 mh->m_flags |= M_PKTHDR; 4197 mh->m_next = NULL; 4198 4199 mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz; 4200 mp->m_data = mp->m_ext.ext_buf; 4201 mp->m_next = NULL; 4202 return; 4203} 4204 4205static __inline void 4206igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4207{ 4208 4209 /* 4210 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 4211 * should be computed by hardware. Also it should not have VLAN tag in 4212 * ethernet header. 4213 */ 4214 if (rxr->lro_enabled && 4215 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 4216 (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && 4217 (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) == 4218 (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) && 4219 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4220 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4221 /* 4222 * Send to the stack if: 4223 ** - LRO not enabled, or 4224 ** - no LRO resources, or 4225 ** - lro enqueue fails 4226 */ 4227 if (rxr->lro.lro_cnt != 0) 4228 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4229 return; 4230 } 4231 IGB_RX_UNLOCK(rxr); 4232 (*ifp->if_input)(ifp, m); 4233 IGB_RX_LOCK(rxr); 4234} 4235 4236/********************************************************************* 4237 * 4238 * This routine executes in interrupt context. It replenishes 4239 * the mbufs in the descriptor and sends data which has been 4240 * dma'ed into host memory to upper layer. 4241 * 4242 * We loop at most count times if count is > 0, or until done if 4243 * count < 0. 4244 * 4245 * Return TRUE if more to clean, FALSE otherwise 4246 *********************************************************************/ 4247static bool 4248igb_rxeof(struct igb_queue *que, int count, int *done) 4249{ 4250 struct adapter *adapter = que->adapter; 4251 struct rx_ring *rxr = que->rxr; 4252 struct ifnet *ifp = adapter->ifp; 4253 struct lro_ctrl *lro = &rxr->lro; 4254 struct lro_entry *queued; 4255 int i, processed = 0, rxdone = 0; 4256 u32 ptype, staterr = 0; 4257 union e1000_adv_rx_desc *cur; 4258 4259 IGB_RX_LOCK(rxr); 4260 /* Sync the ring. */ 4261 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4262 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4263 4264 /* Main clean loop */ 4265 for (i = rxr->next_to_check; count != 0;) { 4266 struct mbuf *sendmp, *mh, *mp; 4267 struct igb_rx_buf *rxbuf; 4268 u16 hlen, plen, hdr, vtag; 4269 bool eop = FALSE; 4270 4271 cur = &rxr->rx_base[i]; 4272 staterr = le32toh(cur->wb.upper.status_error); 4273 if ((staterr & E1000_RXD_STAT_DD) == 0) 4274 break; 4275 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 4276 break; 4277 count--; 4278 sendmp = mh = mp = NULL; 4279 cur->wb.upper.status_error = 0; 4280 rxbuf = &rxr->rx_buffers[i]; 4281 plen = le16toh(cur->wb.upper.length); 4282 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; 4283 vtag = le16toh(cur->wb.upper.vlan); 4284 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4285 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); 4286 4287 /* Make sure all segments of a bad packet are discarded */ 4288 if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) || 4289 (rxr->discard)) { 4290 ifp->if_ierrors++; 4291 ++rxr->rx_discarded; 4292 if (!eop) /* Catch subsequent segs */ 4293 rxr->discard = TRUE; 4294 else 4295 rxr->discard = FALSE; 4296 igb_rx_discard(rxr, i); 4297 goto next_desc; 4298 } 4299 4300 /* 4301 ** The way the hardware is configured to 4302 ** split, it will ONLY use the header buffer 4303 ** when header split is enabled, otherwise we 4304 ** get normal behavior, ie, both header and 4305 ** payload are DMA'd into the payload buffer. 4306 ** 4307 ** The fmp test is to catch the case where a 4308 ** packet spans multiple descriptors, in that 4309 ** case only the first header is valid. 4310 */ 4311 if (rxr->hdr_split && rxr->fmp == NULL) { 4312 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >> 4313 E1000_RXDADV_HDRBUFLEN_SHIFT; 4314 if (hlen > IGB_HDR_BUF) 4315 hlen = IGB_HDR_BUF; 4316 /* Handle the header mbuf */ 4317 mh = rxr->rx_buffers[i].m_head; 4318 mh->m_len = hlen; 4319 /* clear buf info for refresh */ 4320 rxbuf->m_head = NULL; 4321 /* 4322 ** Get the payload length, this 4323 ** could be zero if its a small 4324 ** packet. 4325 */ 4326 if (plen > 0) { 4327 mp = rxr->rx_buffers[i].m_pack; 4328 mp->m_len = plen; 4329 mh->m_next = mp; 4330 /* clear buf info for refresh */ 4331 rxbuf->m_pack = NULL; 4332 rxr->rx_split_packets++; 4333 } 4334 } else { 4335 /* 4336 ** Either no header split, or a 4337 ** secondary piece of a fragmented 4338 ** split packet. 4339 */ 4340 mh = rxr->rx_buffers[i].m_pack; 4341 mh->m_len = plen; 4342 /* clear buf info for refresh */ 4343 rxbuf->m_pack = NULL; 4344 } 4345 4346 ++processed; /* So we know when to refresh */ 4347 4348 /* Initial frame - setup */ 4349 if (rxr->fmp == NULL) { 4350 mh->m_pkthdr.len = mh->m_len; 4351 /* Store the first mbuf */ 4352 rxr->fmp = mh; 4353 rxr->lmp = mh; 4354 if (mp != NULL) { 4355 /* Add payload if split */ 4356 mh->m_pkthdr.len += mp->m_len; 4357 rxr->lmp = mh->m_next; 4358 } 4359 } else { 4360 /* Chain mbuf's together */ 4361 rxr->lmp->m_next = mh; 4362 rxr->lmp = rxr->lmp->m_next; 4363 rxr->fmp->m_pkthdr.len += mh->m_len; 4364 } 4365 4366 if (eop) { 4367 rxr->fmp->m_pkthdr.rcvif = ifp; 4368 ifp->if_ipackets++; 4369 rxr->rx_packets++; 4370 /* capture data for AIM */ 4371 rxr->packets++; 4372 rxr->bytes += rxr->fmp->m_pkthdr.len; 4373 rxr->rx_bytes += rxr->fmp->m_pkthdr.len; 4374 4375 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 4376 igb_rx_checksum(staterr, rxr->fmp, ptype); 4377 4378 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 4379 (staterr & E1000_RXD_STAT_VP) != 0) { 4380 rxr->fmp->m_pkthdr.ether_vtag = vtag; 4381 rxr->fmp->m_flags |= M_VLANTAG; 4382 } 4383#if __FreeBSD_version >= 800000 4384 rxr->fmp->m_pkthdr.flowid = que->msix; 4385 rxr->fmp->m_flags |= M_FLOWID; 4386#endif 4387 sendmp = rxr->fmp; 4388 /* Make sure to set M_PKTHDR. */ 4389 sendmp->m_flags |= M_PKTHDR; 4390 rxr->fmp = NULL; 4391 rxr->lmp = NULL; 4392 } 4393 4394next_desc: 4395 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4396 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4397 4398 /* Advance our pointers to the next descriptor. */ 4399 if (++i == adapter->num_rx_desc) 4400 i = 0; 4401 /* 4402 ** Send to the stack or LRO 4403 */ 4404 if (sendmp != NULL) { 4405 rxr->next_to_check = i; 4406 igb_rx_input(rxr, ifp, sendmp, ptype); 4407 i = rxr->next_to_check; 4408 rxdone++; 4409 } 4410 4411 /* Every 8 descriptors we go to refresh mbufs */ 4412 if (processed == 8) { 4413 igb_refresh_mbufs(rxr, i); 4414 processed = 0; 4415 } 4416 } 4417 4418 /* Catch any remainders */ 4419 if (processed != 0) { 4420 igb_refresh_mbufs(rxr, i); 4421 processed = 0; 4422 } 4423 4424 rxr->next_to_check = i; 4425 4426 /* 4427 * Flush any outstanding LRO work 4428 */ 4429 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4430 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4431 tcp_lro_flush(lro, queued); 4432 } 4433 4434 IGB_RX_UNLOCK(rxr); 4435 4436 if (done != NULL) 4437 *done = rxdone; 4438 4439 /* 4440 ** We still have cleaning to do? 4441 ** Schedule another interrupt if so. 4442 */ 4443 if ((staterr & E1000_RXD_STAT_DD) != 0) 4444 return (TRUE); 4445 4446 return (FALSE); 4447} 4448 4449/********************************************************************* 4450 * 4451 * Verify that the hardware indicated that the checksum is valid. 4452 * Inform the stack about the status of checksum so that stack 4453 * doesn't spend time verifying the checksum. 4454 * 4455 *********************************************************************/ 4456static void 4457igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype) 4458{ 4459 u16 status = (u16)staterr; 4460 u8 errors = (u8) (staterr >> 24); 4461 int sctp; 4462 4463 /* Ignore Checksum bit is set */ 4464 if (status & E1000_RXD_STAT_IXSM) { 4465 mp->m_pkthdr.csum_flags = 0; 4466 return; 4467 } 4468 4469 if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && 4470 (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) 4471 sctp = 1; 4472 else 4473 sctp = 0; 4474 if (status & E1000_RXD_STAT_IPCS) { 4475 /* Did it pass? */ 4476 if (!(errors & E1000_RXD_ERR_IPE)) { 4477 /* IP Checksum Good */ 4478 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4479 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4480 } else 4481 mp->m_pkthdr.csum_flags = 0; 4482 } 4483 4484 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { 4485 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4486#if __FreeBSD_version >= 800000 4487 if (sctp) /* reassign */ 4488 type = CSUM_SCTP_VALID; 4489#endif 4490 /* Did it pass? */ 4491 if (!(errors & E1000_RXD_ERR_TCPE)) { 4492 mp->m_pkthdr.csum_flags |= type; 4493 if (sctp == 0) 4494 mp->m_pkthdr.csum_data = htons(0xffff); 4495 } 4496 } 4497 return; 4498} 4499 4500/* 4501 * This routine is run via an vlan 4502 * config EVENT 4503 */ 4504static void 4505igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4506{ 4507 struct adapter *adapter = ifp->if_softc; 4508 u32 index, bit; 4509 4510 if (ifp->if_softc != arg) /* Not our event */ 4511 return; 4512 4513 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4514 return; 4515 4516 index = (vtag >> 5) & 0x7F; 4517 bit = vtag & 0x1F; 4518 igb_shadow_vfta[index] |= (1 << bit); 4519 ++adapter->num_vlans; 4520 /* Re-init to load the changes */ 4521 igb_init(adapter); 4522} 4523 4524/* 4525 * This routine is run via an vlan 4526 * unconfig EVENT 4527 */ 4528static void 4529igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4530{ 4531 struct adapter *adapter = ifp->if_softc; 4532 u32 index, bit; 4533 4534 if (ifp->if_softc != arg) 4535 return; 4536 4537 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4538 return; 4539 4540 index = (vtag >> 5) & 0x7F; 4541 bit = vtag & 0x1F; 4542 igb_shadow_vfta[index] &= ~(1 << bit); 4543 --adapter->num_vlans; 4544 /* Re-init to load the changes */ 4545 igb_init(adapter); 4546} 4547 4548static void 4549igb_setup_vlan_hw_support(struct adapter *adapter) 4550{ 4551 struct e1000_hw *hw = &adapter->hw; 4552 u32 reg; 4553 4554 /* 4555 ** We get here thru init_locked, meaning 4556 ** a soft reset, this has already cleared 4557 ** the VFTA and other state, so if there 4558 ** have been no vlan's registered do nothing. 4559 */ 4560 if (adapter->num_vlans == 0) 4561 return; 4562 4563 /* 4564 ** A soft reset zero's out the VFTA, so 4565 ** we need to repopulate it now. 4566 */ 4567 for (int i = 0; i < IGB_VFTA_SIZE; i++) 4568 if (igb_shadow_vfta[i] != 0) { 4569 if (hw->mac.type == e1000_vfadapt) 4570 e1000_vfta_set_vf(hw, igb_shadow_vfta[i], TRUE); 4571 else 4572 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, 4573 i, igb_shadow_vfta[i]); 4574 } 4575 4576 if (hw->mac.type == e1000_vfadapt) 4577 e1000_rlpml_set_vf(hw, 4578 adapter->max_frame_size + VLAN_TAG_SIZE); 4579 else { 4580 reg = E1000_READ_REG(hw, E1000_CTRL); 4581 reg |= E1000_CTRL_VME; 4582 E1000_WRITE_REG(hw, E1000_CTRL, reg); 4583 4584 /* Enable the Filter Table */ 4585 reg = E1000_READ_REG(hw, E1000_RCTL); 4586 reg &= ~E1000_RCTL_CFIEN; 4587 reg |= E1000_RCTL_VFE; 4588 E1000_WRITE_REG(hw, E1000_RCTL, reg); 4589 4590 /* Update the frame size */ 4591 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4592 adapter->max_frame_size + VLAN_TAG_SIZE); 4593 } 4594} 4595 4596static void 4597igb_enable_intr(struct adapter *adapter) 4598{ 4599 /* With RSS set up what to auto clear */ 4600 if (adapter->msix_mem) { 4601 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 4602 adapter->eims_mask); 4603 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, 4604 adapter->eims_mask); 4605 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, 4606 adapter->eims_mask); 4607 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 4608 E1000_IMS_LSC); 4609 } else { 4610 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 4611 IMS_ENABLE_MASK); 4612 } 4613 E1000_WRITE_FLUSH(&adapter->hw); 4614 4615 return; 4616} 4617 4618static void 4619igb_disable_intr(struct adapter *adapter) 4620{ 4621 if (adapter->msix_mem) { 4622 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); 4623 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); 4624 } 4625 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); 4626 E1000_WRITE_FLUSH(&adapter->hw); 4627 return; 4628} 4629 4630/* 4631 * Bit of a misnomer, what this really means is 4632 * to enable OS management of the system... aka 4633 * to disable special hardware management features 4634 */ 4635static void 4636igb_init_manageability(struct adapter *adapter) 4637{ 4638 if (adapter->has_manage) { 4639 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4640 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4641 4642 /* disable hardware interception of ARP */ 4643 manc &= ~(E1000_MANC_ARP_EN); 4644 4645 /* enable receiving management packets to the host */ 4646 manc |= E1000_MANC_EN_MNG2HOST; 4647 manc2h |= 1 << 5; /* Mng Port 623 */ 4648 manc2h |= 1 << 6; /* Mng Port 664 */ 4649 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4650 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4651 } 4652} 4653 4654/* 4655 * Give control back to hardware management 4656 * controller if there is one. 4657 */ 4658static void 4659igb_release_manageability(struct adapter *adapter) 4660{ 4661 if (adapter->has_manage) { 4662 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4663 4664 /* re-enable hardware interception of ARP */ 4665 manc |= E1000_MANC_ARP_EN; 4666 manc &= ~E1000_MANC_EN_MNG2HOST; 4667 4668 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4669 } 4670} 4671 4672/* 4673 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 4674 * For ASF and Pass Through versions of f/w this means that 4675 * the driver is loaded. 4676 * 4677 */ 4678static void 4679igb_get_hw_control(struct adapter *adapter) 4680{ 4681 u32 ctrl_ext; 4682 4683 if (adapter->hw.mac.type == e1000_vfadapt) 4684 return; 4685 4686 /* Let firmware know the driver has taken over */ 4687 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4688 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4689 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4690} 4691 4692/* 4693 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 4694 * For ASF and Pass Through versions of f/w this means that the 4695 * driver is no longer loaded. 4696 * 4697 */ 4698static void 4699igb_release_hw_control(struct adapter *adapter) 4700{ 4701 u32 ctrl_ext; 4702 4703 if (adapter->hw.mac.type == e1000_vfadapt) 4704 return; 4705 4706 /* Let firmware taken over control of h/w */ 4707 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4708 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4709 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4710} 4711 4712static int 4713igb_is_valid_ether_addr(uint8_t *addr) 4714{ 4715 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4716 4717 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4718 return (FALSE); 4719 } 4720 4721 return (TRUE); 4722} 4723 4724 4725/* 4726 * Enable PCI Wake On Lan capability 4727 */ 4728static void 4729igb_enable_wakeup(device_t dev) 4730{ 4731 u16 cap, status; 4732 u8 id; 4733 4734 /* First find the capabilities pointer*/ 4735 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 4736 /* Read the PM Capabilities */ 4737 id = pci_read_config(dev, cap, 1); 4738 if (id != PCIY_PMG) /* Something wrong */ 4739 return; 4740 /* OK, we have the power capabilities, so 4741 now get the status register */ 4742 cap += PCIR_POWER_STATUS; 4743 status = pci_read_config(dev, cap, 2); 4744 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4745 pci_write_config(dev, cap, status, 2); 4746 return; 4747} 4748 4749static void 4750igb_led_func(void *arg, int onoff) 4751{ 4752 struct adapter *adapter = arg; 4753 4754 IGB_CORE_LOCK(adapter); 4755 if (onoff) { 4756 e1000_setup_led(&adapter->hw); 4757 e1000_led_on(&adapter->hw); 4758 } else { 4759 e1000_led_off(&adapter->hw); 4760 e1000_cleanup_led(&adapter->hw); 4761 } 4762 IGB_CORE_UNLOCK(adapter); 4763} 4764 4765/********************************************************************** 4766 * 4767 * Update the board statistics counters. 4768 * 4769 **********************************************************************/ 4770static void 4771igb_update_stats_counters(struct adapter *adapter) 4772{ 4773 struct ifnet *ifp; 4774 struct e1000_hw *hw = &adapter->hw; 4775 struct e1000_hw_stats *stats; 4776 4777 /* 4778 ** The virtual function adapter has only a 4779 ** small controlled set of stats, do only 4780 ** those and return. 4781 */ 4782 if (adapter->hw.mac.type == e1000_vfadapt) { 4783 igb_update_vf_stats_counters(adapter); 4784 return; 4785 } 4786 4787 stats = (struct e1000_hw_stats *)adapter->stats; 4788 4789 if(adapter->hw.phy.media_type == e1000_media_type_copper || 4790 (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU)) { 4791 stats->symerrs += 4792 E1000_READ_REG(hw,E1000_SYMERRS); 4793 stats->sec += E1000_READ_REG(hw, E1000_SEC); 4794 } 4795 4796 stats->crcerrs += E1000_READ_REG(hw, E1000_CRCERRS); 4797 stats->mpc += E1000_READ_REG(hw, E1000_MPC); 4798 stats->scc += E1000_READ_REG(hw, E1000_SCC); 4799 stats->ecol += E1000_READ_REG(hw, E1000_ECOL); 4800 4801 stats->mcc += E1000_READ_REG(hw, E1000_MCC); 4802 stats->latecol += E1000_READ_REG(hw, E1000_LATECOL); 4803 stats->colc += E1000_READ_REG(hw, E1000_COLC); 4804 stats->dc += E1000_READ_REG(hw, E1000_DC); 4805 stats->rlec += E1000_READ_REG(hw, E1000_RLEC); 4806 stats->xonrxc += E1000_READ_REG(hw, E1000_XONRXC); 4807 stats->xontxc += E1000_READ_REG(hw, E1000_XONTXC); 4808 stats->xoffrxc += E1000_READ_REG(hw, E1000_XOFFRXC); 4809 stats->xofftxc += E1000_READ_REG(hw, E1000_XOFFTXC); 4810 stats->fcruc += E1000_READ_REG(hw, E1000_FCRUC); 4811 stats->prc64 += E1000_READ_REG(hw, E1000_PRC64); 4812 stats->prc127 += E1000_READ_REG(hw, E1000_PRC127); 4813 stats->prc255 += E1000_READ_REG(hw, E1000_PRC255); 4814 stats->prc511 += E1000_READ_REG(hw, E1000_PRC511); 4815 stats->prc1023 += E1000_READ_REG(hw, E1000_PRC1023); 4816 stats->prc1522 += E1000_READ_REG(hw, E1000_PRC1522); 4817 stats->gprc += E1000_READ_REG(hw, E1000_GPRC); 4818 stats->bprc += E1000_READ_REG(hw, E1000_BPRC); 4819 stats->mprc += E1000_READ_REG(hw, E1000_MPRC); 4820 stats->gptc += E1000_READ_REG(hw, E1000_GPTC); 4821 4822 /* For the 64-bit byte counters the low dword must be read first. */ 4823 /* Both registers clear on the read of the high dword */ 4824 4825 stats->gorc += E1000_READ_REG(hw, E1000_GORCL) + 4826 ((u64)E1000_READ_REG(hw, E1000_GORCH) << 32); 4827 stats->gotc += E1000_READ_REG(hw, E1000_GOTCL) + 4828 ((u64)E1000_READ_REG(hw, E1000_GOTCH) << 32) ; 4829 4830 stats->rnbc += E1000_READ_REG(hw, E1000_RNBC); 4831 stats->ruc += E1000_READ_REG(hw, E1000_RUC); 4832 stats->rfc += E1000_READ_REG(hw, E1000_RFC); 4833 stats->roc += E1000_READ_REG(hw, E1000_ROC); 4834 stats->rjc += E1000_READ_REG(hw, E1000_RJC); 4835 4836 stats->tor += E1000_READ_REG(hw, E1000_TORH); 4837 stats->tot += E1000_READ_REG(hw, E1000_TOTH); 4838 4839 stats->tpr += E1000_READ_REG(hw, E1000_TPR); 4840 stats->tpt += E1000_READ_REG(hw, E1000_TPT); 4841 stats->ptc64 += E1000_READ_REG(hw, E1000_PTC64); 4842 stats->ptc127 += E1000_READ_REG(hw, E1000_PTC127); 4843 stats->ptc255 += E1000_READ_REG(hw, E1000_PTC255); 4844 stats->ptc511 += E1000_READ_REG(hw, E1000_PTC511); 4845 stats->ptc1023 += E1000_READ_REG(hw, E1000_PTC1023); 4846 stats->ptc1522 += E1000_READ_REG(hw, E1000_PTC1522); 4847 stats->mptc += E1000_READ_REG(hw, E1000_MPTC); 4848 stats->bptc += E1000_READ_REG(hw, E1000_BPTC); 4849 4850 /* Interrupt Counts */ 4851 4852 stats->iac += E1000_READ_REG(hw, E1000_IAC); 4853 stats->icrxptc += E1000_READ_REG(hw, E1000_ICRXPTC); 4854 stats->icrxatc += E1000_READ_REG(hw, E1000_ICRXATC); 4855 stats->ictxptc += E1000_READ_REG(hw, E1000_ICTXPTC); 4856 stats->ictxatc += E1000_READ_REG(hw, E1000_ICTXATC); 4857 stats->ictxqec += E1000_READ_REG(hw, E1000_ICTXQEC); 4858 stats->ictxqmtc += E1000_READ_REG(hw, E1000_ICTXQMTC); 4859 stats->icrxdmtc += E1000_READ_REG(hw, E1000_ICRXDMTC); 4860 stats->icrxoc += E1000_READ_REG(hw, E1000_ICRXOC); 4861 4862 /* Host to Card Statistics */ 4863 4864 stats->cbtmpc += E1000_READ_REG(hw, E1000_CBTMPC); 4865 stats->htdpmc += E1000_READ_REG(hw, E1000_HTDPMC); 4866 stats->cbrdpc += E1000_READ_REG(hw, E1000_CBRDPC); 4867 stats->cbrmpc += E1000_READ_REG(hw, E1000_CBRMPC); 4868 stats->rpthc += E1000_READ_REG(hw, E1000_RPTHC); 4869 stats->hgptc += E1000_READ_REG(hw, E1000_HGPTC); 4870 stats->htcbdpc += E1000_READ_REG(hw, E1000_HTCBDPC); 4871 stats->hgorc += (E1000_READ_REG(hw, E1000_HGORCL) + 4872 ((u64)E1000_READ_REG(hw, E1000_HGORCH) << 32)); 4873 stats->hgotc += (E1000_READ_REG(hw, E1000_HGOTCL) + 4874 ((u64)E1000_READ_REG(hw, E1000_HGOTCH) << 32)); 4875 stats->lenerrs += E1000_READ_REG(hw, E1000_LENERRS); 4876 stats->scvpc += E1000_READ_REG(hw, E1000_SCVPC); 4877 stats->hrmpc += E1000_READ_REG(hw, E1000_HRMPC); 4878 4879 stats->algnerrc += E1000_READ_REG(hw, E1000_ALGNERRC); 4880 stats->rxerrc += E1000_READ_REG(hw, E1000_RXERRC); 4881 stats->tncrs += E1000_READ_REG(hw, E1000_TNCRS); 4882 stats->cexterr += E1000_READ_REG(hw, E1000_CEXTERR); 4883 stats->tsctc += E1000_READ_REG(hw, E1000_TSCTC); 4884 stats->tsctfc += E1000_READ_REG(hw, E1000_TSCTFC); 4885 4886 ifp = adapter->ifp; 4887 ifp->if_collisions = stats->colc; 4888 4889 /* Rx Errors */ 4890 ifp->if_ierrors = adapter->dropped_pkts + stats->rxerrc + 4891 stats->crcerrs + stats->algnerrc + 4892 stats->ruc + stats->roc + stats->mpc + stats->cexterr; 4893 4894 /* Tx Errors */ 4895 ifp->if_oerrors = stats->ecol + 4896 stats->latecol + adapter->watchdog_events; 4897 4898 /* Driver specific counters */ 4899 adapter->device_control = E1000_READ_REG(hw, E1000_CTRL); 4900 adapter->rx_control = E1000_READ_REG(hw, E1000_RCTL); 4901 adapter->int_mask = E1000_READ_REG(hw, E1000_IMS); 4902 adapter->eint_mask = E1000_READ_REG(hw, E1000_EIMS); 4903 adapter->packet_buf_alloc_tx = 4904 ((E1000_READ_REG(hw, E1000_PBA) & 0xffff0000) >> 16); 4905 adapter->packet_buf_alloc_rx = 4906 (E1000_READ_REG(hw, E1000_PBA) & 0xffff); 4907} 4908 4909 4910/********************************************************************** 4911 * 4912 * Initialize the VF board statistics counters. 4913 * 4914 **********************************************************************/ 4915static void 4916igb_vf_init_stats(struct adapter *adapter) 4917{ 4918 struct e1000_hw *hw = &adapter->hw; 4919 struct e1000_vf_stats *stats; 4920 4921 stats = (struct e1000_vf_stats *)adapter->stats; 4922 if (stats == NULL) 4923 return; 4924 stats->last_gprc = E1000_READ_REG(hw, E1000_VFGPRC); 4925 stats->last_gorc = E1000_READ_REG(hw, E1000_VFGORC); 4926 stats->last_gptc = E1000_READ_REG(hw, E1000_VFGPTC); 4927 stats->last_gotc = E1000_READ_REG(hw, E1000_VFGOTC); 4928 stats->last_mprc = E1000_READ_REG(hw, E1000_VFMPRC); 4929} 4930 4931/********************************************************************** 4932 * 4933 * Update the VF board statistics counters. 4934 * 4935 **********************************************************************/ 4936static void 4937igb_update_vf_stats_counters(struct adapter *adapter) 4938{ 4939 struct e1000_hw *hw = &adapter->hw; 4940 struct e1000_vf_stats *stats; 4941 4942 if (adapter->link_speed == 0) 4943 return; 4944 4945 stats = (struct e1000_vf_stats *)adapter->stats; 4946 4947 UPDATE_VF_REG(E1000_VFGPRC, 4948 stats->last_gprc, stats->gprc); 4949 UPDATE_VF_REG(E1000_VFGORC, 4950 stats->last_gorc, stats->gorc); 4951 UPDATE_VF_REG(E1000_VFGPTC, 4952 stats->last_gptc, stats->gptc); 4953 UPDATE_VF_REG(E1000_VFGOTC, 4954 stats->last_gotc, stats->gotc); 4955 UPDATE_VF_REG(E1000_VFMPRC, 4956 stats->last_mprc, stats->mprc); 4957} 4958 4959 4960/** igb_sysctl_tdh_handler - Handler function 4961 * Retrieves the TDH value from the hardware 4962 */ 4963static int 4964igb_sysctl_tdh_handler(SYSCTL_HANDLER_ARGS) 4965{ 4966 int error; 4967 4968 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1); 4969 if (!txr) return 0; 4970 4971 unsigned val = E1000_READ_REG(&txr->adapter->hw, E1000_TDH(txr->me)); 4972 error = sysctl_handle_int(oidp, &val, 0, req); 4973 if (error || !req->newptr) 4974 return error; 4975 return 0; 4976} 4977 4978/** igb_sysctl_tdt_handler - Handler function 4979 * Retrieves the TDT value from the hardware 4980 */ 4981static int 4982igb_sysctl_tdt_handler(SYSCTL_HANDLER_ARGS) 4983{ 4984 int error; 4985 4986 struct tx_ring *txr = ((struct tx_ring *)oidp->oid_arg1); 4987 if (!txr) return 0; 4988 4989 unsigned val = E1000_READ_REG(&txr->adapter->hw, E1000_TDT(txr->me)); 4990 error = sysctl_handle_int(oidp, &val, 0, req); 4991 if (error || !req->newptr) 4992 return error; 4993 return 0; 4994} 4995 4996/** igb_sysctl_rdh_handler - Handler function 4997 * Retrieves the RDH value from the hardware 4998 */ 4999static int 5000igb_sysctl_rdh_handler(SYSCTL_HANDLER_ARGS) 5001{ 5002 int error; 5003 5004 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1); 5005 if (!rxr) return 0; 5006 5007 unsigned val = E1000_READ_REG(&rxr->adapter->hw, E1000_RDH(rxr->me)); 5008 error = sysctl_handle_int(oidp, &val, 0, req); 5009 if (error || !req->newptr) 5010 return error; 5011 return 0; 5012} 5013 5014/** igb_sysctl_rdt_handler - Handler function 5015 * Retrieves the RDT value from the hardware 5016 */ 5017static int 5018igb_sysctl_rdt_handler(SYSCTL_HANDLER_ARGS) 5019{ 5020 int error; 5021 5022 struct rx_ring *rxr = ((struct rx_ring *)oidp->oid_arg1); 5023 if (!rxr) return 0; 5024 5025 unsigned val = E1000_READ_REG(&rxr->adapter->hw, E1000_RDT(rxr->me)); 5026 error = sysctl_handle_int(oidp, &val, 0, req); 5027 if (error || !req->newptr) 5028 return error; 5029 return 0; 5030} 5031 5032/* 5033 * Add sysctl variables, one per statistic, to the system. 5034 */ 5035static void 5036igb_add_hw_stats(struct adapter *adapter) 5037{ 5038 5039 device_t dev = adapter->dev; 5040 5041 struct tx_ring *txr = adapter->tx_rings; 5042 struct rx_ring *rxr = adapter->rx_rings; 5043 5044 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 5045 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 5046 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 5047 struct e1000_hw_stats *stats = adapter->stats; 5048 5049 struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node; 5050 struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list; 5051 5052#define QUEUE_NAME_LEN 32 5053 char namebuf[QUEUE_NAME_LEN]; 5054 5055 /* Driver Statistics */ 5056 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "link_irq", 5057 CTLFLAG_RD, &adapter->link_irq, 0, 5058 "Link MSIX IRQ Handled"); 5059 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 5060 CTLFLAG_RD, &adapter->dropped_pkts, 5061 "Driver dropped packets"); 5062 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", 5063 CTLFLAG_RD, &adapter->no_tx_dma_setup, 5064 "Driver tx dma failure in xmit"); 5065 5066 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control", 5067 CTLFLAG_RD, &adapter->device_control, 5068 "Device Control Register"); 5069 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control", 5070 CTLFLAG_RD, &adapter->rx_control, 5071 "Receiver Control Register"); 5072 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask", 5073 CTLFLAG_RD, &adapter->int_mask, 5074 "Interrupt Mask"); 5075 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask", 5076 CTLFLAG_RD, &adapter->eint_mask, 5077 "Extended Interrupt Mask"); 5078 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc", 5079 CTLFLAG_RD, &adapter->packet_buf_alloc_tx, 5080 "Transmit Buffer Packet Allocation"); 5081 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc", 5082 CTLFLAG_RD, &adapter->packet_buf_alloc_rx, 5083 "Receive Buffer Packet Allocation"); 5084 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", 5085 CTLFLAG_RD, &adapter->hw.fc.high_water, 0, 5086 "Flow Control High Watermark"); 5087 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 5088 CTLFLAG_RD, &adapter->hw.fc.low_water, 0, 5089 "Flow Control Low Watermark"); 5090 5091 for (int i = 0; i < adapter->num_queues; i++, txr++) { 5092 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 5093 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 5094 CTLFLAG_RD, NULL, "Queue Name"); 5095 queue_list = SYSCTL_CHILDREN(queue_node); 5096 5097 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_head", 5098 CTLFLAG_RD, txr, sizeof(txr), 5099 igb_sysctl_tdh_handler, "IU", 5100 "Transmit Descriptor Head"); 5101 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "txd_tail", 5102 CTLFLAG_RD, txr, sizeof(txr), 5103 igb_sysctl_tdt_handler, "IU", 5104 "Transmit Descriptor Tail"); 5105 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", 5106 CTLFLAG_RD, &txr->no_desc_avail, 5107 "Queue No Descriptor Available"); 5108 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_packets", 5109 CTLFLAG_RD, &txr->tx_packets, 5110 "Queue Packets Transmitted"); 5111 } 5112 5113 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 5114 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 5115 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 5116 CTLFLAG_RD, NULL, "Queue Name"); 5117 queue_list = SYSCTL_CHILDREN(queue_node); 5118 5119 struct lro_ctrl *lro = &rxr->lro; 5120 5121 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 5122 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 5123 CTLFLAG_RD, NULL, "Queue Name"); 5124 queue_list = SYSCTL_CHILDREN(queue_node); 5125 5126 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_head", 5127 CTLFLAG_RD, rxr, sizeof(rxr), 5128 igb_sysctl_rdh_handler, "IU", 5129 "Receive Descriptor Head"); 5130 SYSCTL_ADD_PROC(ctx, queue_list, OID_AUTO, "rxd_tail", 5131 CTLFLAG_RD, rxr, sizeof(rxr), 5132 igb_sysctl_rdt_handler, "IU", 5133 "Receive Descriptor Tail"); 5134 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets", 5135 CTLFLAG_RD, &rxr->rx_packets, 5136 "Queue Packets Received"); 5137 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes", 5138 CTLFLAG_RD, &rxr->rx_bytes, 5139 "Queue Bytes Received"); 5140 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued", 5141 CTLFLAG_RD, &lro->lro_queued, 0, 5142 "LRO Queued"); 5143 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed", 5144 CTLFLAG_RD, &lro->lro_flushed, 0, 5145 "LRO Flushed"); 5146 } 5147 5148 /* MAC stats get the own sub node */ 5149 5150 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 5151 CTLFLAG_RD, NULL, "MAC Statistics"); 5152 stat_list = SYSCTL_CHILDREN(stat_node); 5153 5154 /* 5155 ** VF adapter has a very limited set of stats 5156 ** since its not managing the metal, so to speak. 5157 */ 5158 if (adapter->hw.mac.type == e1000_vfadapt) { 5159 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 5160 CTLFLAG_RD, &stats->gprc, 5161 "Good Packets Received"); 5162 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 5163 CTLFLAG_RD, &stats->gptc, 5164 "Good Packets Transmitted"); 5165 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 5166 CTLFLAG_RD, &stats->gorc, 5167 "Good Octets Received"); 5168 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octest_txd", 5169 CTLFLAG_RD, &stats->gotc, 5170 "Good Octest Transmitted"); 5171 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 5172 CTLFLAG_RD, &stats->mprc, 5173 "Multicast Packets Received"); 5174 return; 5175 } 5176 5177 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll", 5178 CTLFLAG_RD, &stats->ecol, 5179 "Excessive collisions"); 5180 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll", 5181 CTLFLAG_RD, &stats->scc, 5182 "Single collisions"); 5183 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 5184 CTLFLAG_RD, &stats->mcc, 5185 "Multiple collisions"); 5186 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll", 5187 CTLFLAG_RD, &stats->latecol, 5188 "Late collisions"); 5189 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count", 5190 CTLFLAG_RD, &stats->colc, 5191 "Collision Count"); 5192 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors", 5193 CTLFLAG_RD, &stats->symerrs, 5194 "Symbol Errors"); 5195 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors", 5196 CTLFLAG_RD, &stats->sec, 5197 "Sequence Errors"); 5198 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count", 5199 CTLFLAG_RD, &stats->dc, 5200 "Defer Count"); 5201 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets", 5202 CTLFLAG_RD, &stats->mpc, 5203 "Missed Packets"); 5204 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", 5205 CTLFLAG_RD, &stats->rnbc, 5206 "Receive No Buffers"); 5207 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize", 5208 CTLFLAG_RD, &stats->ruc, 5209 "Receive Undersize"); 5210 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 5211 CTLFLAG_RD, &stats->rfc, 5212 "Fragmented Packets Received "); 5213 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize", 5214 CTLFLAG_RD, &stats->roc, 5215 "Oversized Packets Received"); 5216 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber", 5217 CTLFLAG_RD, &stats->rjc, 5218 "Recevied Jabber"); 5219 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs", 5220 CTLFLAG_RD, &stats->rxerrc, 5221 "Receive Errors"); 5222 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs", 5223 CTLFLAG_RD, &stats->crcerrs, 5224 "CRC errors"); 5225 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs", 5226 CTLFLAG_RD, &stats->algnerrc, 5227 "Alignment Errors"); 5228 /* On 82575 these are collision counts */ 5229 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", 5230 CTLFLAG_RD, &stats->cexterr, 5231 "Collision/Carrier extension errors"); 5232 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_overruns", 5233 CTLFLAG_RD, &adapter->rx_overruns, 5234 "RX overruns"); 5235 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "watchdog_timeouts", 5236 CTLFLAG_RD, &adapter->watchdog_events, 5237 "Watchdog timeouts"); 5238 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 5239 CTLFLAG_RD, &stats->xonrxc, 5240 "XON Received"); 5241 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd", 5242 CTLFLAG_RD, &stats->xontxc, 5243 "XON Transmitted"); 5244 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 5245 CTLFLAG_RD, &stats->xoffrxc, 5246 "XOFF Received"); 5247 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 5248 CTLFLAG_RD, &stats->xofftxc, 5249 "XOFF Transmitted"); 5250 /* Packet Reception Stats */ 5251 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", 5252 CTLFLAG_RD, &stats->tpr, 5253 "Total Packets Received "); 5254 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 5255 CTLFLAG_RD, &stats->gprc, 5256 "Good Packets Received"); 5257 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", 5258 CTLFLAG_RD, &stats->bprc, 5259 "Broadcast Packets Received"); 5260 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 5261 CTLFLAG_RD, &stats->mprc, 5262 "Multicast Packets Received"); 5263 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 5264 CTLFLAG_RD, &stats->prc64, 5265 "64 byte frames received "); 5266 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 5267 CTLFLAG_RD, &stats->prc127, 5268 "65-127 byte frames received"); 5269 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 5270 CTLFLAG_RD, &stats->prc255, 5271 "128-255 byte frames received"); 5272 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 5273 CTLFLAG_RD, &stats->prc511, 5274 "256-511 byte frames received"); 5275 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 5276 CTLFLAG_RD, &stats->prc1023, 5277 "512-1023 byte frames received"); 5278 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 5279 CTLFLAG_RD, &stats->prc1522, 5280 "1023-1522 byte frames received"); 5281 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 5282 CTLFLAG_RD, &stats->gorc, 5283 "Good Octets Received"); 5284 5285 /* Packet Transmission Stats */ 5286 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octest_txd", 5287 CTLFLAG_RD, &stats->gotc, 5288 "Good Octest Transmitted"); 5289 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 5290 CTLFLAG_RD, &stats->tpt, 5291 "Total Packets Transmitted"); 5292 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 5293 CTLFLAG_RD, &stats->gptc, 5294 "Good Packets Transmitted"); 5295 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 5296 CTLFLAG_RD, &stats->bptc, 5297 "Broadcast Packets Transmitted"); 5298 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 5299 CTLFLAG_RD, &stats->mptc, 5300 "Multicast Packets Transmitted"); 5301 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 5302 CTLFLAG_RD, &stats->ptc64, 5303 "64 byte frames transmitted "); 5304 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 5305 CTLFLAG_RD, &stats->ptc127, 5306 "65-127 byte frames transmitted"); 5307 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 5308 CTLFLAG_RD, &stats->ptc255, 5309 "128-255 byte frames transmitted"); 5310 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 5311 CTLFLAG_RD, &stats->ptc511, 5312 "256-511 byte frames transmitted"); 5313 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 5314 CTLFLAG_RD, &stats->ptc1023, 5315 "512-1023 byte frames transmitted"); 5316 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 5317 CTLFLAG_RD, &stats->ptc1522, 5318 "1024-1522 byte frames transmitted"); 5319 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd", 5320 CTLFLAG_RD, &stats->tsctc, 5321 "TSO Contexts Transmitted"); 5322 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", 5323 CTLFLAG_RD, &stats->tsctfc, 5324 "TSO Contexts Failed"); 5325 5326 5327 /* Interrupt Stats */ 5328 5329 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 5330 CTLFLAG_RD, NULL, "Interrupt Statistics"); 5331 int_list = SYSCTL_CHILDREN(int_node); 5332 5333 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts", 5334 CTLFLAG_RD, &stats->iac, 5335 "Interrupt Assertion Count"); 5336 5337 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer", 5338 CTLFLAG_RD, &stats->icrxptc, 5339 "Interrupt Cause Rx Pkt Timer Expire Count"); 5340 5341 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer", 5342 CTLFLAG_RD, &stats->icrxatc, 5343 "Interrupt Cause Rx Abs Timer Expire Count"); 5344 5345 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer", 5346 CTLFLAG_RD, &stats->ictxptc, 5347 "Interrupt Cause Tx Pkt Timer Expire Count"); 5348 5349 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer", 5350 CTLFLAG_RD, &stats->ictxatc, 5351 "Interrupt Cause Tx Abs Timer Expire Count"); 5352 5353 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty", 5354 CTLFLAG_RD, &stats->ictxqec, 5355 "Interrupt Cause Tx Queue Empty Count"); 5356 5357 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh", 5358 CTLFLAG_RD, &stats->ictxqmtc, 5359 "Interrupt Cause Tx Queue Min Thresh Count"); 5360 5361 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", 5362 CTLFLAG_RD, &stats->icrxdmtc, 5363 "Interrupt Cause Rx Desc Min Thresh Count"); 5364 5365 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun", 5366 CTLFLAG_RD, &stats->icrxoc, 5367 "Interrupt Cause Receiver Overrun Count"); 5368 5369 /* Host to Card Stats */ 5370 5371 host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host", 5372 CTLFLAG_RD, NULL, 5373 "Host to Card Statistics"); 5374 5375 host_list = SYSCTL_CHILDREN(host_node); 5376 5377 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt", 5378 CTLFLAG_RD, &stats->cbtmpc, 5379 "Circuit Breaker Tx Packet Count"); 5380 5381 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard", 5382 CTLFLAG_RD, &stats->htdpmc, 5383 "Host Transmit Discarded Packets"); 5384 5385 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt", 5386 CTLFLAG_RD, &stats->rpthc, 5387 "Rx Packets To Host"); 5388 5389 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts", 5390 CTLFLAG_RD, &stats->cbrmpc, 5391 "Circuit Breaker Rx Packet Count"); 5392 5393 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop", 5394 CTLFLAG_RD, &stats->cbrdpc, 5395 "Circuit Breaker Rx Dropped Count"); 5396 5397 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt", 5398 CTLFLAG_RD, &stats->hgptc, 5399 "Host Good Packets Tx Count"); 5400 5401 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop", 5402 CTLFLAG_RD, &stats->htcbdpc, 5403 "Host Tx Circuit Breaker Dropped Count"); 5404 5405 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes", 5406 CTLFLAG_RD, &stats->hgorc, 5407 "Host Good Octets Received Count"); 5408 5409 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes", 5410 CTLFLAG_RD, &stats->hgotc, 5411 "Host Good Octets Transmit Count"); 5412 5413 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors", 5414 CTLFLAG_RD, &stats->lenerrs, 5415 "Length Errors"); 5416 5417 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt", 5418 CTLFLAG_RD, &stats->scvpc, 5419 "SerDes/SGMII Code Violation Pkt Count"); 5420 5421 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed", 5422 CTLFLAG_RD, &stats->hrmpc, 5423 "Header Redirection Missed Packet Count"); 5424} 5425 5426 5427/********************************************************************** 5428 * 5429 * This routine provides a way to dump out the adapter eeprom, 5430 * often a useful debug/service tool. This only dumps the first 5431 * 32 words, stuff that matters is in that extent. 5432 * 5433 **********************************************************************/ 5434static int 5435igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) 5436{ 5437 struct adapter *adapter; 5438 int error; 5439 int result; 5440 5441 result = -1; 5442 error = sysctl_handle_int(oidp, &result, 0, req); 5443 5444 if (error || !req->newptr) 5445 return (error); 5446 5447 /* 5448 * This value will cause a hex dump of the 5449 * first 32 16-bit words of the EEPROM to 5450 * the screen. 5451 */ 5452 if (result == 1) { 5453 adapter = (struct adapter *)arg1; 5454 igb_print_nvm_info(adapter); 5455 } 5456 5457 return (error); 5458} 5459 5460static void 5461igb_print_nvm_info(struct adapter *adapter) 5462{ 5463 u16 eeprom_data; 5464 int i, j, row = 0; 5465 5466 /* Its a bit crude, but it gets the job done */ 5467 printf("\nInterface EEPROM Dump:\n"); 5468 printf("Offset\n0x0000 "); 5469 for (i = 0, j = 0; i < 32; i++, j++) { 5470 if (j == 8) { /* Make the offset block */ 5471 j = 0; ++row; 5472 printf("\n0x00%x0 ",row); 5473 } 5474 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 5475 printf("%04x ", eeprom_data); 5476 } 5477 printf("\n"); 5478} 5479 5480static void 5481igb_add_rx_process_limit(struct adapter *adapter, const char *name, 5482 const char *description, int *limit, int value) 5483{ 5484 *limit = value; 5485 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5486 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5487 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5488} 5489