if_igb.c revision 209512
1/****************************************************************************** 2 3 Copyright (c) 2001-2010, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ 33/*$FreeBSD: head/sys/dev/e1000/if_igb.c 209512 2010-06-24 21:17:58Z gnn $*/ 34 35 36#ifdef HAVE_KERNEL_OPTION_HEADERS 37#include "opt_device_polling.h" 38#include "opt_inet.h" 39#include "opt_altq.h" 40#endif 41 42#include <sys/param.h> 43#include <sys/systm.h> 44#if __FreeBSD_version >= 800000 45#include <sys/buf_ring.h> 46#endif 47#include <sys/bus.h> 48#include <sys/endian.h> 49#include <sys/kernel.h> 50#include <sys/kthread.h> 51#include <sys/malloc.h> 52#include <sys/mbuf.h> 53#include <sys/module.h> 54#include <sys/rman.h> 55#include <sys/socket.h> 56#include <sys/sockio.h> 57#include <sys/sysctl.h> 58#include <sys/taskqueue.h> 59#include <sys/eventhandler.h> 60#include <sys/pcpu.h> 61#include <sys/smp.h> 62#include <machine/smp.h> 63#include <machine/bus.h> 64#include <machine/resource.h> 65 66#include <net/bpf.h> 67#include <net/ethernet.h> 68#include <net/if.h> 69#include <net/if_arp.h> 70#include <net/if_dl.h> 71#include <net/if_media.h> 72 73#include <net/if_types.h> 74#include <net/if_vlan_var.h> 75 76#include <netinet/in_systm.h> 77#include <netinet/in.h> 78#include <netinet/if_ether.h> 79#include <netinet/ip.h> 80#include <netinet/ip6.h> 81#include <netinet/tcp.h> 82#include <netinet/tcp_lro.h> 83#include <netinet/udp.h> 84 85#include <machine/in_cksum.h> 86#include <dev/led/led.h> 87#include <dev/pci/pcivar.h> 88#include <dev/pci/pcireg.h> 89 90#include "e1000_api.h" 91#include "e1000_82575.h" 92#include "if_igb.h" 93 94/********************************************************************* 95 * Set this to one to display debug statistics 96 *********************************************************************/ 97int igb_display_debug_stats = 0; 98 99/********************************************************************* 100 * Driver version: 101 *********************************************************************/ 102char igb_driver_version[] = "version - 1.9.6"; 103 104 105/********************************************************************* 106 * PCI Device ID Table 107 * 108 * Used by probe to select devices to load on 109 * Last field stores an index into e1000_strings 110 * Last entry must be all 0s 111 * 112 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 113 *********************************************************************/ 114 115static igb_vendor_info_t igb_vendor_info_array[] = 116{ 117 { 0x8086, E1000_DEV_ID_82575EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 118 { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, 119 PCI_ANY_ID, PCI_ANY_ID, 0}, 120 { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, 121 PCI_ANY_ID, PCI_ANY_ID, 0}, 122 { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0}, 123 { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0}, 124 { 0x8086, E1000_DEV_ID_82576_NS_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 125 { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 126 { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 127 { 0x8086, E1000_DEV_ID_82576_SERDES_QUAD, 128 PCI_ANY_ID, PCI_ANY_ID, 0}, 129 { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER, 130 PCI_ANY_ID, PCI_ANY_ID, 0}, 131 { 0x8086, E1000_DEV_ID_82580_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 132 { 0x8086, E1000_DEV_ID_82580_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 133 { 0x8086, E1000_DEV_ID_82580_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 134 { 0x8086, E1000_DEV_ID_82580_SGMII, PCI_ANY_ID, PCI_ANY_ID, 0}, 135 { 0x8086, E1000_DEV_ID_82580_COPPER_DUAL, 136 PCI_ANY_ID, PCI_ANY_ID, 0}, 137 /* required last entry */ 138 { 0, 0, 0, 0, 0} 139}; 140 141/********************************************************************* 142 * Table of branding strings for all supported NICs. 143 *********************************************************************/ 144 145static char *igb_strings[] = { 146 "Intel(R) PRO/1000 Network Connection" 147}; 148 149/********************************************************************* 150 * Function prototypes 151 *********************************************************************/ 152static int igb_probe(device_t); 153static int igb_attach(device_t); 154static int igb_detach(device_t); 155static int igb_shutdown(device_t); 156static int igb_suspend(device_t); 157static int igb_resume(device_t); 158static void igb_start(struct ifnet *); 159static void igb_start_locked(struct tx_ring *, struct ifnet *ifp); 160#if __FreeBSD_version >= 800000 161static int igb_mq_start(struct ifnet *, struct mbuf *); 162static int igb_mq_start_locked(struct ifnet *, 163 struct tx_ring *, struct mbuf *); 164static void igb_qflush(struct ifnet *); 165#endif 166static int igb_ioctl(struct ifnet *, u_long, caddr_t); 167static void igb_init(void *); 168static void igb_init_locked(struct adapter *); 169static void igb_stop(void *); 170static void igb_media_status(struct ifnet *, struct ifmediareq *); 171static int igb_media_change(struct ifnet *); 172static void igb_identify_hardware(struct adapter *); 173static int igb_allocate_pci_resources(struct adapter *); 174static int igb_allocate_msix(struct adapter *); 175static int igb_allocate_legacy(struct adapter *); 176static int igb_setup_msix(struct adapter *); 177static void igb_free_pci_resources(struct adapter *); 178static void igb_local_timer(void *); 179static void igb_reset(struct adapter *); 180static void igb_setup_interface(device_t, struct adapter *); 181static int igb_allocate_queues(struct adapter *); 182static void igb_configure_queues(struct adapter *); 183 184static int igb_allocate_transmit_buffers(struct tx_ring *); 185static void igb_setup_transmit_structures(struct adapter *); 186static void igb_setup_transmit_ring(struct tx_ring *); 187static void igb_initialize_transmit_units(struct adapter *); 188static void igb_free_transmit_structures(struct adapter *); 189static void igb_free_transmit_buffers(struct tx_ring *); 190 191static int igb_allocate_receive_buffers(struct rx_ring *); 192static int igb_setup_receive_structures(struct adapter *); 193static int igb_setup_receive_ring(struct rx_ring *); 194static void igb_initialize_receive_units(struct adapter *); 195static void igb_free_receive_structures(struct adapter *); 196static void igb_free_receive_buffers(struct rx_ring *); 197static void igb_free_receive_ring(struct rx_ring *); 198 199static void igb_enable_intr(struct adapter *); 200static void igb_disable_intr(struct adapter *); 201static void igb_update_stats_counters(struct adapter *); 202static bool igb_txeof(struct tx_ring *); 203 204static __inline void igb_rx_discard(struct rx_ring *, int); 205static __inline void igb_rx_input(struct rx_ring *, 206 struct ifnet *, struct mbuf *, u32); 207 208static bool igb_rxeof(struct igb_queue *, int, int *); 209static void igb_rx_checksum(u32, struct mbuf *, u32); 210static int igb_tx_ctx_setup(struct tx_ring *, struct mbuf *); 211static bool igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 212static void igb_set_promisc(struct adapter *); 213static void igb_disable_promisc(struct adapter *); 214static void igb_set_multi(struct adapter *); 215static void igb_update_link_status(struct adapter *); 216static void igb_refresh_mbufs(struct rx_ring *, int); 217 218static void igb_register_vlan(void *, struct ifnet *, u16); 219static void igb_unregister_vlan(void *, struct ifnet *, u16); 220static void igb_setup_vlan_hw_support(struct adapter *); 221 222static int igb_xmit(struct tx_ring *, struct mbuf **); 223static int igb_dma_malloc(struct adapter *, bus_size_t, 224 struct igb_dma_alloc *, int); 225static void igb_dma_free(struct adapter *, struct igb_dma_alloc *); 226static int igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS); 227static void igb_print_nvm_info(struct adapter *); 228static int igb_is_valid_ether_addr(u8 *); 229static void igb_add_hw_stats(struct adapter *adapter); 230/* Management and WOL Support */ 231static void igb_init_manageability(struct adapter *); 232static void igb_release_manageability(struct adapter *); 233static void igb_get_hw_control(struct adapter *); 234static void igb_release_hw_control(struct adapter *); 235static void igb_enable_wakeup(device_t); 236static void igb_led_func(void *, int); 237 238static int igb_irq_fast(void *); 239static void igb_add_rx_process_limit(struct adapter *, const char *, 240 const char *, int *, int); 241static void igb_handle_que(void *context, int pending); 242static void igb_handle_link(void *context, int pending); 243 244/* These are MSIX only irq handlers */ 245static void igb_msix_que(void *); 246static void igb_msix_link(void *); 247 248#ifdef DEVICE_POLLING 249static poll_handler_t igb_poll; 250#endif /* POLLING */ 251 252/********************************************************************* 253 * FreeBSD Device Interface Entry Points 254 *********************************************************************/ 255 256static device_method_t igb_methods[] = { 257 /* Device interface */ 258 DEVMETHOD(device_probe, igb_probe), 259 DEVMETHOD(device_attach, igb_attach), 260 DEVMETHOD(device_detach, igb_detach), 261 DEVMETHOD(device_shutdown, igb_shutdown), 262 DEVMETHOD(device_suspend, igb_suspend), 263 DEVMETHOD(device_resume, igb_resume), 264 {0, 0} 265}; 266 267static driver_t igb_driver = { 268 "igb", igb_methods, sizeof(struct adapter), 269}; 270 271static devclass_t igb_devclass; 272DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); 273MODULE_DEPEND(igb, pci, 1, 1, 1); 274MODULE_DEPEND(igb, ether, 1, 1, 1); 275 276/********************************************************************* 277 * Tunable default values. 278 *********************************************************************/ 279 280/* Descriptor defaults */ 281static int igb_rxd = IGB_DEFAULT_RXD; 282static int igb_txd = IGB_DEFAULT_TXD; 283TUNABLE_INT("hw.igb.rxd", &igb_rxd); 284TUNABLE_INT("hw.igb.txd", &igb_txd); 285 286/* 287** AIM: Adaptive Interrupt Moderation 288** which means that the interrupt rate 289** is varied over time based on the 290** traffic for that interrupt vector 291*/ 292static int igb_enable_aim = TRUE; 293TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim); 294 295/* 296 * MSIX should be the default for best performance, 297 * but this allows it to be forced off for testing. 298 */ 299static int igb_enable_msix = 1; 300TUNABLE_INT("hw.igb.enable_msix", &igb_enable_msix); 301 302/* 303 * Header split has seemed to be beneficial in 304 * many circumstances tested, however there have 305 * been some stability issues, so the default is 306 * off. 307 */ 308static bool igb_header_split = FALSE; 309TUNABLE_INT("hw.igb.hdr_split", &igb_header_split); 310 311/* 312** This will autoconfigure based on 313** the number of CPUs if left at 0. 314*/ 315static int igb_num_queues = 0; 316TUNABLE_INT("hw.igb.num_queues", &igb_num_queues); 317 318/* How many packets rxeof tries to clean at a time */ 319static int igb_rx_process_limit = 100; 320TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit); 321 322/* Flow control setting - default to FULL */ 323static int igb_fc_setting = e1000_fc_full; 324TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting); 325 326/* 327** Shadow VFTA table, this is needed because 328** the real filter table gets cleared during 329** a soft reset and the driver needs to be able 330** to repopulate it. 331*/ 332static u32 igb_shadow_vfta[IGB_VFTA_SIZE]; 333 334 335/********************************************************************* 336 * Device identification routine 337 * 338 * igb_probe determines if the driver should be loaded on 339 * adapter based on PCI vendor/device id of the adapter. 340 * 341 * return BUS_PROBE_DEFAULT on success, positive on failure 342 *********************************************************************/ 343 344static int 345igb_probe(device_t dev) 346{ 347 char adapter_name[60]; 348 uint16_t pci_vendor_id = 0; 349 uint16_t pci_device_id = 0; 350 uint16_t pci_subvendor_id = 0; 351 uint16_t pci_subdevice_id = 0; 352 igb_vendor_info_t *ent; 353 354 INIT_DEBUGOUT("igb_probe: begin"); 355 356 pci_vendor_id = pci_get_vendor(dev); 357 if (pci_vendor_id != IGB_VENDOR_ID) 358 return (ENXIO); 359 360 pci_device_id = pci_get_device(dev); 361 pci_subvendor_id = pci_get_subvendor(dev); 362 pci_subdevice_id = pci_get_subdevice(dev); 363 364 ent = igb_vendor_info_array; 365 while (ent->vendor_id != 0) { 366 if ((pci_vendor_id == ent->vendor_id) && 367 (pci_device_id == ent->device_id) && 368 369 ((pci_subvendor_id == ent->subvendor_id) || 370 (ent->subvendor_id == PCI_ANY_ID)) && 371 372 ((pci_subdevice_id == ent->subdevice_id) || 373 (ent->subdevice_id == PCI_ANY_ID))) { 374 sprintf(adapter_name, "%s %s", 375 igb_strings[ent->index], 376 igb_driver_version); 377 device_set_desc_copy(dev, adapter_name); 378 return (BUS_PROBE_DEFAULT); 379 } 380 ent++; 381 } 382 383 return (ENXIO); 384} 385 386/********************************************************************* 387 * Device initialization routine 388 * 389 * The attach entry point is called when the driver is being loaded. 390 * This routine identifies the type of hardware, allocates all resources 391 * and initializes the hardware. 392 * 393 * return 0 on success, positive on failure 394 *********************************************************************/ 395 396static int 397igb_attach(device_t dev) 398{ 399 struct adapter *adapter; 400 int error = 0; 401 u16 eeprom_data; 402 403 INIT_DEBUGOUT("igb_attach: begin"); 404 405 adapter = device_get_softc(dev); 406 adapter->dev = adapter->osdep.dev = dev; 407 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 408 409 /* SYSCTL stuff */ 410 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 411 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 412 OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 413 igb_sysctl_nvm_info, "I", "NVM Information"); 414 415 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 416 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 417 OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW, 418 &igb_fc_setting, 0, "Flow Control"); 419 420 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 421 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 422 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 423 &igb_enable_aim, 1, "Interrupt Moderation"); 424 425 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 426 427 /* Determine hardware and mac info */ 428 igb_identify_hardware(adapter); 429 430 /* Setup PCI resources */ 431 if (igb_allocate_pci_resources(adapter)) { 432 device_printf(dev, "Allocation of PCI resources failed\n"); 433 error = ENXIO; 434 goto err_pci; 435 } 436 437 /* Do Shared Code initialization */ 438 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 439 device_printf(dev, "Setup of Shared code failed\n"); 440 error = ENXIO; 441 goto err_pci; 442 } 443 444 e1000_get_bus_info(&adapter->hw); 445 446 /* Sysctls for limiting the amount of work done in the taskqueue */ 447 igb_add_rx_process_limit(adapter, "rx_processing_limit", 448 "max number of rx packets to process", &adapter->rx_process_limit, 449 igb_rx_process_limit); 450 451 /* 452 * Validate number of transmit and receive descriptors. It 453 * must not exceed hardware maximum, and must be multiple 454 * of E1000_DBA_ALIGN. 455 */ 456 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 || 457 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) { 458 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 459 IGB_DEFAULT_TXD, igb_txd); 460 adapter->num_tx_desc = IGB_DEFAULT_TXD; 461 } else 462 adapter->num_tx_desc = igb_txd; 463 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 || 464 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) { 465 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 466 IGB_DEFAULT_RXD, igb_rxd); 467 adapter->num_rx_desc = IGB_DEFAULT_RXD; 468 } else 469 adapter->num_rx_desc = igb_rxd; 470 471 adapter->hw.mac.autoneg = DO_AUTO_NEG; 472 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 473 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 474 475 /* Copper options */ 476 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 477 adapter->hw.phy.mdix = AUTO_ALL_MODES; 478 adapter->hw.phy.disable_polarity_correction = FALSE; 479 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE; 480 } 481 482 /* 483 * Set the frame limits assuming 484 * standard ethernet sized frames. 485 */ 486 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 487 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 488 489 /* 490 ** Allocate and Setup Queues 491 */ 492 if (igb_allocate_queues(adapter)) { 493 error = ENOMEM; 494 goto err_pci; 495 } 496 497 /* 498 ** Start from a known state, this is 499 ** important in reading the nvm and 500 ** mac from that. 501 */ 502 e1000_reset_hw(&adapter->hw); 503 504 /* Make sure we have a good EEPROM before we read from it */ 505 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 506 /* 507 ** Some PCI-E parts fail the first check due to 508 ** the link being in sleep state, call it again, 509 ** if it fails a second time its a real issue. 510 */ 511 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 512 device_printf(dev, 513 "The EEPROM Checksum Is Not Valid\n"); 514 error = EIO; 515 goto err_late; 516 } 517 } 518 519 /* 520 ** Copy the permanent MAC address out of the EEPROM 521 */ 522 if (e1000_read_mac_addr(&adapter->hw) < 0) { 523 device_printf(dev, "EEPROM read error while reading MAC" 524 " address\n"); 525 error = EIO; 526 goto err_late; 527 } 528 /* Check its sanity */ 529 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) { 530 device_printf(dev, "Invalid MAC address\n"); 531 error = EIO; 532 goto err_late; 533 } 534 535 /* 536 ** Configure Interrupts 537 */ 538 if ((adapter->msix > 1) && (igb_enable_msix)) 539 error = igb_allocate_msix(adapter); 540 else /* MSI or Legacy */ 541 error = igb_allocate_legacy(adapter); 542 if (error) 543 goto err_late; 544 545 /* Setup OS specific network interface */ 546 igb_setup_interface(dev, adapter); 547 548 /* Now get a good starting state */ 549 igb_reset(adapter); 550 551 /* Initialize statistics */ 552 igb_update_stats_counters(adapter); 553 554 adapter->hw.mac.get_link_status = 1; 555 igb_update_link_status(adapter); 556 557 /* Indicate SOL/IDER usage */ 558 if (e1000_check_reset_block(&adapter->hw)) 559 device_printf(dev, 560 "PHY reset is blocked due to SOL/IDER session.\n"); 561 562 /* Determine if we have to control management hardware */ 563 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 564 565 /* 566 * Setup Wake-on-Lan 567 */ 568 /* APME bit in EEPROM is mapped to WUC.APME */ 569 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME; 570 if (eeprom_data) 571 adapter->wol = E1000_WUFC_MAG; 572 573 /* Register for VLAN events */ 574 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 575 igb_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 576 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 577 igb_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 578 579 igb_add_hw_stats(adapter); 580 581 /* Tell the stack that the interface is not active */ 582 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 583 584 adapter->led_dev = led_create(igb_led_func, adapter, 585 device_get_nameunit(dev)); 586 587 INIT_DEBUGOUT("igb_attach: end"); 588 589 return (0); 590 591err_late: 592 igb_free_transmit_structures(adapter); 593 igb_free_receive_structures(adapter); 594 igb_release_hw_control(adapter); 595err_pci: 596 igb_free_pci_resources(adapter); 597 IGB_CORE_LOCK_DESTROY(adapter); 598 599 return (error); 600} 601 602/********************************************************************* 603 * Device removal routine 604 * 605 * The detach entry point is called when the driver is being removed. 606 * This routine stops the adapter and deallocates all the resources 607 * that were allocated for driver operation. 608 * 609 * return 0 on success, positive on failure 610 *********************************************************************/ 611 612static int 613igb_detach(device_t dev) 614{ 615 struct adapter *adapter = device_get_softc(dev); 616 struct ifnet *ifp = adapter->ifp; 617 618 INIT_DEBUGOUT("igb_detach: begin"); 619 620 /* Make sure VLANS are not using driver */ 621 if (adapter->ifp->if_vlantrunk != NULL) { 622 device_printf(dev,"Vlan in use, detach first\n"); 623 return (EBUSY); 624 } 625 626 if (adapter->led_dev != NULL) 627 led_destroy(adapter->led_dev); 628 629#ifdef DEVICE_POLLING 630 if (ifp->if_capenable & IFCAP_POLLING) 631 ether_poll_deregister(ifp); 632#endif 633 634 IGB_CORE_LOCK(adapter); 635 adapter->in_detach = 1; 636 igb_stop(adapter); 637 IGB_CORE_UNLOCK(adapter); 638 639 e1000_phy_hw_reset(&adapter->hw); 640 641 /* Give control back to firmware */ 642 igb_release_manageability(adapter); 643 igb_release_hw_control(adapter); 644 645 if (adapter->wol) { 646 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 647 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 648 igb_enable_wakeup(dev); 649 } 650 651 /* Unregister VLAN events */ 652 if (adapter->vlan_attach != NULL) 653 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 654 if (adapter->vlan_detach != NULL) 655 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 656 657 ether_ifdetach(adapter->ifp); 658 659 callout_drain(&adapter->timer); 660 661 igb_free_pci_resources(adapter); 662 bus_generic_detach(dev); 663 if_free(ifp); 664 665 igb_free_transmit_structures(adapter); 666 igb_free_receive_structures(adapter); 667 668 IGB_CORE_LOCK_DESTROY(adapter); 669 670 return (0); 671} 672 673/********************************************************************* 674 * 675 * Shutdown entry point 676 * 677 **********************************************************************/ 678 679static int 680igb_shutdown(device_t dev) 681{ 682 return igb_suspend(dev); 683} 684 685/* 686 * Suspend/resume device methods. 687 */ 688static int 689igb_suspend(device_t dev) 690{ 691 struct adapter *adapter = device_get_softc(dev); 692 693 IGB_CORE_LOCK(adapter); 694 695 igb_stop(adapter); 696 697 igb_release_manageability(adapter); 698 igb_release_hw_control(adapter); 699 700 if (adapter->wol) { 701 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 702 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 703 igb_enable_wakeup(dev); 704 } 705 706 IGB_CORE_UNLOCK(adapter); 707 708 return bus_generic_suspend(dev); 709} 710 711static int 712igb_resume(device_t dev) 713{ 714 struct adapter *adapter = device_get_softc(dev); 715 struct ifnet *ifp = adapter->ifp; 716 717 IGB_CORE_LOCK(adapter); 718 igb_init_locked(adapter); 719 igb_init_manageability(adapter); 720 721 if ((ifp->if_flags & IFF_UP) && 722 (ifp->if_drv_flags & IFF_DRV_RUNNING)) 723 igb_start(ifp); 724 725 IGB_CORE_UNLOCK(adapter); 726 727 return bus_generic_resume(dev); 728} 729 730 731/********************************************************************* 732 * Transmit entry point 733 * 734 * igb_start is called by the stack to initiate a transmit. 735 * The driver will remain in this routine as long as there are 736 * packets to transmit and transmit resources are available. 737 * In case resources are not available stack is notified and 738 * the packet is requeued. 739 **********************************************************************/ 740 741static void 742igb_start_locked(struct tx_ring *txr, struct ifnet *ifp) 743{ 744 struct adapter *adapter = ifp->if_softc; 745 struct mbuf *m_head; 746 747 IGB_TX_LOCK_ASSERT(txr); 748 749 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 750 IFF_DRV_RUNNING) 751 return; 752 if (!adapter->link_active) 753 return; 754 755 /* Call cleanup if number of TX descriptors low */ 756 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) 757 igb_txeof(txr); 758 759 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 760 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 761 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 762 break; 763 } 764 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 765 if (m_head == NULL) 766 break; 767 /* 768 * Encapsulation can modify our pointer, and or make it 769 * NULL on failure. In that event, we can't requeue. 770 */ 771 if (igb_xmit(txr, &m_head)) { 772 if (m_head == NULL) 773 break; 774 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 775 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 776 break; 777 } 778 779 /* Send a copy of the frame to the BPF listener */ 780 ETHER_BPF_MTAP(ifp, m_head); 781 782 /* Set watchdog on */ 783 txr->watchdog_time = ticks; 784 txr->watchdog_check = TRUE; 785 } 786} 787 788/* 789 * Legacy TX driver routine, called from the 790 * stack, always uses tx[0], and spins for it. 791 * Should not be used with multiqueue tx 792 */ 793static void 794igb_start(struct ifnet *ifp) 795{ 796 struct adapter *adapter = ifp->if_softc; 797 struct tx_ring *txr = adapter->tx_rings; 798 799 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 800 IGB_TX_LOCK(txr); 801 igb_start_locked(txr, ifp); 802 IGB_TX_UNLOCK(txr); 803 } 804 return; 805} 806 807#if __FreeBSD_version >= 800000 808/* 809** Multiqueue Transmit driver 810** 811*/ 812static int 813igb_mq_start(struct ifnet *ifp, struct mbuf *m) 814{ 815 struct adapter *adapter = ifp->if_softc; 816 struct igb_queue *que; 817 struct tx_ring *txr; 818 int i = 0, err = 0; 819 820 /* Which queue to use */ 821 if ((m->m_flags & M_FLOWID) != 0) 822 i = m->m_pkthdr.flowid % adapter->num_queues; 823 824 txr = &adapter->tx_rings[i]; 825 que = &adapter->queues[i]; 826 827 if (IGB_TX_TRYLOCK(txr)) { 828 err = igb_mq_start_locked(ifp, txr, m); 829 IGB_TX_UNLOCK(txr); 830 } else { 831 err = drbr_enqueue(ifp, txr->br, m); 832 taskqueue_enqueue(que->tq, &que->que_task); 833 } 834 835 return (err); 836} 837 838static int 839igb_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 840{ 841 struct adapter *adapter = txr->adapter; 842 struct mbuf *next; 843 int err = 0, enq; 844 845 IGB_TX_LOCK_ASSERT(txr); 846 847 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 848 IFF_DRV_RUNNING || adapter->link_active == 0) { 849 if (m != NULL) 850 err = drbr_enqueue(ifp, txr->br, m); 851 return (err); 852 } 853 854 /* Call cleanup if number of TX descriptors low */ 855 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) 856 igb_txeof(txr); 857 858 enq = 0; 859 if (m == NULL) { 860 next = drbr_dequeue(ifp, txr->br); 861 } else if (drbr_needs_enqueue(ifp, txr->br)) { 862 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 863 return (err); 864 next = drbr_dequeue(ifp, txr->br); 865 } else 866 next = m; 867 868 /* Process the queue */ 869 while (next != NULL) { 870 if ((err = igb_xmit(txr, &next)) != 0) { 871 if (next != NULL) 872 err = drbr_enqueue(ifp, txr->br, next); 873 break; 874 } 875 enq++; 876 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 877 ETHER_BPF_MTAP(ifp, next); 878 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 879 break; 880 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 881 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 882 break; 883 } 884 next = drbr_dequeue(ifp, txr->br); 885 } 886 if (enq > 0) { 887 /* Set the watchdog */ 888 txr->watchdog_check = TRUE; 889 txr->watchdog_time = ticks; 890 } 891 return (err); 892} 893 894/* 895** Flush all ring buffers 896*/ 897static void 898igb_qflush(struct ifnet *ifp) 899{ 900 struct adapter *adapter = ifp->if_softc; 901 struct tx_ring *txr = adapter->tx_rings; 902 struct mbuf *m; 903 904 for (int i = 0; i < adapter->num_queues; i++, txr++) { 905 IGB_TX_LOCK(txr); 906 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 907 m_freem(m); 908 IGB_TX_UNLOCK(txr); 909 } 910 if_qflush(ifp); 911} 912#endif /* __FreeBSD_version >= 800000 */ 913 914/********************************************************************* 915 * Ioctl entry point 916 * 917 * igb_ioctl is called when the user wants to configure the 918 * interface. 919 * 920 * return 0 on success, positive on failure 921 **********************************************************************/ 922 923static int 924igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 925{ 926 struct adapter *adapter = ifp->if_softc; 927 struct ifreq *ifr = (struct ifreq *)data; 928#ifdef INET 929 struct ifaddr *ifa = (struct ifaddr *)data; 930#endif 931 int error = 0; 932 933 if (adapter->in_detach) 934 return (error); 935 936 switch (command) { 937 case SIOCSIFADDR: 938#ifdef INET 939 if (ifa->ifa_addr->sa_family == AF_INET) { 940 /* 941 * XXX 942 * Since resetting hardware takes a very long time 943 * and results in link renegotiation we only 944 * initialize the hardware only when it is absolutely 945 * required. 946 */ 947 ifp->if_flags |= IFF_UP; 948 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 949 IGB_CORE_LOCK(adapter); 950 igb_init_locked(adapter); 951 IGB_CORE_UNLOCK(adapter); 952 } 953 if (!(ifp->if_flags & IFF_NOARP)) 954 arp_ifinit(ifp, ifa); 955 } else 956#endif 957 error = ether_ioctl(ifp, command, data); 958 break; 959 case SIOCSIFMTU: 960 { 961 int max_frame_size; 962 963 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 964 965 IGB_CORE_LOCK(adapter); 966 max_frame_size = 9234; 967 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 968 ETHER_CRC_LEN) { 969 IGB_CORE_UNLOCK(adapter); 970 error = EINVAL; 971 break; 972 } 973 974 ifp->if_mtu = ifr->ifr_mtu; 975 adapter->max_frame_size = 976 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 977 igb_init_locked(adapter); 978 IGB_CORE_UNLOCK(adapter); 979 break; 980 } 981 case SIOCSIFFLAGS: 982 IOCTL_DEBUGOUT("ioctl rcv'd:\ 983 SIOCSIFFLAGS (Set Interface Flags)"); 984 IGB_CORE_LOCK(adapter); 985 if (ifp->if_flags & IFF_UP) { 986 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 987 if ((ifp->if_flags ^ adapter->if_flags) & 988 (IFF_PROMISC | IFF_ALLMULTI)) { 989 igb_disable_promisc(adapter); 990 igb_set_promisc(adapter); 991 } 992 } else 993 igb_init_locked(adapter); 994 } else 995 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 996 igb_stop(adapter); 997 adapter->if_flags = ifp->if_flags; 998 IGB_CORE_UNLOCK(adapter); 999 break; 1000 case SIOCADDMULTI: 1001 case SIOCDELMULTI: 1002 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 1003 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1004 IGB_CORE_LOCK(adapter); 1005 igb_disable_intr(adapter); 1006 igb_set_multi(adapter); 1007#ifdef DEVICE_POLLING 1008 if (!(ifp->if_capenable & IFCAP_POLLING)) 1009#endif 1010 igb_enable_intr(adapter); 1011 IGB_CORE_UNLOCK(adapter); 1012 } 1013 break; 1014 case SIOCSIFMEDIA: 1015 /* Check SOL/IDER usage */ 1016 IGB_CORE_LOCK(adapter); 1017 if (e1000_check_reset_block(&adapter->hw)) { 1018 IGB_CORE_UNLOCK(adapter); 1019 device_printf(adapter->dev, "Media change is" 1020 " blocked due to SOL/IDER session.\n"); 1021 break; 1022 } 1023 IGB_CORE_UNLOCK(adapter); 1024 case SIOCGIFMEDIA: 1025 IOCTL_DEBUGOUT("ioctl rcv'd: \ 1026 SIOCxIFMEDIA (Get/Set Interface Media)"); 1027 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1028 break; 1029 case SIOCSIFCAP: 1030 { 1031 int mask, reinit; 1032 1033 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 1034 reinit = 0; 1035 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 1036#ifdef DEVICE_POLLING 1037 if (mask & IFCAP_POLLING) { 1038 if (ifr->ifr_reqcap & IFCAP_POLLING) { 1039 error = ether_poll_register(igb_poll, ifp); 1040 if (error) 1041 return (error); 1042 IGB_CORE_LOCK(adapter); 1043 igb_disable_intr(adapter); 1044 ifp->if_capenable |= IFCAP_POLLING; 1045 IGB_CORE_UNLOCK(adapter); 1046 } else { 1047 error = ether_poll_deregister(ifp); 1048 /* Enable interrupt even in error case */ 1049 IGB_CORE_LOCK(adapter); 1050 igb_enable_intr(adapter); 1051 ifp->if_capenable &= ~IFCAP_POLLING; 1052 IGB_CORE_UNLOCK(adapter); 1053 } 1054 } 1055#endif 1056 if (mask & IFCAP_HWCSUM) { 1057 ifp->if_capenable ^= IFCAP_HWCSUM; 1058 reinit = 1; 1059 } 1060 if (mask & IFCAP_TSO4) { 1061 ifp->if_capenable ^= IFCAP_TSO4; 1062 reinit = 1; 1063 } 1064 if (mask & IFCAP_VLAN_HWTAGGING) { 1065 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 1066 reinit = 1; 1067 } 1068 if (mask & IFCAP_VLAN_HWFILTER) { 1069 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 1070 reinit = 1; 1071 } 1072 if (mask & IFCAP_LRO) { 1073 ifp->if_capenable ^= IFCAP_LRO; 1074 reinit = 1; 1075 } 1076 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 1077 igb_init(adapter); 1078 VLAN_CAPABILITIES(ifp); 1079 break; 1080 } 1081 1082 default: 1083 error = ether_ioctl(ifp, command, data); 1084 break; 1085 } 1086 1087 return (error); 1088} 1089 1090 1091/********************************************************************* 1092 * Init entry point 1093 * 1094 * This routine is used in two ways. It is used by the stack as 1095 * init entry point in network interface structure. It is also used 1096 * by the driver as a hw/sw initialization routine to get to a 1097 * consistent state. 1098 * 1099 * return 0 on success, positive on failure 1100 **********************************************************************/ 1101 1102static void 1103igb_init_locked(struct adapter *adapter) 1104{ 1105 struct ifnet *ifp = adapter->ifp; 1106 device_t dev = adapter->dev; 1107 1108 INIT_DEBUGOUT("igb_init: begin"); 1109 1110 IGB_CORE_LOCK_ASSERT(adapter); 1111 1112 igb_disable_intr(adapter); 1113 callout_stop(&adapter->timer); 1114 1115 /* Get the latest mac address, User can use a LAA */ 1116 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1117 ETHER_ADDR_LEN); 1118 1119 /* Put the address into the Receive Address Array */ 1120 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1121 1122 igb_reset(adapter); 1123 igb_update_link_status(adapter); 1124 1125 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1126 1127 /* Use real VLAN Filter support? */ 1128 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { 1129 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) 1130 /* Use real VLAN Filter support */ 1131 igb_setup_vlan_hw_support(adapter); 1132 else { 1133 u32 ctrl; 1134 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1135 ctrl |= E1000_CTRL_VME; 1136 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1137 } 1138 } 1139 1140 /* Set hardware offload abilities */ 1141 ifp->if_hwassist = 0; 1142 if (ifp->if_capenable & IFCAP_TXCSUM) { 1143 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1144#if __FreeBSD_version >= 800000 1145 if (adapter->hw.mac.type == e1000_82576) 1146 ifp->if_hwassist |= CSUM_SCTP; 1147#endif 1148 } 1149 1150 if (ifp->if_capenable & IFCAP_TSO4) 1151 ifp->if_hwassist |= CSUM_TSO; 1152 1153 /* Configure for OS presence */ 1154 igb_init_manageability(adapter); 1155 1156 /* Prepare transmit descriptors and buffers */ 1157 igb_setup_transmit_structures(adapter); 1158 igb_initialize_transmit_units(adapter); 1159 1160 /* Setup Multicast table */ 1161 igb_set_multi(adapter); 1162 1163 /* 1164 ** Figure out the desired mbuf pool 1165 ** for doing jumbo/packetsplit 1166 */ 1167 if (ifp->if_mtu > ETHERMTU) 1168 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1169 else 1170 adapter->rx_mbuf_sz = MCLBYTES; 1171 1172 /* Prepare receive descriptors and buffers */ 1173 if (igb_setup_receive_structures(adapter)) { 1174 device_printf(dev, "Could not setup receive structures\n"); 1175 return; 1176 } 1177 igb_initialize_receive_units(adapter); 1178 1179 /* Don't lose promiscuous settings */ 1180 igb_set_promisc(adapter); 1181 1182 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1183 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1184 1185 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1186 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1187 1188 if (adapter->msix > 1) /* Set up queue routing */ 1189 igb_configure_queues(adapter); 1190 1191 /* Set up VLAN tag offload and filter */ 1192 igb_setup_vlan_hw_support(adapter); 1193 1194 /* this clears any pending interrupts */ 1195 E1000_READ_REG(&adapter->hw, E1000_ICR); 1196#ifdef DEVICE_POLLING 1197 /* 1198 * Only enable interrupts if we are not polling, make sure 1199 * they are off otherwise. 1200 */ 1201 if (ifp->if_capenable & IFCAP_POLLING) 1202 igb_disable_intr(adapter); 1203 else 1204#endif /* DEVICE_POLLING */ 1205 { 1206 igb_enable_intr(adapter); 1207 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); 1208 } 1209 1210 /* Don't reset the phy next time init gets called */ 1211 adapter->hw.phy.reset_disable = TRUE; 1212} 1213 1214static void 1215igb_init(void *arg) 1216{ 1217 struct adapter *adapter = arg; 1218 1219 IGB_CORE_LOCK(adapter); 1220 igb_init_locked(adapter); 1221 IGB_CORE_UNLOCK(adapter); 1222} 1223 1224 1225static void 1226igb_handle_que(void *context, int pending) 1227{ 1228 struct igb_queue *que = context; 1229 struct adapter *adapter = que->adapter; 1230 struct tx_ring *txr = que->txr; 1231 struct ifnet *ifp = adapter->ifp; 1232 1233 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1234 bool more; 1235 1236 more = igb_rxeof(que, -1, NULL); 1237 1238 IGB_TX_LOCK(txr); 1239 if (igb_txeof(txr)) 1240 more = TRUE; 1241#if __FreeBSD_version >= 800000 1242 if (!drbr_empty(ifp, txr->br)) 1243 igb_mq_start_locked(ifp, txr, NULL); 1244#else 1245 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1246 igb_start_locked(txr, ifp); 1247#endif 1248 IGB_TX_UNLOCK(txr); 1249 if (more) { 1250 taskqueue_enqueue(que->tq, &que->que_task); 1251 return; 1252 } 1253 } 1254 1255#ifdef DEVICE_POLLING 1256 if (ifp->if_capenable & IFCAP_POLLING) 1257 return; 1258#endif 1259 /* Reenable this interrupt */ 1260 if (que->eims) 1261 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); 1262 else 1263 igb_enable_intr(adapter); 1264} 1265 1266/* Deal with link in a sleepable context */ 1267static void 1268igb_handle_link(void *context, int pending) 1269{ 1270 struct adapter *adapter = context; 1271 1272 adapter->hw.mac.get_link_status = 1; 1273 igb_update_link_status(adapter); 1274} 1275 1276/********************************************************************* 1277 * 1278 * MSI/Legacy Deferred 1279 * Interrupt Service routine 1280 * 1281 *********************************************************************/ 1282static int 1283igb_irq_fast(void *arg) 1284{ 1285 struct adapter *adapter = arg; 1286 struct igb_queue *que = adapter->queues; 1287 u32 reg_icr; 1288 1289 1290 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1291 1292 /* Hot eject? */ 1293 if (reg_icr == 0xffffffff) 1294 return FILTER_STRAY; 1295 1296 /* Definitely not our interrupt. */ 1297 if (reg_icr == 0x0) 1298 return FILTER_STRAY; 1299 1300 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1301 return FILTER_STRAY; 1302 1303 /* 1304 * Mask interrupts until the taskqueue is finished running. This is 1305 * cheap, just assume that it is needed. This also works around the 1306 * MSI message reordering errata on certain systems. 1307 */ 1308 igb_disable_intr(adapter); 1309 taskqueue_enqueue(que->tq, &que->que_task); 1310 1311 /* Link status change */ 1312 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 1313 taskqueue_enqueue(que->tq, &adapter->link_task); 1314 1315 if (reg_icr & E1000_ICR_RXO) 1316 adapter->rx_overruns++; 1317 return FILTER_HANDLED; 1318} 1319 1320#ifdef DEVICE_POLLING 1321/********************************************************************* 1322 * 1323 * Legacy polling routine : if using this code you MUST be sure that 1324 * multiqueue is not defined, ie, set igb_num_queues to 1. 1325 * 1326 *********************************************************************/ 1327#if __FreeBSD_version >= 800000 1328#define POLL_RETURN_COUNT(a) (a) 1329static int 1330#else 1331#define POLL_RETURN_COUNT(a) 1332static void 1333#endif 1334igb_poll(struct ifnet *ifp, enum poll_cmd cmd, int count) 1335{ 1336 struct adapter *adapter = ifp->if_softc; 1337 struct igb_queue *que = adapter->queues; 1338 struct tx_ring *txr = adapter->tx_rings; 1339 u32 reg_icr, rx_done = 0; 1340 u32 loop = IGB_MAX_LOOP; 1341 bool more; 1342 1343 IGB_CORE_LOCK(adapter); 1344 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 1345 IGB_CORE_UNLOCK(adapter); 1346 return POLL_RETURN_COUNT(rx_done); 1347 } 1348 1349 if (cmd == POLL_AND_CHECK_STATUS) { 1350 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1351 /* Link status change */ 1352 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) 1353 igb_handle_link(adapter, 0); 1354 1355 if (reg_icr & E1000_ICR_RXO) 1356 adapter->rx_overruns++; 1357 } 1358 IGB_CORE_UNLOCK(adapter); 1359 1360 igb_rxeof(que, count, &rx_done); 1361 1362 IGB_TX_LOCK(txr); 1363 do { 1364 more = igb_txeof(txr); 1365 } while (loop-- && more); 1366#if __FreeBSD_version >= 800000 1367 if (!drbr_empty(ifp, txr->br)) 1368 igb_mq_start_locked(ifp, txr, NULL); 1369#else 1370 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1371 igb_start_locked(txr, ifp); 1372#endif 1373 IGB_TX_UNLOCK(txr); 1374 return POLL_RETURN_COUNT(rx_done); 1375} 1376#endif /* DEVICE_POLLING */ 1377 1378/********************************************************************* 1379 * 1380 * MSIX TX Interrupt Service routine 1381 * 1382 **********************************************************************/ 1383static void 1384igb_msix_que(void *arg) 1385{ 1386 struct igb_queue *que = arg; 1387 struct adapter *adapter = que->adapter; 1388 struct tx_ring *txr = que->txr; 1389 struct rx_ring *rxr = que->rxr; 1390 u32 newitr = 0; 1391 bool more_tx, more_rx; 1392 1393 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, que->eims); 1394 ++que->irqs; 1395 1396 IGB_TX_LOCK(txr); 1397 more_tx = igb_txeof(txr); 1398 IGB_TX_UNLOCK(txr); 1399 1400 more_rx = igb_rxeof(que, adapter->rx_process_limit, NULL); 1401 1402 if (igb_enable_aim == FALSE) 1403 goto no_calc; 1404 /* 1405 ** Do Adaptive Interrupt Moderation: 1406 ** - Write out last calculated setting 1407 ** - Calculate based on average size over 1408 ** the last interval. 1409 */ 1410 if (que->eitr_setting) 1411 E1000_WRITE_REG(&adapter->hw, 1412 E1000_EITR(que->msix), que->eitr_setting); 1413 1414 que->eitr_setting = 0; 1415 1416 /* Idle, do nothing */ 1417 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1418 goto no_calc; 1419 1420 /* Used half Default if sub-gig */ 1421 if (adapter->link_speed != 1000) 1422 newitr = IGB_DEFAULT_ITR / 2; 1423 else { 1424 if ((txr->bytes) && (txr->packets)) 1425 newitr = txr->bytes/txr->packets; 1426 if ((rxr->bytes) && (rxr->packets)) 1427 newitr = max(newitr, 1428 (rxr->bytes / rxr->packets)); 1429 newitr += 24; /* account for hardware frame, crc */ 1430 /* set an upper boundary */ 1431 newitr = min(newitr, 3000); 1432 /* Be nice to the mid range */ 1433 if ((newitr > 300) && (newitr < 1200)) 1434 newitr = (newitr / 3); 1435 else 1436 newitr = (newitr / 2); 1437 } 1438 newitr &= 0x7FFC; /* Mask invalid bits */ 1439 if (adapter->hw.mac.type == e1000_82575) 1440 newitr |= newitr << 16; 1441 else 1442 newitr |= E1000_EITR_CNT_IGNR; 1443 1444 /* save for next interrupt */ 1445 que->eitr_setting = newitr; 1446 1447 /* Reset state */ 1448 txr->bytes = 0; 1449 txr->packets = 0; 1450 rxr->bytes = 0; 1451 rxr->packets = 0; 1452 1453no_calc: 1454 /* Schedule a clean task if needed*/ 1455 if (more_tx || more_rx) 1456 taskqueue_enqueue(que->tq, &que->que_task); 1457 else 1458 /* Reenable this interrupt */ 1459 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, que->eims); 1460 return; 1461} 1462 1463 1464/********************************************************************* 1465 * 1466 * MSIX Link Interrupt Service routine 1467 * 1468 **********************************************************************/ 1469 1470static void 1471igb_msix_link(void *arg) 1472{ 1473 struct adapter *adapter = arg; 1474 u32 icr; 1475 1476 ++adapter->link_irq; 1477 icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1478 if (!(icr & E1000_ICR_LSC)) 1479 goto spurious; 1480 igb_handle_link(adapter, 0); 1481 1482spurious: 1483 /* Rearm */ 1484 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); 1485 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); 1486 return; 1487} 1488 1489 1490/********************************************************************* 1491 * 1492 * Media Ioctl callback 1493 * 1494 * This routine is called whenever the user queries the status of 1495 * the interface using ifconfig. 1496 * 1497 **********************************************************************/ 1498static void 1499igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1500{ 1501 struct adapter *adapter = ifp->if_softc; 1502 u_char fiber_type = IFM_1000_SX; 1503 1504 INIT_DEBUGOUT("igb_media_status: begin"); 1505 1506 IGB_CORE_LOCK(adapter); 1507 igb_update_link_status(adapter); 1508 1509 ifmr->ifm_status = IFM_AVALID; 1510 ifmr->ifm_active = IFM_ETHER; 1511 1512 if (!adapter->link_active) { 1513 IGB_CORE_UNLOCK(adapter); 1514 return; 1515 } 1516 1517 ifmr->ifm_status |= IFM_ACTIVE; 1518 1519 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1520 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) 1521 ifmr->ifm_active |= fiber_type | IFM_FDX; 1522 else { 1523 switch (adapter->link_speed) { 1524 case 10: 1525 ifmr->ifm_active |= IFM_10_T; 1526 break; 1527 case 100: 1528 ifmr->ifm_active |= IFM_100_TX; 1529 break; 1530 case 1000: 1531 ifmr->ifm_active |= IFM_1000_T; 1532 break; 1533 } 1534 if (adapter->link_duplex == FULL_DUPLEX) 1535 ifmr->ifm_active |= IFM_FDX; 1536 else 1537 ifmr->ifm_active |= IFM_HDX; 1538 } 1539 IGB_CORE_UNLOCK(adapter); 1540} 1541 1542/********************************************************************* 1543 * 1544 * Media Ioctl callback 1545 * 1546 * This routine is called when the user changes speed/duplex using 1547 * media/mediopt option with ifconfig. 1548 * 1549 **********************************************************************/ 1550static int 1551igb_media_change(struct ifnet *ifp) 1552{ 1553 struct adapter *adapter = ifp->if_softc; 1554 struct ifmedia *ifm = &adapter->media; 1555 1556 INIT_DEBUGOUT("igb_media_change: begin"); 1557 1558 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1559 return (EINVAL); 1560 1561 IGB_CORE_LOCK(adapter); 1562 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1563 case IFM_AUTO: 1564 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1565 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1566 break; 1567 case IFM_1000_LX: 1568 case IFM_1000_SX: 1569 case IFM_1000_T: 1570 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1571 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1572 break; 1573 case IFM_100_TX: 1574 adapter->hw.mac.autoneg = FALSE; 1575 adapter->hw.phy.autoneg_advertised = 0; 1576 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1577 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1578 else 1579 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1580 break; 1581 case IFM_10_T: 1582 adapter->hw.mac.autoneg = FALSE; 1583 adapter->hw.phy.autoneg_advertised = 0; 1584 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1585 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1586 else 1587 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1588 break; 1589 default: 1590 device_printf(adapter->dev, "Unsupported media type\n"); 1591 } 1592 1593 /* As the speed/duplex settings my have changed we need to 1594 * reset the PHY. 1595 */ 1596 adapter->hw.phy.reset_disable = FALSE; 1597 1598 igb_init_locked(adapter); 1599 IGB_CORE_UNLOCK(adapter); 1600 1601 return (0); 1602} 1603 1604 1605/********************************************************************* 1606 * 1607 * This routine maps the mbufs to Advanced TX descriptors. 1608 * used by the 82575 adapter. 1609 * 1610 **********************************************************************/ 1611 1612static int 1613igb_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1614{ 1615 struct adapter *adapter = txr->adapter; 1616 bus_dma_segment_t segs[IGB_MAX_SCATTER]; 1617 bus_dmamap_t map; 1618 struct igb_tx_buffer *tx_buffer, *tx_buffer_mapped; 1619 union e1000_adv_tx_desc *txd = NULL; 1620 struct mbuf *m_head; 1621 u32 olinfo_status = 0, cmd_type_len = 0; 1622 int nsegs, i, j, error, first, last = 0; 1623 u32 hdrlen = 0; 1624 1625 m_head = *m_headp; 1626 1627 1628 /* Set basic descriptor constants */ 1629 cmd_type_len |= E1000_ADVTXD_DTYP_DATA; 1630 cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT; 1631 if (m_head->m_flags & M_VLANTAG) 1632 cmd_type_len |= E1000_ADVTXD_DCMD_VLE; 1633 1634 /* 1635 * Force a cleanup if number of TX descriptors 1636 * available hits the threshold 1637 */ 1638 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) { 1639 igb_txeof(txr); 1640 /* Now do we at least have a minimal? */ 1641 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 1642 txr->no_desc_avail++; 1643 return (ENOBUFS); 1644 } 1645 } 1646 1647 /* 1648 * Map the packet for DMA. 1649 * 1650 * Capture the first descriptor index, 1651 * this descriptor will have the index 1652 * of the EOP which is the only one that 1653 * now gets a DONE bit writeback. 1654 */ 1655 first = txr->next_avail_desc; 1656 tx_buffer = &txr->tx_buffers[first]; 1657 tx_buffer_mapped = tx_buffer; 1658 map = tx_buffer->map; 1659 1660 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1661 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1662 1663 if (error == EFBIG) { 1664 struct mbuf *m; 1665 1666 m = m_defrag(*m_headp, M_DONTWAIT); 1667 if (m == NULL) { 1668 adapter->mbuf_defrag_failed++; 1669 m_freem(*m_headp); 1670 *m_headp = NULL; 1671 return (ENOBUFS); 1672 } 1673 *m_headp = m; 1674 1675 /* Try it again */ 1676 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1677 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1678 1679 if (error == ENOMEM) { 1680 adapter->no_tx_dma_setup++; 1681 return (error); 1682 } else if (error != 0) { 1683 adapter->no_tx_dma_setup++; 1684 m_freem(*m_headp); 1685 *m_headp = NULL; 1686 return (error); 1687 } 1688 } else if (error == ENOMEM) { 1689 adapter->no_tx_dma_setup++; 1690 return (error); 1691 } else if (error != 0) { 1692 adapter->no_tx_dma_setup++; 1693 m_freem(*m_headp); 1694 *m_headp = NULL; 1695 return (error); 1696 } 1697 1698 /* Check again to be sure we have enough descriptors */ 1699 if (nsegs > (txr->tx_avail - 2)) { 1700 txr->no_desc_avail++; 1701 bus_dmamap_unload(txr->txtag, map); 1702 return (ENOBUFS); 1703 } 1704 m_head = *m_headp; 1705 1706 /* 1707 * Set up the context descriptor: 1708 * used when any hardware offload is done. 1709 * This includes CSUM, VLAN, and TSO. It 1710 * will use the first descriptor. 1711 */ 1712 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1713 if (igb_tso_setup(txr, m_head, &hdrlen)) { 1714 cmd_type_len |= E1000_ADVTXD_DCMD_TSE; 1715 olinfo_status |= E1000_TXD_POPTS_IXSM << 8; 1716 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 1717 } else 1718 return (ENXIO); 1719 } else if (igb_tx_ctx_setup(txr, m_head)) 1720 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 1721 1722 /* Calculate payload length */ 1723 olinfo_status |= ((m_head->m_pkthdr.len - hdrlen) 1724 << E1000_ADVTXD_PAYLEN_SHIFT); 1725 1726 /* 82575 needs the queue index added */ 1727 if (adapter->hw.mac.type == e1000_82575) 1728 olinfo_status |= txr->me << 4; 1729 1730 /* Set up our transmit descriptors */ 1731 i = txr->next_avail_desc; 1732 for (j = 0; j < nsegs; j++) { 1733 bus_size_t seg_len; 1734 bus_addr_t seg_addr; 1735 1736 tx_buffer = &txr->tx_buffers[i]; 1737 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i]; 1738 seg_addr = segs[j].ds_addr; 1739 seg_len = segs[j].ds_len; 1740 1741 txd->read.buffer_addr = htole64(seg_addr); 1742 txd->read.cmd_type_len = htole32(cmd_type_len | seg_len); 1743 txd->read.olinfo_status = htole32(olinfo_status); 1744 last = i; 1745 if (++i == adapter->num_tx_desc) 1746 i = 0; 1747 tx_buffer->m_head = NULL; 1748 tx_buffer->next_eop = -1; 1749 } 1750 1751 txr->next_avail_desc = i; 1752 txr->tx_avail -= nsegs; 1753 1754 tx_buffer->m_head = m_head; 1755 tx_buffer_mapped->map = tx_buffer->map; 1756 tx_buffer->map = map; 1757 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1758 1759 /* 1760 * Last Descriptor of Packet 1761 * needs End Of Packet (EOP) 1762 * and Report Status (RS) 1763 */ 1764 txd->read.cmd_type_len |= 1765 htole32(E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_RS); 1766 /* 1767 * Keep track in the first buffer which 1768 * descriptor will be written back 1769 */ 1770 tx_buffer = &txr->tx_buffers[first]; 1771 tx_buffer->next_eop = last; 1772 txr->watchdog_time = ticks; 1773 1774 /* 1775 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 1776 * that this frame is available to transmit. 1777 */ 1778 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1779 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1780 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); 1781 ++txr->tx_packets; 1782 1783 return (0); 1784 1785} 1786 1787static void 1788igb_set_promisc(struct adapter *adapter) 1789{ 1790 struct ifnet *ifp = adapter->ifp; 1791 uint32_t reg_rctl; 1792 1793 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1794 1795 if (ifp->if_flags & IFF_PROMISC) { 1796 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1797 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1798 } else if (ifp->if_flags & IFF_ALLMULTI) { 1799 reg_rctl |= E1000_RCTL_MPE; 1800 reg_rctl &= ~E1000_RCTL_UPE; 1801 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1802 } 1803} 1804 1805static void 1806igb_disable_promisc(struct adapter *adapter) 1807{ 1808 uint32_t reg_rctl; 1809 1810 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1811 1812 reg_rctl &= (~E1000_RCTL_UPE); 1813 reg_rctl &= (~E1000_RCTL_MPE); 1814 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1815} 1816 1817 1818/********************************************************************* 1819 * Multicast Update 1820 * 1821 * This routine is called whenever multicast address list is updated. 1822 * 1823 **********************************************************************/ 1824 1825static void 1826igb_set_multi(struct adapter *adapter) 1827{ 1828 struct ifnet *ifp = adapter->ifp; 1829 struct ifmultiaddr *ifma; 1830 u32 reg_rctl = 0; 1831 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_ADDR_LEN]; 1832 1833 int mcnt = 0; 1834 1835 IOCTL_DEBUGOUT("igb_set_multi: begin"); 1836 1837#if __FreeBSD_version < 800000 1838 IF_ADDR_LOCK(ifp); 1839#else 1840 if_maddr_rlock(ifp); 1841#endif 1842 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1843 if (ifma->ifma_addr->sa_family != AF_LINK) 1844 continue; 1845 1846 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 1847 break; 1848 1849 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1850 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 1851 mcnt++; 1852 } 1853#if __FreeBSD_version < 800000 1854 IF_ADDR_UNLOCK(ifp); 1855#else 1856 if_maddr_runlock(ifp); 1857#endif 1858 1859 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 1860 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1861 reg_rctl |= E1000_RCTL_MPE; 1862 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1863 } else 1864 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 1865} 1866 1867 1868/********************************************************************* 1869 * Timer routine: 1870 * This routine checks for link status, 1871 * updates statistics, and does the watchdog. 1872 * 1873 **********************************************************************/ 1874 1875static void 1876igb_local_timer(void *arg) 1877{ 1878 struct adapter *adapter = arg; 1879 device_t dev = adapter->dev; 1880 struct tx_ring *txr = adapter->tx_rings; 1881 1882 1883 IGB_CORE_LOCK_ASSERT(adapter); 1884 1885 igb_update_link_status(adapter); 1886 igb_update_stats_counters(adapter); 1887 1888 /* 1889 ** Watchdog: check for time since any descriptor was cleaned 1890 */ 1891 for (int i = 0; i < adapter->num_queues; i++, txr++) { 1892 if (txr->watchdog_check == FALSE) 1893 continue; 1894 if ((ticks - txr->watchdog_time) > IGB_WATCHDOG) 1895 goto timeout; 1896 } 1897 1898 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1899 return; 1900 1901timeout: 1902 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1903 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1904 E1000_READ_REG(&adapter->hw, E1000_TDH(txr->me)), 1905 E1000_READ_REG(&adapter->hw, E1000_TDT(txr->me))); 1906 device_printf(dev,"TX(%d) desc avail = %d," 1907 "Next TX to Clean = %d\n", 1908 txr->me, txr->tx_avail, txr->next_to_clean); 1909 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1910 adapter->watchdog_events++; 1911 igb_init_locked(adapter); 1912} 1913 1914static void 1915igb_update_link_status(struct adapter *adapter) 1916{ 1917 struct e1000_hw *hw = &adapter->hw; 1918 struct ifnet *ifp = adapter->ifp; 1919 device_t dev = adapter->dev; 1920 struct tx_ring *txr = adapter->tx_rings; 1921 u32 link_check = 0; 1922 1923 /* Get the cached link value or read for real */ 1924 switch (hw->phy.media_type) { 1925 case e1000_media_type_copper: 1926 if (hw->mac.get_link_status) { 1927 /* Do the work to read phy */ 1928 e1000_check_for_link(hw); 1929 link_check = !hw->mac.get_link_status; 1930 } else 1931 link_check = TRUE; 1932 break; 1933 case e1000_media_type_fiber: 1934 e1000_check_for_link(hw); 1935 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 1936 E1000_STATUS_LU); 1937 break; 1938 case e1000_media_type_internal_serdes: 1939 e1000_check_for_link(hw); 1940 link_check = adapter->hw.mac.serdes_has_link; 1941 break; 1942 default: 1943 case e1000_media_type_unknown: 1944 break; 1945 } 1946 1947 /* Now we check if a transition has happened */ 1948 if (link_check && (adapter->link_active == 0)) { 1949 e1000_get_speed_and_duplex(&adapter->hw, 1950 &adapter->link_speed, &adapter->link_duplex); 1951 if (bootverbose) 1952 device_printf(dev, "Link is up %d Mbps %s\n", 1953 adapter->link_speed, 1954 ((adapter->link_duplex == FULL_DUPLEX) ? 1955 "Full Duplex" : "Half Duplex")); 1956 adapter->link_active = 1; 1957 ifp->if_baudrate = adapter->link_speed * 1000000; 1958 /* This can sleep */ 1959 if_link_state_change(ifp, LINK_STATE_UP); 1960 } else if (!link_check && (adapter->link_active == 1)) { 1961 ifp->if_baudrate = adapter->link_speed = 0; 1962 adapter->link_duplex = 0; 1963 if (bootverbose) 1964 device_printf(dev, "Link is Down\n"); 1965 adapter->link_active = 0; 1966 /* This can sleep */ 1967 if_link_state_change(ifp, LINK_STATE_DOWN); 1968 /* Turn off watchdogs */ 1969 for (int i = 0; i < adapter->num_queues; i++, txr++) 1970 txr->watchdog_check = FALSE; 1971 } 1972} 1973 1974/********************************************************************* 1975 * 1976 * This routine disables all traffic on the adapter by issuing a 1977 * global reset on the MAC and deallocates TX/RX buffers. 1978 * 1979 **********************************************************************/ 1980 1981static void 1982igb_stop(void *arg) 1983{ 1984 struct adapter *adapter = arg; 1985 struct ifnet *ifp = adapter->ifp; 1986 struct tx_ring *txr = adapter->tx_rings; 1987 1988 IGB_CORE_LOCK_ASSERT(adapter); 1989 1990 INIT_DEBUGOUT("igb_stop: begin"); 1991 1992 igb_disable_intr(adapter); 1993 1994 callout_stop(&adapter->timer); 1995 1996 /* Tell the stack that the interface is no longer active */ 1997 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1998 1999 /* Unarm watchdog timer. */ 2000 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2001 IGB_TX_LOCK(txr); 2002 txr->watchdog_check = FALSE; 2003 IGB_TX_UNLOCK(txr); 2004 } 2005 2006 e1000_reset_hw(&adapter->hw); 2007 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 2008 2009 e1000_led_off(&adapter->hw); 2010 e1000_cleanup_led(&adapter->hw); 2011} 2012 2013 2014/********************************************************************* 2015 * 2016 * Determine hardware revision. 2017 * 2018 **********************************************************************/ 2019static void 2020igb_identify_hardware(struct adapter *adapter) 2021{ 2022 device_t dev = adapter->dev; 2023 2024 /* Make sure our PCI config space has the necessary stuff set */ 2025 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2026 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2027 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2028 device_printf(dev, "Memory Access and/or Bus Master bits " 2029 "were not set!\n"); 2030 adapter->hw.bus.pci_cmd_word |= 2031 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2032 pci_write_config(dev, PCIR_COMMAND, 2033 adapter->hw.bus.pci_cmd_word, 2); 2034 } 2035 2036 /* Save off the information about this board */ 2037 adapter->hw.vendor_id = pci_get_vendor(dev); 2038 adapter->hw.device_id = pci_get_device(dev); 2039 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2040 adapter->hw.subsystem_vendor_id = 2041 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2042 adapter->hw.subsystem_device_id = 2043 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2044 2045 /* Do Shared Code Init and Setup */ 2046 if (e1000_set_mac_type(&adapter->hw)) { 2047 device_printf(dev, "Setup init failure\n"); 2048 return; 2049 } 2050} 2051 2052static int 2053igb_allocate_pci_resources(struct adapter *adapter) 2054{ 2055 device_t dev = adapter->dev; 2056 int rid; 2057 2058 rid = PCIR_BAR(0); 2059 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2060 &rid, RF_ACTIVE); 2061 if (adapter->pci_mem == NULL) { 2062 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2063 return (ENXIO); 2064 } 2065 adapter->osdep.mem_bus_space_tag = 2066 rman_get_bustag(adapter->pci_mem); 2067 adapter->osdep.mem_bus_space_handle = 2068 rman_get_bushandle(adapter->pci_mem); 2069 adapter->hw.hw_addr = (u8 *)&adapter->osdep.mem_bus_space_handle; 2070 2071 adapter->num_queues = 1; /* Defaults for Legacy or MSI */ 2072 2073 /* This will setup either MSI/X or MSI */ 2074 adapter->msix = igb_setup_msix(adapter); 2075 adapter->hw.back = &adapter->osdep; 2076 2077 return (0); 2078} 2079 2080/********************************************************************* 2081 * 2082 * Setup the Legacy or MSI Interrupt handler 2083 * 2084 **********************************************************************/ 2085static int 2086igb_allocate_legacy(struct adapter *adapter) 2087{ 2088 device_t dev = adapter->dev; 2089 struct igb_queue *que = adapter->queues; 2090 int error, rid = 0; 2091 2092 /* Turn off all interrupts */ 2093 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2094 2095 /* MSI RID is 1 */ 2096 if (adapter->msix == 1) 2097 rid = 1; 2098 2099 /* We allocate a single interrupt resource */ 2100 adapter->res = bus_alloc_resource_any(dev, 2101 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2102 if (adapter->res == NULL) { 2103 device_printf(dev, "Unable to allocate bus resource: " 2104 "interrupt\n"); 2105 return (ENXIO); 2106 } 2107 2108 /* 2109 * Try allocating a fast interrupt and the associated deferred 2110 * processing contexts. 2111 */ 2112 TASK_INIT(&que->que_task, 0, igb_handle_que, que); 2113 /* Make tasklet for deferred link handling */ 2114 TASK_INIT(&adapter->link_task, 0, igb_handle_link, adapter); 2115 que->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, 2116 taskqueue_thread_enqueue, &que->tq); 2117 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s taskq", 2118 device_get_nameunit(adapter->dev)); 2119 if ((error = bus_setup_intr(dev, adapter->res, 2120 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, NULL, 2121 adapter, &adapter->tag)) != 0) { 2122 device_printf(dev, "Failed to register fast interrupt " 2123 "handler: %d\n", error); 2124 taskqueue_free(que->tq); 2125 que->tq = NULL; 2126 return (error); 2127 } 2128 2129 return (0); 2130} 2131 2132 2133/********************************************************************* 2134 * 2135 * Setup the MSIX Queue Interrupt handlers: 2136 * 2137 **********************************************************************/ 2138static int 2139igb_allocate_msix(struct adapter *adapter) 2140{ 2141 device_t dev = adapter->dev; 2142 struct igb_queue *que = adapter->queues; 2143 int error, rid, vector = 0; 2144 2145 2146 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 2147 rid = vector +1; 2148 que->res = bus_alloc_resource_any(dev, 2149 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2150 if (que->res == NULL) { 2151 device_printf(dev, 2152 "Unable to allocate bus resource: " 2153 "MSIX Queue Interrupt\n"); 2154 return (ENXIO); 2155 } 2156 error = bus_setup_intr(dev, que->res, 2157 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2158 igb_msix_que, que, &que->tag); 2159 if (error) { 2160 que->res = NULL; 2161 device_printf(dev, "Failed to register Queue handler"); 2162 return (error); 2163 } 2164#if __FreeBSD_version >= 800504 2165 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 2166#endif 2167 que->msix = vector; 2168 if (adapter->hw.mac.type == e1000_82575) 2169 que->eims = E1000_EICR_TX_QUEUE0 << i; 2170 else 2171 que->eims = 1 << vector; 2172 /* 2173 ** Bind the msix vector, and thus the 2174 ** rings to the corresponding cpu. 2175 */ 2176 if (adapter->num_queues > 1) 2177 bus_bind_intr(dev, que->res, i); 2178 /* Make tasklet for deferred handling */ 2179 TASK_INIT(&que->que_task, 0, igb_handle_que, que); 2180 que->tq = taskqueue_create_fast("igb_que", M_NOWAIT, 2181 taskqueue_thread_enqueue, &que->tq); 2182 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 2183 device_get_nameunit(adapter->dev)); 2184 } 2185 2186 /* And Link */ 2187 rid = vector + 1; 2188 adapter->res = bus_alloc_resource_any(dev, 2189 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 2190 if (adapter->res == NULL) { 2191 device_printf(dev, 2192 "Unable to allocate bus resource: " 2193 "MSIX Link Interrupt\n"); 2194 return (ENXIO); 2195 } 2196 if ((error = bus_setup_intr(dev, adapter->res, 2197 INTR_TYPE_NET | INTR_MPSAFE, NULL, 2198 igb_msix_link, adapter, &adapter->tag)) != 0) { 2199 device_printf(dev, "Failed to register Link handler"); 2200 return (error); 2201 } 2202#if __FreeBSD_version >= 800504 2203 bus_describe_intr(dev, adapter->res, adapter->tag, "link"); 2204#endif 2205 adapter->linkvec = vector; 2206 2207 return (0); 2208} 2209 2210 2211static void 2212igb_configure_queues(struct adapter *adapter) 2213{ 2214 struct e1000_hw *hw = &adapter->hw; 2215 struct igb_queue *que; 2216 u32 tmp, ivar = 0; 2217 u32 newitr = IGB_DEFAULT_ITR; 2218 2219 /* First turn on RSS capability */ 2220 if (adapter->hw.mac.type > e1000_82575) 2221 E1000_WRITE_REG(hw, E1000_GPIE, 2222 E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | 2223 E1000_GPIE_PBA | E1000_GPIE_NSICR); 2224 2225 /* Turn on MSIX */ 2226 switch (adapter->hw.mac.type) { 2227 case e1000_82580: 2228 /* RX entries */ 2229 for (int i = 0; i < adapter->num_queues; i++) { 2230 u32 index = i >> 1; 2231 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2232 que = &adapter->queues[i]; 2233 if (i & 1) { 2234 ivar &= 0xFF00FFFF; 2235 ivar |= (que->msix | E1000_IVAR_VALID) << 16; 2236 } else { 2237 ivar &= 0xFFFFFF00; 2238 ivar |= que->msix | E1000_IVAR_VALID; 2239 } 2240 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2241 } 2242 /* TX entries */ 2243 for (int i = 0; i < adapter->num_queues; i++) { 2244 u32 index = i >> 1; 2245 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2246 que = &adapter->queues[i]; 2247 if (i & 1) { 2248 ivar &= 0x00FFFFFF; 2249 ivar |= (que->msix | E1000_IVAR_VALID) << 24; 2250 } else { 2251 ivar &= 0xFFFF00FF; 2252 ivar |= (que->msix | E1000_IVAR_VALID) << 8; 2253 } 2254 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2255 adapter->eims_mask |= que->eims; 2256 } 2257 2258 /* And for the link interrupt */ 2259 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; 2260 adapter->link_mask = 1 << adapter->linkvec; 2261 adapter->eims_mask |= adapter->link_mask; 2262 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 2263 break; 2264 case e1000_82576: 2265 /* RX entries */ 2266 for (int i = 0; i < adapter->num_queues; i++) { 2267 u32 index = i & 0x7; /* Each IVAR has two entries */ 2268 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2269 que = &adapter->queues[i]; 2270 if (i < 8) { 2271 ivar &= 0xFFFFFF00; 2272 ivar |= que->msix | E1000_IVAR_VALID; 2273 } else { 2274 ivar &= 0xFF00FFFF; 2275 ivar |= (que->msix | E1000_IVAR_VALID) << 16; 2276 } 2277 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2278 adapter->eims_mask |= que->eims; 2279 } 2280 /* TX entries */ 2281 for (int i = 0; i < adapter->num_queues; i++) { 2282 u32 index = i & 0x7; /* Each IVAR has two entries */ 2283 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2284 que = &adapter->queues[i]; 2285 if (i < 8) { 2286 ivar &= 0xFFFF00FF; 2287 ivar |= (que->msix | E1000_IVAR_VALID) << 8; 2288 } else { 2289 ivar &= 0x00FFFFFF; 2290 ivar |= (que->msix | E1000_IVAR_VALID) << 24; 2291 } 2292 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2293 adapter->eims_mask |= que->eims; 2294 } 2295 2296 /* And for the link interrupt */ 2297 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; 2298 adapter->link_mask = 1 << adapter->linkvec; 2299 adapter->eims_mask |= adapter->link_mask; 2300 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 2301 break; 2302 2303 case e1000_82575: 2304 /* enable MSI-X support*/ 2305 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); 2306 tmp |= E1000_CTRL_EXT_PBA_CLR; 2307 /* Auto-Mask interrupts upon ICR read. */ 2308 tmp |= E1000_CTRL_EXT_EIAME; 2309 tmp |= E1000_CTRL_EXT_IRCA; 2310 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); 2311 2312 /* Queues */ 2313 for (int i = 0; i < adapter->num_queues; i++) { 2314 que = &adapter->queues[i]; 2315 tmp = E1000_EICR_RX_QUEUE0 << i; 2316 tmp |= E1000_EICR_TX_QUEUE0 << i; 2317 que->eims = tmp; 2318 E1000_WRITE_REG_ARRAY(hw, E1000_MSIXBM(0), 2319 i, que->eims); 2320 adapter->eims_mask |= que->eims; 2321 } 2322 2323 /* Link */ 2324 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), 2325 E1000_EIMS_OTHER); 2326 adapter->link_mask |= E1000_EIMS_OTHER; 2327 adapter->eims_mask |= adapter->link_mask; 2328 default: 2329 break; 2330 } 2331 2332 /* Set the starting interrupt rate */ 2333 if (hw->mac.type == e1000_82575) 2334 newitr |= newitr << 16; 2335 else 2336 newitr |= E1000_EITR_CNT_IGNR; 2337 2338 for (int i = 0; i < adapter->num_queues; i++) { 2339 que = &adapter->queues[i]; 2340 E1000_WRITE_REG(hw, E1000_EITR(que->msix), newitr); 2341 } 2342 2343 return; 2344} 2345 2346 2347static void 2348igb_free_pci_resources(struct adapter *adapter) 2349{ 2350 struct igb_queue *que = adapter->queues; 2351 device_t dev = adapter->dev; 2352 int rid; 2353 2354 /* 2355 ** There is a slight possibility of a failure mode 2356 ** in attach that will result in entering this function 2357 ** before interrupt resources have been initialized, and 2358 ** in that case we do not want to execute the loops below 2359 ** We can detect this reliably by the state of the adapter 2360 ** res pointer. 2361 */ 2362 if (adapter->res == NULL) 2363 goto mem; 2364 2365 /* 2366 * First release all the interrupt resources: 2367 */ 2368 for (int i = 0; i < adapter->num_queues; i++, que++) { 2369 rid = que->msix + 1; 2370 if (que->tag != NULL) { 2371 bus_teardown_intr(dev, que->res, que->tag); 2372 que->tag = NULL; 2373 } 2374 if (que->res != NULL) 2375 bus_release_resource(dev, 2376 SYS_RES_IRQ, rid, que->res); 2377 } 2378 2379 /* Clean the Legacy or Link interrupt last */ 2380 if (adapter->linkvec) /* we are doing MSIX */ 2381 rid = adapter->linkvec + 1; 2382 else 2383 (adapter->msix != 0) ? (rid = 1):(rid = 0); 2384 2385 if (adapter->tag != NULL) { 2386 bus_teardown_intr(dev, adapter->res, adapter->tag); 2387 adapter->tag = NULL; 2388 } 2389 if (adapter->res != NULL) 2390 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 2391 2392mem: 2393 if (adapter->msix) 2394 pci_release_msi(dev); 2395 2396 if (adapter->msix_mem != NULL) 2397 bus_release_resource(dev, SYS_RES_MEMORY, 2398 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2399 2400 if (adapter->pci_mem != NULL) 2401 bus_release_resource(dev, SYS_RES_MEMORY, 2402 PCIR_BAR(0), adapter->pci_mem); 2403 2404} 2405 2406/* 2407 * Setup Either MSI/X or MSI 2408 */ 2409static int 2410igb_setup_msix(struct adapter *adapter) 2411{ 2412 device_t dev = adapter->dev; 2413 int rid, want, queues, msgs; 2414 2415 /* tuneable override */ 2416 if (igb_enable_msix == 0) 2417 goto msi; 2418 2419 /* First try MSI/X */ 2420 rid = PCIR_BAR(IGB_MSIX_BAR); 2421 adapter->msix_mem = bus_alloc_resource_any(dev, 2422 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2423 if (!adapter->msix_mem) { 2424 /* May not be enabled */ 2425 device_printf(adapter->dev, 2426 "Unable to map MSIX table \n"); 2427 goto msi; 2428 } 2429 2430 msgs = pci_msix_count(dev); 2431 if (msgs == 0) { /* system has msix disabled */ 2432 bus_release_resource(dev, SYS_RES_MEMORY, 2433 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2434 adapter->msix_mem = NULL; 2435 goto msi; 2436 } 2437 2438 /* Figure out a reasonable auto config value */ 2439 queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus; 2440 2441 /* Manual override */ 2442 if (igb_num_queues != 0) 2443 queues = igb_num_queues; 2444 2445 /* Can have max of 4 queues on 82575 */ 2446 if ((adapter->hw.mac.type == e1000_82575) && (queues > 4)) 2447 queues = 4; 2448 2449 /* 2450 ** One vector (RX/TX pair) per queue 2451 ** plus an additional for Link interrupt 2452 */ 2453 want = queues + 1; 2454 if (msgs >= want) 2455 msgs = want; 2456 else { 2457 device_printf(adapter->dev, 2458 "MSIX Configuration Problem, " 2459 "%d vectors configured, but %d queues wanted!\n", 2460 msgs, want); 2461 return (ENXIO); 2462 } 2463 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2464 device_printf(adapter->dev, 2465 "Using MSIX interrupts with %d vectors\n", msgs); 2466 adapter->num_queues = queues; 2467 return (msgs); 2468 } 2469msi: 2470 msgs = pci_msi_count(dev); 2471 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2472 device_printf(adapter->dev,"Using MSI interrupt\n"); 2473 return (msgs); 2474} 2475 2476/********************************************************************* 2477 * 2478 * Set up an fresh starting state 2479 * 2480 **********************************************************************/ 2481static void 2482igb_reset(struct adapter *adapter) 2483{ 2484 device_t dev = adapter->dev; 2485 struct e1000_hw *hw = &adapter->hw; 2486 struct e1000_fc_info *fc = &hw->fc; 2487 struct ifnet *ifp = adapter->ifp; 2488 u32 pba = 0; 2489 u16 hwm; 2490 2491 INIT_DEBUGOUT("igb_reset: begin"); 2492 2493 /* Let the firmware know the OS is in control */ 2494 igb_get_hw_control(adapter); 2495 2496 /* 2497 * Packet Buffer Allocation (PBA) 2498 * Writing PBA sets the receive portion of the buffer 2499 * the remainder is used for the transmit buffer. 2500 */ 2501 switch (hw->mac.type) { 2502 case e1000_82575: 2503 pba = E1000_PBA_32K; 2504 break; 2505 case e1000_82576: 2506 pba = E1000_PBA_64K; 2507 break; 2508 case e1000_82580: 2509 pba = E1000_PBA_35K; 2510 default: 2511 break; 2512 } 2513 2514 /* Special needs in case of Jumbo frames */ 2515 if ((hw->mac.type == e1000_82575) && (ifp->if_mtu > ETHERMTU)) { 2516 u32 tx_space, min_tx, min_rx; 2517 pba = E1000_READ_REG(hw, E1000_PBA); 2518 tx_space = pba >> 16; 2519 pba &= 0xffff; 2520 min_tx = (adapter->max_frame_size + 2521 sizeof(struct e1000_tx_desc) - ETHERNET_FCS_SIZE) * 2; 2522 min_tx = roundup2(min_tx, 1024); 2523 min_tx >>= 10; 2524 min_rx = adapter->max_frame_size; 2525 min_rx = roundup2(min_rx, 1024); 2526 min_rx >>= 10; 2527 if (tx_space < min_tx && 2528 ((min_tx - tx_space) < pba)) { 2529 pba = pba - (min_tx - tx_space); 2530 /* 2531 * if short on rx space, rx wins 2532 * and must trump tx adjustment 2533 */ 2534 if (pba < min_rx) 2535 pba = min_rx; 2536 } 2537 E1000_WRITE_REG(hw, E1000_PBA, pba); 2538 } 2539 2540 INIT_DEBUGOUT1("igb_init: pba=%dK",pba); 2541 2542 /* 2543 * These parameters control the automatic generation (Tx) and 2544 * response (Rx) to Ethernet PAUSE frames. 2545 * - High water mark should allow for at least two frames to be 2546 * received after sending an XOFF. 2547 * - Low water mark works best when it is very near the high water mark. 2548 * This allows the receiver to restart by sending XON when it has 2549 * drained a bit. 2550 */ 2551 hwm = min(((pba << 10) * 9 / 10), 2552 ((pba << 10) - 2 * adapter->max_frame_size)); 2553 2554 if (hw->mac.type < e1000_82576) { 2555 fc->high_water = hwm & 0xFFF8; /* 8-byte granularity */ 2556 fc->low_water = fc->high_water - 8; 2557 } else { 2558 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */ 2559 fc->low_water = fc->high_water - 16; 2560 } 2561 2562 fc->pause_time = IGB_FC_PAUSE_TIME; 2563 fc->send_xon = TRUE; 2564 2565 /* Set Flow control, use the tunable location if sane */ 2566 if ((igb_fc_setting >= 0) || (igb_fc_setting < 4)) 2567 fc->requested_mode = igb_fc_setting; 2568 else 2569 fc->requested_mode = e1000_fc_none; 2570 2571 fc->current_mode = fc->requested_mode; 2572 2573 /* Issue a global reset */ 2574 e1000_reset_hw(hw); 2575 E1000_WRITE_REG(hw, E1000_WUC, 0); 2576 2577 if (e1000_init_hw(hw) < 0) 2578 device_printf(dev, "Hardware Initialization Failed\n"); 2579 2580 if (hw->mac.type == e1000_82580) { 2581 u32 reg; 2582 2583 hwm = (pba << 10) - (2 * adapter->max_frame_size); 2584 /* 2585 * 0x80000000 - enable DMA COAL 2586 * 0x10000000 - use L0s as low power 2587 * 0x20000000 - use L1 as low power 2588 * X << 16 - exit dma coal when rx data exceeds X kB 2589 * Y - upper limit to stay in dma coal in units of 32usecs 2590 */ 2591 E1000_WRITE_REG(hw, E1000_DMACR, 2592 0xA0000006 | ((hwm << 6) & 0x00FF0000)); 2593 2594 /* set hwm to PBA - 2 * max frame size */ 2595 E1000_WRITE_REG(hw, E1000_FCRTC, hwm); 2596 /* 2597 * This sets the time to wait before requesting transition to 2598 * low power state to number of usecs needed to receive 1 512 2599 * byte frame at gigabit line rate 2600 */ 2601 E1000_WRITE_REG(hw, E1000_DMCTLX, 4); 2602 2603 /* free space in tx packet buffer to wake from DMA coal */ 2604 E1000_WRITE_REG(hw, E1000_DMCTXTH, 2605 (20480 - (2 * adapter->max_frame_size)) >> 6); 2606 2607 /* make low power state decision controlled by DMA coal */ 2608 reg = E1000_READ_REG(hw, E1000_PCIEMISC); 2609 E1000_WRITE_REG(hw, E1000_PCIEMISC, 2610 reg | E1000_PCIEMISC_LX_DECISION); 2611 } 2612 2613 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 2614 e1000_get_phy_info(hw); 2615 e1000_check_for_link(hw); 2616 return; 2617} 2618 2619/********************************************************************* 2620 * 2621 * Setup networking device structure and register an interface. 2622 * 2623 **********************************************************************/ 2624static void 2625igb_setup_interface(device_t dev, struct adapter *adapter) 2626{ 2627 struct ifnet *ifp; 2628 2629 INIT_DEBUGOUT("igb_setup_interface: begin"); 2630 2631 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2632 if (ifp == NULL) 2633 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 2634 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2635 ifp->if_mtu = ETHERMTU; 2636 ifp->if_init = igb_init; 2637 ifp->if_softc = adapter; 2638 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2639 ifp->if_ioctl = igb_ioctl; 2640 ifp->if_start = igb_start; 2641#if __FreeBSD_version >= 800000 2642 ifp->if_transmit = igb_mq_start; 2643 ifp->if_qflush = igb_qflush; 2644#endif 2645 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 2646 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 2647 IFQ_SET_READY(&ifp->if_snd); 2648 2649 ether_ifattach(ifp, adapter->hw.mac.addr); 2650 2651 ifp->if_capabilities = ifp->if_capenable = 0; 2652 2653 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_MTU; 2654 ifp->if_capabilities |= IFCAP_TSO4; 2655 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 2656 if (igb_header_split) 2657 ifp->if_capabilities |= IFCAP_LRO; 2658 2659 ifp->if_capenable = ifp->if_capabilities; 2660#ifdef DEVICE_POLLING 2661 ifp->if_capabilities |= IFCAP_POLLING; 2662#endif 2663 2664 /* 2665 * Tell the upper layer(s) we 2666 * support full VLAN capability. 2667 */ 2668 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2669 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2670 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2671 2672 /* 2673 ** Dont turn this on by default, if vlans are 2674 ** created on another pseudo device (eg. lagg) 2675 ** then vlan events are not passed thru, breaking 2676 ** operation, but with HW FILTER off it works. If 2677 ** using vlans directly on the em driver you can 2678 ** enable this and get full hardware tag filtering. 2679 */ 2680 ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2681 2682 /* 2683 * Specify the media types supported by this adapter and register 2684 * callbacks to update media and link information 2685 */ 2686 ifmedia_init(&adapter->media, IFM_IMASK, 2687 igb_media_change, igb_media_status); 2688 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2689 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2690 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 2691 0, NULL); 2692 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 2693 } else { 2694 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2695 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2696 0, NULL); 2697 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2698 0, NULL); 2699 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2700 0, NULL); 2701 if (adapter->hw.phy.type != e1000_phy_ife) { 2702 ifmedia_add(&adapter->media, 2703 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2704 ifmedia_add(&adapter->media, 2705 IFM_ETHER | IFM_1000_T, 0, NULL); 2706 } 2707 } 2708 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2709 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2710} 2711 2712 2713/* 2714 * Manage DMA'able memory. 2715 */ 2716static void 2717igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 2718{ 2719 if (error) 2720 return; 2721 *(bus_addr_t *) arg = segs[0].ds_addr; 2722} 2723 2724static int 2725igb_dma_malloc(struct adapter *adapter, bus_size_t size, 2726 struct igb_dma_alloc *dma, int mapflags) 2727{ 2728 int error; 2729 2730 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 2731 IGB_DBA_ALIGN, 0, /* alignment, bounds */ 2732 BUS_SPACE_MAXADDR, /* lowaddr */ 2733 BUS_SPACE_MAXADDR, /* highaddr */ 2734 NULL, NULL, /* filter, filterarg */ 2735 size, /* maxsize */ 2736 1, /* nsegments */ 2737 size, /* maxsegsize */ 2738 0, /* flags */ 2739 NULL, /* lockfunc */ 2740 NULL, /* lockarg */ 2741 &dma->dma_tag); 2742 if (error) { 2743 device_printf(adapter->dev, 2744 "%s: bus_dma_tag_create failed: %d\n", 2745 __func__, error); 2746 goto fail_0; 2747 } 2748 2749 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 2750 BUS_DMA_NOWAIT, &dma->dma_map); 2751 if (error) { 2752 device_printf(adapter->dev, 2753 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 2754 __func__, (uintmax_t)size, error); 2755 goto fail_2; 2756 } 2757 2758 dma->dma_paddr = 0; 2759 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2760 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 2761 if (error || dma->dma_paddr == 0) { 2762 device_printf(adapter->dev, 2763 "%s: bus_dmamap_load failed: %d\n", 2764 __func__, error); 2765 goto fail_3; 2766 } 2767 2768 return (0); 2769 2770fail_3: 2771 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2772fail_2: 2773 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2774 bus_dma_tag_destroy(dma->dma_tag); 2775fail_0: 2776 dma->dma_map = NULL; 2777 dma->dma_tag = NULL; 2778 2779 return (error); 2780} 2781 2782static void 2783igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma) 2784{ 2785 if (dma->dma_tag == NULL) 2786 return; 2787 if (dma->dma_map != NULL) { 2788 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2789 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2790 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2791 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2792 dma->dma_map = NULL; 2793 } 2794 bus_dma_tag_destroy(dma->dma_tag); 2795 dma->dma_tag = NULL; 2796} 2797 2798 2799/********************************************************************* 2800 * 2801 * Allocate memory for the transmit and receive rings, and then 2802 * the descriptors associated with each, called only once at attach. 2803 * 2804 **********************************************************************/ 2805static int 2806igb_allocate_queues(struct adapter *adapter) 2807{ 2808 device_t dev = adapter->dev; 2809 struct igb_queue *que = NULL; 2810 struct tx_ring *txr = NULL; 2811 struct rx_ring *rxr = NULL; 2812 int rsize, tsize, error = E1000_SUCCESS; 2813 int txconf = 0, rxconf = 0; 2814 2815 /* First allocate the top level queue structs */ 2816 if (!(adapter->queues = 2817 (struct igb_queue *) malloc(sizeof(struct igb_queue) * 2818 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2819 device_printf(dev, "Unable to allocate queue memory\n"); 2820 error = ENOMEM; 2821 goto fail; 2822 } 2823 2824 /* Next allocate the TX ring struct memory */ 2825 if (!(adapter->tx_rings = 2826 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2827 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2828 device_printf(dev, "Unable to allocate TX ring memory\n"); 2829 error = ENOMEM; 2830 goto tx_fail; 2831 } 2832 2833 /* Now allocate the RX */ 2834 if (!(adapter->rx_rings = 2835 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2836 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2837 device_printf(dev, "Unable to allocate RX ring memory\n"); 2838 error = ENOMEM; 2839 goto rx_fail; 2840 } 2841 2842 tsize = roundup2(adapter->num_tx_desc * 2843 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN); 2844 /* 2845 * Now set up the TX queues, txconf is needed to handle the 2846 * possibility that things fail midcourse and we need to 2847 * undo memory gracefully 2848 */ 2849 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2850 /* Set up some basics */ 2851 txr = &adapter->tx_rings[i]; 2852 txr->adapter = adapter; 2853 txr->me = i; 2854 2855 /* Initialize the TX lock */ 2856 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2857 device_get_nameunit(dev), txr->me); 2858 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2859 2860 if (igb_dma_malloc(adapter, tsize, 2861 &txr->txdma, BUS_DMA_NOWAIT)) { 2862 device_printf(dev, 2863 "Unable to allocate TX Descriptor memory\n"); 2864 error = ENOMEM; 2865 goto err_tx_desc; 2866 } 2867 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; 2868 bzero((void *)txr->tx_base, tsize); 2869 2870 /* Now allocate transmit buffers for the ring */ 2871 if (igb_allocate_transmit_buffers(txr)) { 2872 device_printf(dev, 2873 "Critical Failure setting up transmit buffers\n"); 2874 error = ENOMEM; 2875 goto err_tx_desc; 2876 } 2877#if __FreeBSD_version >= 800000 2878 /* Allocate a buf ring */ 2879 txr->br = buf_ring_alloc(IGB_BR_SIZE, M_DEVBUF, 2880 M_WAITOK, &txr->tx_mtx); 2881#endif 2882 } 2883 2884 /* 2885 * Next the RX queues... 2886 */ 2887 rsize = roundup2(adapter->num_rx_desc * 2888 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 2889 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2890 rxr = &adapter->rx_rings[i]; 2891 rxr->adapter = adapter; 2892 rxr->me = i; 2893 2894 /* Initialize the RX lock */ 2895 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2896 device_get_nameunit(dev), txr->me); 2897 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2898 2899 if (igb_dma_malloc(adapter, rsize, 2900 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2901 device_printf(dev, 2902 "Unable to allocate RxDescriptor memory\n"); 2903 error = ENOMEM; 2904 goto err_rx_desc; 2905 } 2906 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2907 bzero((void *)rxr->rx_base, rsize); 2908 2909 /* Allocate receive buffers for the ring*/ 2910 if (igb_allocate_receive_buffers(rxr)) { 2911 device_printf(dev, 2912 "Critical Failure setting up receive buffers\n"); 2913 error = ENOMEM; 2914 goto err_rx_desc; 2915 } 2916 } 2917 2918 /* 2919 ** Finally set up the queue holding structs 2920 */ 2921 for (int i = 0; i < adapter->num_queues; i++) { 2922 que = &adapter->queues[i]; 2923 que->adapter = adapter; 2924 que->txr = &adapter->tx_rings[i]; 2925 que->rxr = &adapter->rx_rings[i]; 2926 } 2927 2928 return (0); 2929 2930err_rx_desc: 2931 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2932 igb_dma_free(adapter, &rxr->rxdma); 2933err_tx_desc: 2934 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2935 igb_dma_free(adapter, &txr->txdma); 2936 free(adapter->rx_rings, M_DEVBUF); 2937rx_fail: 2938#if __FreeBSD_version >= 800000 2939 buf_ring_free(txr->br, M_DEVBUF); 2940#endif 2941 free(adapter->tx_rings, M_DEVBUF); 2942tx_fail: 2943 free(adapter->queues, M_DEVBUF); 2944fail: 2945 return (error); 2946} 2947 2948/********************************************************************* 2949 * 2950 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2951 * the information needed to transmit a packet on the wire. This is 2952 * called only once at attach, setup is done every reset. 2953 * 2954 **********************************************************************/ 2955static int 2956igb_allocate_transmit_buffers(struct tx_ring *txr) 2957{ 2958 struct adapter *adapter = txr->adapter; 2959 device_t dev = adapter->dev; 2960 struct igb_tx_buffer *txbuf; 2961 int error, i; 2962 2963 /* 2964 * Setup DMA descriptor areas. 2965 */ 2966 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 2967 1, 0, /* alignment, bounds */ 2968 BUS_SPACE_MAXADDR, /* lowaddr */ 2969 BUS_SPACE_MAXADDR, /* highaddr */ 2970 NULL, NULL, /* filter, filterarg */ 2971 IGB_TSO_SIZE, /* maxsize */ 2972 IGB_MAX_SCATTER, /* nsegments */ 2973 PAGE_SIZE, /* maxsegsize */ 2974 0, /* flags */ 2975 NULL, /* lockfunc */ 2976 NULL, /* lockfuncarg */ 2977 &txr->txtag))) { 2978 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2979 goto fail; 2980 } 2981 2982 if (!(txr->tx_buffers = 2983 (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) * 2984 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2985 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2986 error = ENOMEM; 2987 goto fail; 2988 } 2989 2990 /* Create the descriptor buffer dma maps */ 2991 txbuf = txr->tx_buffers; 2992 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2993 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2994 if (error != 0) { 2995 device_printf(dev, "Unable to create TX DMA map\n"); 2996 goto fail; 2997 } 2998 } 2999 3000 return 0; 3001fail: 3002 /* We free all, it handles case where we are in the middle */ 3003 igb_free_transmit_structures(adapter); 3004 return (error); 3005} 3006 3007/********************************************************************* 3008 * 3009 * Initialize a transmit ring. 3010 * 3011 **********************************************************************/ 3012static void 3013igb_setup_transmit_ring(struct tx_ring *txr) 3014{ 3015 struct adapter *adapter = txr->adapter; 3016 struct igb_tx_buffer *txbuf; 3017 int i; 3018 3019 /* Clear the old descriptor contents */ 3020 IGB_TX_LOCK(txr); 3021 bzero((void *)txr->tx_base, 3022 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc); 3023 /* Reset indices */ 3024 txr->next_avail_desc = 0; 3025 txr->next_to_clean = 0; 3026 3027 /* Free any existing tx buffers. */ 3028 txbuf = txr->tx_buffers; 3029 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 3030 if (txbuf->m_head != NULL) { 3031 bus_dmamap_sync(txr->txtag, txbuf->map, 3032 BUS_DMASYNC_POSTWRITE); 3033 bus_dmamap_unload(txr->txtag, txbuf->map); 3034 m_freem(txbuf->m_head); 3035 txbuf->m_head = NULL; 3036 } 3037 /* clear the watch index */ 3038 txbuf->next_eop = -1; 3039 } 3040 3041 /* Set number of descriptors available */ 3042 txr->tx_avail = adapter->num_tx_desc; 3043 3044 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3045 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3046 IGB_TX_UNLOCK(txr); 3047} 3048 3049/********************************************************************* 3050 * 3051 * Initialize all transmit rings. 3052 * 3053 **********************************************************************/ 3054static void 3055igb_setup_transmit_structures(struct adapter *adapter) 3056{ 3057 struct tx_ring *txr = adapter->tx_rings; 3058 3059 for (int i = 0; i < adapter->num_queues; i++, txr++) 3060 igb_setup_transmit_ring(txr); 3061 3062 return; 3063} 3064 3065/********************************************************************* 3066 * 3067 * Enable transmit unit. 3068 * 3069 **********************************************************************/ 3070static void 3071igb_initialize_transmit_units(struct adapter *adapter) 3072{ 3073 struct tx_ring *txr = adapter->tx_rings; 3074 struct e1000_hw *hw = &adapter->hw; 3075 u32 tctl, txdctl; 3076 3077 INIT_DEBUGOUT("igb_initialize_transmit_units: begin"); 3078 3079 /* Setup the Tx Descriptor Rings */ 3080 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3081 u64 bus_addr = txr->txdma.dma_paddr; 3082 3083 E1000_WRITE_REG(hw, E1000_TDLEN(i), 3084 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 3085 E1000_WRITE_REG(hw, E1000_TDBAH(i), 3086 (uint32_t)(bus_addr >> 32)); 3087 E1000_WRITE_REG(hw, E1000_TDBAL(i), 3088 (uint32_t)bus_addr); 3089 3090 /* Setup the HW Tx Head and Tail descriptor pointers */ 3091 E1000_WRITE_REG(hw, E1000_TDT(i), 0); 3092 E1000_WRITE_REG(hw, E1000_TDH(i), 0); 3093 3094 HW_DEBUGOUT2("Base = %x, Length = %x\n", 3095 E1000_READ_REG(hw, E1000_TDBAL(i)), 3096 E1000_READ_REG(hw, E1000_TDLEN(i))); 3097 3098 txr->watchdog_check = FALSE; 3099 3100 txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i)); 3101 txdctl |= IGB_TX_PTHRESH; 3102 txdctl |= IGB_TX_HTHRESH << 8; 3103 txdctl |= IGB_TX_WTHRESH << 16; 3104 txdctl |= E1000_TXDCTL_QUEUE_ENABLE; 3105 E1000_WRITE_REG(hw, E1000_TXDCTL(i), txdctl); 3106 } 3107 3108 /* Program the Transmit Control Register */ 3109 tctl = E1000_READ_REG(hw, E1000_TCTL); 3110 tctl &= ~E1000_TCTL_CT; 3111 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 3112 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 3113 3114 e1000_config_collision_dist(hw); 3115 3116 /* This write will effectively turn on the transmit unit. */ 3117 E1000_WRITE_REG(hw, E1000_TCTL, tctl); 3118} 3119 3120/********************************************************************* 3121 * 3122 * Free all transmit rings. 3123 * 3124 **********************************************************************/ 3125static void 3126igb_free_transmit_structures(struct adapter *adapter) 3127{ 3128 struct tx_ring *txr = adapter->tx_rings; 3129 3130 for (int i = 0; i < adapter->num_queues; i++, txr++) { 3131 IGB_TX_LOCK(txr); 3132 igb_free_transmit_buffers(txr); 3133 igb_dma_free(adapter, &txr->txdma); 3134 IGB_TX_UNLOCK(txr); 3135 IGB_TX_LOCK_DESTROY(txr); 3136 } 3137 free(adapter->tx_rings, M_DEVBUF); 3138} 3139 3140/********************************************************************* 3141 * 3142 * Free transmit ring related data structures. 3143 * 3144 **********************************************************************/ 3145static void 3146igb_free_transmit_buffers(struct tx_ring *txr) 3147{ 3148 struct adapter *adapter = txr->adapter; 3149 struct igb_tx_buffer *tx_buffer; 3150 int i; 3151 3152 INIT_DEBUGOUT("free_transmit_ring: begin"); 3153 3154 if (txr->tx_buffers == NULL) 3155 return; 3156 3157 tx_buffer = txr->tx_buffers; 3158 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3159 if (tx_buffer->m_head != NULL) { 3160 bus_dmamap_sync(txr->txtag, tx_buffer->map, 3161 BUS_DMASYNC_POSTWRITE); 3162 bus_dmamap_unload(txr->txtag, 3163 tx_buffer->map); 3164 m_freem(tx_buffer->m_head); 3165 tx_buffer->m_head = NULL; 3166 if (tx_buffer->map != NULL) { 3167 bus_dmamap_destroy(txr->txtag, 3168 tx_buffer->map); 3169 tx_buffer->map = NULL; 3170 } 3171 } else if (tx_buffer->map != NULL) { 3172 bus_dmamap_unload(txr->txtag, 3173 tx_buffer->map); 3174 bus_dmamap_destroy(txr->txtag, 3175 tx_buffer->map); 3176 tx_buffer->map = NULL; 3177 } 3178 } 3179#if __FreeBSD_version >= 800000 3180 if (txr->br != NULL) 3181 buf_ring_free(txr->br, M_DEVBUF); 3182#endif 3183 if (txr->tx_buffers != NULL) { 3184 free(txr->tx_buffers, M_DEVBUF); 3185 txr->tx_buffers = NULL; 3186 } 3187 if (txr->txtag != NULL) { 3188 bus_dma_tag_destroy(txr->txtag); 3189 txr->txtag = NULL; 3190 } 3191 return; 3192} 3193 3194/********************************************************************** 3195 * 3196 * Setup work for hardware segmentation offload (TSO) 3197 * 3198 **********************************************************************/ 3199static boolean_t 3200igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen) 3201{ 3202 struct adapter *adapter = txr->adapter; 3203 struct e1000_adv_tx_context_desc *TXD; 3204 struct igb_tx_buffer *tx_buffer; 3205 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3206 u32 mss_l4len_idx = 0; 3207 u16 vtag = 0; 3208 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 3209 struct ether_vlan_header *eh; 3210 struct ip *ip; 3211 struct tcphdr *th; 3212 3213 3214 /* 3215 * Determine where frame payload starts. 3216 * Jump over vlan headers if already present 3217 */ 3218 eh = mtod(mp, struct ether_vlan_header *); 3219 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 3220 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3221 else 3222 ehdrlen = ETHER_HDR_LEN; 3223 3224 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3225 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3226 return FALSE; 3227 3228 /* Only supports IPV4 for now */ 3229 ctxd = txr->next_avail_desc; 3230 tx_buffer = &txr->tx_buffers[ctxd]; 3231 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3232 3233 ip = (struct ip *)(mp->m_data + ehdrlen); 3234 if (ip->ip_p != IPPROTO_TCP) 3235 return FALSE; /* 0 */ 3236 ip->ip_sum = 0; 3237 ip_hlen = ip->ip_hl << 2; 3238 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3239 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3240 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3241 tcp_hlen = th->th_off << 2; 3242 /* 3243 * Calculate header length, this is used 3244 * in the transmit desc in igb_xmit 3245 */ 3246 *hdrlen = ehdrlen + ip_hlen + tcp_hlen; 3247 3248 /* VLAN MACLEN IPLEN */ 3249 if (mp->m_flags & M_VLANTAG) { 3250 vtag = htole16(mp->m_pkthdr.ether_vtag); 3251 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3252 } 3253 3254 vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT); 3255 vlan_macip_lens |= ip_hlen; 3256 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3257 3258 /* ADV DTYPE TUCMD */ 3259 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3260 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3261 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3262 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3263 3264 /* MSS L4LEN IDX */ 3265 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT); 3266 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); 3267 /* 82575 needs the queue index added */ 3268 if (adapter->hw.mac.type == e1000_82575) 3269 mss_l4len_idx |= txr->me << 4; 3270 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3271 3272 TXD->seqnum_seed = htole32(0); 3273 tx_buffer->m_head = NULL; 3274 tx_buffer->next_eop = -1; 3275 3276 if (++ctxd == adapter->num_tx_desc) 3277 ctxd = 0; 3278 3279 txr->tx_avail--; 3280 txr->next_avail_desc = ctxd; 3281 return TRUE; 3282} 3283 3284 3285/********************************************************************* 3286 * 3287 * Context Descriptor setup for VLAN or CSUM 3288 * 3289 **********************************************************************/ 3290 3291static bool 3292igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3293{ 3294 struct adapter *adapter = txr->adapter; 3295 struct e1000_adv_tx_context_desc *TXD; 3296 struct igb_tx_buffer *tx_buffer; 3297 u32 vlan_macip_lens, type_tucmd_mlhl, mss_l4len_idx; 3298 struct ether_vlan_header *eh; 3299 struct ip *ip = NULL; 3300 struct ip6_hdr *ip6; 3301 int ehdrlen, ctxd, ip_hlen = 0; 3302 u16 etype, vtag = 0; 3303 u8 ipproto = 0; 3304 bool offload = TRUE; 3305 3306 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 3307 offload = FALSE; 3308 3309 vlan_macip_lens = type_tucmd_mlhl = mss_l4len_idx = 0; 3310 ctxd = txr->next_avail_desc; 3311 tx_buffer = &txr->tx_buffers[ctxd]; 3312 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3313 3314 /* 3315 ** In advanced descriptors the vlan tag must 3316 ** be placed into the context descriptor, thus 3317 ** we need to be here just for that setup. 3318 */ 3319 if (mp->m_flags & M_VLANTAG) { 3320 vtag = htole16(mp->m_pkthdr.ether_vtag); 3321 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3322 } else if (offload == FALSE) 3323 return FALSE; 3324 3325 /* 3326 * Determine where frame payload starts. 3327 * Jump over vlan headers if already present, 3328 * helpful for QinQ too. 3329 */ 3330 eh = mtod(mp, struct ether_vlan_header *); 3331 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3332 etype = ntohs(eh->evl_proto); 3333 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3334 } else { 3335 etype = ntohs(eh->evl_encap_proto); 3336 ehdrlen = ETHER_HDR_LEN; 3337 } 3338 3339 /* Set the ether header length */ 3340 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; 3341 3342 switch (etype) { 3343 case ETHERTYPE_IP: 3344 ip = (struct ip *)(mp->m_data + ehdrlen); 3345 ip_hlen = ip->ip_hl << 2; 3346 if (mp->m_len < ehdrlen + ip_hlen) { 3347 offload = FALSE; 3348 break; 3349 } 3350 ipproto = ip->ip_p; 3351 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3352 break; 3353 case ETHERTYPE_IPV6: 3354 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3355 ip_hlen = sizeof(struct ip6_hdr); 3356 if (mp->m_len < ehdrlen + ip_hlen) 3357 return (FALSE); 3358 ipproto = ip6->ip6_nxt; 3359 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; 3360 break; 3361 default: 3362 offload = FALSE; 3363 break; 3364 } 3365 3366 vlan_macip_lens |= ip_hlen; 3367 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3368 3369 switch (ipproto) { 3370 case IPPROTO_TCP: 3371 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 3372 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3373 break; 3374 case IPPROTO_UDP: 3375 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 3376 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; 3377 break; 3378#if __FreeBSD_version >= 800000 3379 case IPPROTO_SCTP: 3380 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 3381 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; 3382 break; 3383#endif 3384 default: 3385 offload = FALSE; 3386 break; 3387 } 3388 3389 /* 82575 needs the queue index added */ 3390 if (adapter->hw.mac.type == e1000_82575) 3391 mss_l4len_idx = txr->me << 4; 3392 3393 /* Now copy bits into descriptor */ 3394 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3395 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3396 TXD->seqnum_seed = htole32(0); 3397 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3398 3399 tx_buffer->m_head = NULL; 3400 tx_buffer->next_eop = -1; 3401 3402 /* We've consumed the first desc, adjust counters */ 3403 if (++ctxd == adapter->num_tx_desc) 3404 ctxd = 0; 3405 txr->next_avail_desc = ctxd; 3406 --txr->tx_avail; 3407 3408 return (offload); 3409} 3410 3411 3412/********************************************************************** 3413 * 3414 * Examine each tx_buffer in the used queue. If the hardware is done 3415 * processing the packet then free associated resources. The 3416 * tx_buffer is put back on the free queue. 3417 * 3418 * TRUE return means there's work in the ring to clean, FALSE its empty. 3419 **********************************************************************/ 3420static bool 3421igb_txeof(struct tx_ring *txr) 3422{ 3423 struct adapter *adapter = txr->adapter; 3424 int first, last, done; 3425 struct igb_tx_buffer *tx_buffer; 3426 struct e1000_tx_desc *tx_desc, *eop_desc; 3427 struct ifnet *ifp = adapter->ifp; 3428 3429 IGB_TX_LOCK_ASSERT(txr); 3430 3431 if (txr->tx_avail == adapter->num_tx_desc) 3432 return FALSE; 3433 3434 first = txr->next_to_clean; 3435 tx_desc = &txr->tx_base[first]; 3436 tx_buffer = &txr->tx_buffers[first]; 3437 last = tx_buffer->next_eop; 3438 eop_desc = &txr->tx_base[last]; 3439 3440 /* 3441 * What this does is get the index of the 3442 * first descriptor AFTER the EOP of the 3443 * first packet, that way we can do the 3444 * simple comparison on the inner while loop. 3445 */ 3446 if (++last == adapter->num_tx_desc) 3447 last = 0; 3448 done = last; 3449 3450 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3451 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3452 3453 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3454 /* We clean the range of the packet */ 3455 while (first != done) { 3456 tx_desc->upper.data = 0; 3457 tx_desc->lower.data = 0; 3458 tx_desc->buffer_addr = 0; 3459 ++txr->tx_avail; 3460 3461 if (tx_buffer->m_head) { 3462 txr->bytes += 3463 tx_buffer->m_head->m_pkthdr.len; 3464 bus_dmamap_sync(txr->txtag, 3465 tx_buffer->map, 3466 BUS_DMASYNC_POSTWRITE); 3467 bus_dmamap_unload(txr->txtag, 3468 tx_buffer->map); 3469 3470 m_freem(tx_buffer->m_head); 3471 tx_buffer->m_head = NULL; 3472 } 3473 tx_buffer->next_eop = -1; 3474 txr->watchdog_time = ticks; 3475 3476 if (++first == adapter->num_tx_desc) 3477 first = 0; 3478 3479 tx_buffer = &txr->tx_buffers[first]; 3480 tx_desc = &txr->tx_base[first]; 3481 } 3482 ++txr->packets; 3483 ++ifp->if_opackets; 3484 /* See if we can continue to the next packet */ 3485 last = tx_buffer->next_eop; 3486 if (last != -1) { 3487 eop_desc = &txr->tx_base[last]; 3488 /* Get new done point */ 3489 if (++last == adapter->num_tx_desc) last = 0; 3490 done = last; 3491 } else 3492 break; 3493 } 3494 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3495 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3496 3497 txr->next_to_clean = first; 3498 3499 /* 3500 * If we have enough room, clear IFF_DRV_OACTIVE 3501 * to tell the stack that it is OK to send packets. 3502 */ 3503 if (txr->tx_avail > IGB_TX_CLEANUP_THRESHOLD) { 3504 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3505 /* All clean, turn off the watchdog */ 3506 if (txr->tx_avail == adapter->num_tx_desc) { 3507 txr->watchdog_check = FALSE; 3508 return FALSE; 3509 } 3510 } 3511 3512 return (TRUE); 3513} 3514 3515 3516/********************************************************************* 3517 * 3518 * Refresh mbuf buffers for RX descriptor rings 3519 * - now keeps its own state so discards due to resource 3520 * exhaustion are unnecessary, if an mbuf cannot be obtained 3521 * it just returns, keeping its placeholder, thus it can simply 3522 * be recalled to try again. 3523 * 3524 **********************************************************************/ 3525static void 3526igb_refresh_mbufs(struct rx_ring *rxr, int limit) 3527{ 3528 struct adapter *adapter = rxr->adapter; 3529 bus_dma_segment_t hseg[1]; 3530 bus_dma_segment_t pseg[1]; 3531 struct igb_rx_buf *rxbuf; 3532 struct mbuf *mh, *mp; 3533 int i, nsegs, error, cleaned; 3534 3535 i = rxr->next_to_refresh; 3536 cleaned = -1; /* Signify no completions */ 3537 while (i != limit) { 3538 rxbuf = &rxr->rx_buffers[i]; 3539 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 3540 mh = m_gethdr(M_DONTWAIT, MT_DATA); 3541 if (mh == NULL) 3542 goto update; 3543 mh->m_pkthdr.len = mh->m_len = MHLEN; 3544 mh->m_len = MHLEN; 3545 mh->m_flags |= M_PKTHDR; 3546 m_adj(mh, ETHER_ALIGN); 3547 /* Get the memory mapping */ 3548 error = bus_dmamap_load_mbuf_sg(rxr->htag, 3549 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); 3550 if (error != 0) { 3551 printf("GET BUF: dmamap load" 3552 " failure - %d\n", error); 3553 m_free(mh); 3554 goto update; 3555 } 3556 rxbuf->m_head = mh; 3557 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3558 BUS_DMASYNC_PREREAD); 3559 rxr->rx_base[i].read.hdr_addr = 3560 htole64(hseg[0].ds_addr); 3561 } 3562 3563 if (rxbuf->m_pack == NULL) { 3564 mp = m_getjcl(M_DONTWAIT, MT_DATA, 3565 M_PKTHDR, adapter->rx_mbuf_sz); 3566 if (mp == NULL) 3567 goto update; 3568 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3569 /* Get the memory mapping */ 3570 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 3571 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); 3572 if (error != 0) { 3573 printf("GET BUF: dmamap load" 3574 " failure - %d\n", error); 3575 m_free(mp); 3576 goto update; 3577 } 3578 rxbuf->m_pack = mp; 3579 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3580 BUS_DMASYNC_PREREAD); 3581 rxr->rx_base[i].read.pkt_addr = 3582 htole64(pseg[0].ds_addr); 3583 } 3584 3585 cleaned = i; 3586 /* Calculate next index */ 3587 if (++i == adapter->num_rx_desc) 3588 i = 0; 3589 /* This is the work marker for refresh */ 3590 rxr->next_to_refresh = i; 3591 } 3592update: 3593 if (cleaned != -1) /* If we refreshed some, bump tail */ 3594 E1000_WRITE_REG(&adapter->hw, 3595 E1000_RDT(rxr->me), cleaned); 3596 return; 3597} 3598 3599 3600/********************************************************************* 3601 * 3602 * Allocate memory for rx_buffer structures. Since we use one 3603 * rx_buffer per received packet, the maximum number of rx_buffer's 3604 * that we'll need is equal to the number of receive descriptors 3605 * that we've allocated. 3606 * 3607 **********************************************************************/ 3608static int 3609igb_allocate_receive_buffers(struct rx_ring *rxr) 3610{ 3611 struct adapter *adapter = rxr->adapter; 3612 device_t dev = adapter->dev; 3613 struct igb_rx_buf *rxbuf; 3614 int i, bsize, error; 3615 3616 bsize = sizeof(struct igb_rx_buf) * adapter->num_rx_desc; 3617 if (!(rxr->rx_buffers = 3618 (struct igb_rx_buf *) malloc(bsize, 3619 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3620 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3621 error = ENOMEM; 3622 goto fail; 3623 } 3624 3625 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 3626 1, 0, /* alignment, bounds */ 3627 BUS_SPACE_MAXADDR, /* lowaddr */ 3628 BUS_SPACE_MAXADDR, /* highaddr */ 3629 NULL, NULL, /* filter, filterarg */ 3630 MSIZE, /* maxsize */ 3631 1, /* nsegments */ 3632 MSIZE, /* maxsegsize */ 3633 0, /* flags */ 3634 NULL, /* lockfunc */ 3635 NULL, /* lockfuncarg */ 3636 &rxr->htag))) { 3637 device_printf(dev, "Unable to create RX DMA tag\n"); 3638 goto fail; 3639 } 3640 3641 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), 3642 1, 0, /* alignment, bounds */ 3643 BUS_SPACE_MAXADDR, /* lowaddr */ 3644 BUS_SPACE_MAXADDR, /* highaddr */ 3645 NULL, NULL, /* filter, filterarg */ 3646 MJUMPAGESIZE, /* maxsize */ 3647 1, /* nsegments */ 3648 MJUMPAGESIZE, /* maxsegsize */ 3649 0, /* flags */ 3650 NULL, /* lockfunc */ 3651 NULL, /* lockfuncarg */ 3652 &rxr->ptag))) { 3653 device_printf(dev, "Unable to create RX payload DMA tag\n"); 3654 goto fail; 3655 } 3656 3657 for (i = 0; i < adapter->num_rx_desc; i++) { 3658 rxbuf = &rxr->rx_buffers[i]; 3659 error = bus_dmamap_create(rxr->htag, 3660 BUS_DMA_NOWAIT, &rxbuf->hmap); 3661 if (error) { 3662 device_printf(dev, 3663 "Unable to create RX head DMA maps\n"); 3664 goto fail; 3665 } 3666 error = bus_dmamap_create(rxr->ptag, 3667 BUS_DMA_NOWAIT, &rxbuf->pmap); 3668 if (error) { 3669 device_printf(dev, 3670 "Unable to create RX packet DMA maps\n"); 3671 goto fail; 3672 } 3673 } 3674 3675 return (0); 3676 3677fail: 3678 /* Frees all, but can handle partial completion */ 3679 igb_free_receive_structures(adapter); 3680 return (error); 3681} 3682 3683 3684static void 3685igb_free_receive_ring(struct rx_ring *rxr) 3686{ 3687 struct adapter *adapter; 3688 struct igb_rx_buf *rxbuf; 3689 int i; 3690 3691 adapter = rxr->adapter; 3692 for (i = 0; i < adapter->num_rx_desc; i++) { 3693 rxbuf = &rxr->rx_buffers[i]; 3694 if (rxbuf->m_head != NULL) { 3695 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3696 BUS_DMASYNC_POSTREAD); 3697 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 3698 rxbuf->m_head->m_flags |= M_PKTHDR; 3699 m_freem(rxbuf->m_head); 3700 } 3701 if (rxbuf->m_pack != NULL) { 3702 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3703 BUS_DMASYNC_POSTREAD); 3704 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 3705 rxbuf->m_pack->m_flags |= M_PKTHDR; 3706 m_freem(rxbuf->m_pack); 3707 } 3708 rxbuf->m_head = NULL; 3709 rxbuf->m_pack = NULL; 3710 } 3711} 3712 3713 3714/********************************************************************* 3715 * 3716 * Initialize a receive ring and its buffers. 3717 * 3718 **********************************************************************/ 3719static int 3720igb_setup_receive_ring(struct rx_ring *rxr) 3721{ 3722 struct adapter *adapter; 3723 struct ifnet *ifp; 3724 device_t dev; 3725 struct igb_rx_buf *rxbuf; 3726 bus_dma_segment_t pseg[1], hseg[1]; 3727 struct lro_ctrl *lro = &rxr->lro; 3728 int rsize, nsegs, error = 0; 3729 3730 adapter = rxr->adapter; 3731 dev = adapter->dev; 3732 ifp = adapter->ifp; 3733 3734 /* Clear the ring contents */ 3735 IGB_RX_LOCK(rxr); 3736 rsize = roundup2(adapter->num_rx_desc * 3737 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 3738 bzero((void *)rxr->rx_base, rsize); 3739 3740 /* 3741 ** Free current RX buffer structures and their mbufs 3742 */ 3743 igb_free_receive_ring(rxr); 3744 3745 /* Configure for header split? */ 3746 if (igb_header_split) 3747 rxr->hdr_split = TRUE; 3748 3749 /* Now replenish the ring mbufs */ 3750 for (int j = 0; j != adapter->num_rx_desc; ++j) { 3751 struct mbuf *mh, *mp; 3752 3753 rxbuf = &rxr->rx_buffers[j]; 3754 if (rxr->hdr_split == FALSE) 3755 goto skip_head; 3756 3757 /* First the header */ 3758 rxbuf->m_head = m_gethdr(M_DONTWAIT, MT_DATA); 3759 if (rxbuf->m_head == NULL) { 3760 error = ENOBUFS; 3761 goto fail; 3762 } 3763 m_adj(rxbuf->m_head, ETHER_ALIGN); 3764 mh = rxbuf->m_head; 3765 mh->m_len = mh->m_pkthdr.len = MHLEN; 3766 mh->m_flags |= M_PKTHDR; 3767 /* Get the memory mapping */ 3768 error = bus_dmamap_load_mbuf_sg(rxr->htag, 3769 rxbuf->hmap, rxbuf->m_head, hseg, 3770 &nsegs, BUS_DMA_NOWAIT); 3771 if (error != 0) /* Nothing elegant to do here */ 3772 goto fail; 3773 bus_dmamap_sync(rxr->htag, 3774 rxbuf->hmap, BUS_DMASYNC_PREREAD); 3775 /* Update descriptor */ 3776 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 3777 3778skip_head: 3779 /* Now the payload cluster */ 3780 rxbuf->m_pack = m_getjcl(M_DONTWAIT, MT_DATA, 3781 M_PKTHDR, adapter->rx_mbuf_sz); 3782 if (rxbuf->m_pack == NULL) { 3783 error = ENOBUFS; 3784 goto fail; 3785 } 3786 mp = rxbuf->m_pack; 3787 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 3788 /* Get the memory mapping */ 3789 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 3790 rxbuf->pmap, mp, pseg, 3791 &nsegs, BUS_DMA_NOWAIT); 3792 if (error != 0) 3793 goto fail; 3794 bus_dmamap_sync(rxr->ptag, 3795 rxbuf->pmap, BUS_DMASYNC_PREREAD); 3796 /* Update descriptor */ 3797 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 3798 } 3799 3800 /* Setup our descriptor indices */ 3801 rxr->next_to_check = 0; 3802 rxr->next_to_refresh = 0; 3803 rxr->lro_enabled = FALSE; 3804 rxr->rx_split_packets = 0; 3805 rxr->rx_bytes = 0; 3806 3807 rxr->fmp = NULL; 3808 rxr->lmp = NULL; 3809 rxr->discard = FALSE; 3810 3811 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3812 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3813 3814 /* 3815 ** Now set up the LRO interface, we 3816 ** also only do head split when LRO 3817 ** is enabled, since so often they 3818 ** are undesireable in similar setups. 3819 */ 3820 if (ifp->if_capenable & IFCAP_LRO) { 3821 int err = tcp_lro_init(lro); 3822 if (err) { 3823 device_printf(dev, "LRO Initialization failed!\n"); 3824 goto fail; 3825 } 3826 INIT_DEBUGOUT("RX LRO Initialized\n"); 3827 rxr->lro_enabled = TRUE; 3828 lro->ifp = adapter->ifp; 3829 } 3830 3831 IGB_RX_UNLOCK(rxr); 3832 return (0); 3833 3834fail: 3835 igb_free_receive_ring(rxr); 3836 IGB_RX_UNLOCK(rxr); 3837 return (error); 3838} 3839 3840/********************************************************************* 3841 * 3842 * Initialize all receive rings. 3843 * 3844 **********************************************************************/ 3845static int 3846igb_setup_receive_structures(struct adapter *adapter) 3847{ 3848 struct rx_ring *rxr = adapter->rx_rings; 3849 int i; 3850 3851 for (i = 0; i < adapter->num_queues; i++, rxr++) 3852 if (igb_setup_receive_ring(rxr)) 3853 goto fail; 3854 3855 return (0); 3856fail: 3857 /* 3858 * Free RX buffers allocated so far, we will only handle 3859 * the rings that completed, the failing case will have 3860 * cleaned up for itself. 'i' is the endpoint. 3861 */ 3862 for (int j = 0; j > i; ++j) { 3863 rxr = &adapter->rx_rings[i]; 3864 igb_free_receive_ring(rxr); 3865 } 3866 3867 return (ENOBUFS); 3868} 3869 3870/********************************************************************* 3871 * 3872 * Enable receive unit. 3873 * 3874 **********************************************************************/ 3875static void 3876igb_initialize_receive_units(struct adapter *adapter) 3877{ 3878 struct rx_ring *rxr = adapter->rx_rings; 3879 struct ifnet *ifp = adapter->ifp; 3880 struct e1000_hw *hw = &adapter->hw; 3881 u32 rctl, rxcsum, psize, srrctl = 0; 3882 3883 INIT_DEBUGOUT("igb_initialize_receive_unit: begin"); 3884 3885 /* 3886 * Make sure receives are disabled while setting 3887 * up the descriptor ring 3888 */ 3889 rctl = E1000_READ_REG(hw, E1000_RCTL); 3890 E1000_WRITE_REG(hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3891 3892 /* 3893 ** Set up for header split 3894 */ 3895 if (rxr->hdr_split) { 3896 /* Use a standard mbuf for the header */ 3897 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; 3898 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3899 } else 3900 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 3901 3902 /* 3903 ** Set up for jumbo frames 3904 */ 3905 if (ifp->if_mtu > ETHERMTU) { 3906 rctl |= E1000_RCTL_LPE; 3907 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 3908 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 3909 3910 /* Set maximum packet len */ 3911 psize = adapter->max_frame_size; 3912 /* are we on a vlan? */ 3913 if (adapter->ifp->if_vlantrunk != NULL) 3914 psize += VLAN_TAG_SIZE; 3915 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); 3916 } else { 3917 rctl &= ~E1000_RCTL_LPE; 3918 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 3919 rctl |= E1000_RCTL_SZ_2048; 3920 } 3921 3922 /* Setup the Base and Length of the Rx Descriptor Rings */ 3923 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3924 u64 bus_addr = rxr->rxdma.dma_paddr; 3925 u32 rxdctl; 3926 3927 E1000_WRITE_REG(hw, E1000_RDLEN(i), 3928 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 3929 E1000_WRITE_REG(hw, E1000_RDBAH(i), 3930 (uint32_t)(bus_addr >> 32)); 3931 E1000_WRITE_REG(hw, E1000_RDBAL(i), 3932 (uint32_t)bus_addr); 3933 E1000_WRITE_REG(hw, E1000_SRRCTL(i), srrctl); 3934 /* Enable this Queue */ 3935 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); 3936 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 3937 rxdctl &= 0xFFF00000; 3938 rxdctl |= IGB_RX_PTHRESH; 3939 rxdctl |= IGB_RX_HTHRESH << 8; 3940 rxdctl |= IGB_RX_WTHRESH << 16; 3941 E1000_WRITE_REG(hw, E1000_RXDCTL(i), rxdctl); 3942 } 3943 3944 /* 3945 ** Setup for RX MultiQueue 3946 */ 3947 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 3948 if (adapter->num_queues >1) { 3949 u32 random[10], mrqc, shift = 0; 3950 union igb_reta { 3951 u32 dword; 3952 u8 bytes[4]; 3953 } reta; 3954 3955 arc4rand(&random, sizeof(random), 0); 3956 if (adapter->hw.mac.type == e1000_82575) 3957 shift = 6; 3958 /* Warning FM follows */ 3959 for (int i = 0; i < 128; i++) { 3960 reta.bytes[i & 3] = 3961 (i % adapter->num_queues) << shift; 3962 if ((i & 3) == 3) 3963 E1000_WRITE_REG(hw, 3964 E1000_RETA(i >> 2), reta.dword); 3965 } 3966 /* Now fill in hash table */ 3967 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 3968 for (int i = 0; i < 10; i++) 3969 E1000_WRITE_REG_ARRAY(hw, 3970 E1000_RSSRK(0), i, random[i]); 3971 3972 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 3973 E1000_MRQC_RSS_FIELD_IPV4_TCP); 3974 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | 3975 E1000_MRQC_RSS_FIELD_IPV6_TCP); 3976 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | 3977 E1000_MRQC_RSS_FIELD_IPV6_UDP); 3978 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 3979 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 3980 3981 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 3982 3983 /* 3984 ** NOTE: Receive Full-Packet Checksum Offload 3985 ** is mutually exclusive with Multiqueue. However 3986 ** this is not the same as TCP/IP checksums which 3987 ** still work. 3988 */ 3989 rxcsum |= E1000_RXCSUM_PCSD; 3990#if __FreeBSD_version >= 800000 3991 /* For SCTP Offload */ 3992 if ((hw->mac.type == e1000_82576) 3993 && (ifp->if_capenable & IFCAP_RXCSUM)) 3994 rxcsum |= E1000_RXCSUM_CRCOFL; 3995#endif 3996 } else { 3997 /* Non RSS setup */ 3998 if (ifp->if_capenable & IFCAP_RXCSUM) { 3999 rxcsum |= E1000_RXCSUM_IPPCSE; 4000#if __FreeBSD_version >= 800000 4001 if (adapter->hw.mac.type == e1000_82576) 4002 rxcsum |= E1000_RXCSUM_CRCOFL; 4003#endif 4004 } else 4005 rxcsum &= ~E1000_RXCSUM_TUOFL; 4006 } 4007 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 4008 4009 /* Setup the Receive Control Register */ 4010 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 4011 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 4012 E1000_RCTL_RDMTS_HALF | 4013 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 4014 /* Strip CRC bytes. */ 4015 rctl |= E1000_RCTL_SECRC; 4016 /* Make sure VLAN Filters are off */ 4017 rctl &= ~E1000_RCTL_VFE; 4018 /* Don't store bad packets */ 4019 rctl &= ~E1000_RCTL_SBP; 4020 4021 /* Enable Receives */ 4022 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 4023 4024 /* 4025 * Setup the HW Rx Head and Tail Descriptor Pointers 4026 * - needs to be after enable 4027 */ 4028 for (int i = 0; i < adapter->num_queues; i++) { 4029 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 4030 E1000_WRITE_REG(hw, E1000_RDT(i), 4031 adapter->num_rx_desc - 1); 4032 } 4033 return; 4034} 4035 4036/********************************************************************* 4037 * 4038 * Free receive rings. 4039 * 4040 **********************************************************************/ 4041static void 4042igb_free_receive_structures(struct adapter *adapter) 4043{ 4044 struct rx_ring *rxr = adapter->rx_rings; 4045 4046 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4047 struct lro_ctrl *lro = &rxr->lro; 4048 igb_free_receive_buffers(rxr); 4049 tcp_lro_free(lro); 4050 igb_dma_free(adapter, &rxr->rxdma); 4051 } 4052 4053 free(adapter->rx_rings, M_DEVBUF); 4054} 4055 4056/********************************************************************* 4057 * 4058 * Free receive ring data structures. 4059 * 4060 **********************************************************************/ 4061static void 4062igb_free_receive_buffers(struct rx_ring *rxr) 4063{ 4064 struct adapter *adapter = rxr->adapter; 4065 struct igb_rx_buf *rxbuf; 4066 int i; 4067 4068 INIT_DEBUGOUT("free_receive_structures: begin"); 4069 4070 /* Cleanup any existing buffers */ 4071 if (rxr->rx_buffers != NULL) { 4072 for (i = 0; i < adapter->num_rx_desc; i++) { 4073 rxbuf = &rxr->rx_buffers[i]; 4074 if (rxbuf->m_head != NULL) { 4075 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 4076 BUS_DMASYNC_POSTREAD); 4077 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 4078 rxbuf->m_head->m_flags |= M_PKTHDR; 4079 m_freem(rxbuf->m_head); 4080 } 4081 if (rxbuf->m_pack != NULL) { 4082 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 4083 BUS_DMASYNC_POSTREAD); 4084 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 4085 rxbuf->m_pack->m_flags |= M_PKTHDR; 4086 m_freem(rxbuf->m_pack); 4087 } 4088 rxbuf->m_head = NULL; 4089 rxbuf->m_pack = NULL; 4090 if (rxbuf->hmap != NULL) { 4091 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 4092 rxbuf->hmap = NULL; 4093 } 4094 if (rxbuf->pmap != NULL) { 4095 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 4096 rxbuf->pmap = NULL; 4097 } 4098 } 4099 if (rxr->rx_buffers != NULL) { 4100 free(rxr->rx_buffers, M_DEVBUF); 4101 rxr->rx_buffers = NULL; 4102 } 4103 } 4104 4105 if (rxr->htag != NULL) { 4106 bus_dma_tag_destroy(rxr->htag); 4107 rxr->htag = NULL; 4108 } 4109 if (rxr->ptag != NULL) { 4110 bus_dma_tag_destroy(rxr->ptag); 4111 rxr->ptag = NULL; 4112 } 4113} 4114 4115static __inline void 4116igb_rx_discard(struct rx_ring *rxr, int i) 4117{ 4118 struct adapter *adapter = rxr->adapter; 4119 struct igb_rx_buf *rbuf; 4120 struct mbuf *mh, *mp; 4121 4122 rbuf = &rxr->rx_buffers[i]; 4123 if (rxr->fmp != NULL) { 4124 rxr->fmp->m_flags |= M_PKTHDR; 4125 m_freem(rxr->fmp); 4126 rxr->fmp = NULL; 4127 rxr->lmp = NULL; 4128 } 4129 4130 mh = rbuf->m_head; 4131 mp = rbuf->m_pack; 4132 4133 /* Reuse loaded DMA map and just update mbuf chain */ 4134 mh->m_len = MHLEN; 4135 mh->m_flags |= M_PKTHDR; 4136 mh->m_next = NULL; 4137 4138 mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz; 4139 mp->m_data = mp->m_ext.ext_buf; 4140 mp->m_next = NULL; 4141 return; 4142} 4143 4144static __inline void 4145igb_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 4146{ 4147 4148 /* 4149 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 4150 * should be computed by hardware. Also it should not have VLAN tag in 4151 * ethernet header. 4152 */ 4153 if (rxr->lro_enabled && 4154 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 4155 (ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && 4156 (ptype & (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP)) == 4157 (E1000_RXDADV_PKTTYPE_IPV4 | E1000_RXDADV_PKTTYPE_TCP) && 4158 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 4159 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 4160 /* 4161 * Send to the stack if: 4162 ** - LRO not enabled, or 4163 ** - no LRO resources, or 4164 ** - lro enqueue fails 4165 */ 4166 if (rxr->lro.lro_cnt != 0) 4167 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 4168 return; 4169 } 4170 IGB_RX_UNLOCK(rxr); 4171 (*ifp->if_input)(ifp, m); 4172 IGB_RX_LOCK(rxr); 4173} 4174 4175/********************************************************************* 4176 * 4177 * This routine executes in interrupt context. It replenishes 4178 * the mbufs in the descriptor and sends data which has been 4179 * dma'ed into host memory to upper layer. 4180 * 4181 * We loop at most count times if count is > 0, or until done if 4182 * count < 0. 4183 * 4184 * Return TRUE if more to clean, FALSE otherwise 4185 *********************************************************************/ 4186static bool 4187igb_rxeof(struct igb_queue *que, int count, int *done) 4188{ 4189 struct adapter *adapter = que->adapter; 4190 struct rx_ring *rxr = que->rxr; 4191 struct ifnet *ifp = adapter->ifp; 4192 struct lro_ctrl *lro = &rxr->lro; 4193 struct lro_entry *queued; 4194 int i, processed = 0, rxdone = 0; 4195 u32 ptype, staterr = 0; 4196 union e1000_adv_rx_desc *cur; 4197 4198 IGB_RX_LOCK(rxr); 4199 /* Sync the ring. */ 4200 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4201 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 4202 4203 /* Main clean loop */ 4204 for (i = rxr->next_to_check; count != 0;) { 4205 struct mbuf *sendmp, *mh, *mp; 4206 struct igb_rx_buf *rxbuf; 4207 u16 hlen, plen, hdr, vtag; 4208 bool eop = FALSE; 4209 4210 cur = &rxr->rx_base[i]; 4211 staterr = le32toh(cur->wb.upper.status_error); 4212 if ((staterr & E1000_RXD_STAT_DD) == 0) 4213 break; 4214 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 4215 break; 4216 count--; 4217 sendmp = mh = mp = NULL; 4218 cur->wb.upper.status_error = 0; 4219 rxbuf = &rxr->rx_buffers[i]; 4220 plen = le16toh(cur->wb.upper.length); 4221 ptype = le32toh(cur->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; 4222 vtag = le16toh(cur->wb.upper.vlan); 4223 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 4224 eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); 4225 4226 /* Make sure all segments of a bad packet are discarded */ 4227 if (((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0) || 4228 (rxr->discard)) { 4229 ifp->if_ierrors++; 4230 ++rxr->rx_discarded; 4231 if (!eop) /* Catch subsequent segs */ 4232 rxr->discard = TRUE; 4233 else 4234 rxr->discard = FALSE; 4235 igb_rx_discard(rxr, i); 4236 goto next_desc; 4237 } 4238 4239 /* 4240 ** The way the hardware is configured to 4241 ** split, it will ONLY use the header buffer 4242 ** when header split is enabled, otherwise we 4243 ** get normal behavior, ie, both header and 4244 ** payload are DMA'd into the payload buffer. 4245 ** 4246 ** The fmp test is to catch the case where a 4247 ** packet spans multiple descriptors, in that 4248 ** case only the first header is valid. 4249 */ 4250 if (rxr->hdr_split && rxr->fmp == NULL) { 4251 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >> 4252 E1000_RXDADV_HDRBUFLEN_SHIFT; 4253 if (hlen > IGB_HDR_BUF) 4254 hlen = IGB_HDR_BUF; 4255 /* Handle the header mbuf */ 4256 mh = rxr->rx_buffers[i].m_head; 4257 mh->m_len = hlen; 4258 /* clear buf info for refresh */ 4259 rxbuf->m_head = NULL; 4260 /* 4261 ** Get the payload length, this 4262 ** could be zero if its a small 4263 ** packet. 4264 */ 4265 if (plen > 0) { 4266 mp = rxr->rx_buffers[i].m_pack; 4267 mp->m_len = plen; 4268 mh->m_next = mp; 4269 /* clear buf info for refresh */ 4270 rxbuf->m_pack = NULL; 4271 rxr->rx_split_packets++; 4272 } 4273 } else { 4274 /* 4275 ** Either no header split, or a 4276 ** secondary piece of a fragmented 4277 ** split packet. 4278 */ 4279 mh = rxr->rx_buffers[i].m_pack; 4280 mh->m_len = plen; 4281 /* clear buf info for refresh */ 4282 rxbuf->m_pack = NULL; 4283 } 4284 4285 ++processed; /* So we know when to refresh */ 4286 4287 /* Initial frame - setup */ 4288 if (rxr->fmp == NULL) { 4289 mh->m_pkthdr.len = mh->m_len; 4290 /* Store the first mbuf */ 4291 rxr->fmp = mh; 4292 rxr->lmp = mh; 4293 if (mp != NULL) { 4294 /* Add payload if split */ 4295 mh->m_pkthdr.len += mp->m_len; 4296 rxr->lmp = mh->m_next; 4297 } 4298 } else { 4299 /* Chain mbuf's together */ 4300 rxr->lmp->m_next = mh; 4301 rxr->lmp = rxr->lmp->m_next; 4302 rxr->fmp->m_pkthdr.len += mh->m_len; 4303 } 4304 4305 if (eop) { 4306 rxr->fmp->m_pkthdr.rcvif = ifp; 4307 ifp->if_ipackets++; 4308 rxr->rx_packets++; 4309 /* capture data for AIM */ 4310 rxr->packets++; 4311 rxr->bytes += rxr->fmp->m_pkthdr.len; 4312 rxr->rx_bytes += rxr->fmp->m_pkthdr.len; 4313 4314 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 4315 igb_rx_checksum(staterr, rxr->fmp, ptype); 4316 4317 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 4318 (staterr & E1000_RXD_STAT_VP) != 0) { 4319 rxr->fmp->m_pkthdr.ether_vtag = vtag; 4320 rxr->fmp->m_flags |= M_VLANTAG; 4321 } 4322#if __FreeBSD_version >= 800000 4323 rxr->fmp->m_pkthdr.flowid = que->msix; 4324 rxr->fmp->m_flags |= M_FLOWID; 4325#endif 4326 sendmp = rxr->fmp; 4327 /* Make sure to set M_PKTHDR. */ 4328 sendmp->m_flags |= M_PKTHDR; 4329 rxr->fmp = NULL; 4330 rxr->lmp = NULL; 4331 } 4332 4333next_desc: 4334 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4335 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4336 4337 /* Advance our pointers to the next descriptor. */ 4338 if (++i == adapter->num_rx_desc) 4339 i = 0; 4340 /* 4341 ** Send to the stack or LRO 4342 */ 4343 if (sendmp != NULL) { 4344 rxr->next_to_check = i; 4345 igb_rx_input(rxr, ifp, sendmp, ptype); 4346 i = rxr->next_to_check; 4347 rxdone++; 4348 } 4349 4350 /* Every 8 descriptors we go to refresh mbufs */ 4351 if (processed == 8) { 4352 igb_refresh_mbufs(rxr, i); 4353 processed = 0; 4354 } 4355 } 4356 4357 /* Catch any remainders */ 4358 if (processed != 0) { 4359 igb_refresh_mbufs(rxr, i); 4360 processed = 0; 4361 } 4362 4363 rxr->next_to_check = i; 4364 4365 /* 4366 * Flush any outstanding LRO work 4367 */ 4368 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 4369 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4370 tcp_lro_flush(lro, queued); 4371 } 4372 4373 IGB_RX_UNLOCK(rxr); 4374 4375 if (done != NULL) 4376 *done = rxdone; 4377 4378 /* 4379 ** We still have cleaning to do? 4380 ** Schedule another interrupt if so. 4381 */ 4382 if ((staterr & E1000_RXD_STAT_DD) != 0) 4383 return (TRUE); 4384 4385 return (FALSE); 4386} 4387 4388/********************************************************************* 4389 * 4390 * Verify that the hardware indicated that the checksum is valid. 4391 * Inform the stack about the status of checksum so that stack 4392 * doesn't spend time verifying the checksum. 4393 * 4394 *********************************************************************/ 4395static void 4396igb_rx_checksum(u32 staterr, struct mbuf *mp, u32 ptype) 4397{ 4398 u16 status = (u16)staterr; 4399 u8 errors = (u8) (staterr >> 24); 4400 int sctp; 4401 4402 /* Ignore Checksum bit is set */ 4403 if (status & E1000_RXD_STAT_IXSM) { 4404 mp->m_pkthdr.csum_flags = 0; 4405 return; 4406 } 4407 4408 if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && 4409 (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) 4410 sctp = 1; 4411 else 4412 sctp = 0; 4413 if (status & E1000_RXD_STAT_IPCS) { 4414 /* Did it pass? */ 4415 if (!(errors & E1000_RXD_ERR_IPE)) { 4416 /* IP Checksum Good */ 4417 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4418 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4419 } else 4420 mp->m_pkthdr.csum_flags = 0; 4421 } 4422 4423 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { 4424 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4425#if __FreeBSD_version >= 800000 4426 if (sctp) /* reassign */ 4427 type = CSUM_SCTP_VALID; 4428#endif 4429 /* Did it pass? */ 4430 if (!(errors & E1000_RXD_ERR_TCPE)) { 4431 mp->m_pkthdr.csum_flags |= type; 4432 if (sctp == 0) 4433 mp->m_pkthdr.csum_data = htons(0xffff); 4434 } 4435 } 4436 return; 4437} 4438 4439/* 4440 * This routine is run via an vlan 4441 * config EVENT 4442 */ 4443static void 4444igb_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4445{ 4446 struct adapter *adapter = ifp->if_softc; 4447 u32 index, bit; 4448 4449 if (ifp->if_softc != arg) /* Not our event */ 4450 return; 4451 4452 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4453 return; 4454 4455 index = (vtag >> 5) & 0x7F; 4456 bit = vtag & 0x1F; 4457 igb_shadow_vfta[index] |= (1 << bit); 4458 ++adapter->num_vlans; 4459 /* Re-init to load the changes */ 4460 igb_init(adapter); 4461} 4462 4463/* 4464 * This routine is run via an vlan 4465 * unconfig EVENT 4466 */ 4467static void 4468igb_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 4469{ 4470 struct adapter *adapter = ifp->if_softc; 4471 u32 index, bit; 4472 4473 if (ifp->if_softc != arg) 4474 return; 4475 4476 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 4477 return; 4478 4479 index = (vtag >> 5) & 0x7F; 4480 bit = vtag & 0x1F; 4481 igb_shadow_vfta[index] &= ~(1 << bit); 4482 --adapter->num_vlans; 4483 /* Re-init to load the changes */ 4484 igb_init(adapter); 4485} 4486 4487static void 4488igb_setup_vlan_hw_support(struct adapter *adapter) 4489{ 4490 struct e1000_hw *hw = &adapter->hw; 4491 u32 reg; 4492 4493 /* 4494 ** We get here thru init_locked, meaning 4495 ** a soft reset, this has already cleared 4496 ** the VFTA and other state, so if there 4497 ** have been no vlan's registered do nothing. 4498 */ 4499 if (adapter->num_vlans == 0) 4500 return; 4501 4502 /* 4503 ** A soft reset zero's out the VFTA, so 4504 ** we need to repopulate it now. 4505 */ 4506 for (int i = 0; i < IGB_VFTA_SIZE; i++) 4507 if (igb_shadow_vfta[i] != 0) 4508 E1000_WRITE_REG_ARRAY(hw, E1000_VFTA, 4509 i, igb_shadow_vfta[i]); 4510 4511 reg = E1000_READ_REG(hw, E1000_CTRL); 4512 reg |= E1000_CTRL_VME; 4513 E1000_WRITE_REG(hw, E1000_CTRL, reg); 4514 4515 /* Enable the Filter Table */ 4516 reg = E1000_READ_REG(hw, E1000_RCTL); 4517 reg &= ~E1000_RCTL_CFIEN; 4518 reg |= E1000_RCTL_VFE; 4519 E1000_WRITE_REG(hw, E1000_RCTL, reg); 4520 4521 /* Update the frame size */ 4522 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4523 adapter->max_frame_size + VLAN_TAG_SIZE); 4524} 4525 4526static void 4527igb_enable_intr(struct adapter *adapter) 4528{ 4529 /* With RSS set up what to auto clear */ 4530 if (adapter->msix_mem) { 4531 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 4532 adapter->eims_mask); 4533 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, 4534 adapter->eims_mask); 4535 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, 4536 adapter->eims_mask); 4537 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 4538 E1000_IMS_LSC); 4539 } else { 4540 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 4541 IMS_ENABLE_MASK); 4542 } 4543 E1000_WRITE_FLUSH(&adapter->hw); 4544 4545 return; 4546} 4547 4548static void 4549igb_disable_intr(struct adapter *adapter) 4550{ 4551 if (adapter->msix_mem) { 4552 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); 4553 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); 4554 } 4555 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); 4556 E1000_WRITE_FLUSH(&adapter->hw); 4557 return; 4558} 4559 4560/* 4561 * Bit of a misnomer, what this really means is 4562 * to enable OS management of the system... aka 4563 * to disable special hardware management features 4564 */ 4565static void 4566igb_init_manageability(struct adapter *adapter) 4567{ 4568 if (adapter->has_manage) { 4569 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4570 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4571 4572 /* disable hardware interception of ARP */ 4573 manc &= ~(E1000_MANC_ARP_EN); 4574 4575 /* enable receiving management packets to the host */ 4576 manc |= E1000_MANC_EN_MNG2HOST; 4577 manc2h |= 1 << 5; /* Mng Port 623 */ 4578 manc2h |= 1 << 6; /* Mng Port 664 */ 4579 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4580 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4581 } 4582} 4583 4584/* 4585 * Give control back to hardware management 4586 * controller if there is one. 4587 */ 4588static void 4589igb_release_manageability(struct adapter *adapter) 4590{ 4591 if (adapter->has_manage) { 4592 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4593 4594 /* re-enable hardware interception of ARP */ 4595 manc |= E1000_MANC_ARP_EN; 4596 manc &= ~E1000_MANC_EN_MNG2HOST; 4597 4598 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4599 } 4600} 4601 4602/* 4603 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 4604 * For ASF and Pass Through versions of f/w this means that 4605 * the driver is loaded. 4606 * 4607 */ 4608static void 4609igb_get_hw_control(struct adapter *adapter) 4610{ 4611 u32 ctrl_ext; 4612 4613 /* Let firmware know the driver has taken over */ 4614 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4615 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4616 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4617} 4618 4619/* 4620 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 4621 * For ASF and Pass Through versions of f/w this means that the 4622 * driver is no longer loaded. 4623 * 4624 */ 4625static void 4626igb_release_hw_control(struct adapter *adapter) 4627{ 4628 u32 ctrl_ext; 4629 4630 /* Let firmware taken over control of h/w */ 4631 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4632 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4633 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4634} 4635 4636static int 4637igb_is_valid_ether_addr(uint8_t *addr) 4638{ 4639 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4640 4641 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4642 return (FALSE); 4643 } 4644 4645 return (TRUE); 4646} 4647 4648 4649/* 4650 * Enable PCI Wake On Lan capability 4651 */ 4652static void 4653igb_enable_wakeup(device_t dev) 4654{ 4655 u16 cap, status; 4656 u8 id; 4657 4658 /* First find the capabilities pointer*/ 4659 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 4660 /* Read the PM Capabilities */ 4661 id = pci_read_config(dev, cap, 1); 4662 if (id != PCIY_PMG) /* Something wrong */ 4663 return; 4664 /* OK, we have the power capabilities, so 4665 now get the status register */ 4666 cap += PCIR_POWER_STATUS; 4667 status = pci_read_config(dev, cap, 2); 4668 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4669 pci_write_config(dev, cap, status, 2); 4670 return; 4671} 4672 4673static void 4674igb_led_func(void *arg, int onoff) 4675{ 4676 struct adapter *adapter = arg; 4677 4678 IGB_CORE_LOCK(adapter); 4679 if (onoff) { 4680 e1000_setup_led(&adapter->hw); 4681 e1000_led_on(&adapter->hw); 4682 } else { 4683 e1000_led_off(&adapter->hw); 4684 e1000_cleanup_led(&adapter->hw); 4685 } 4686 IGB_CORE_UNLOCK(adapter); 4687} 4688 4689/********************************************************************** 4690 * 4691 * Update the board statistics counters. 4692 * 4693 **********************************************************************/ 4694static void 4695igb_update_stats_counters(struct adapter *adapter) 4696{ 4697 struct ifnet *ifp; 4698 4699 if (adapter->hw.phy.media_type == e1000_media_type_copper || 4700 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 4701 adapter->stats.symerrs += 4702 E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 4703 adapter->stats.sec += 4704 E1000_READ_REG(&adapter->hw, E1000_SEC); 4705 } 4706 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 4707 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 4708 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 4709 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 4710 4711 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 4712 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 4713 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 4714 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 4715 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 4716 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 4717 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 4718 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 4719 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 4720 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 4721 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 4722 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 4723 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 4724 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 4725 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 4726 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 4727 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 4728 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 4729 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 4730 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 4731 4732 /* For the 64-bit byte counters the low dword must be read first. */ 4733 /* Both registers clear on the read of the high dword */ 4734 4735 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCL) + 4736 ((u64)E1000_READ_REG(&adapter->hw, E1000_GORCH) << 32); 4737 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCL) + 4738 ((u64)E1000_READ_REG(&adapter->hw, E1000_GOTCH) << 32) ; 4739 4740 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 4741 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 4742 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 4743 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 4744 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 4745 4746 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 4747 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 4748 4749 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 4750 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 4751 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 4752 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 4753 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 4754 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 4755 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 4756 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 4757 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 4758 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 4759 4760 /* Interrupt Counts */ 4761 4762 adapter->stats.iac += E1000_READ_REG(&adapter->hw, E1000_IAC); 4763 adapter->stats.icrxptc += E1000_READ_REG(&adapter->hw, E1000_ICRXPTC); 4764 adapter->stats.icrxatc += E1000_READ_REG(&adapter->hw, E1000_ICRXATC); 4765 adapter->stats.ictxptc += E1000_READ_REG(&adapter->hw, E1000_ICTXPTC); 4766 adapter->stats.ictxatc += E1000_READ_REG(&adapter->hw, E1000_ICTXATC); 4767 adapter->stats.ictxqec += E1000_READ_REG(&adapter->hw, E1000_ICTXQEC); 4768 adapter->stats.ictxqmtc += E1000_READ_REG(&adapter->hw, E1000_ICTXQMTC); 4769 adapter->stats.icrxdmtc += E1000_READ_REG(&adapter->hw, E1000_ICRXDMTC); 4770 adapter->stats.icrxoc += E1000_READ_REG(&adapter->hw, E1000_ICRXOC); 4771 4772 /* Host to Card Statistics */ 4773 4774 adapter->stats.cbtmpc += E1000_READ_REG(&adapter->hw, E1000_CBTMPC); 4775 adapter->stats.htdpmc += E1000_READ_REG(&adapter->hw, E1000_HTDPMC); 4776 adapter->stats.cbrdpc += E1000_READ_REG(&adapter->hw, E1000_CBRDPC); 4777 adapter->stats.cbrmpc += E1000_READ_REG(&adapter->hw, E1000_CBRMPC); 4778 adapter->stats.rpthc += E1000_READ_REG(&adapter->hw, E1000_RPTHC); 4779 adapter->stats.hgptc += E1000_READ_REG(&adapter->hw, E1000_HGPTC); 4780 adapter->stats.htcbdpc += E1000_READ_REG(&adapter->hw, E1000_HTCBDPC); 4781 adapter->stats.hgorc += (E1000_READ_REG(&adapter->hw, E1000_HGORCL) + 4782 ((u64)E1000_READ_REG(&adapter->hw, 4783 E1000_HGORCH) << 32)); 4784 4785 adapter->stats.hgotc += (E1000_READ_REG(&adapter->hw, E1000_HGOTCL) + 4786 ((u64)E1000_READ_REG(&adapter->hw, 4787 E1000_HGOTCH) << 32)); 4788 adapter->stats.lenerrs += E1000_READ_REG(&adapter->hw, E1000_LENERRS); 4789 adapter->stats.scvpc += E1000_READ_REG(&adapter->hw, E1000_SCVPC); 4790 adapter->stats.hrmpc += E1000_READ_REG(&adapter->hw, E1000_HRMPC); 4791 4792 adapter->stats.algnerrc += 4793 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 4794 adapter->stats.rxerrc += 4795 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 4796 adapter->stats.tncrs += 4797 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 4798 adapter->stats.cexterr += 4799 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 4800 adapter->stats.tsctc += 4801 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 4802 adapter->stats.tsctfc += 4803 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 4804 ifp = adapter->ifp; 4805 4806 ifp->if_collisions = adapter->stats.colc; 4807 4808 /* Rx Errors */ 4809 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 4810 adapter->stats.crcerrs + adapter->stats.algnerrc + 4811 adapter->stats.ruc + adapter->stats.roc + 4812 adapter->stats.mpc + adapter->stats.cexterr; 4813 4814 /* Tx Errors */ 4815 ifp->if_oerrors = adapter->stats.ecol + 4816 adapter->stats.latecol + adapter->watchdog_events; 4817 4818 /* Driver specific counters */ 4819 adapter->device_control = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4820 adapter->rx_control = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4821 adapter->int_mask = E1000_READ_REG(&adapter->hw, E1000_IMS); 4822 adapter->eint_mask = E1000_READ_REG(&adapter->hw, E1000_EIMS); 4823 adapter->packet_buf_alloc_tx = ((E1000_READ_REG(&adapter->hw, E1000_PBA) 4824 & 0xffff0000) >> 16); 4825 4826 adapter->packet_buf_alloc_rx = (E1000_READ_REG(&adapter->hw, E1000_PBA) 4827 & 0xffff); 4828 4829} 4830 4831 4832/* 4833 * Add sysctl variables, one per statistic, to the system. 4834 */ 4835static void 4836igb_add_hw_stats(struct adapter *adapter) 4837{ 4838 4839 device_t dev = adapter->dev; 4840 4841 struct tx_ring *txr = adapter->tx_rings; 4842 struct rx_ring *rxr = adapter->rx_rings; 4843 4844 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 4845 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 4846 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 4847 struct e1000_hw_stats *stats = &adapter->stats; 4848 4849 struct sysctl_oid *stat_node, *queue_node, *int_node, *host_node; 4850 struct sysctl_oid_list *stat_list, *queue_list, *int_list, *host_list; 4851 4852#define QUEUE_NAME_LEN 32 4853 char namebuf[QUEUE_NAME_LEN]; 4854 4855 /* Driver Statistics */ 4856 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "link_irq", 4857 CTLFLAG_RD, &adapter->link_irq, 0, 4858 "Link MSIX IRQ Handled"); 4859 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dropped", 4860 CTLFLAG_RD, &adapter->dropped_pkts, 4861 "Driver dropped packets"); 4862 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_dma_fail", 4863 CTLFLAG_RD, &adapter->no_tx_dma_setup, 4864 "Driver tx dma failure in xmit"); 4865 4866 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "device_control", 4867 CTLFLAG_RD, &adapter->device_control, 4868 "Device Control Register"); 4869 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_control", 4870 CTLFLAG_RD, &adapter->rx_control, 4871 "Receiver Control Register"); 4872 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "interrupt_mask", 4873 CTLFLAG_RD, &adapter->int_mask, 4874 "Interrupt Mask"); 4875 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "extended_int_mask", 4876 CTLFLAG_RD, &adapter->eint_mask, 4877 "Extended Interrupt Mask"); 4878 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "tx_buf_alloc", 4879 CTLFLAG_RD, &adapter->packet_buf_alloc_tx, 4880 "Transmit Buffer Packet Allocation"); 4881 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "rx_buf_alloc", 4882 CTLFLAG_RD, &adapter->packet_buf_alloc_rx, 4883 "Receive Buffer Packet Allocation"); 4884 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_high_water", 4885 CTLFLAG_RD, &adapter->hw.fc.high_water, 0, 4886 "Flow Control High Watermark"); 4887 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "fc_low_water", 4888 CTLFLAG_RD, &adapter->hw.fc.low_water, 0, 4889 "Flow Control Low Watermark"); 4890 4891 for (int i = 0; i < adapter->num_queues; i++, txr++) { 4892 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 4893 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 4894 CTLFLAG_RD, NULL, "Queue Name"); 4895 queue_list = SYSCTL_CHILDREN(queue_node); 4896 4897 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "txd_head", 4898 CTLFLAG_RD, &txr->tdh, 0, 4899 "Transmit Descriptor Head"); 4900 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "txd_tail", 4901 CTLFLAG_RD, &txr->tdt, 0, 4902 "Transmit Descriptor Tail"); 4903 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "no_desc_avail", 4904 CTLFLAG_RD, &txr->no_desc_avail, 4905 "Queue No Descriptor Available"); 4906 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "tx_packets", 4907 CTLFLAG_RD, &txr->tx_packets, 4908 "Queue Packets Transmitted"); 4909 } 4910 4911 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 4912 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 4913 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 4914 CTLFLAG_RD, NULL, "Queue Name"); 4915 queue_list = SYSCTL_CHILDREN(queue_node); 4916 4917 struct lro_ctrl *lro = &rxr->lro; 4918 4919 snprintf(namebuf, QUEUE_NAME_LEN, "queue%d", i); 4920 queue_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, namebuf, 4921 CTLFLAG_RD, NULL, "Queue Name"); 4922 queue_list = SYSCTL_CHILDREN(queue_node); 4923 4924 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rxd_head", 4925 CTLFLAG_RD, &rxr->rdh, 0, 4926 "Receive Descriptor Head"); 4927 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "rxd_tail", 4928 CTLFLAG_RD, &rxr->rdt, 0, 4929 "Receive Descriptor Tail"); 4930 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_packets", 4931 CTLFLAG_RD, &rxr->rx_packets, 4932 "Queue Packets Received"); 4933 SYSCTL_ADD_QUAD(ctx, queue_list, OID_AUTO, "rx_bytes", 4934 CTLFLAG_RD, &rxr->rx_bytes, 4935 "Queue Bytes Received"); 4936 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_queued", 4937 CTLFLAG_RD, &lro->lro_queued, 0, 4938 "LRO Queued"); 4939 SYSCTL_ADD_UINT(ctx, queue_list, OID_AUTO, "lro_flushed", 4940 CTLFLAG_RD, &lro->lro_flushed, 0, 4941 "LRO Flushed"); 4942 } 4943 4944 /* MAC stats get the own sub node */ 4945 4946 stat_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "mac_stats", 4947 CTLFLAG_RD, NULL, "MAC Statistics"); 4948 stat_list = SYSCTL_CHILDREN(stat_node); 4949 4950 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "excess_coll", 4951 CTLFLAG_RD, &stats->ecol, 4952 "Excessive collisions"); 4953 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "single_coll", 4954 CTLFLAG_RD, &stats->scc, 4955 "Single collisions"); 4956 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "multiple_coll", 4957 CTLFLAG_RD, &stats->mcc, 4958 "Multiple collisions"); 4959 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "late_coll", 4960 CTLFLAG_RD, &stats->latecol, 4961 "Late collisions"); 4962 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "collision_count", 4963 CTLFLAG_RD, &stats->colc, 4964 "Collision Count"); 4965 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "symbol_errors", 4966 CTLFLAG_RD, &adapter->stats.symerrs, 4967 "Symbol Errors"); 4968 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "sequence_errors", 4969 CTLFLAG_RD, &adapter->stats.sec, 4970 "Sequence Errors"); 4971 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "defer_count", 4972 CTLFLAG_RD, &adapter->stats.dc, 4973 "Defer Count"); 4974 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "missed_packets", 4975 CTLFLAG_RD, &adapter->stats.mpc, 4976 "Missed Packets"); 4977 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_no_buff", 4978 CTLFLAG_RD, &adapter->stats.rnbc, 4979 "Receive No Buffers"); 4980 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_undersize", 4981 CTLFLAG_RD, &adapter->stats.ruc, 4982 "Receive Undersize"); 4983 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_fragmented", 4984 CTLFLAG_RD, &adapter->stats.rfc, 4985 "Fragmented Packets Received "); 4986 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_oversize", 4987 CTLFLAG_RD, &adapter->stats.roc, 4988 "Oversized Packets Received"); 4989 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_jabber", 4990 CTLFLAG_RD, &adapter->stats.rjc, 4991 "Recevied Jabber"); 4992 4993 /* RLEC is inaccurate on some hardware, calculate our own. */ 4994/* SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_len_errs", */ 4995/* CTLFLAG_RD, adapter->stats.roc + adapter->stats.ruc, */ 4996/* "Receive Length Errors"); */ 4997 4998 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "recv_errs", 4999 CTLFLAG_RD, &adapter->stats.rxerrc, 5000 "Receive Errors"); 5001 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "crc_errs", 5002 CTLFLAG_RD, &adapter->stats.crcerrs, 5003 "CRC errors"); 5004 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "alignment_errs", 5005 CTLFLAG_RD, &adapter->stats.algnerrc, 5006 "Alignment Errors"); 5007 /* On 82575 these are collision counts */ 5008 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "coll_ext_errs", 5009 CTLFLAG_RD, &adapter->stats.cexterr, 5010 "Collision/Carrier extension errors"); 5011 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_overruns", 5012 CTLFLAG_RD, &adapter->rx_overruns, 5013 "RX overruns"); 5014 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "watchdog_timeouts", 5015 CTLFLAG_RD, &adapter->watchdog_events, 5016 "Watchdog timeouts"); 5017 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_recvd", 5018 CTLFLAG_RD, &adapter->stats.xonrxc, 5019 "XON Received"); 5020 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xon_txd", 5021 CTLFLAG_RD, &adapter->stats.xontxc, 5022 "XON Transmitted"); 5023 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_recvd", 5024 CTLFLAG_RD, &adapter->stats.xoffrxc, 5025 "XOFF Received"); 5026 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "xoff_txd", 5027 CTLFLAG_RD, &adapter->stats.xofftxc, 5028 "XOFF Transmitted"); 5029 /* Packet Reception Stats */ 5030 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_recvd", 5031 CTLFLAG_RD, &adapter->stats.tpr, 5032 "Total Packets Received "); 5033 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_recvd", 5034 CTLFLAG_RD, &adapter->stats.gprc, 5035 "Good Packets Received"); 5036 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_recvd", 5037 CTLFLAG_RD, &adapter->stats.bprc, 5038 "Broadcast Packets Received"); 5039 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_recvd", 5040 CTLFLAG_RD, &adapter->stats.mprc, 5041 "Multicast Packets Received"); 5042 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_64", 5043 CTLFLAG_RD, &adapter->stats.prc64, 5044 "64 byte frames received "); 5045 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_65_127", 5046 CTLFLAG_RD, &adapter->stats.prc127, 5047 "65-127 byte frames received"); 5048 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_128_255", 5049 CTLFLAG_RD, &adapter->stats.prc255, 5050 "128-255 byte frames received"); 5051 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_256_511", 5052 CTLFLAG_RD, &adapter->stats.prc511, 5053 "256-511 byte frames received"); 5054 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_512_1023", 5055 CTLFLAG_RD, &adapter->stats.prc1023, 5056 "512-1023 byte frames received"); 5057 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "rx_frames_1024_1522", 5058 CTLFLAG_RD, &adapter->stats.prc1522, 5059 "1023-1522 byte frames received"); 5060 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octets_recvd", 5061 CTLFLAG_RD, &adapter->stats.gorc, 5062 "Good Octets Received"); 5063 5064 /* Packet Transmission Stats */ 5065 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_octest_txd", 5066 CTLFLAG_RD, &adapter->stats.gotc, 5067 "Good Octest Transmitted"); 5068 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "total_pkts_txd", 5069 CTLFLAG_RD, &adapter->stats.tpt, 5070 "Total Packets Transmitted"); 5071 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "good_pkts_txd", 5072 CTLFLAG_RD, &adapter->stats.gptc, 5073 "Good Packets Transmitted"); 5074 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "bcast_pkts_txd", 5075 CTLFLAG_RD, &adapter->stats.bptc, 5076 "Broadcast Packets Transmitted"); 5077 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "mcast_pkts_txd", 5078 CTLFLAG_RD, &adapter->stats.mptc, 5079 "Multicast Packets Transmitted"); 5080 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_64", 5081 CTLFLAG_RD, &adapter->stats.ptc64, 5082 "64 byte frames transmitted "); 5083 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_65_127", 5084 CTLFLAG_RD, &adapter->stats.ptc127, 5085 "65-127 byte frames transmitted"); 5086 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_128_255", 5087 CTLFLAG_RD, &adapter->stats.ptc255, 5088 "128-255 byte frames transmitted"); 5089 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_256_511", 5090 CTLFLAG_RD, &adapter->stats.ptc511, 5091 "256-511 byte frames transmitted"); 5092 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_512_1023", 5093 CTLFLAG_RD, &adapter->stats.ptc1023, 5094 "512-1023 byte frames transmitted"); 5095 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tx_frames_1024_1522", 5096 CTLFLAG_RD, &adapter->stats.ptc1522, 5097 "1024-1522 byte frames transmitted"); 5098 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_txd", 5099 CTLFLAG_RD, &adapter->stats.tsctc, 5100 "TSO Contexts Transmitted"); 5101 SYSCTL_ADD_QUAD(ctx, stat_list, OID_AUTO, "tso_ctx_fail", 5102 CTLFLAG_RD, &adapter->stats.tsctfc, 5103 "TSO Contexts Failed"); 5104 5105 5106 /* Interrupt Stats */ 5107 5108 int_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "interrupts", 5109 CTLFLAG_RD, NULL, "Interrupt Statistics"); 5110 int_list = SYSCTL_CHILDREN(int_node); 5111 5112 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "asserts", 5113 CTLFLAG_RD, &adapter->stats.iac, 5114 "Interrupt Assertion Count"); 5115 5116 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_pkt_timer", 5117 CTLFLAG_RD, &adapter->stats.icrxptc, 5118 "Interrupt Cause Rx Pkt Timer Expire Count"); 5119 5120 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_abs_timer", 5121 CTLFLAG_RD, &adapter->stats.icrxatc, 5122 "Interrupt Cause Rx Abs Timer Expire Count"); 5123 5124 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_pkt_timer", 5125 CTLFLAG_RD, &adapter->stats.ictxptc, 5126 "Interrupt Cause Tx Pkt Timer Expire Count"); 5127 5128 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_abs_timer", 5129 CTLFLAG_RD, &adapter->stats.ictxatc, 5130 "Interrupt Cause Tx Abs Timer Expire Count"); 5131 5132 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_empty", 5133 CTLFLAG_RD, &adapter->stats.ictxqec, 5134 "Interrupt Cause Tx Queue Empty Count"); 5135 5136 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "tx_queue_min_thresh", 5137 CTLFLAG_RD, &adapter->stats.ictxqmtc, 5138 "Interrupt Cause Tx Queue Min Thresh Count"); 5139 5140 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_desc_min_thresh", 5141 CTLFLAG_RD, &adapter->stats.icrxdmtc, 5142 "Interrupt Cause Rx Desc Min Thresh Count"); 5143 5144 SYSCTL_ADD_QUAD(ctx, int_list, OID_AUTO, "rx_overrun", 5145 CTLFLAG_RD, &adapter->stats.icrxoc, 5146 "Interrupt Cause Receiver Overrun Count"); 5147 5148 /* Host to Card Stats */ 5149 5150 host_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "host", 5151 CTLFLAG_RD, NULL, 5152 "Host to Card Statistics"); 5153 5154 host_list = SYSCTL_CHILDREN(host_node); 5155 5156 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt", 5157 CTLFLAG_RD, &adapter->stats.cbtmpc, 5158 "Circuit Breaker Tx Packet Count"); 5159 5160 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "host_tx_pkt_discard", 5161 CTLFLAG_RD, &adapter->stats.htdpmc, 5162 "Host Transmit Discarded Packets"); 5163 5164 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_pkt", 5165 CTLFLAG_RD, &adapter->stats.rpthc, 5166 "Rx Packets To Host"); 5167 5168 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkts", 5169 CTLFLAG_RD, &adapter->stats.cbrmpc, 5170 "Circuit Breaker Rx Packet Count"); 5171 5172 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_rx_pkt_drop", 5173 CTLFLAG_RD, &adapter->stats.cbrdpc, 5174 "Circuit Breaker Rx Dropped Count"); 5175 5176 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_pkt", 5177 CTLFLAG_RD, &adapter->stats.hgptc, 5178 "Host Good Packets Tx Count"); 5179 5180 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "breaker_tx_pkt_drop", 5181 CTLFLAG_RD, &adapter->stats.htcbdpc, 5182 "Host Tx Circuit Breaker Dropped Count"); 5183 5184 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "rx_good_bytes", 5185 CTLFLAG_RD, &adapter->stats.hgorc, 5186 "Host Good Octets Received Count"); 5187 5188 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "tx_good_bytes", 5189 CTLFLAG_RD, &adapter->stats.hgotc, 5190 "Host Good Octets Transmit Count"); 5191 5192 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "length_errors", 5193 CTLFLAG_RD, &adapter->stats.lenerrs, 5194 "Length Errors"); 5195 5196 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "serdes_violation_pkt", 5197 CTLFLAG_RD, &adapter->stats.scvpc, 5198 "SerDes/SGMII Code Violation Pkt Count"); 5199 5200 SYSCTL_ADD_QUAD(ctx, host_list, OID_AUTO, "header_redir_missed", 5201 CTLFLAG_RD, &adapter->stats.hrmpc, 5202 "Header Redirection Missed Packet Count"); 5203 5204 5205} 5206/********************************************************************** 5207 * 5208 * This routine provides a way to dump out the adapter eeprom, 5209 * often a useful debug/service tool. This only dumps the first 5210 * 32 words, stuff that matters is in that extent. 5211 * 5212 **********************************************************************/ 5213static int 5214igb_sysctl_nvm_info(SYSCTL_HANDLER_ARGS) 5215{ 5216 struct adapter *adapter; 5217 int error; 5218 int result; 5219 5220 result = -1; 5221 error = sysctl_handle_int(oidp, &result, 0, req); 5222 5223 if (error || !req->newptr) 5224 return (error); 5225 5226 /* 5227 * This value will cause a hex dump of the 5228 * first 32 16-bit words of the EEPROM to 5229 * the screen. 5230 */ 5231 if (result == 1) { 5232 adapter = (struct adapter *)arg1; 5233 igb_print_nvm_info(adapter); 5234 } 5235 5236 return (error); 5237} 5238 5239static void 5240igb_print_nvm_info(struct adapter *adapter) 5241{ 5242 u16 eeprom_data; 5243 int i, j, row = 0; 5244 5245 /* Its a bit crude, but it gets the job done */ 5246 printf("\nInterface EEPROM Dump:\n"); 5247 printf("Offset\n0x0000 "); 5248 for (i = 0, j = 0; i < 32; i++, j++) { 5249 if (j == 8) { /* Make the offset block */ 5250 j = 0; ++row; 5251 printf("\n0x00%x0 ",row); 5252 } 5253 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 5254 printf("%04x ", eeprom_data); 5255 } 5256 printf("\n"); 5257} 5258 5259static void 5260igb_add_rx_process_limit(struct adapter *adapter, const char *name, 5261 const char *description, int *limit, int value) 5262{ 5263 *limit = value; 5264 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 5265 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 5266 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 5267} 5268