if_igb.c revision 190872
121308Sache/****************************************************************************** 221308Sache 321308Sache Copyright (c) 2001-2009, Intel Corporation 421308Sache All rights reserved. 5157184Sache 621308Sache Redistribution and use in source and binary forms, with or without 721308Sache modification, are permitted provided that the following conditions are met: 821308Sache 921308Sache 1. Redistributions of source code must retain the above copyright notice, 1021308Sache this list of conditions and the following disclaimer. 1121308Sache 1221308Sache 2. Redistributions in binary form must reproduce the above copyright 1358310Sache notice, this list of conditions and the following disclaimer in the 1421308Sache documentation and/or other materials provided with the distribution. 1521308Sache 1621308Sache 3. Neither the name of the Intel Corporation nor the names of its 1721308Sache contributors may be used to endorse or promote products derived from 1821308Sache this software without specific prior written permission. 1921308Sache 2021308Sache THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 2121308Sache AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2221308Sache IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2321308Sache ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 2458310Sache LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 2521308Sache CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 2621308Sache SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 2721308Sache INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2821308Sache CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 2921308Sache ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 3021308Sache POSSIBILITY OF SUCH DAMAGE. 3121308Sache 3221308Sache******************************************************************************/ 3375406Sache/*$FreeBSD: head/sys/dev/e1000/if_igb.c 190872 2009-04-10 00:05:46Z jfv $*/ 3475406Sache 3521308Sache#ifdef HAVE_KERNEL_OPTION_HEADERS 3621308Sache#include "opt_device_polling.h" 3721308Sache#endif 3821308Sache 3921308Sache#include <sys/param.h> 4021308Sache#include <sys/systm.h> 41157184Sache#include <sys/bus.h> 42157184Sache#include <sys/endian.h> 43157184Sache#include <sys/kernel.h> 44157184Sache#include <sys/kthread.h> 45157184Sache#include <sys/malloc.h> 4621308Sache#include <sys/mbuf.h> 4721308Sache#include <sys/module.h> 4821308Sache#include <sys/rman.h> 4921308Sache#include <sys/socket.h> 5021308Sache#include <sys/sockio.h> 5121308Sache#include <sys/sysctl.h> 5221308Sache#include <sys/taskqueue.h> 5321308Sache#include <sys/eventhandler.h> 5421308Sache#include <sys/pcpu.h> 5521308Sache#ifdef IGB_TIMESYNC 5621308Sache#include <sys/ioccom.h> 5721308Sache#include <sys/time.h> 5821308Sache#endif 5921308Sache#include <machine/bus.h> 6021308Sache#include <machine/resource.h> 6121308Sache 6221308Sache#include <net/bpf.h> 6321308Sache#include <net/ethernet.h> 6421308Sache#include <net/if.h> 6521308Sache#include <net/if_arp.h> 6621308Sache#include <net/if_dl.h> 6721308Sache#include <net/if_media.h> 6821308Sache 6921308Sache#include <net/if_types.h> 7021308Sache#include <net/if_vlan_var.h> 7121308Sache 7221308Sache#include <netinet/in_systm.h> 7321308Sache#include <netinet/in.h> 7421308Sache#include <netinet/if_ether.h> 7521308Sache#include <netinet/ip.h> 7621308Sache#include <netinet/ip6.h> 7721308Sache#include <netinet/tcp.h> 7821308Sache#include <netinet/tcp_lro.h> 7921308Sache#include <netinet/udp.h> 80119610Sache 81119610Sache#include <machine/in_cksum.h> 8221308Sache#include <dev/pci/pcivar.h> 8321308Sache#include <dev/pci/pcireg.h> 84136644Sache 85119610Sache#include "e1000_api.h" 86119610Sache#include "e1000_82575.h" 87119610Sache#include "if_igb.h" 88119610Sache 89119610Sache/********************************************************************* 9021308Sache * Set this to one to display debug statistics 9121308Sache *********************************************************************/ 9221308Sacheint igb_display_debug_stats = 0; 9321308Sache 9421308Sache/********************************************************************* 9521308Sache * Driver version: 96119610Sache *********************************************************************/ 97119610Sachechar igb_driver_version[] = "version - 1.5.2"; 98119610Sache 99119610Sache 100119610Sache/********************************************************************* 101119610Sache * PCI Device ID Table 102119610Sache * 10321308Sache * Used by probe to select devices to load on 10421308Sache * Last field stores an index into e1000_strings 10521308Sache * Last entry must be all 0s 10621308Sache * 10721308Sache * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 10821308Sache *********************************************************************/ 10921308Sache 11075406Sachestatic igb_vendor_info_t igb_vendor_info_array[] = 11121308Sache{ 11221308Sache { 0x8086, E1000_DEV_ID_82575EB_COPPER, PCI_ANY_ID, PCI_ANY_ID, 0}, 11375406Sache { 0x8086, E1000_DEV_ID_82575EB_FIBER_SERDES, 11421308Sache PCI_ANY_ID, PCI_ANY_ID, 0}, 11521308Sache { 0x8086, E1000_DEV_ID_82575GB_QUAD_COPPER, 11621308Sache PCI_ANY_ID, PCI_ANY_ID, 0}, 117119610Sache { 0x8086, E1000_DEV_ID_82576, PCI_ANY_ID, PCI_ANY_ID, 0}, 11821308Sache { 0x8086, E1000_DEV_ID_82576_NS, PCI_ANY_ID, PCI_ANY_ID, 0}, 11921308Sache { 0x8086, E1000_DEV_ID_82576_FIBER, PCI_ANY_ID, PCI_ANY_ID, 0}, 12021308Sache { 0x8086, E1000_DEV_ID_82576_SERDES, PCI_ANY_ID, PCI_ANY_ID, 0}, 12121308Sache { 0x8086, E1000_DEV_ID_82576_QUAD_COPPER, 12221308Sache PCI_ANY_ID, PCI_ANY_ID, 0}, 12321308Sache /* required last entry */ 12421308Sache { 0, 0, 0, 0, 0} 12521308Sache}; 12621308Sache 12721308Sache/********************************************************************* 12821308Sache * Table of branding strings for all supported NICs. 12921308Sache *********************************************************************/ 13021308Sache 13121308Sachestatic char *igb_strings[] = { 13221308Sache "Intel(R) PRO/1000 Network Connection" 13321308Sache}; 13421308Sache 135119610Sache/********************************************************************* 136119610Sache * Function prototypes 137119610Sache *********************************************************************/ 138119610Sachestatic int igb_probe(device_t); 13921308Sachestatic int igb_attach(device_t); 14021308Sachestatic int igb_detach(device_t); 14121308Sachestatic int igb_shutdown(device_t); 14221308Sachestatic int igb_suspend(device_t); 14321308Sachestatic int igb_resume(device_t); 14421308Sachestatic void igb_start(struct ifnet *); 14558310Sachestatic void igb_start_locked(struct tx_ring *, struct ifnet *ifp); 14658310Sachestatic int igb_ioctl(struct ifnet *, u_long, caddr_t); 14721308Sachestatic void igb_watchdog(struct adapter *); 14821308Sachestatic void igb_init(void *); 14921308Sachestatic void igb_init_locked(struct adapter *); 15021308Sachestatic void igb_stop(void *); 15121308Sachestatic void igb_media_status(struct ifnet *, struct ifmediareq *); 15221308Sachestatic int igb_media_change(struct ifnet *); 153119610Sachestatic void igb_identify_hardware(struct adapter *); 154119610Sachestatic int igb_allocate_pci_resources(struct adapter *); 155119610Sachestatic int igb_allocate_msix(struct adapter *); 156119610Sachestatic int igb_allocate_legacy(struct adapter *); 15721308Sache#if __FreeBSD_version >= 602105 15821308Sachestatic int igb_setup_msix(struct adapter *); 15921308Sache#endif 16021308Sachestatic void igb_free_pci_resources(struct adapter *); 161static void igb_local_timer(void *); 162static int igb_hardware_init(struct adapter *); 163static void igb_setup_interface(device_t, struct adapter *); 164static int igb_allocate_queues(struct adapter *); 165static void igb_configure_queues(struct adapter *); 166 167static int igb_allocate_transmit_buffers(struct tx_ring *); 168static void igb_setup_transmit_structures(struct adapter *); 169static void igb_setup_transmit_ring(struct tx_ring *); 170static void igb_initialize_transmit_units(struct adapter *); 171static void igb_free_transmit_structures(struct adapter *); 172static void igb_free_transmit_buffers(struct tx_ring *); 173 174static int igb_allocate_receive_buffers(struct rx_ring *); 175static int igb_setup_receive_structures(struct adapter *); 176static int igb_setup_receive_ring(struct rx_ring *); 177static void igb_initialize_receive_units(struct adapter *); 178static void igb_free_receive_structures(struct adapter *); 179static void igb_free_receive_buffers(struct rx_ring *); 180 181static void igb_enable_intr(struct adapter *); 182static void igb_disable_intr(struct adapter *); 183static void igb_update_stats_counters(struct adapter *); 184static bool igb_txeof(struct tx_ring *); 185static bool igb_rxeof(struct rx_ring *, int); 186static void igb_rx_checksum(u32, struct mbuf *, bool); 187static int igb_tx_ctx_setup(struct tx_ring *, struct mbuf *); 188static bool igb_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 189static void igb_set_promisc(struct adapter *); 190static void igb_disable_promisc(struct adapter *); 191static void igb_set_multi(struct adapter *); 192static void igb_print_hw_stats(struct adapter *); 193static void igb_update_link_status(struct adapter *); 194static int igb_get_buf(struct rx_ring *, int, u8); 195static void igb_register_vlan(void *, struct ifnet *, u16); 196static void igb_unregister_vlan(void *, struct ifnet *, u16); 197static int igb_xmit(struct tx_ring *, struct mbuf **); 198static int igb_dma_malloc(struct adapter *, bus_size_t, 199 struct igb_dma_alloc *, int); 200static void igb_dma_free(struct adapter *, struct igb_dma_alloc *); 201static void igb_print_debug_info(struct adapter *); 202static void igb_print_nvm_info(struct adapter *); 203static int igb_is_valid_ether_addr(u8 *); 204static int igb_sysctl_stats(SYSCTL_HANDLER_ARGS); 205static int igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS); 206/* Management and WOL Support */ 207static void igb_init_manageability(struct adapter *); 208static void igb_release_manageability(struct adapter *); 209static void igb_get_hw_control(struct adapter *); 210static void igb_release_hw_control(struct adapter *); 211static void igb_enable_wakeup(device_t); 212 213#ifdef IGB_TIMESYNC 214/* Precision Time sync support */ 215static int igb_tsync_init(struct adapter *); 216static void igb_tsync_disable(struct adapter *); 217#endif 218 219#if __FreeBSD_version > 700000 220static int igb_irq_fast(void *); 221#else 222static void igb_irq_fast(void *); 223#endif 224 225static void igb_add_rx_process_limit(struct adapter *, const char *, 226 const char *, int *, int); 227static void igb_handle_rxtx(void *context, int pending); 228static void igb_handle_tx(void *context, int pending); 229static void igb_handle_rx(void *context, int pending); 230 231#if __FreeBSD_version >= 602105 232/* These are MSIX only irq handlers */ 233static void igb_msix_rx(void *); 234static void igb_msix_tx(void *); 235static void igb_msix_link(void *); 236#endif 237 238/* Adaptive Interrupt Moderation */ 239static void igb_update_aim(struct rx_ring *); 240 241/********************************************************************* 242 * FreeBSD Device Interface Entry Points 243 *********************************************************************/ 244 245static device_method_t igb_methods[] = { 246 /* Device interface */ 247 DEVMETHOD(device_probe, igb_probe), 248 DEVMETHOD(device_attach, igb_attach), 249 DEVMETHOD(device_detach, igb_detach), 250 DEVMETHOD(device_shutdown, igb_shutdown), 251 DEVMETHOD(device_suspend, igb_suspend), 252 DEVMETHOD(device_resume, igb_resume), 253 {0, 0} 254}; 255 256static driver_t igb_driver = { 257 "igb", igb_methods, sizeof(struct adapter), 258}; 259 260static devclass_t igb_devclass; 261DRIVER_MODULE(igb, pci, igb_driver, igb_devclass, 0, 0); 262MODULE_DEPEND(igb, pci, 1, 1, 1); 263MODULE_DEPEND(igb, ether, 1, 1, 1); 264 265/********************************************************************* 266 * Tunable default values. 267 *********************************************************************/ 268 269/* Descriptor defaults */ 270static int igb_rxd = IGB_DEFAULT_RXD; 271static int igb_txd = IGB_DEFAULT_TXD; 272TUNABLE_INT("hw.igb.rxd", &igb_rxd); 273TUNABLE_INT("hw.igb.txd", &igb_txd); 274 275/* 276** These parameters are used in Adaptive 277** Interrupt Moderation. The value is set 278** into EITR and controls the interrupt 279** frequency. A variable static scheme can 280** be created by changing the assigned value 281** of igb_ave_latency to the desired value, 282** and then set igb_enable_aim to FALSE. 283** This will result in all EITR registers 284** getting set to that value statically. 285*/ 286static int igb_enable_aim = TRUE; 287TUNABLE_INT("hw.igb.enable_aim", &igb_enable_aim); 288static int igb_low_latency = IGB_LOW_LATENCY; 289TUNABLE_INT("hw.igb.low_latency", &igb_low_latency); 290static int igb_ave_latency = IGB_AVE_LATENCY; 291TUNABLE_INT("hw.igb.ave_latency", &igb_ave_latency); 292static int igb_bulk_latency = IGB_BULK_LATENCY; 293TUNABLE_INT("hw.igb.bulk_latency", &igb_bulk_latency); 294 295/* 296** IF YOU CHANGE THESE: be sure and change IGB_MSIX_VEC in 297** if_igb.h to match. These can be autoconfigured if set to 298** 0, it will then be based on number of cpus. 299*/ 300static int igb_tx_queues = 1; 301static int igb_rx_queues = 1; 302TUNABLE_INT("hw.igb.tx_queues", &igb_tx_queues); 303TUNABLE_INT("hw.igb.rx_queues", &igb_rx_queues); 304 305/* How many packets rxeof tries to clean at a time */ 306static int igb_rx_process_limit = 100; 307TUNABLE_INT("hw.igb.rx_process_limit", &igb_rx_process_limit); 308 309/* Flow control setting - default to FULL */ 310static int igb_fc_setting = e1000_fc_full; 311TUNABLE_INT("hw.igb.fc_setting", &igb_fc_setting); 312 313/* 314 * Should the driver do LRO on the RX end 315 * this can be toggled on the fly, but the 316 * interface must be reset (down/up) for it 317 * to take effect. 318 */ 319static int igb_enable_lro = 1; 320TUNABLE_INT("hw.igb.enable_lro", &igb_enable_lro); 321 322/* 323 * Enable RX Header Split 324 */ 325static int igb_rx_hdr_split = 1; 326TUNABLE_INT("hw.igb.rx_hdr_split", &igb_rx_hdr_split); 327 328extern int mp_ncpus; 329/********************************************************************* 330 * Device identification routine 331 * 332 * igb_probe determines if the driver should be loaded on 333 * adapter based on PCI vendor/device id of the adapter. 334 * 335 * return BUS_PROBE_DEFAULT on success, positive on failure 336 *********************************************************************/ 337 338static int 339igb_probe(device_t dev) 340{ 341 char adapter_name[60]; 342 uint16_t pci_vendor_id = 0; 343 uint16_t pci_device_id = 0; 344 uint16_t pci_subvendor_id = 0; 345 uint16_t pci_subdevice_id = 0; 346 igb_vendor_info_t *ent; 347 348 INIT_DEBUGOUT("igb_probe: begin"); 349 350 pci_vendor_id = pci_get_vendor(dev); 351 if (pci_vendor_id != IGB_VENDOR_ID) 352 return (ENXIO); 353 354 pci_device_id = pci_get_device(dev); 355 pci_subvendor_id = pci_get_subvendor(dev); 356 pci_subdevice_id = pci_get_subdevice(dev); 357 358 ent = igb_vendor_info_array; 359 while (ent->vendor_id != 0) { 360 if ((pci_vendor_id == ent->vendor_id) && 361 (pci_device_id == ent->device_id) && 362 363 ((pci_subvendor_id == ent->subvendor_id) || 364 (ent->subvendor_id == PCI_ANY_ID)) && 365 366 ((pci_subdevice_id == ent->subdevice_id) || 367 (ent->subdevice_id == PCI_ANY_ID))) { 368 sprintf(adapter_name, "%s %s", 369 igb_strings[ent->index], 370 igb_driver_version); 371 device_set_desc_copy(dev, adapter_name); 372 return (BUS_PROBE_DEFAULT); 373 } 374 ent++; 375 } 376 377 return (ENXIO); 378} 379 380/********************************************************************* 381 * Device initialization routine 382 * 383 * The attach entry point is called when the driver is being loaded. 384 * This routine identifies the type of hardware, allocates all resources 385 * and initializes the hardware. 386 * 387 * return 0 on success, positive on failure 388 *********************************************************************/ 389 390static int 391igb_attach(device_t dev) 392{ 393 struct adapter *adapter; 394 int error = 0; 395 u16 eeprom_data; 396 397 INIT_DEBUGOUT("igb_attach: begin"); 398 399 adapter = device_get_softc(dev); 400 adapter->dev = adapter->osdep.dev = dev; 401 IGB_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 402 403 /* SYSCTL stuff */ 404 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 405 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 406 OID_AUTO, "debug", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 407 igb_sysctl_debug_info, "I", "Debug Information"); 408 409 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 410 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 411 OID_AUTO, "stats", CTLTYPE_INT|CTLFLAG_RW, adapter, 0, 412 igb_sysctl_stats, "I", "Statistics"); 413 414 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 415 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 416 OID_AUTO, "flow_control", CTLTYPE_INT|CTLFLAG_RW, 417 &igb_fc_setting, 0, "Flow Control"); 418 419 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 420 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 421 OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW, 422 &igb_enable_lro, 0, "Large Receive Offload"); 423 424 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 425 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 426 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 427 &igb_enable_aim, 1, "Interrupt Moderation"); 428 429 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 430 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 431 OID_AUTO, "low_latency", CTLTYPE_INT|CTLFLAG_RW, 432 &igb_low_latency, 1, "Low Latency"); 433 434 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 435 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 436 OID_AUTO, "ave_latency", CTLTYPE_INT|CTLFLAG_RW, 437 &igb_ave_latency, 1, "Average Latency"); 438 439 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 440 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 441 OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW, 442 &igb_bulk_latency, 1, "Bulk Latency"); 443 444 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 445 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 446 OID_AUTO, "hdr_split", CTLTYPE_INT|CTLFLAG_RW, 447 &igb_rx_hdr_split, 0, "RX Header Split"); 448 449 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 450 451 /* Determine hardware and mac info */ 452 igb_identify_hardware(adapter); 453 454 /* Setup PCI resources */ 455 if (igb_allocate_pci_resources(adapter)) { 456 device_printf(dev, "Allocation of PCI resources failed\n"); 457 error = ENXIO; 458 goto err_pci; 459 } 460 461 /* Do Shared Code initialization */ 462 if (e1000_setup_init_funcs(&adapter->hw, TRUE)) { 463 device_printf(dev, "Setup of Shared code failed\n"); 464 error = ENXIO; 465 goto err_pci; 466 } 467 468 e1000_get_bus_info(&adapter->hw); 469 470 /* Sysctls for limiting the amount of work done in the taskqueue */ 471 igb_add_rx_process_limit(adapter, "rx_processing_limit", 472 "max number of rx packets to process", &adapter->rx_process_limit, 473 igb_rx_process_limit); 474 475 /* 476 * Validate number of transmit and receive descriptors. It 477 * must not exceed hardware maximum, and must be multiple 478 * of E1000_DBA_ALIGN. 479 */ 480 if (((igb_txd * sizeof(struct e1000_tx_desc)) % IGB_DBA_ALIGN) != 0 || 481 (igb_txd > IGB_MAX_TXD) || (igb_txd < IGB_MIN_TXD)) { 482 device_printf(dev, "Using %d TX descriptors instead of %d!\n", 483 IGB_DEFAULT_TXD, igb_txd); 484 adapter->num_tx_desc = IGB_DEFAULT_TXD; 485 } else 486 adapter->num_tx_desc = igb_txd; 487 if (((igb_rxd * sizeof(struct e1000_rx_desc)) % IGB_DBA_ALIGN) != 0 || 488 (igb_rxd > IGB_MAX_RXD) || (igb_rxd < IGB_MIN_RXD)) { 489 device_printf(dev, "Using %d RX descriptors instead of %d!\n", 490 IGB_DEFAULT_RXD, igb_rxd); 491 adapter->num_rx_desc = IGB_DEFAULT_RXD; 492 } else 493 adapter->num_rx_desc = igb_rxd; 494 495 adapter->hw.mac.autoneg = DO_AUTO_NEG; 496 adapter->hw.phy.autoneg_wait_to_complete = FALSE; 497 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 498 499 /* Copper options */ 500 if (adapter->hw.phy.media_type == e1000_media_type_copper) { 501 adapter->hw.phy.mdix = AUTO_ALL_MODES; 502 adapter->hw.phy.disable_polarity_correction = FALSE; 503 adapter->hw.phy.ms_type = IGB_MASTER_SLAVE; 504 } 505 506 /* 507 * Set the frame limits assuming 508 * standard ethernet sized frames. 509 */ 510 adapter->max_frame_size = ETHERMTU + ETHER_HDR_LEN + ETHERNET_FCS_SIZE; 511 adapter->min_frame_size = ETH_ZLEN + ETHERNET_FCS_SIZE; 512 513 /* 514 ** Allocate and Setup Queues 515 */ 516 if (igb_allocate_queues(adapter)) { 517 error = ENOMEM; 518 goto err_pci; 519 } 520 521 /* 522 ** Start from a known state, this is 523 ** important in reading the nvm and 524 ** mac from that. 525 */ 526 e1000_reset_hw(&adapter->hw); 527 528 /* Make sure we have a good EEPROM before we read from it */ 529 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 530 /* 531 ** Some PCI-E parts fail the first check due to 532 ** the link being in sleep state, call it again, 533 ** if it fails a second time its a real issue. 534 */ 535 if (e1000_validate_nvm_checksum(&adapter->hw) < 0) { 536 device_printf(dev, 537 "The EEPROM Checksum Is Not Valid\n"); 538 error = EIO; 539 goto err_late; 540 } 541 } 542 543 /* 544 ** Copy the permanent MAC address out of the EEPROM 545 */ 546 if (e1000_read_mac_addr(&adapter->hw) < 0) { 547 device_printf(dev, "EEPROM read error while reading MAC" 548 " address\n"); 549 error = EIO; 550 goto err_late; 551 } 552 /* Check its sanity */ 553 if (!igb_is_valid_ether_addr(adapter->hw.mac.addr)) { 554 device_printf(dev, "Invalid MAC address\n"); 555 error = EIO; 556 goto err_late; 557 } 558 559 /* Now Initialize the hardware */ 560 if (igb_hardware_init(adapter)) { 561 device_printf(dev, "Unable to initialize the hardware\n"); 562 error = EIO; 563 goto err_late; 564 } 565 566 /* 567 ** Configure Interrupts 568 */ 569 if (adapter->msix > 1) /* MSIX */ 570 error = igb_allocate_msix(adapter); 571 else /* MSI or Legacy */ 572 error = igb_allocate_legacy(adapter); 573 if (error) 574 goto err_late; 575 576 /* Setup OS specific network interface */ 577 igb_setup_interface(dev, adapter); 578 579 /* Initialize statistics */ 580 igb_update_stats_counters(adapter); 581 582 adapter->hw.mac.get_link_status = 1; 583 igb_update_link_status(adapter); 584 585 /* Indicate SOL/IDER usage */ 586 if (e1000_check_reset_block(&adapter->hw)) 587 device_printf(dev, 588 "PHY reset is blocked due to SOL/IDER session.\n"); 589 590 /* Determine if we have to control management hardware */ 591 adapter->has_manage = e1000_enable_mng_pass_thru(&adapter->hw); 592 593 /* 594 * Setup Wake-on-Lan 595 */ 596 /* APME bit in EEPROM is mapped to WUC.APME */ 597 eeprom_data = E1000_READ_REG(&adapter->hw, E1000_WUC) & E1000_WUC_APME; 598 if (eeprom_data) 599 adapter->wol = E1000_WUFC_MAG; 600 601 /* Register for VLAN events */ 602 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 603 igb_register_vlan, 0, EVENTHANDLER_PRI_FIRST); 604 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 605 igb_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST); 606 607 /* Tell the stack that the interface is not active */ 608 adapter->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 609 610 INIT_DEBUGOUT("igb_attach: end"); 611 612 return (0); 613 614err_late: 615 igb_free_transmit_structures(adapter); 616 igb_free_receive_structures(adapter); 617 igb_release_hw_control(adapter); 618err_pci: 619 igb_free_pci_resources(adapter); 620 IGB_CORE_LOCK_DESTROY(adapter); 621 622 return (error); 623} 624 625/********************************************************************* 626 * Device removal routine 627 * 628 * The detach entry point is called when the driver is being removed. 629 * This routine stops the adapter and deallocates all the resources 630 * that were allocated for driver operation. 631 * 632 * return 0 on success, positive on failure 633 *********************************************************************/ 634 635static int 636igb_detach(device_t dev) 637{ 638 struct adapter *adapter = device_get_softc(dev); 639 struct ifnet *ifp = adapter->ifp; 640 641 INIT_DEBUGOUT("igb_detach: begin"); 642 643 /* Make sure VLANS are not using driver */ 644#if __FreeBSD_version >= 700000 645 if (adapter->ifp->if_vlantrunk != NULL) { 646#else 647 if (adapter->ifp->if_nvlans != 0) { 648#endif 649 device_printf(dev,"Vlan in use, detach first\n"); 650 return (EBUSY); 651 } 652 653 IGB_CORE_LOCK(adapter); 654 adapter->in_detach = 1; 655 igb_stop(adapter); 656 IGB_CORE_UNLOCK(adapter); 657 658 e1000_phy_hw_reset(&adapter->hw); 659 660 /* Give control back to firmware */ 661 igb_release_manageability(adapter); 662 igb_release_hw_control(adapter); 663 664 if (adapter->wol) { 665 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 666 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 667 igb_enable_wakeup(dev); 668 } 669 670 /* Unregister VLAN events */ 671 if (adapter->vlan_attach != NULL) 672 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 673 if (adapter->vlan_detach != NULL) 674 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 675 676 ether_ifdetach(adapter->ifp); 677 678 callout_drain(&adapter->timer); 679 680 igb_free_pci_resources(adapter); 681 bus_generic_detach(dev); 682 if_free(ifp); 683 684 igb_free_transmit_structures(adapter); 685 igb_free_receive_structures(adapter); 686 687 IGB_CORE_LOCK_DESTROY(adapter); 688 689 return (0); 690} 691 692/********************************************************************* 693 * 694 * Shutdown entry point 695 * 696 **********************************************************************/ 697 698static int 699igb_shutdown(device_t dev) 700{ 701 return igb_suspend(dev); 702} 703 704/* 705 * Suspend/resume device methods. 706 */ 707static int 708igb_suspend(device_t dev) 709{ 710 struct adapter *adapter = device_get_softc(dev); 711 712 IGB_CORE_LOCK(adapter); 713 714 igb_stop(adapter); 715 716 igb_release_manageability(adapter); 717 igb_release_hw_control(adapter); 718 719 if (adapter->wol) { 720 E1000_WRITE_REG(&adapter->hw, E1000_WUC, E1000_WUC_PME_EN); 721 E1000_WRITE_REG(&adapter->hw, E1000_WUFC, adapter->wol); 722 igb_enable_wakeup(dev); 723 } 724 725 IGB_CORE_UNLOCK(adapter); 726 727 return bus_generic_suspend(dev); 728} 729 730static int 731igb_resume(device_t dev) 732{ 733 struct adapter *adapter = device_get_softc(dev); 734 struct ifnet *ifp = adapter->ifp; 735 736 IGB_CORE_LOCK(adapter); 737 igb_init_locked(adapter); 738 igb_init_manageability(adapter); 739 740 if ((ifp->if_flags & IFF_UP) && 741 (ifp->if_drv_flags & IFF_DRV_RUNNING)) 742 igb_start(ifp); 743 744 IGB_CORE_UNLOCK(adapter); 745 746 return bus_generic_resume(dev); 747} 748 749 750/********************************************************************* 751 * Transmit entry point 752 * 753 * igb_start is called by the stack to initiate a transmit. 754 * The driver will remain in this routine as long as there are 755 * packets to transmit and transmit resources are available. 756 * In case resources are not available stack is notified and 757 * the packet is requeued. 758 **********************************************************************/ 759 760static void 761igb_start_locked(struct tx_ring *txr, struct ifnet *ifp) 762{ 763 struct adapter *adapter = ifp->if_softc; 764 struct mbuf *m_head; 765 766 IGB_TX_LOCK_ASSERT(txr); 767 768 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 769 IFF_DRV_RUNNING) 770 return; 771 if (!adapter->link_active) 772 return; 773 774 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 775 776 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 777 if (m_head == NULL) 778 break; 779 /* 780 * Encapsulation can modify our pointer, and or make it 781 * NULL on failure. In that event, we can't requeue. 782 */ 783 if (igb_xmit(txr, &m_head)) { 784 if (m_head == NULL) 785 break; 786 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 787 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 788 break; 789 } 790 791 /* Send a copy of the frame to the BPF listener */ 792 ETHER_BPF_MTAP(ifp, m_head); 793 794 /* Set timeout in case hardware has problems transmitting. */ 795 txr->watchdog_timer = IGB_TX_TIMEOUT; 796 } 797} 798 799static void 800igb_start(struct ifnet *ifp) 801{ 802 struct adapter *adapter = ifp->if_softc; 803 struct tx_ring *txr; 804 u32 queue = 0; 805 806 /* 807 ** This is really just here for testing 808 ** TX multiqueue, ultimately what is 809 ** needed is the flow support in the stack 810 ** and appropriate logic here to deal with 811 ** it. -jfv 812 */ 813 if (adapter->num_tx_queues > 1) 814 queue = (curcpu % adapter->num_tx_queues); 815 816 txr = &adapter->tx_rings[queue]; 817 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 818 IGB_TX_LOCK(txr); 819 igb_start_locked(txr, ifp); 820 IGB_TX_UNLOCK(txr); 821 } 822} 823 824/********************************************************************* 825 * Ioctl entry point 826 * 827 * igb_ioctl is called when the user wants to configure the 828 * interface. 829 * 830 * return 0 on success, positive on failure 831 **********************************************************************/ 832 833static int 834igb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 835{ 836 struct adapter *adapter = ifp->if_softc; 837 struct ifreq *ifr = (struct ifreq *)data; 838#ifdef INET 839 struct ifaddr *ifa = (struct ifaddr *)data; 840#endif 841 int error = 0; 842 843 if (adapter->in_detach) 844 return (error); 845 846 switch (command) { 847 case SIOCSIFADDR: 848#ifdef INET 849 if (ifa->ifa_addr->sa_family == AF_INET) { 850 /* 851 * XXX 852 * Since resetting hardware takes a very long time 853 * and results in link renegotiation we only 854 * initialize the hardware only when it is absolutely 855 * required. 856 */ 857 ifp->if_flags |= IFF_UP; 858 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 859 IGB_CORE_LOCK(adapter); 860 igb_init_locked(adapter); 861 IGB_CORE_UNLOCK(adapter); 862 } 863 arp_ifinit(ifp, ifa); 864 } else 865#endif 866 error = ether_ioctl(ifp, command, data); 867 break; 868 case SIOCSIFMTU: 869 { 870 int max_frame_size; 871 872 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFMTU (Set Interface MTU)"); 873 874 IGB_CORE_LOCK(adapter); 875 max_frame_size = 9234; 876 if (ifr->ifr_mtu > max_frame_size - ETHER_HDR_LEN - 877 ETHER_CRC_LEN) { 878 IGB_CORE_UNLOCK(adapter); 879 error = EINVAL; 880 break; 881 } 882 883 ifp->if_mtu = ifr->ifr_mtu; 884 adapter->max_frame_size = 885 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 886 igb_init_locked(adapter); 887 IGB_CORE_UNLOCK(adapter); 888 break; 889 } 890 case SIOCSIFFLAGS: 891 IOCTL_DEBUGOUT("ioctl rcv'd:\ 892 SIOCSIFFLAGS (Set Interface Flags)"); 893 IGB_CORE_LOCK(adapter); 894 if (ifp->if_flags & IFF_UP) { 895 if ((ifp->if_drv_flags & IFF_DRV_RUNNING)) { 896 if ((ifp->if_flags ^ adapter->if_flags) & 897 (IFF_PROMISC | IFF_ALLMULTI)) { 898 igb_disable_promisc(adapter); 899 igb_set_promisc(adapter); 900 } 901 } else 902 igb_init_locked(adapter); 903 } else 904 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 905 igb_stop(adapter); 906 adapter->if_flags = ifp->if_flags; 907 IGB_CORE_UNLOCK(adapter); 908 break; 909 case SIOCADDMULTI: 910 case SIOCDELMULTI: 911 IOCTL_DEBUGOUT("ioctl rcv'd: SIOC(ADD|DEL)MULTI"); 912 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 913 IGB_CORE_LOCK(adapter); 914 igb_disable_intr(adapter); 915 igb_set_multi(adapter); 916 igb_enable_intr(adapter); 917 IGB_CORE_UNLOCK(adapter); 918 } 919 break; 920 case SIOCSIFMEDIA: 921 /* Check SOL/IDER usage */ 922 IGB_CORE_LOCK(adapter); 923 if (e1000_check_reset_block(&adapter->hw)) { 924 IGB_CORE_UNLOCK(adapter); 925 device_printf(adapter->dev, "Media change is" 926 " blocked due to SOL/IDER session.\n"); 927 break; 928 } 929 IGB_CORE_UNLOCK(adapter); 930 case SIOCGIFMEDIA: 931 IOCTL_DEBUGOUT("ioctl rcv'd: \ 932 SIOCxIFMEDIA (Get/Set Interface Media)"); 933 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 934 break; 935 case SIOCSIFCAP: 936 { 937 int mask, reinit; 938 939 IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFCAP (Set Capabilities)"); 940 reinit = 0; 941 mask = ifr->ifr_reqcap ^ ifp->if_capenable; 942 if (mask & IFCAP_HWCSUM) { 943 ifp->if_capenable ^= IFCAP_HWCSUM; 944 reinit = 1; 945 } 946 if (mask & IFCAP_TSO4) { 947 ifp->if_capenable ^= IFCAP_TSO4; 948 reinit = 1; 949 } 950 if (mask & IFCAP_VLAN_HWTAGGING) { 951 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 952 reinit = 1; 953 } 954 if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) 955 igb_init(adapter); 956#if __FreeBSD_version >= 700000 957 VLAN_CAPABILITIES(ifp); 958#endif 959 break; 960 } 961 962#ifdef IGB_TIMESYNC 963 /* 964 ** IOCTL support for Precision Time (IEEE 1588) Support 965 */ 966 case IGB_TIMESYNC_READTS: 967 { 968 u32 rx_ctl, tx_ctl; 969 struct igb_tsync_read *tdata; 970 971 tdata = (struct igb_tsync_read *) ifr->ifr_data; 972 973 if (tdata->read_current_time) { 974 getnanotime(&tdata->system_time); 975 tdata->network_time = E1000_READ_REG(&adapter->hw, 976 E1000_SYSTIML); 977 tdata->network_time |= 978 (u64)E1000_READ_REG(&adapter->hw, 979 E1000_SYSTIMH ) << 32; 980 } 981 982 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 983 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 984 985 if (rx_ctl & 0x1) { 986 u32 tmp; 987 unsigned char *tmp_cp; 988 989 tdata->rx_valid = 1; 990 tdata->rx_stamp = E1000_READ_REG(&adapter->hw, E1000_RXSTMPL); 991 tdata->rx_stamp |= (u64)E1000_READ_REG(&adapter->hw, 992 E1000_RXSTMPH) << 32; 993 994 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRL); 995 tmp_cp = (unsigned char *) &tmp; 996 tdata->srcid[0] = tmp_cp[0]; 997 tdata->srcid[1] = tmp_cp[1]; 998 tdata->srcid[2] = tmp_cp[2]; 999 tdata->srcid[3] = tmp_cp[3]; 1000 tmp = E1000_READ_REG(&adapter->hw, E1000_RXSATRH); 1001 tmp_cp = (unsigned char *) &tmp; 1002 tdata->srcid[4] = tmp_cp[0]; 1003 tdata->srcid[5] = tmp_cp[1]; 1004 tdata->seqid = tmp >> 16; 1005 tdata->seqid = htons(tdata->seqid); 1006 } else 1007 tdata->rx_valid = 0; 1008 1009 if (tx_ctl & 0x1) { 1010 tdata->tx_valid = 1; 1011 tdata->tx_stamp = E1000_READ_REG(&adapter->hw, E1000_TXSTMPL); 1012 tdata->tx_stamp |= (u64) E1000_READ_REG(&adapter->hw, 1013 E1000_TXSTMPH) << 32; 1014 } else 1015 tdata->tx_valid = 0; 1016 1017 return (0); 1018 } 1019#endif /* IGB_TIMESYNC */ 1020 1021 default: 1022 error = ether_ioctl(ifp, command, data); 1023 break; 1024 } 1025 1026 return (error); 1027} 1028 1029/********************************************************************* 1030 * Watchdog timer: 1031 * 1032 * This routine is called from the local timer every second. 1033 * As long as transmit descriptors are being cleaned the value 1034 * is non-zero and we do nothing. Reaching 0 indicates a tx hang 1035 * and we then reset the device. 1036 * 1037 **********************************************************************/ 1038 1039static void 1040igb_watchdog(struct adapter *adapter) 1041{ 1042 struct tx_ring *txr = adapter->tx_rings; 1043 bool tx_hang = FALSE; 1044 1045 IGB_CORE_LOCK_ASSERT(adapter); 1046 1047 /* 1048 ** The timer is set to 5 every time start() queues a packet. 1049 ** Then txeof keeps resetting it as long as it cleans at 1050 ** least one descriptor. 1051 ** Finally, anytime all descriptors are clean the timer is 1052 ** set to 0. 1053 ** 1054 ** With TX Multiqueue we need to check every queue's timer, 1055 ** if any time out we do the reset. 1056 */ 1057 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 1058 IGB_TX_LOCK(txr); 1059 if (txr->watchdog_timer == 0 || 1060 (--txr->watchdog_timer)) { 1061 IGB_TX_UNLOCK(txr); 1062 continue; 1063 } else { 1064 tx_hang = TRUE; 1065 IGB_TX_UNLOCK(txr); 1066 break; 1067 } 1068 } 1069 if (tx_hang == FALSE) 1070 return; 1071 1072 /* If we are in this routine because of pause frames, then 1073 * don't reset the hardware. 1074 */ 1075 if (E1000_READ_REG(&adapter->hw, E1000_STATUS) & 1076 E1000_STATUS_TXOFF) { 1077 txr = adapter->tx_rings; /* reset pointer */ 1078 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 1079 IGB_TX_LOCK(txr); 1080 txr->watchdog_timer = IGB_TX_TIMEOUT; 1081 IGB_TX_UNLOCK(txr); 1082 } 1083 return; 1084 } 1085 1086 if (e1000_check_for_link(&adapter->hw) == 0) 1087 device_printf(adapter->dev, "watchdog timeout -- resetting\n"); 1088 1089 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 1090 device_printf(adapter->dev, "Queue(%d) tdh = %d, tdt = %d\n", 1091 i, E1000_READ_REG(&adapter->hw, E1000_TDH(i)), 1092 E1000_READ_REG(&adapter->hw, E1000_TDT(i))); 1093 device_printf(adapter->dev, "Queue(%d) desc avail = %d," 1094 " Next Desc to Clean = %d\n", i, txr->tx_avail, 1095 txr->next_to_clean); 1096 } 1097 1098 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1099 adapter->watchdog_events++; 1100 1101 igb_init_locked(adapter); 1102} 1103 1104/********************************************************************* 1105 * Init entry point 1106 * 1107 * This routine is used in two ways. It is used by the stack as 1108 * init entry point in network interface structure. It is also used 1109 * by the driver as a hw/sw initialization routine to get to a 1110 * consistent state. 1111 * 1112 * return 0 on success, positive on failure 1113 **********************************************************************/ 1114 1115static void 1116igb_init_locked(struct adapter *adapter) 1117{ 1118 struct rx_ring *rxr = adapter->rx_rings; 1119 struct tx_ring *txr = adapter->tx_rings; 1120 struct ifnet *ifp = adapter->ifp; 1121 device_t dev = adapter->dev; 1122 u32 pba = 0; 1123 1124 INIT_DEBUGOUT("igb_init: begin"); 1125 1126 IGB_CORE_LOCK_ASSERT(adapter); 1127 1128 igb_stop(adapter); 1129 1130 /* 1131 * Packet Buffer Allocation (PBA) 1132 * Writing PBA sets the receive portion of the buffer 1133 * the remainder is used for the transmit buffer. 1134 */ 1135 if (adapter->hw.mac.type == e1000_82575) { 1136 INIT_DEBUGOUT1("igb_init: pba=%dK",pba); 1137 pba = E1000_PBA_32K; /* 32K for Rx, 16K for Tx */ 1138 E1000_WRITE_REG(&adapter->hw, E1000_PBA, pba); 1139 } 1140 1141 /* Get the latest mac address, User can use a LAA */ 1142 bcopy(IF_LLADDR(adapter->ifp), adapter->hw.mac.addr, 1143 ETHER_ADDR_LEN); 1144 1145 /* Put the address into the Receive Address Array */ 1146 e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0); 1147 1148 /* Initialize the hardware */ 1149 if (igb_hardware_init(adapter)) { 1150 device_printf(dev, "Unable to initialize the hardware\n"); 1151 return; 1152 } 1153 igb_update_link_status(adapter); 1154 1155 E1000_WRITE_REG(&adapter->hw, E1000_VET, ETHERTYPE_VLAN); 1156 1157 if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) && 1158 ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)) { 1159 u32 ctrl; 1160 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 1161 ctrl |= E1000_CTRL_VME; 1162 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 1163 } 1164 1165 /* Set hardware offload abilities */ 1166 ifp->if_hwassist = 0; 1167 if (ifp->if_capenable & IFCAP_TXCSUM) { 1168 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 1169#if __FreeBSD_version >= 800000 1170 if (adapter->hw.mac.type == e1000_82576) 1171 ifp->if_hwassist |= CSUM_SCTP; 1172#endif 1173 } 1174 1175 if (ifp->if_capenable & IFCAP_TSO4) 1176 ifp->if_hwassist |= CSUM_TSO; 1177 1178 /* Configure for OS presence */ 1179 igb_init_manageability(adapter); 1180 1181 /* Prepare transmit descriptors and buffers */ 1182 igb_setup_transmit_structures(adapter); 1183 igb_initialize_transmit_units(adapter); 1184 1185 /* Setup Multicast table */ 1186 igb_set_multi(adapter); 1187 1188 /* 1189 ** Figure out the desired mbuf pool 1190 ** for doing jumbo/packetsplit 1191 */ 1192 if (ifp->if_mtu > ETHERMTU) 1193 adapter->rx_mbuf_sz = MJUMPAGESIZE; 1194 else 1195 adapter->rx_mbuf_sz = MCLBYTES; 1196 1197 /* Prepare receive descriptors and buffers */ 1198 if (igb_setup_receive_structures(adapter)) { 1199 device_printf(dev, "Could not setup receive structures\n"); 1200 igb_stop(adapter); 1201 return; 1202 } 1203 igb_initialize_receive_units(adapter); 1204 1205 /* Don't lose promiscuous settings */ 1206 igb_set_promisc(adapter); 1207 1208 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1209 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1210 1211 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1212 e1000_clear_hw_cntrs_base_generic(&adapter->hw); 1213 1214 if (adapter->msix > 1) /* Set up queue routing */ 1215 igb_configure_queues(adapter); 1216 1217 /* Set default RX interrupt moderation */ 1218 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 1219 E1000_WRITE_REG(&adapter->hw, 1220 E1000_EITR(rxr->msix), igb_ave_latency); 1221 rxr->eitr_setting = igb_ave_latency; 1222 } 1223 1224 /* Set TX interrupt rate & reset TX watchdog */ 1225 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 1226 E1000_WRITE_REG(&adapter->hw, 1227 E1000_EITR(txr->msix), igb_ave_latency); 1228 txr->watchdog_timer = FALSE; 1229 } 1230 1231 /* this clears any pending interrupts */ 1232 E1000_READ_REG(&adapter->hw, E1000_ICR); 1233 igb_enable_intr(adapter); 1234 E1000_WRITE_REG(&adapter->hw, E1000_ICS, E1000_ICS_LSC); 1235 1236#ifdef IGB_TIMESYNC 1237 /* Initialize IEEE 1588 Time sync if available */ 1238 if (adapter->hw.mac.type == e1000_82576) 1239 igb_tsync_init(adapter); 1240#endif 1241 1242 /* Don't reset the phy next time init gets called */ 1243 adapter->hw.phy.reset_disable = TRUE; 1244} 1245 1246static void 1247igb_init(void *arg) 1248{ 1249 struct adapter *adapter = arg; 1250 1251 IGB_CORE_LOCK(adapter); 1252 igb_init_locked(adapter); 1253 IGB_CORE_UNLOCK(adapter); 1254} 1255 1256 1257static void 1258igb_handle_rxtx(void *context, int pending) 1259{ 1260 struct adapter *adapter = context; 1261 struct tx_ring *txr = adapter->tx_rings; 1262 struct rx_ring *rxr = adapter->rx_rings; 1263 struct ifnet *ifp; 1264 1265 ifp = adapter->ifp; 1266 1267 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1268 if (igb_rxeof(rxr, adapter->rx_process_limit)) 1269 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1270 IGB_TX_LOCK(txr); 1271 igb_txeof(txr); 1272 1273 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1274 igb_start_locked(txr, ifp); 1275 IGB_TX_UNLOCK(txr); 1276 } 1277 1278 igb_enable_intr(adapter); 1279} 1280 1281static void 1282igb_handle_rx(void *context, int pending) 1283{ 1284 struct rx_ring *rxr = context; 1285 struct adapter *adapter = rxr->adapter; 1286 struct ifnet *ifp = adapter->ifp; 1287 1288 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 1289 if (igb_rxeof(rxr, adapter->rx_process_limit) != 0) 1290 /* More to clean, schedule another task */ 1291 taskqueue_enqueue(adapter->tq, &rxr->rx_task); 1292 1293} 1294 1295static void 1296igb_handle_tx(void *context, int pending) 1297{ 1298 struct tx_ring *txr = context; 1299 struct adapter *adapter = txr->adapter; 1300 struct ifnet *ifp = adapter->ifp; 1301 1302 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 1303 IGB_TX_LOCK(txr); 1304 igb_txeof(txr); 1305 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1306 igb_start_locked(txr, ifp); 1307 IGB_TX_UNLOCK(txr); 1308 } 1309} 1310 1311 1312/********************************************************************* 1313 * 1314 * MSI/Legacy Deferred 1315 * Interrupt Service routine 1316 * 1317 *********************************************************************/ 1318#if __FreeBSD_version < 700000 1319static void 1320#else 1321static int 1322#endif 1323igb_irq_fast(void *arg) 1324{ 1325 struct adapter *adapter = arg; 1326 uint32_t reg_icr; 1327 1328 1329 reg_icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1330 1331 /* Hot eject? */ 1332 if (reg_icr == 0xffffffff) 1333 return FILTER_STRAY; 1334 1335 /* Definitely not our interrupt. */ 1336 if (reg_icr == 0x0) 1337 return FILTER_STRAY; 1338 1339 if ((reg_icr & E1000_ICR_INT_ASSERTED) == 0) 1340 return FILTER_STRAY; 1341 1342 /* 1343 * Mask interrupts until the taskqueue is finished running. This is 1344 * cheap, just assume that it is needed. This also works around the 1345 * MSI message reordering errata on certain systems. 1346 */ 1347 igb_disable_intr(adapter); 1348 taskqueue_enqueue(adapter->tq, &adapter->rxtx_task); 1349 1350 /* Link status change */ 1351 if (reg_icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) { 1352 adapter->hw.mac.get_link_status = 1; 1353 igb_update_link_status(adapter); 1354 } 1355 1356 if (reg_icr & E1000_ICR_RXO) 1357 adapter->rx_overruns++; 1358 return FILTER_HANDLED; 1359} 1360 1361 1362#if __FreeBSD_version >= 602105 1363/********************************************************************* 1364 * 1365 * MSIX TX Interrupt Service routine 1366 * 1367 **********************************************************************/ 1368static void 1369igb_msix_tx(void *arg) 1370{ 1371 struct tx_ring *txr = arg; 1372 struct adapter *adapter = txr->adapter; 1373 u32 loop = IGB_MAX_LOOP; 1374 bool more; 1375 1376 ++txr->tx_irq; 1377 IGB_TX_LOCK(txr); 1378 1379 do { 1380 more = igb_txeof(txr); 1381 } while (loop-- && more); 1382 1383 IGB_TX_UNLOCK(txr); 1384 1385 /* Schedule a clean task */ 1386 taskqueue_enqueue(adapter->tq, &txr->tx_task); 1387 1388 /* Reenable this interrupt */ 1389 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, txr->eims); 1390 return; 1391} 1392 1393/********************************************************************* 1394 * 1395 * MSIX RX Interrupt Service routine 1396 * 1397 **********************************************************************/ 1398 1399static void 1400igb_msix_rx(void *arg) 1401{ 1402 struct rx_ring *rxr = arg; 1403 struct adapter *adapter = rxr->adapter; 1404 u32 loop = IGB_MAX_LOOP; 1405 bool more; 1406 1407 ++rxr->rx_irq; 1408 do { 1409 more = igb_rxeof(rxr, adapter->rx_process_limit); 1410 } while (loop-- && more); 1411 1412 /* Update interrupt rate */ 1413 if (igb_enable_aim == TRUE) 1414 igb_update_aim(rxr); 1415 1416 /* Schedule another clean */ 1417 taskqueue_enqueue(adapter->tq, &rxr->rx_task); 1418 1419 /* Reenable this interrupt */ 1420 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, rxr->eims); 1421 return; 1422} 1423 1424 1425/********************************************************************* 1426 * 1427 * MSIX Link Interrupt Service routine 1428 * 1429 **********************************************************************/ 1430 1431static void 1432igb_msix_link(void *arg) 1433{ 1434 struct adapter *adapter = arg; 1435 u32 icr; 1436 1437 ++adapter->link_irq; 1438 icr = E1000_READ_REG(&adapter->hw, E1000_ICR); 1439 if (!(icr & E1000_ICR_LSC)) 1440 goto spurious; 1441 adapter->hw.mac.get_link_status = 1; 1442 igb_update_link_status(adapter); 1443 1444spurious: 1445 /* Rearm */ 1446 E1000_WRITE_REG(&adapter->hw, E1000_IMS, E1000_IMS_LSC); 1447 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, adapter->link_mask); 1448 return; 1449} 1450#endif 1451 1452 1453/* 1454** Routine to adjust the RX EITR value based on traffic, 1455** its a simple three state model, but seems to help. 1456** 1457** Note that the three EITR values are tuneable using 1458** sysctl in real time. The feature can be effectively 1459** nullified by setting them equal. 1460*/ 1461#define BULK_THRESHOLD 10000 1462#define AVE_THRESHOLD 1600 1463 1464static void 1465igb_update_aim(struct rx_ring *rxr) 1466{ 1467 struct adapter *adapter = rxr->adapter; 1468 u32 olditr, newitr; 1469 1470 /* Update interrupt moderation based on traffic */ 1471 olditr = rxr->eitr_setting; 1472 newitr = olditr; 1473 1474 /* Idle, don't change setting */ 1475 if (rxr->bytes == 0) 1476 return; 1477 1478 if (olditr == igb_low_latency) { 1479 if (rxr->bytes > AVE_THRESHOLD) 1480 newitr = igb_ave_latency; 1481 } else if (olditr == igb_ave_latency) { 1482 if (rxr->bytes < AVE_THRESHOLD) 1483 newitr = igb_low_latency; 1484 else if (rxr->bytes > BULK_THRESHOLD) 1485 newitr = igb_bulk_latency; 1486 } else if (olditr == igb_bulk_latency) { 1487 if (rxr->bytes < BULK_THRESHOLD) 1488 newitr = igb_ave_latency; 1489 } 1490 1491 if (olditr != newitr) { 1492 /* Change interrupt rate */ 1493 rxr->eitr_setting = newitr; 1494 E1000_WRITE_REG(&adapter->hw, E1000_EITR(rxr->me), 1495 newitr | (newitr << 16)); 1496 } 1497 1498 rxr->bytes = 0; 1499 return; 1500} 1501 1502 1503/********************************************************************* 1504 * 1505 * Media Ioctl callback 1506 * 1507 * This routine is called whenever the user queries the status of 1508 * the interface using ifconfig. 1509 * 1510 **********************************************************************/ 1511static void 1512igb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr) 1513{ 1514 struct adapter *adapter = ifp->if_softc; 1515 u_char fiber_type = IFM_1000_SX; 1516 1517 INIT_DEBUGOUT("igb_media_status: begin"); 1518 1519 IGB_CORE_LOCK(adapter); 1520 igb_update_link_status(adapter); 1521 1522 ifmr->ifm_status = IFM_AVALID; 1523 ifmr->ifm_active = IFM_ETHER; 1524 1525 if (!adapter->link_active) { 1526 IGB_CORE_UNLOCK(adapter); 1527 return; 1528 } 1529 1530 ifmr->ifm_status |= IFM_ACTIVE; 1531 1532 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 1533 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) 1534 ifmr->ifm_active |= fiber_type | IFM_FDX; 1535 else { 1536 switch (adapter->link_speed) { 1537 case 10: 1538 ifmr->ifm_active |= IFM_10_T; 1539 break; 1540 case 100: 1541 ifmr->ifm_active |= IFM_100_TX; 1542 break; 1543 case 1000: 1544 ifmr->ifm_active |= IFM_1000_T; 1545 break; 1546 } 1547 if (adapter->link_duplex == FULL_DUPLEX) 1548 ifmr->ifm_active |= IFM_FDX; 1549 else 1550 ifmr->ifm_active |= IFM_HDX; 1551 } 1552 IGB_CORE_UNLOCK(adapter); 1553} 1554 1555/********************************************************************* 1556 * 1557 * Media Ioctl callback 1558 * 1559 * This routine is called when the user changes speed/duplex using 1560 * media/mediopt option with ifconfig. 1561 * 1562 **********************************************************************/ 1563static int 1564igb_media_change(struct ifnet *ifp) 1565{ 1566 struct adapter *adapter = ifp->if_softc; 1567 struct ifmedia *ifm = &adapter->media; 1568 1569 INIT_DEBUGOUT("igb_media_change: begin"); 1570 1571 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1572 return (EINVAL); 1573 1574 IGB_CORE_LOCK(adapter); 1575 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1576 case IFM_AUTO: 1577 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1578 adapter->hw.phy.autoneg_advertised = AUTONEG_ADV_DEFAULT; 1579 break; 1580 case IFM_1000_LX: 1581 case IFM_1000_SX: 1582 case IFM_1000_T: 1583 adapter->hw.mac.autoneg = DO_AUTO_NEG; 1584 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL; 1585 break; 1586 case IFM_100_TX: 1587 adapter->hw.mac.autoneg = FALSE; 1588 adapter->hw.phy.autoneg_advertised = 0; 1589 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1590 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_FULL; 1591 else 1592 adapter->hw.mac.forced_speed_duplex = ADVERTISE_100_HALF; 1593 break; 1594 case IFM_10_T: 1595 adapter->hw.mac.autoneg = FALSE; 1596 adapter->hw.phy.autoneg_advertised = 0; 1597 if ((ifm->ifm_media & IFM_GMASK) == IFM_FDX) 1598 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_FULL; 1599 else 1600 adapter->hw.mac.forced_speed_duplex = ADVERTISE_10_HALF; 1601 break; 1602 default: 1603 device_printf(adapter->dev, "Unsupported media type\n"); 1604 } 1605 1606 /* As the speed/duplex settings my have changed we need to 1607 * reset the PHY. 1608 */ 1609 adapter->hw.phy.reset_disable = FALSE; 1610 1611 igb_init_locked(adapter); 1612 IGB_CORE_UNLOCK(adapter); 1613 1614 return (0); 1615} 1616 1617 1618/********************************************************************* 1619 * 1620 * This routine maps the mbufs to Advanced TX descriptors. 1621 * used by the 82575 adapter. 1622 * 1623 **********************************************************************/ 1624 1625static int 1626igb_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1627{ 1628 struct adapter *adapter = txr->adapter; 1629 bus_dma_segment_t segs[IGB_MAX_SCATTER]; 1630 bus_dmamap_t map; 1631 struct igb_tx_buffer *tx_buffer, *tx_buffer_mapped; 1632 union e1000_adv_tx_desc *txd = NULL; 1633 struct mbuf *m_head; 1634 u32 olinfo_status = 0, cmd_type_len = 0; 1635 int nsegs, i, j, error, first, last = 0; 1636 u32 hdrlen = 0, offload = 0; 1637 1638 m_head = *m_headp; 1639 1640 1641 /* Set basic descriptor constants */ 1642 cmd_type_len |= E1000_ADVTXD_DTYP_DATA; 1643 cmd_type_len |= E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT; 1644 if (m_head->m_flags & M_VLANTAG) 1645 cmd_type_len |= E1000_ADVTXD_DCMD_VLE; 1646 1647 /* 1648 * Force a cleanup if number of TX descriptors 1649 * available hits the threshold 1650 */ 1651 if (txr->tx_avail <= IGB_TX_CLEANUP_THRESHOLD) { 1652 igb_txeof(txr); 1653 /* Now do we at least have a minimal? */ 1654 if (txr->tx_avail <= IGB_TX_OP_THRESHOLD) { 1655 txr->no_desc_avail++; 1656 return (ENOBUFS); 1657 } 1658 } 1659 1660 /* 1661 * Map the packet for DMA. 1662 * 1663 * Capture the first descriptor index, 1664 * this descriptor will have the index 1665 * of the EOP which is the only one that 1666 * now gets a DONE bit writeback. 1667 */ 1668 first = txr->next_avail_desc; 1669 tx_buffer = &txr->tx_buffers[first]; 1670 tx_buffer_mapped = tx_buffer; 1671 map = tx_buffer->map; 1672 1673 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1674 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1675 1676 if (error == EFBIG) { 1677 struct mbuf *m; 1678 1679 m = m_defrag(*m_headp, M_DONTWAIT); 1680 if (m == NULL) { 1681 adapter->mbuf_defrag_failed++; 1682 m_freem(*m_headp); 1683 *m_headp = NULL; 1684 return (ENOBUFS); 1685 } 1686 *m_headp = m; 1687 1688 /* Try it again */ 1689 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1690 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1691 1692 if (error == ENOMEM) { 1693 adapter->no_tx_dma_setup++; 1694 return (error); 1695 } else if (error != 0) { 1696 adapter->no_tx_dma_setup++; 1697 m_freem(*m_headp); 1698 *m_headp = NULL; 1699 return (error); 1700 } 1701 } else if (error == ENOMEM) { 1702 adapter->no_tx_dma_setup++; 1703 return (error); 1704 } else if (error != 0) { 1705 adapter->no_tx_dma_setup++; 1706 m_freem(*m_headp); 1707 *m_headp = NULL; 1708 return (error); 1709 } 1710 1711 /* Check again to be sure we have enough descriptors */ 1712 if (nsegs > (txr->tx_avail - 2)) { 1713 txr->no_desc_avail++; 1714 bus_dmamap_unload(txr->txtag, map); 1715 return (ENOBUFS); 1716 } 1717 m_head = *m_headp; 1718 1719 /* 1720 * Set up the context descriptor: 1721 * used when any hardware offload is done. 1722 * This includes CSUM, VLAN, and TSO. It 1723 * will use the first descriptor. 1724 */ 1725 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1726 if (igb_tso_setup(txr, m_head, &hdrlen)) { 1727 cmd_type_len |= E1000_ADVTXD_DCMD_TSE; 1728 olinfo_status |= E1000_TXD_POPTS_IXSM << 8; 1729 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 1730 } else 1731 return (ENXIO); 1732 } else 1733 /* Do all other context descriptor setup */ 1734 offload = igb_tx_ctx_setup(txr, m_head); 1735 if (offload == TRUE) 1736 olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 1737#ifdef IGB_TIMESYNC 1738 if (offload == IGB_TIMESTAMP) 1739 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP; 1740#endif 1741 /* Calculate payload length */ 1742 olinfo_status |= ((m_head->m_pkthdr.len - hdrlen) 1743 << E1000_ADVTXD_PAYLEN_SHIFT); 1744 1745 /* Set up our transmit descriptors */ 1746 i = txr->next_avail_desc; 1747 for (j = 0; j < nsegs; j++) { 1748 bus_size_t seg_len; 1749 bus_addr_t seg_addr; 1750 1751 tx_buffer = &txr->tx_buffers[i]; 1752 txd = (union e1000_adv_tx_desc *)&txr->tx_base[i]; 1753 seg_addr = segs[j].ds_addr; 1754 seg_len = segs[j].ds_len; 1755 1756 txd->read.buffer_addr = htole64(seg_addr); 1757 txd->read.cmd_type_len = htole32( 1758 adapter->txd_cmd | cmd_type_len | seg_len); 1759 txd->read.olinfo_status = htole32(olinfo_status); 1760 last = i; 1761 if (++i == adapter->num_tx_desc) 1762 i = 0; 1763 tx_buffer->m_head = NULL; 1764 tx_buffer->next_eop = -1; 1765 } 1766 1767 txr->next_avail_desc = i; 1768 txr->tx_avail -= nsegs; 1769 1770 tx_buffer->m_head = m_head; 1771 tx_buffer_mapped->map = tx_buffer->map; 1772 tx_buffer->map = map; 1773 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1774 1775 /* 1776 * Last Descriptor of Packet 1777 * needs End Of Packet (EOP) 1778 * and Report Status (RS) 1779 */ 1780 txd->read.cmd_type_len |= 1781 htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 1782 /* 1783 * Keep track in the first buffer which 1784 * descriptor will be written back 1785 */ 1786 tx_buffer = &txr->tx_buffers[first]; 1787 tx_buffer->next_eop = last; 1788 1789 /* 1790 * Advance the Transmit Descriptor Tail (TDT), this tells the E1000 1791 * that this frame is available to transmit. 1792 */ 1793 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1794 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1795 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), i); 1796 ++txr->tx_packets; 1797 1798 return (0); 1799 1800} 1801 1802static void 1803igb_set_promisc(struct adapter *adapter) 1804{ 1805 struct ifnet *ifp = adapter->ifp; 1806 uint32_t reg_rctl; 1807 1808 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1809 1810 if (ifp->if_flags & IFF_PROMISC) { 1811 reg_rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE); 1812 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1813 } else if (ifp->if_flags & IFF_ALLMULTI) { 1814 reg_rctl |= E1000_RCTL_MPE; 1815 reg_rctl &= ~E1000_RCTL_UPE; 1816 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1817 } 1818} 1819 1820static void 1821igb_disable_promisc(struct adapter *adapter) 1822{ 1823 uint32_t reg_rctl; 1824 1825 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1826 1827 reg_rctl &= (~E1000_RCTL_UPE); 1828 reg_rctl &= (~E1000_RCTL_MPE); 1829 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1830} 1831 1832 1833/********************************************************************* 1834 * Multicast Update 1835 * 1836 * This routine is called whenever multicast address list is updated. 1837 * 1838 **********************************************************************/ 1839 1840static void 1841igb_set_multi(struct adapter *adapter) 1842{ 1843 struct ifnet *ifp = adapter->ifp; 1844 struct ifmultiaddr *ifma; 1845 u32 reg_rctl = 0; 1846 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * ETH_ADDR_LEN]; 1847 1848 int mcnt = 0; 1849 1850 IOCTL_DEBUGOUT("igb_set_multi: begin"); 1851 1852 IF_ADDR_LOCK(ifp); 1853 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1854 if (ifma->ifma_addr->sa_family != AF_LINK) 1855 continue; 1856 1857 if (mcnt == MAX_NUM_MULTICAST_ADDRESSES) 1858 break; 1859 1860 bcopy(LLADDR((struct sockaddr_dl *)ifma->ifma_addr), 1861 &mta[mcnt * ETH_ADDR_LEN], ETH_ADDR_LEN); 1862 mcnt++; 1863 } 1864 IF_ADDR_UNLOCK(ifp); 1865 1866 if (mcnt >= MAX_NUM_MULTICAST_ADDRESSES) { 1867 reg_rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 1868 reg_rctl |= E1000_RCTL_MPE; 1869 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, reg_rctl); 1870 } else 1871 e1000_update_mc_addr_list(&adapter->hw, mta, mcnt); 1872} 1873 1874 1875/********************************************************************* 1876 * Timer routine 1877 * 1878 * This routine checks for link status and updates statistics. 1879 * 1880 **********************************************************************/ 1881 1882static void 1883igb_local_timer(void *arg) 1884{ 1885 struct adapter *adapter = arg; 1886 struct ifnet *ifp = adapter->ifp; 1887 1888 IGB_CORE_LOCK_ASSERT(adapter); 1889 1890 igb_update_link_status(adapter); 1891 igb_update_stats_counters(adapter); 1892 1893 if (igb_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) 1894 igb_print_hw_stats(adapter); 1895 1896 /* 1897 * Each second we check the watchdog to 1898 * protect against hardware hangs. 1899 */ 1900 igb_watchdog(adapter); 1901 1902 /* Trigger an RX interrupt on all queues */ 1903 E1000_WRITE_REG(&adapter->hw, E1000_EICS, adapter->rx_mask); 1904 1905 callout_reset(&adapter->timer, hz, igb_local_timer, adapter); 1906 1907} 1908 1909static void 1910igb_update_link_status(struct adapter *adapter) 1911{ 1912 struct e1000_hw *hw = &adapter->hw; 1913 struct ifnet *ifp = adapter->ifp; 1914 device_t dev = adapter->dev; 1915 struct tx_ring *txr = adapter->tx_rings; 1916 u32 link_check = 0; 1917 1918 /* Get the cached link value or read for real */ 1919 switch (hw->phy.media_type) { 1920 case e1000_media_type_copper: 1921 if (hw->mac.get_link_status) { 1922 /* Do the work to read phy */ 1923 e1000_check_for_link(hw); 1924 link_check = !hw->mac.get_link_status; 1925 } else 1926 link_check = TRUE; 1927 break; 1928 case e1000_media_type_fiber: 1929 e1000_check_for_link(hw); 1930 link_check = (E1000_READ_REG(hw, E1000_STATUS) & 1931 E1000_STATUS_LU); 1932 break; 1933 case e1000_media_type_internal_serdes: 1934 e1000_check_for_link(hw); 1935 link_check = adapter->hw.mac.serdes_has_link; 1936 break; 1937 default: 1938 case e1000_media_type_unknown: 1939 break; 1940 } 1941 1942 /* Now we check if a transition has happened */ 1943 if (link_check && (adapter->link_active == 0)) { 1944 e1000_get_speed_and_duplex(&adapter->hw, 1945 &adapter->link_speed, &adapter->link_duplex); 1946 if (bootverbose) 1947 device_printf(dev, "Link is up %d Mbps %s\n", 1948 adapter->link_speed, 1949 ((adapter->link_duplex == FULL_DUPLEX) ? 1950 "Full Duplex" : "Half Duplex")); 1951 adapter->link_active = 1; 1952 ifp->if_baudrate = adapter->link_speed * 1000000; 1953 if_link_state_change(ifp, LINK_STATE_UP); 1954 } else if (!link_check && (adapter->link_active == 1)) { 1955 ifp->if_baudrate = adapter->link_speed = 0; 1956 adapter->link_duplex = 0; 1957 if (bootverbose) 1958 device_printf(dev, "Link is Down\n"); 1959 adapter->link_active = 0; 1960 if_link_state_change(ifp, LINK_STATE_DOWN); 1961 /* Turn off watchdogs */ 1962 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) 1963 txr->watchdog_timer = FALSE; 1964 } 1965} 1966 1967/********************************************************************* 1968 * 1969 * This routine disables all traffic on the adapter by issuing a 1970 * global reset on the MAC and deallocates TX/RX buffers. 1971 * 1972 **********************************************************************/ 1973 1974static void 1975igb_stop(void *arg) 1976{ 1977 struct adapter *adapter = arg; 1978 struct ifnet *ifp = adapter->ifp; 1979 1980 IGB_CORE_LOCK_ASSERT(adapter); 1981 1982 INIT_DEBUGOUT("igb_stop: begin"); 1983 1984 igb_disable_intr(adapter); 1985 1986 callout_stop(&adapter->timer); 1987 1988 /* Tell the stack that the interface is no longer active */ 1989 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1990 1991#ifdef IGB_TIMESYNC 1992 /* Disable IEEE 1588 Time sync */ 1993 if (adapter->hw.mac.type == e1000_82576) 1994 igb_tsync_disable(adapter); 1995#endif 1996 1997 e1000_reset_hw(&adapter->hw); 1998 E1000_WRITE_REG(&adapter->hw, E1000_WUC, 0); 1999} 2000 2001 2002/********************************************************************* 2003 * 2004 * Determine hardware revision. 2005 * 2006 **********************************************************************/ 2007static void 2008igb_identify_hardware(struct adapter *adapter) 2009{ 2010 device_t dev = adapter->dev; 2011 2012 /* Make sure our PCI config space has the necessary stuff set */ 2013 adapter->hw.bus.pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 2014 if (!((adapter->hw.bus.pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 2015 (adapter->hw.bus.pci_cmd_word & PCIM_CMD_MEMEN))) { 2016 device_printf(dev, "Memory Access and/or Bus Master bits " 2017 "were not set!\n"); 2018 adapter->hw.bus.pci_cmd_word |= 2019 (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 2020 pci_write_config(dev, PCIR_COMMAND, 2021 adapter->hw.bus.pci_cmd_word, 2); 2022 } 2023 2024 /* Save off the information about this board */ 2025 adapter->hw.vendor_id = pci_get_vendor(dev); 2026 adapter->hw.device_id = pci_get_device(dev); 2027 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 2028 adapter->hw.subsystem_vendor_id = 2029 pci_read_config(dev, PCIR_SUBVEND_0, 2); 2030 adapter->hw.subsystem_device_id = 2031 pci_read_config(dev, PCIR_SUBDEV_0, 2); 2032 2033 /* Do Shared Code Init and Setup */ 2034 if (e1000_set_mac_type(&adapter->hw)) { 2035 device_printf(dev, "Setup init failure\n"); 2036 return; 2037 } 2038} 2039 2040static int 2041igb_allocate_pci_resources(struct adapter *adapter) 2042{ 2043 device_t dev = adapter->dev; 2044 int rid, error = 0; 2045 2046 rid = PCIR_BAR(0); 2047 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 2048 &rid, RF_ACTIVE); 2049 if (adapter->pci_mem == NULL) { 2050 device_printf(dev, "Unable to allocate bus resource: memory\n"); 2051 return (ENXIO); 2052 } 2053 adapter->osdep.mem_bus_space_tag = 2054 rman_get_bustag(adapter->pci_mem); 2055 adapter->osdep.mem_bus_space_handle = 2056 rman_get_bushandle(adapter->pci_mem); 2057 adapter->hw.hw_addr = (uint8_t *)&adapter->osdep.mem_bus_space_handle; 2058 2059 /* 2060 ** Init the resource arrays 2061 */ 2062 for (int i = 0; i < IGB_MSIX_VEC; i++) { 2063 adapter->rid[i] = i + 1; /* MSI/X RID starts at 1 */ 2064 adapter->tag[i] = NULL; 2065 adapter->res[i] = NULL; 2066 } 2067 2068 adapter->num_tx_queues = 1; /* Defaults for Legacy or MSI */ 2069 adapter->num_rx_queues = 1; 2070 2071#if __FreeBSD_version >= 602105 2072 /* This will setup either MSI/X or MSI */ 2073 adapter->msix = igb_setup_msix(adapter); 2074#endif 2075 adapter->hw.back = &adapter->osdep; 2076 2077 return (error); 2078} 2079 2080/********************************************************************* 2081 * 2082 * Setup the Legacy or MSI Interrupt handler 2083 * 2084 **********************************************************************/ 2085static int 2086igb_allocate_legacy(struct adapter *adapter) 2087{ 2088 device_t dev = adapter->dev; 2089 int error; 2090 2091 /* Turn off all interrupts */ 2092 E1000_WRITE_REG(&adapter->hw, E1000_IMC, 0xffffffff); 2093 2094 /* Legacy RID at 0 */ 2095 if (adapter->msix == 0) 2096 adapter->rid[0] = 0; 2097 2098 /* We allocate a single interrupt resource */ 2099 adapter->res[0] = bus_alloc_resource_any(dev, 2100 SYS_RES_IRQ, &adapter->rid[0], RF_SHAREABLE | RF_ACTIVE); 2101 if (adapter->res[0] == NULL) { 2102 device_printf(dev, "Unable to allocate bus resource: " 2103 "interrupt\n"); 2104 return (ENXIO); 2105 } 2106 2107 /* 2108 * Try allocating a fast interrupt and the associated deferred 2109 * processing contexts. 2110 */ 2111 TASK_INIT(&adapter->rxtx_task, 0, igb_handle_rxtx, adapter); 2112 adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, 2113 taskqueue_thread_enqueue, &adapter->tq); 2114 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2115 device_get_nameunit(adapter->dev)); 2116 if ((error = bus_setup_intr(dev, adapter->res[0], 2117 INTR_TYPE_NET | INTR_MPSAFE, igb_irq_fast, 2118#if __FreeBSD_version >= 700000 2119 NULL, 2120#endif 2121 adapter, &adapter->tag[0])) != 0) { 2122 device_printf(dev, "Failed to register fast interrupt " 2123 "handler: %d\n", error); 2124 taskqueue_free(adapter->tq); 2125 adapter->tq = NULL; 2126 return (error); 2127 } 2128 2129 return (0); 2130} 2131 2132 2133#if __FreeBSD_version >= 602105 2134/********************************************************************* 2135 * 2136 * Setup the MSIX Interrupt handlers: 2137 * 2138 **********************************************************************/ 2139static int 2140igb_allocate_msix(struct adapter *adapter) 2141{ 2142 device_t dev = adapter->dev; 2143 struct tx_ring *txr = adapter->tx_rings; 2144 struct rx_ring *rxr = adapter->rx_rings; 2145 int error, vector = 0; 2146 2147 /* 2148 * Setup the interrupt handlers 2149 */ 2150 2151 /* TX Setup */ 2152 for (int i = 0; i < adapter->num_tx_queues; i++, vector++, txr++) { 2153 adapter->res[vector] = bus_alloc_resource_any(dev, 2154 SYS_RES_IRQ, &adapter->rid[vector], 2155 RF_SHAREABLE | RF_ACTIVE); 2156 if (adapter->res[vector] == NULL) { 2157 device_printf(dev, 2158 "Unable to allocate bus resource: " 2159 "MSIX TX Interrupt\n"); 2160 return (ENXIO); 2161 } 2162 error = bus_setup_intr(dev, adapter->res[vector], 2163 INTR_TYPE_NET | INTR_MPSAFE, 2164#if __FreeBSD_version >= 700000 2165 NULL, 2166#endif 2167 igb_msix_tx, txr, &adapter->tag[vector]); 2168 if (error) { 2169 adapter->res[vector] = NULL; 2170 device_printf(dev, "Failed to register TX handler"); 2171 return (error); 2172 } 2173 /* Make tasklet for deferred handling - one per queue */ 2174 TASK_INIT(&txr->tx_task, 0, igb_handle_tx, txr); 2175 if (adapter->hw.mac.type == e1000_82575) { 2176 txr->eims = E1000_EICR_TX_QUEUE0 << i; 2177 /* MSIXBM registers start at 0 */ 2178 txr->msix = adapter->rid[vector] - 1; 2179 } else { 2180 txr->eims = 1 << vector; 2181 txr->msix = vector; 2182 } 2183 } 2184 2185 /* RX Setup */ 2186 for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rxr++) { 2187 adapter->res[vector] = bus_alloc_resource_any(dev, 2188 SYS_RES_IRQ, &adapter->rid[vector], 2189 RF_SHAREABLE | RF_ACTIVE); 2190 if (adapter->res[vector] == NULL) { 2191 device_printf(dev, 2192 "Unable to allocate bus resource: " 2193 "MSIX RX Interrupt\n"); 2194 return (ENXIO); 2195 } 2196 error = bus_setup_intr(dev, adapter->res[vector], 2197 INTR_TYPE_NET | INTR_MPSAFE, 2198#if __FreeBSD_version >= 700000 2199 NULL, 2200#endif 2201 igb_msix_rx, rxr, &adapter->tag[vector]); 2202 if (error) { 2203 adapter->res[vector] = NULL; 2204 device_printf(dev, "Failed to register RX handler"); 2205 return (error); 2206 } 2207 /* Make tasklet for deferred handling - one per queue */ 2208 TASK_INIT(&rxr->rx_task, 0, igb_handle_rx, rxr); 2209 if (adapter->hw.mac.type == e1000_82575) { 2210 rxr->eims = E1000_EICR_RX_QUEUE0 << i; 2211 rxr->msix = adapter->rid[vector] - 1; 2212 } else { 2213 rxr->eims = 1 << vector; 2214 rxr->msix = vector; 2215 } 2216 /* Get a mask for local timer */ 2217 adapter->rx_mask |= rxr->eims; 2218 } 2219 2220 /* And Link */ 2221 adapter->res[vector] = bus_alloc_resource_any(dev, 2222 SYS_RES_IRQ, &adapter->rid[vector], 2223 RF_SHAREABLE | RF_ACTIVE); 2224 if (adapter->res[vector] == NULL) { 2225 device_printf(dev, 2226 "Unable to allocate bus resource: " 2227 "MSIX Link Interrupt\n"); 2228 return (ENXIO); 2229 } 2230 if ((error = bus_setup_intr(dev, adapter->res[vector], 2231 INTR_TYPE_NET | INTR_MPSAFE, 2232#if __FreeBSD_version >= 700000 2233 NULL, 2234#endif 2235 igb_msix_link, adapter, &adapter->tag[vector])) != 0) { 2236 device_printf(dev, "Failed to register Link handler"); 2237 return (error); 2238 } 2239 if (adapter->hw.mac.type == e1000_82575) 2240 adapter->linkvec = adapter->rid[vector] - 1; 2241 else 2242 adapter->linkvec = vector; 2243 2244 adapter->tq = taskqueue_create_fast("igb_taskq", M_NOWAIT, 2245 taskqueue_thread_enqueue, &adapter->tq); 2246 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s taskq", 2247 device_get_nameunit(adapter->dev)); 2248 2249 return (0); 2250} 2251#else /* FreeBSD 6.1/2 */ 2252static int 2253igb_allocate_msix(struct adapter *adapter) 2254{ 2255 return (1); 2256} 2257#endif 2258 2259 2260static void 2261igb_configure_queues(struct adapter *adapter) 2262{ 2263 struct e1000_hw *hw = &adapter->hw; 2264 struct tx_ring *txr; 2265 struct rx_ring *rxr; 2266 2267 /* Turn on MSIX */ 2268 /* 2269 ** 82576 uses IVARs to route MSI/X 2270 ** interrupts, its not very intuitive, 2271 ** study the code carefully :) 2272 */ 2273 if (adapter->hw.mac.type == e1000_82576) { 2274 u32 ivar = 0; 2275 /* First turn on the capability */ 2276 E1000_WRITE_REG(hw, E1000_GPIE, 2277 E1000_GPIE_MSIX_MODE | 2278 E1000_GPIE_EIAME | 2279 E1000_GPIE_PBA | E1000_GPIE_NSICR); 2280 /* RX */ 2281 for (int i = 0; i < adapter->num_rx_queues; i++) { 2282 u32 index = i & 0x7; /* Each IVAR has two entries */ 2283 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2284 rxr = &adapter->rx_rings[i]; 2285 if (i < 8) { 2286 ivar &= 0xFFFFFF00; 2287 ivar |= rxr->msix | E1000_IVAR_VALID; 2288 } else { 2289 ivar &= 0xFF00FFFF; 2290 ivar |= (rxr->msix | E1000_IVAR_VALID) << 16; 2291 } 2292 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2293 adapter->eims_mask |= rxr->eims; 2294 } 2295 /* TX */ 2296 for (int i = 0; i < adapter->num_tx_queues; i++) { 2297 u32 index = i & 0x7; /* Each IVAR has two entries */ 2298 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 2299 txr = &adapter->tx_rings[i]; 2300 if (i < 8) { 2301 ivar &= 0xFFFF00FF; 2302 ivar |= (txr->msix | E1000_IVAR_VALID) << 8; 2303 } else { 2304 ivar &= 0x00FFFFFF; 2305 ivar |= (txr->msix | E1000_IVAR_VALID) << 24; 2306 } 2307 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 2308 adapter->eims_mask |= txr->eims; 2309 } 2310 2311 /* And for the link interrupt */ 2312 ivar = (adapter->linkvec | E1000_IVAR_VALID) << 8; 2313 adapter->link_mask = 1 << adapter->linkvec; 2314 adapter->eims_mask |= adapter->link_mask; 2315 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 2316 } else 2317 { /* 82575 */ 2318 int tmp; 2319 2320 /* enable MSI-X PBA support*/ 2321 tmp = E1000_READ_REG(hw, E1000_CTRL_EXT); 2322 tmp |= E1000_CTRL_EXT_PBA_CLR; 2323 /* Auto-Mask interrupts upon ICR read. */ 2324 tmp |= E1000_CTRL_EXT_EIAME; 2325 tmp |= E1000_CTRL_EXT_IRCA; 2326 E1000_WRITE_REG(hw, E1000_CTRL_EXT, tmp); 2327 2328 /* TX */ 2329 for (int i = 0; i < adapter->num_tx_queues; i++) { 2330 txr = &adapter->tx_rings[i]; 2331 E1000_WRITE_REG(hw, E1000_MSIXBM(txr->msix), 2332 txr->eims); 2333 adapter->eims_mask |= txr->eims; 2334 } 2335 2336 /* RX */ 2337 for (int i = 0; i < adapter->num_rx_queues; i++) { 2338 rxr = &adapter->rx_rings[i]; 2339 E1000_WRITE_REG(hw, E1000_MSIXBM(rxr->msix), 2340 rxr->eims); 2341 adapter->eims_mask |= rxr->eims; 2342 } 2343 2344 /* Link */ 2345 E1000_WRITE_REG(hw, E1000_MSIXBM(adapter->linkvec), 2346 E1000_EIMS_OTHER); 2347 adapter->link_mask |= E1000_EIMS_OTHER; 2348 adapter->eims_mask |= adapter->link_mask; 2349 } 2350 return; 2351} 2352 2353 2354static void 2355igb_free_pci_resources(struct adapter *adapter) 2356{ 2357 device_t dev = adapter->dev; 2358 2359 /* Make sure the for loop below runs once */ 2360 if (adapter->msix == 0) 2361 adapter->msix = 1; 2362 2363 /* 2364 * First release all the interrupt resources: 2365 * notice that since these are just kept 2366 * in an array we can do the same logic 2367 * whether its MSIX or just legacy. 2368 */ 2369 for (int i = 0; i < adapter->msix; i++) { 2370 if (adapter->tag[i] != NULL) { 2371 bus_teardown_intr(dev, adapter->res[i], 2372 adapter->tag[i]); 2373 adapter->tag[i] = NULL; 2374 } 2375 if (adapter->res[i] != NULL) { 2376 bus_release_resource(dev, SYS_RES_IRQ, 2377 adapter->rid[i], adapter->res[i]); 2378 } 2379 } 2380 2381#if __FreeBSD_version >= 602105 2382 if (adapter->msix) 2383 pci_release_msi(dev); 2384 2385 if (adapter->msix_mem != NULL) 2386 bus_release_resource(dev, SYS_RES_MEMORY, 2387 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2388#endif 2389 2390 if (adapter->pci_mem != NULL) 2391 bus_release_resource(dev, SYS_RES_MEMORY, 2392 PCIR_BAR(0), adapter->pci_mem); 2393 2394} 2395 2396#if __FreeBSD_version >= 602105 2397/* 2398 * Setup Either MSI/X or MSI 2399 */ 2400static int 2401igb_setup_msix(struct adapter *adapter) 2402{ 2403 device_t dev = adapter->dev; 2404 int rid, want, queues, msgs; 2405 2406 /* First try MSI/X */ 2407 rid = PCIR_BAR(IGB_MSIX_BAR); 2408 adapter->msix_mem = bus_alloc_resource_any(dev, 2409 SYS_RES_MEMORY, &rid, RF_ACTIVE); 2410 if (!adapter->msix_mem) { 2411 /* May not be enabled */ 2412 device_printf(adapter->dev, 2413 "Unable to map MSIX table \n"); 2414 goto msi; 2415 } 2416 2417 msgs = pci_msix_count(dev); 2418 if (msgs == 0) { /* system has msix disabled */ 2419 bus_release_resource(dev, SYS_RES_MEMORY, 2420 PCIR_BAR(IGB_MSIX_BAR), adapter->msix_mem); 2421 adapter->msix_mem = NULL; 2422 goto msi; 2423 } 2424 2425 /* Limit by the number set in header */ 2426 if (msgs > IGB_MSIX_VEC) 2427 msgs = IGB_MSIX_VEC; 2428 2429 /* Figure out a reasonable auto config value */ 2430 queues = (mp_ncpus > ((msgs-1)/2)) ? (msgs-1)/2 : mp_ncpus; 2431 2432 if (igb_tx_queues == 0) 2433 igb_tx_queues = queues; 2434 if (igb_rx_queues == 0) 2435 igb_rx_queues = queues; 2436 want = igb_tx_queues + igb_rx_queues + 1; 2437 if (msgs >= want) 2438 msgs = want; 2439 else { 2440 device_printf(adapter->dev, 2441 "MSIX Configuration Problem, " 2442 "%d vectors configured, but %d queues wanted!\n", 2443 msgs, want); 2444 return (ENXIO); 2445 } 2446 if ((msgs) && pci_alloc_msix(dev, &msgs) == 0) { 2447 device_printf(adapter->dev, 2448 "Using MSIX interrupts with %d vectors\n", msgs); 2449 adapter->num_tx_queues = igb_tx_queues; 2450 adapter->num_rx_queues = igb_rx_queues; 2451 return (msgs); 2452 } 2453msi: 2454 msgs = pci_msi_count(dev); 2455 if (msgs == 1 && pci_alloc_msi(dev, &msgs) == 0) 2456 device_printf(adapter->dev,"Using MSI interrupt\n"); 2457 return (msgs); 2458} 2459#endif /* __FreeBSD_version >= 602105 */ 2460 2461/********************************************************************* 2462 * 2463 * Initialize the hardware to a configuration 2464 * as specified by the adapter structure. 2465 * 2466 **********************************************************************/ 2467static int 2468igb_hardware_init(struct adapter *adapter) 2469{ 2470 device_t dev = adapter->dev; 2471 u32 rx_buffer_size; 2472 2473 INIT_DEBUGOUT("igb_hardware_init: begin"); 2474 2475 /* Issue a global reset */ 2476 e1000_reset_hw(&adapter->hw); 2477 2478 /* Let the firmware know the OS is in control */ 2479 igb_get_hw_control(adapter); 2480 2481 /* 2482 * These parameters control the automatic generation (Tx) and 2483 * response (Rx) to Ethernet PAUSE frames. 2484 * - High water mark should allow for at least two frames to be 2485 * received after sending an XOFF. 2486 * - Low water mark works best when it is very near the high water mark. 2487 * This allows the receiver to restart by sending XON when it has 2488 * drained a bit. Here we use an arbitary value of 1500 which will 2489 * restart after one full frame is pulled from the buffer. There 2490 * could be several smaller frames in the buffer and if so they will 2491 * not trigger the XON until their total number reduces the buffer 2492 * by 1500. 2493 * - The pause time is fairly large at 1000 x 512ns = 512 usec. 2494 */ 2495 if (adapter->hw.mac.type == e1000_82576) 2496 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, 2497 E1000_RXPBS) & 0xffff) << 10 ); 2498 else 2499 rx_buffer_size = ((E1000_READ_REG(&adapter->hw, 2500 E1000_PBA) & 0xffff) << 10 ); 2501 2502 adapter->hw.fc.high_water = rx_buffer_size - 2503 roundup2(adapter->max_frame_size, 1024); 2504 adapter->hw.fc.low_water = adapter->hw.fc.high_water - 1500; 2505 2506 adapter->hw.fc.pause_time = IGB_FC_PAUSE_TIME; 2507 adapter->hw.fc.send_xon = TRUE; 2508 2509 /* Set Flow control, use the tunable location if sane */ 2510 if ((igb_fc_setting >= 0) || (igb_fc_setting < 4)) 2511 adapter->hw.fc.requested_mode = igb_fc_setting; 2512 else 2513 adapter->hw.fc.requested_mode = e1000_fc_none; 2514 2515 if (e1000_init_hw(&adapter->hw) < 0) { 2516 device_printf(dev, "Hardware Initialization Failed\n"); 2517 return (EIO); 2518 } 2519 2520 e1000_check_for_link(&adapter->hw); 2521 2522 return (0); 2523} 2524 2525/********************************************************************* 2526 * 2527 * Setup networking device structure and register an interface. 2528 * 2529 **********************************************************************/ 2530static void 2531igb_setup_interface(device_t dev, struct adapter *adapter) 2532{ 2533 struct ifnet *ifp; 2534 2535 INIT_DEBUGOUT("igb_setup_interface: begin"); 2536 2537 ifp = adapter->ifp = if_alloc(IFT_ETHER); 2538 if (ifp == NULL) 2539 panic("%s: can not if_alloc()", device_get_nameunit(dev)); 2540 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 2541 ifp->if_mtu = ETHERMTU; 2542 ifp->if_init = igb_init; 2543 ifp->if_softc = adapter; 2544 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 2545 ifp->if_ioctl = igb_ioctl; 2546 ifp->if_start = igb_start; 2547 IFQ_SET_MAXLEN(&ifp->if_snd, adapter->num_tx_desc - 1); 2548 ifp->if_snd.ifq_drv_maxlen = adapter->num_tx_desc - 1; 2549 IFQ_SET_READY(&ifp->if_snd); 2550 2551 ether_ifattach(ifp, adapter->hw.mac.addr); 2552 2553 ifp->if_capabilities = ifp->if_capenable = 0; 2554 2555 ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM; 2556 ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_VLAN_MTU; 2557 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 2558 ifp->if_capenable = ifp->if_capabilities; 2559 2560 /* 2561 * Tell the upper layer(s) we support long frames. 2562 */ 2563 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2564 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER; 2565 ifp->if_capabilities |= IFCAP_VLAN_MTU; 2566 ifp->if_capenable |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWFILTER; 2567 ifp->if_capenable |= IFCAP_VLAN_MTU; 2568 2569 /* 2570 * Specify the media types supported by this adapter and register 2571 * callbacks to update media and link information 2572 */ 2573 ifmedia_init(&adapter->media, IFM_IMASK, 2574 igb_media_change, igb_media_status); 2575 if ((adapter->hw.phy.media_type == e1000_media_type_fiber) || 2576 (adapter->hw.phy.media_type == e1000_media_type_internal_serdes)) { 2577 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX | IFM_FDX, 2578 0, NULL); 2579 ifmedia_add(&adapter->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 2580 } else { 2581 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T, 0, NULL); 2582 ifmedia_add(&adapter->media, IFM_ETHER | IFM_10_T | IFM_FDX, 2583 0, NULL); 2584 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX, 2585 0, NULL); 2586 ifmedia_add(&adapter->media, IFM_ETHER | IFM_100_TX | IFM_FDX, 2587 0, NULL); 2588 if (adapter->hw.phy.type != e1000_phy_ife) { 2589 ifmedia_add(&adapter->media, 2590 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2591 ifmedia_add(&adapter->media, 2592 IFM_ETHER | IFM_1000_T, 0, NULL); 2593 } 2594 } 2595 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2596 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2597} 2598 2599 2600/* 2601 * Manage DMA'able memory. 2602 */ 2603static void 2604igb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 2605{ 2606 if (error) 2607 return; 2608 *(bus_addr_t *) arg = segs[0].ds_addr; 2609} 2610 2611static int 2612igb_dma_malloc(struct adapter *adapter, bus_size_t size, 2613 struct igb_dma_alloc *dma, int mapflags) 2614{ 2615 int error; 2616 2617 error = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 2618 IGB_DBA_ALIGN, 0, /* alignment, bounds */ 2619 BUS_SPACE_MAXADDR, /* lowaddr */ 2620 BUS_SPACE_MAXADDR, /* highaddr */ 2621 NULL, NULL, /* filter, filterarg */ 2622 size, /* maxsize */ 2623 1, /* nsegments */ 2624 size, /* maxsegsize */ 2625 0, /* flags */ 2626 NULL, /* lockfunc */ 2627 NULL, /* lockarg */ 2628 &dma->dma_tag); 2629 if (error) { 2630 device_printf(adapter->dev, 2631 "%s: bus_dma_tag_create failed: %d\n", 2632 __func__, error); 2633 goto fail_0; 2634 } 2635 2636 error = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, 2637 BUS_DMA_NOWAIT, &dma->dma_map); 2638 if (error) { 2639 device_printf(adapter->dev, 2640 "%s: bus_dmamem_alloc(%ju) failed: %d\n", 2641 __func__, (uintmax_t)size, error); 2642 goto fail_2; 2643 } 2644 2645 dma->dma_paddr = 0; 2646 error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2647 size, igb_dmamap_cb, &dma->dma_paddr, mapflags | BUS_DMA_NOWAIT); 2648 if (error || dma->dma_paddr == 0) { 2649 device_printf(adapter->dev, 2650 "%s: bus_dmamap_load failed: %d\n", 2651 __func__, error); 2652 goto fail_3; 2653 } 2654 2655 return (0); 2656 2657fail_3: 2658 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2659fail_2: 2660 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2661 bus_dma_tag_destroy(dma->dma_tag); 2662fail_0: 2663 dma->dma_map = NULL; 2664 dma->dma_tag = NULL; 2665 2666 return (error); 2667} 2668 2669static void 2670igb_dma_free(struct adapter *adapter, struct igb_dma_alloc *dma) 2671{ 2672 if (dma->dma_tag == NULL) 2673 return; 2674 if (dma->dma_map != NULL) { 2675 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 2676 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2677 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2678 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 2679 dma->dma_map = NULL; 2680 } 2681 bus_dma_tag_destroy(dma->dma_tag); 2682 dma->dma_tag = NULL; 2683} 2684 2685 2686/********************************************************************* 2687 * 2688 * Allocate memory for the transmit and receive rings, and then 2689 * the descriptors associated with each, called only once at attach. 2690 * 2691 **********************************************************************/ 2692static int 2693igb_allocate_queues(struct adapter *adapter) 2694{ 2695 device_t dev = adapter->dev; 2696 struct tx_ring *txr; 2697 struct rx_ring *rxr; 2698 int rsize, tsize, error = E1000_SUCCESS; 2699 int txconf = 0, rxconf = 0; 2700 2701 /* First allocate the TX ring struct memory */ 2702 if (!(adapter->tx_rings = 2703 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2704 adapter->num_tx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2705 device_printf(dev, "Unable to allocate TX ring memory\n"); 2706 error = ENOMEM; 2707 goto fail; 2708 } 2709 txr = adapter->tx_rings; 2710 2711 /* Next allocate the RX */ 2712 if (!(adapter->rx_rings = 2713 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2714 adapter->num_rx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2715 device_printf(dev, "Unable to allocate RX ring memory\n"); 2716 error = ENOMEM; 2717 goto rx_fail; 2718 } 2719 rxr = adapter->rx_rings; 2720 2721 tsize = roundup2(adapter->num_tx_desc * 2722 sizeof(union e1000_adv_tx_desc), IGB_DBA_ALIGN); 2723 /* 2724 * Now set up the TX queues, txconf is needed to handle the 2725 * possibility that things fail midcourse and we need to 2726 * undo memory gracefully 2727 */ 2728 for (int i = 0; i < adapter->num_tx_queues; i++, txconf++) { 2729 /* Set up some basics */ 2730 txr = &adapter->tx_rings[i]; 2731 txr->adapter = adapter; 2732 txr->me = i; 2733 2734 /* Initialize the TX lock */ 2735 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2736 device_get_nameunit(dev), txr->me); 2737 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2738 2739 if (igb_dma_malloc(adapter, tsize, 2740 &txr->txdma, BUS_DMA_NOWAIT)) { 2741 device_printf(dev, 2742 "Unable to allocate TX Descriptor memory\n"); 2743 error = ENOMEM; 2744 goto err_tx_desc; 2745 } 2746 txr->tx_base = (struct e1000_tx_desc *)txr->txdma.dma_vaddr; 2747 bzero((void *)txr->tx_base, tsize); 2748 2749 /* Now allocate transmit buffers for the ring */ 2750 if (igb_allocate_transmit_buffers(txr)) { 2751 device_printf(dev, 2752 "Critical Failure setting up transmit buffers\n"); 2753 error = ENOMEM; 2754 goto err_tx_desc; 2755 } 2756 2757 } 2758 2759 /* 2760 * Next the RX queues... 2761 */ 2762 rsize = roundup2(adapter->num_rx_desc * 2763 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 2764 for (int i = 0; i < adapter->num_rx_queues; i++, rxconf++) { 2765 rxr = &adapter->rx_rings[i]; 2766 rxr->adapter = adapter; 2767 rxr->me = i; 2768 2769 /* Initialize the RX lock */ 2770 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2771 device_get_nameunit(dev), txr->me); 2772 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2773 2774 if (igb_dma_malloc(adapter, rsize, 2775 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2776 device_printf(dev, 2777 "Unable to allocate RxDescriptor memory\n"); 2778 error = ENOMEM; 2779 goto err_rx_desc; 2780 } 2781 rxr->rx_base = (union e1000_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2782 bzero((void *)rxr->rx_base, rsize); 2783 2784 /* Allocate receive buffers for the ring*/ 2785 if (igb_allocate_receive_buffers(rxr)) { 2786 device_printf(dev, 2787 "Critical Failure setting up receive buffers\n"); 2788 error = ENOMEM; 2789 goto err_rx_desc; 2790 } 2791 } 2792 2793 return (0); 2794 2795err_rx_desc: 2796 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2797 igb_dma_free(adapter, &rxr->rxdma); 2798err_tx_desc: 2799 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2800 igb_dma_free(adapter, &txr->txdma); 2801 free(adapter->rx_rings, M_DEVBUF); 2802rx_fail: 2803 free(adapter->tx_rings, M_DEVBUF); 2804fail: 2805 return (error); 2806} 2807 2808/********************************************************************* 2809 * 2810 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2811 * the information needed to transmit a packet on the wire. This is 2812 * called only once at attach, setup is done every reset. 2813 * 2814 **********************************************************************/ 2815static int 2816igb_allocate_transmit_buffers(struct tx_ring *txr) 2817{ 2818 struct adapter *adapter = txr->adapter; 2819 device_t dev = adapter->dev; 2820 struct igb_tx_buffer *txbuf; 2821 int error, i; 2822 2823 /* 2824 * Setup DMA descriptor areas. 2825 */ 2826 if ((error = bus_dma_tag_create(NULL, /* parent */ 2827 PAGE_SIZE, 0, /* alignment, bounds */ 2828 BUS_SPACE_MAXADDR, /* lowaddr */ 2829 BUS_SPACE_MAXADDR, /* highaddr */ 2830 NULL, NULL, /* filter, filterarg */ 2831 IGB_TSO_SIZE, /* maxsize */ 2832 IGB_MAX_SCATTER, /* nsegments */ 2833 PAGE_SIZE, /* maxsegsize */ 2834 0, /* flags */ 2835 NULL, /* lockfunc */ 2836 NULL, /* lockfuncarg */ 2837 &txr->txtag))) { 2838 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2839 goto fail; 2840 } 2841 2842 if (!(txr->tx_buffers = 2843 (struct igb_tx_buffer *) malloc(sizeof(struct igb_tx_buffer) * 2844 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2845 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2846 error = ENOMEM; 2847 goto fail; 2848 } 2849 2850 /* Create the descriptor buffer dma maps */ 2851 txbuf = txr->tx_buffers; 2852 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2853 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2854 if (error != 0) { 2855 device_printf(dev, "Unable to create TX DMA map\n"); 2856 goto fail; 2857 } 2858 } 2859 2860 return 0; 2861fail: 2862 /* We free all, it handles case where we are in the middle */ 2863 igb_free_transmit_structures(adapter); 2864 return (error); 2865} 2866 2867/********************************************************************* 2868 * 2869 * Initialize a transmit ring. 2870 * 2871 **********************************************************************/ 2872static void 2873igb_setup_transmit_ring(struct tx_ring *txr) 2874{ 2875 struct adapter *adapter = txr->adapter; 2876 struct igb_tx_buffer *txbuf; 2877 int i; 2878 2879 /* Clear the old ring contents */ 2880 bzero((void *)txr->tx_base, 2881 (sizeof(union e1000_adv_tx_desc)) * adapter->num_tx_desc); 2882 /* Reset indices */ 2883 txr->next_avail_desc = 0; 2884 txr->next_to_clean = 0; 2885 2886 /* Free any existing tx buffers. */ 2887 txbuf = txr->tx_buffers; 2888 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2889 if (txbuf->m_head != NULL) { 2890 bus_dmamap_sync(txr->txtag, txbuf->map, 2891 BUS_DMASYNC_POSTWRITE); 2892 bus_dmamap_unload(txr->txtag, txbuf->map); 2893 m_freem(txbuf->m_head); 2894 txbuf->m_head = NULL; 2895 } 2896 /* clear the watch index */ 2897 txbuf->next_eop = -1; 2898 } 2899 2900 /* Set number of descriptors available */ 2901 txr->tx_avail = adapter->num_tx_desc; 2902 2903 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2904 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2905 2906} 2907 2908/********************************************************************* 2909 * 2910 * Initialize all transmit rings. 2911 * 2912 **********************************************************************/ 2913static void 2914igb_setup_transmit_structures(struct adapter *adapter) 2915{ 2916 struct tx_ring *txr = adapter->tx_rings; 2917 2918 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) 2919 igb_setup_transmit_ring(txr); 2920 2921 return; 2922} 2923 2924/********************************************************************* 2925 * 2926 * Enable transmit unit. 2927 * 2928 **********************************************************************/ 2929static void 2930igb_initialize_transmit_units(struct adapter *adapter) 2931{ 2932 struct tx_ring *txr = adapter->tx_rings; 2933 u32 tctl, txdctl; 2934 2935 INIT_DEBUGOUT("igb_initialize_transmit_units: begin"); 2936 2937 /* Setup the Base and Length of the Tx Descriptor Rings */ 2938 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 2939 u64 bus_addr = txr->txdma.dma_paddr; 2940 2941 E1000_WRITE_REG(&adapter->hw, E1000_TDLEN(i), 2942 adapter->num_tx_desc * sizeof(struct e1000_tx_desc)); 2943 E1000_WRITE_REG(&adapter->hw, E1000_TDBAH(i), 2944 (uint32_t)(bus_addr >> 32)); 2945 E1000_WRITE_REG(&adapter->hw, E1000_TDBAL(i), 2946 (uint32_t)bus_addr); 2947 2948 /* Setup the HW Tx Head and Tail descriptor pointers */ 2949 E1000_WRITE_REG(&adapter->hw, E1000_TDT(i), 0); 2950 E1000_WRITE_REG(&adapter->hw, E1000_TDH(i), 0); 2951 2952 HW_DEBUGOUT2("Base = %x, Length = %x\n", 2953 E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)), 2954 E1000_READ_REG(&adapter->hw, E1000_TDLEN(i))); 2955 2956 /* Setup Transmit Descriptor Base Settings */ 2957 adapter->txd_cmd = E1000_TXD_CMD_IFCS; 2958 2959 txdctl = E1000_READ_REG(&adapter->hw, E1000_TXDCTL(i)); 2960 txdctl |= E1000_TXDCTL_QUEUE_ENABLE; 2961 E1000_WRITE_REG(&adapter->hw, E1000_TXDCTL(i), txdctl); 2962 } 2963 2964 /* Program the Transmit Control Register */ 2965 tctl = E1000_READ_REG(&adapter->hw, E1000_TCTL); 2966 tctl &= ~E1000_TCTL_CT; 2967 tctl |= (E1000_TCTL_PSP | E1000_TCTL_RTLC | E1000_TCTL_EN | 2968 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT)); 2969 2970 e1000_config_collision_dist(&adapter->hw); 2971 2972 /* This write will effectively turn on the transmit unit. */ 2973 E1000_WRITE_REG(&adapter->hw, E1000_TCTL, tctl); 2974 2975} 2976 2977/********************************************************************* 2978 * 2979 * Free all transmit rings. 2980 * 2981 **********************************************************************/ 2982static void 2983igb_free_transmit_structures(struct adapter *adapter) 2984{ 2985 struct tx_ring *txr = adapter->tx_rings; 2986 2987 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 2988 IGB_TX_LOCK(txr); 2989 igb_free_transmit_buffers(txr); 2990 igb_dma_free(adapter, &txr->txdma); 2991 IGB_TX_UNLOCK(txr); 2992 IGB_TX_LOCK_DESTROY(txr); 2993 } 2994 free(adapter->tx_rings, M_DEVBUF); 2995} 2996 2997/********************************************************************* 2998 * 2999 * Free transmit ring related data structures. 3000 * 3001 **********************************************************************/ 3002static void 3003igb_free_transmit_buffers(struct tx_ring *txr) 3004{ 3005 struct adapter *adapter = txr->adapter; 3006 struct igb_tx_buffer *tx_buffer; 3007 int i; 3008 3009 INIT_DEBUGOUT("free_transmit_ring: begin"); 3010 3011 if (txr->tx_buffers == NULL) 3012 return; 3013 3014 tx_buffer = txr->tx_buffers; 3015 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 3016 if (tx_buffer->m_head != NULL) { 3017 bus_dmamap_sync(txr->txtag, tx_buffer->map, 3018 BUS_DMASYNC_POSTWRITE); 3019 bus_dmamap_unload(txr->txtag, 3020 tx_buffer->map); 3021 m_freem(tx_buffer->m_head); 3022 tx_buffer->m_head = NULL; 3023 if (tx_buffer->map != NULL) { 3024 bus_dmamap_destroy(txr->txtag, 3025 tx_buffer->map); 3026 tx_buffer->map = NULL; 3027 } 3028 } else if (tx_buffer->map != NULL) { 3029 bus_dmamap_unload(txr->txtag, 3030 tx_buffer->map); 3031 bus_dmamap_destroy(txr->txtag, 3032 tx_buffer->map); 3033 tx_buffer->map = NULL; 3034 } 3035 } 3036 3037 if (txr->tx_buffers != NULL) { 3038 free(txr->tx_buffers, M_DEVBUF); 3039 txr->tx_buffers = NULL; 3040 } 3041 if (txr->txtag != NULL) { 3042 bus_dma_tag_destroy(txr->txtag); 3043 txr->txtag = NULL; 3044 } 3045 return; 3046} 3047 3048#if __FreeBSD_version >= 700000 3049/********************************************************************** 3050 * 3051 * Setup work for hardware segmentation offload (TSO) on 3052 * adapters using advanced tx descriptors (82575) 3053 * 3054 **********************************************************************/ 3055static boolean_t 3056igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen) 3057{ 3058 struct adapter *adapter = txr->adapter; 3059 struct e1000_adv_tx_context_desc *TXD; 3060 struct igb_tx_buffer *tx_buffer; 3061 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3062 u32 mss_l4len_idx = 0; 3063 u16 vtag = 0; 3064 int ctxd, ehdrlen, ip_hlen, tcp_hlen; 3065 struct ether_vlan_header *eh; 3066 struct ip *ip; 3067 struct tcphdr *th; 3068 3069 3070 /* 3071 * Determine where frame payload starts. 3072 * Jump over vlan headers if already present 3073 */ 3074 eh = mtod(mp, struct ether_vlan_header *); 3075 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 3076 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3077 else 3078 ehdrlen = ETHER_HDR_LEN; 3079 3080 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 3081 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 3082 return FALSE; 3083 3084 /* Only supports IPV4 for now */ 3085 ctxd = txr->next_avail_desc; 3086 tx_buffer = &txr->tx_buffers[ctxd]; 3087 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3088 3089 ip = (struct ip *)(mp->m_data + ehdrlen); 3090 if (ip->ip_p != IPPROTO_TCP) 3091 return FALSE; /* 0 */ 3092 ip->ip_sum = 0; 3093 ip_hlen = ip->ip_hl << 2; 3094 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 3095 th->th_sum = in_pseudo(ip->ip_src.s_addr, 3096 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 3097 tcp_hlen = th->th_off << 2; 3098 /* 3099 * Calculate header length, this is used 3100 * in the transmit desc in igb_xmit 3101 */ 3102 *hdrlen = ehdrlen + ip_hlen + tcp_hlen; 3103 3104 /* VLAN MACLEN IPLEN */ 3105 if (mp->m_flags & M_VLANTAG) { 3106 vtag = htole16(mp->m_pkthdr.ether_vtag); 3107 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3108 } 3109 3110 vlan_macip_lens |= (ehdrlen << E1000_ADVTXD_MACLEN_SHIFT); 3111 vlan_macip_lens |= ip_hlen; 3112 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3113 3114 /* ADV DTYPE TUCMD */ 3115 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3116 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3117 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3118 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3119 3120 /* MSS L4LEN IDX */ 3121 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << E1000_ADVTXD_MSS_SHIFT); 3122 mss_l4len_idx |= (tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); 3123 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 3124 3125 TXD->seqnum_seed = htole32(0); 3126 tx_buffer->m_head = NULL; 3127 tx_buffer->next_eop = -1; 3128 3129 if (++ctxd == adapter->num_tx_desc) 3130 ctxd = 0; 3131 3132 txr->tx_avail--; 3133 txr->next_avail_desc = ctxd; 3134 return TRUE; 3135} 3136#else /* fake out for 6.2 */ 3137static boolean_t 3138igb_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *hdrlen) 3139{ 3140 return (FALSE); 3141} 3142#endif 3143 3144/********************************************************************* 3145 * 3146 * Context Descriptor setup for VLAN or CSUM 3147 * 3148 **********************************************************************/ 3149 3150static int 3151igb_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 3152{ 3153 struct adapter *adapter = txr->adapter; 3154 struct e1000_adv_tx_context_desc *TXD; 3155 struct igb_tx_buffer *tx_buffer; 3156 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 3157 struct ether_vlan_header *eh; 3158 struct ip *ip = NULL; 3159 struct ip6_hdr *ip6; 3160 int ehdrlen, ip_hlen = 0; 3161 u16 etype; 3162 u8 ipproto = 0; 3163 bool offload = FALSE; 3164#if __FreeBSD_version >= 700000 3165 u16 vtag = 0; 3166#else 3167 struct m_tag *mtag; 3168#endif 3169 3170 int ctxd = txr->next_avail_desc; 3171 tx_buffer = &txr->tx_buffers[ctxd]; 3172 TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[ctxd]; 3173 3174 if (mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) 3175 offload = TRUE; 3176 3177 /* 3178 ** In advanced descriptors the vlan tag must 3179 ** be placed into the descriptor itself. 3180 */ 3181#if __FreeBSD_version < 700000 3182 mtag = VLAN_OUTPUT_TAG(ifp, mp); 3183 if (mtag != NULL) { 3184 vlan_macip_lens |= 3185 htole16(VLAN_TAG_VALUE(mtag)) << E1000_ADVTXD_VLAN_SHIFT; 3186 offload = TRUE; 3187 } 3188#else 3189 if (mp->m_flags & M_VLANTAG) { 3190 vtag = htole16(mp->m_pkthdr.ether_vtag); 3191 vlan_macip_lens |= (vtag << E1000_ADVTXD_VLAN_SHIFT); 3192 offload = TRUE; 3193 } 3194#endif 3195 /* 3196 * Determine where frame payload starts. 3197 * Jump over vlan headers if already present, 3198 * helpful for QinQ too. 3199 */ 3200 eh = mtod(mp, struct ether_vlan_header *); 3201 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 3202 etype = ntohs(eh->evl_proto); 3203 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 3204 } else { 3205 etype = ntohs(eh->evl_encap_proto); 3206 ehdrlen = ETHER_HDR_LEN; 3207 } 3208 3209 /* Set the ether header length */ 3210 vlan_macip_lens |= ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; 3211 3212 switch (etype) { 3213 case ETHERTYPE_IP: 3214 ip = (struct ip *)(mp->m_data + ehdrlen); 3215 ip_hlen = ip->ip_hl << 2; 3216 if (mp->m_len < ehdrlen + ip_hlen) 3217 return FALSE; 3218 ipproto = ip->ip_p; 3219 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 3220 break; 3221 case ETHERTYPE_IPV6: 3222 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 3223 ip_hlen = sizeof(struct ip6_hdr); 3224 if (mp->m_len < ehdrlen + ip_hlen) 3225 return FALSE; /* failure */ 3226 ipproto = ip6->ip6_nxt; 3227 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; 3228 break; 3229#ifdef IGB_TIMESYNC 3230 case ETHERTYPE_IEEE1588: 3231 return (IGB_TIMESTAMP); 3232#endif 3233 default: 3234 return (FALSE); 3235 } 3236 3237 vlan_macip_lens |= ip_hlen; 3238 type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 3239 3240 switch (ipproto) { 3241 case IPPROTO_TCP: 3242 if (mp->m_pkthdr.csum_flags & CSUM_TCP) { 3243 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 3244 offload = TRUE; 3245 } 3246 break; 3247 case IPPROTO_UDP: 3248 { 3249#ifdef IGB_TIMESYNC 3250 void *hdr = (caddr_t) ip + ip_hlen; 3251 struct udphdr *uh = (struct udphdr *)hdr; 3252 3253 if (uh->uh_dport == htons(TSYNC_PORT)) 3254 return (IGB_TIMESTAMP); 3255#endif 3256 if (mp->m_pkthdr.csum_flags & CSUM_UDP) { 3257 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; 3258 offload = TRUE; 3259 } 3260 break; 3261 } 3262#if __FreeBSD_version >= 800000 3263 case IPPROTO_SCTP: 3264 { 3265 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) { 3266 type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; 3267 offload = TRUE; 3268 } 3269 break; 3270 } 3271#endif 3272 default: 3273 return (FALSE); 3274 } 3275 3276 if (offload != TRUE) 3277 return (FALSE); 3278 3279 /* Now copy bits into descriptor */ 3280 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 3281 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 3282 TXD->seqnum_seed = htole32(0); 3283 TXD->mss_l4len_idx = htole32(0); 3284 3285 tx_buffer->m_head = NULL; 3286 tx_buffer->next_eop = -1; 3287 3288 /* We've consumed the first desc, adjust counters */ 3289 if (++ctxd == adapter->num_tx_desc) 3290 ctxd = 0; 3291 txr->next_avail_desc = ctxd; 3292 --txr->tx_avail; 3293 3294 return (TRUE); 3295} 3296 3297 3298/********************************************************************** 3299 * 3300 * Examine each tx_buffer in the used queue. If the hardware is done 3301 * processing the packet then free associated resources. The 3302 * tx_buffer is put back on the free queue. 3303 * 3304 * TRUE return means there's work in the ring to clean, FALSE its empty. 3305 **********************************************************************/ 3306static bool 3307igb_txeof(struct tx_ring *txr) 3308{ 3309 struct adapter *adapter = txr->adapter; 3310 int first, last, done, num_avail; 3311 struct igb_tx_buffer *tx_buffer; 3312 struct e1000_tx_desc *tx_desc, *eop_desc; 3313 struct ifnet *ifp = adapter->ifp; 3314 3315 IGB_TX_LOCK_ASSERT(txr); 3316 3317 if (txr->tx_avail == adapter->num_tx_desc) 3318 return FALSE; 3319 3320 num_avail = txr->tx_avail; 3321 first = txr->next_to_clean; 3322 tx_desc = &txr->tx_base[first]; 3323 tx_buffer = &txr->tx_buffers[first]; 3324 last = tx_buffer->next_eop; 3325 eop_desc = &txr->tx_base[last]; 3326 3327 /* 3328 * What this does is get the index of the 3329 * first descriptor AFTER the EOP of the 3330 * first packet, that way we can do the 3331 * simple comparison on the inner while loop. 3332 */ 3333 if (++last == adapter->num_tx_desc) 3334 last = 0; 3335 done = last; 3336 3337 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3338 BUS_DMASYNC_POSTREAD); 3339 3340 while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 3341 /* We clean the range of the packet */ 3342 while (first != done) { 3343 tx_desc->upper.data = 0; 3344 tx_desc->lower.data = 0; 3345 tx_desc->buffer_addr = 0; 3346 num_avail++; 3347 3348 if (tx_buffer->m_head) { 3349 ifp->if_opackets++; 3350 bus_dmamap_sync(txr->txtag, 3351 tx_buffer->map, 3352 BUS_DMASYNC_POSTWRITE); 3353 bus_dmamap_unload(txr->txtag, 3354 tx_buffer->map); 3355 3356 m_freem(tx_buffer->m_head); 3357 tx_buffer->m_head = NULL; 3358 } 3359 tx_buffer->next_eop = -1; 3360 3361 if (++first == adapter->num_tx_desc) 3362 first = 0; 3363 3364 tx_buffer = &txr->tx_buffers[first]; 3365 tx_desc = &txr->tx_base[first]; 3366 } 3367 /* See if we can continue to the next packet */ 3368 last = tx_buffer->next_eop; 3369 if (last != -1) { 3370 eop_desc = &txr->tx_base[last]; 3371 /* Get new done point */ 3372 if (++last == adapter->num_tx_desc) last = 0; 3373 done = last; 3374 } else 3375 break; 3376 } 3377 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 3378 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3379 3380 txr->next_to_clean = first; 3381 3382 /* 3383 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack 3384 * that it is OK to send packets. 3385 * If there are no pending descriptors, clear the timeout. Otherwise, 3386 * if some descriptors have been freed, restart the timeout. 3387 */ 3388 if (num_avail > IGB_TX_CLEANUP_THRESHOLD) { 3389 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 3390 /* All clean, turn off the timer */ 3391 if (num_avail == adapter->num_tx_desc) { 3392 txr->watchdog_timer = 0; 3393 txr->tx_avail = num_avail; 3394 return FALSE; 3395 } 3396 /* Some cleaned, reset the timer */ 3397 else if (num_avail != txr->tx_avail) 3398 txr->watchdog_timer = IGB_TX_TIMEOUT; 3399 } 3400 txr->tx_avail = num_avail; 3401 return TRUE; 3402} 3403 3404 3405/********************************************************************* 3406 * 3407 * Setup descriptor buffer(s) from system mbuf buffer pools. 3408 * i - designates the ring index 3409 * clean - tells the function whether to update 3410 * the header, the packet buffer, or both. 3411 * 3412 **********************************************************************/ 3413static int 3414igb_get_buf(struct rx_ring *rxr, int i, u8 clean) 3415{ 3416 struct adapter *adapter = rxr->adapter; 3417 struct mbuf *mh, *mp; 3418 bus_dma_segment_t seg[2]; 3419 bus_dmamap_t map; 3420 struct igb_rx_buffer *rx_buffer; 3421 int error, nsegs; 3422 int merr = 0; 3423 3424 3425 rx_buffer = &rxr->rx_buffers[i]; 3426 3427 /* First get our header and payload mbuf */ 3428 if (clean & IGB_CLEAN_HEADER) { 3429 mh = m_gethdr(M_DONTWAIT, MT_DATA); 3430 if (mh == NULL) 3431 goto remap; 3432 } else /* reuse */ 3433 mh = rxr->rx_buffers[i].m_head; 3434 3435 mh->m_len = MHLEN; 3436 mh->m_flags |= M_PKTHDR; 3437 3438 if (clean & IGB_CLEAN_PAYLOAD) { 3439 mp = m_getjcl(M_DONTWAIT, MT_DATA, 3440 M_PKTHDR, adapter->rx_mbuf_sz); 3441 if (mp == NULL) 3442 goto remap; 3443 mp->m_len = adapter->rx_mbuf_sz; 3444 mp->m_flags &= ~M_PKTHDR; 3445 } else { /* reusing */ 3446 mp = rxr->rx_buffers[i].m_pack; 3447 mp->m_len = adapter->rx_mbuf_sz; 3448 mp->m_flags &= ~M_PKTHDR; 3449 } 3450 /* 3451 ** Need to create a chain for the following 3452 ** dmamap call at this point. 3453 */ 3454 mh->m_next = mp; 3455 mh->m_pkthdr.len = mh->m_len + mp->m_len; 3456 3457 /* Get the memory mapping */ 3458 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3459 rxr->rx_spare_map, mh, seg, &nsegs, BUS_DMA_NOWAIT); 3460 if (error != 0) { 3461 printf("GET BUF: dmamap load failure - %d\n", error); 3462 m_free(mh); 3463 return (error); 3464 } 3465 3466 /* Unload old mapping and update buffer struct */ 3467 if (rx_buffer->m_head != NULL) 3468 bus_dmamap_unload(rxr->rxtag, rx_buffer->map); 3469 map = rx_buffer->map; 3470 rx_buffer->map = rxr->rx_spare_map; 3471 rxr->rx_spare_map = map; 3472 rx_buffer->m_head = mh; 3473 rx_buffer->m_pack = mp; 3474 bus_dmamap_sync(rxr->rxtag, 3475 rx_buffer->map, BUS_DMASYNC_PREREAD); 3476 3477 /* Update descriptor */ 3478 rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr); 3479 rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr); 3480 3481 return (0); 3482 3483 /* 3484 ** If we get here, we have an mbuf resource 3485 ** issue, so we discard the incoming packet 3486 ** and attempt to reuse existing mbufs next 3487 ** pass thru the ring, but to do so we must 3488 ** fix up the descriptor which had the address 3489 ** clobbered with writeback info. 3490 */ 3491remap: 3492 adapter->mbuf_header_failed++; 3493 merr = ENOBUFS; 3494 /* Is there a reusable buffer? */ 3495 mh = rxr->rx_buffers[i].m_head; 3496 if (mh == NULL) /* Nope, init error */ 3497 return (merr); 3498 mp = rxr->rx_buffers[i].m_pack; 3499 if (mp == NULL) /* Nope, init error */ 3500 return (merr); 3501 /* Get our old mapping */ 3502 rx_buffer = &rxr->rx_buffers[i]; 3503 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3504 rx_buffer->map, mh, seg, &nsegs, BUS_DMA_NOWAIT); 3505 if (error != 0) { 3506 /* We really have a problem */ 3507 m_free(mh); 3508 return (error); 3509 } 3510 /* Now fix the descriptor as needed */ 3511 rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr); 3512 rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr); 3513 return (merr); 3514} 3515 3516 3517/********************************************************************* 3518 * 3519 * Allocate memory for rx_buffer structures. Since we use one 3520 * rx_buffer per received packet, the maximum number of rx_buffer's 3521 * that we'll need is equal to the number of receive descriptors 3522 * that we've allocated. 3523 * 3524 **********************************************************************/ 3525static int 3526igb_allocate_receive_buffers(struct rx_ring *rxr) 3527{ 3528 struct adapter *adapter = rxr->adapter; 3529 device_t dev = adapter->dev; 3530 struct igb_rx_buffer *rxbuf; 3531 int i, bsize, error; 3532 3533 bsize = sizeof(struct igb_rx_buffer) * adapter->num_rx_desc; 3534 if (!(rxr->rx_buffers = 3535 (struct igb_rx_buffer *) malloc(bsize, 3536 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3537 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3538 error = ENOMEM; 3539 goto fail; 3540 } 3541 3542 /* 3543 ** The tag is made to accomodate the largest buffer size 3544 ** with packet split (hence the two segments, even though 3545 ** it may not always use this. 3546 */ 3547 if ((error = bus_dma_tag_create(NULL, /* parent */ 3548 PAGE_SIZE, 0, /* alignment, bounds */ 3549 BUS_SPACE_MAXADDR, /* lowaddr */ 3550 BUS_SPACE_MAXADDR, /* highaddr */ 3551 NULL, NULL, /* filter, filterarg */ 3552 MJUM16BYTES, /* maxsize */ 3553 2, /* nsegments */ 3554 MJUMPAGESIZE, /* maxsegsize */ 3555 0, /* flags */ 3556 NULL, /* lockfunc */ 3557 NULL, /* lockfuncarg */ 3558 &rxr->rxtag))) { 3559 device_printf(dev, "Unable to create RX DMA tag\n"); 3560 goto fail; 3561 } 3562 3563 /* Create the spare map (used by getbuf) */ 3564 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3565 &rxr->rx_spare_map); 3566 if (error) { 3567 device_printf(dev, 3568 "%s: bus_dmamap_create header spare failed: %d\n", 3569 __func__, error); 3570 goto fail; 3571 } 3572 3573 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3574 rxbuf = &rxr->rx_buffers[i]; 3575 error = bus_dmamap_create(rxr->rxtag, 3576 BUS_DMA_NOWAIT, &rxbuf->map); 3577 if (error) { 3578 device_printf(dev, "Unable to create RX DMA maps\n"); 3579 goto fail; 3580 } 3581 } 3582 3583 return (0); 3584 3585fail: 3586 /* Frees all, but can handle partial completion */ 3587 igb_free_receive_structures(adapter); 3588 return (error); 3589} 3590 3591/********************************************************************* 3592 * 3593 * Initialize a receive ring and its buffers. 3594 * 3595 **********************************************************************/ 3596static int 3597igb_setup_receive_ring(struct rx_ring *rxr) 3598{ 3599 struct adapter *adapter; 3600 device_t dev; 3601 struct igb_rx_buffer *rxbuf; 3602 struct lro_ctrl *lro = &rxr->lro; 3603 int j, rsize; 3604 3605 adapter = rxr->adapter; 3606 dev = adapter->dev; 3607 3608 /* Clear the ring contents */ 3609 rsize = roundup2(adapter->num_rx_desc * 3610 sizeof(union e1000_adv_rx_desc), IGB_DBA_ALIGN); 3611 bzero((void *)rxr->rx_base, rsize); 3612 3613 /* 3614 ** Free current RX buffer structures and their mbufs 3615 */ 3616 for (int i = 0; i < adapter->num_rx_desc; i++) { 3617 rxbuf = &rxr->rx_buffers[i]; 3618 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3619 BUS_DMASYNC_POSTREAD); 3620 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3621 if (rxbuf->m_head) { 3622 rxbuf->m_head->m_next = rxbuf->m_pack; 3623 m_freem(rxbuf->m_head); 3624 } 3625 rxbuf->m_head = NULL; 3626 rxbuf->m_pack = NULL; 3627 } 3628 3629 /* Next replenish the ring */ 3630 for (j = 0; j < adapter->num_rx_desc; j++) { 3631 if (igb_get_buf(rxr, j, IGB_CLEAN_BOTH) == ENOBUFS) { 3632 rxr->rx_buffers[j].m_head = NULL; 3633 rxr->rx_buffers[j].m_pack = NULL; 3634 rxr->rx_base[j].read.hdr_addr = 0; 3635 rxr->rx_base[j].read.pkt_addr = 0; 3636 goto fail; 3637 } 3638 } 3639 3640 /* Setup our descriptor indices */ 3641 rxr->next_to_check = 0; 3642 rxr->last_cleaned = 0; 3643 3644 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3645 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3646 3647 /* Now set up the LRO interface */ 3648 if (igb_enable_lro) { 3649 int err = tcp_lro_init(lro); 3650 if (err) { 3651 device_printf(dev,"LRO Initialization failed!\n"); 3652 goto fail; 3653 } 3654 INIT_DEBUGOUT("RX LRO Initialized\n"); 3655 lro->ifp = adapter->ifp; 3656 } 3657 3658 return (0); 3659fail: 3660 /* 3661 * We need to clean up any buffers allocated 3662 * so far, 'j' is the failing index. 3663 */ 3664 for (int i = 0; i < j; i++) { 3665 rxbuf = &rxr->rx_buffers[i]; 3666 if (rxbuf->m_head != NULL) { 3667 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3668 BUS_DMASYNC_POSTREAD); 3669 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3670 m_freem(rxbuf->m_head); 3671 rxbuf->m_head = NULL; 3672 } 3673 } 3674 return (ENOBUFS); 3675} 3676 3677/********************************************************************* 3678 * 3679 * Initialize all receive rings. 3680 * 3681 **********************************************************************/ 3682static int 3683igb_setup_receive_structures(struct adapter *adapter) 3684{ 3685 struct rx_ring *rxr = adapter->rx_rings; 3686 int i, j; 3687 3688 for (i = 0; i < adapter->num_rx_queues; i++, rxr++) 3689 if (igb_setup_receive_ring(rxr)) 3690 goto fail; 3691 3692 return (0); 3693fail: 3694 /* 3695 * Free RX buffers allocated so far, we will only handle 3696 * the rings that completed, the failing case will have 3697 * cleaned up for itself. The value of 'i' will be the 3698 * failed ring so we must pre-decrement it. 3699 */ 3700 rxr = adapter->rx_rings; 3701 for (--i; i > 0; i--, rxr++) { 3702 for (j = 0; j < adapter->num_rx_desc; j++) { 3703 struct igb_rx_buffer *rxbuf; 3704 rxbuf = &rxr->rx_buffers[j]; 3705 if (rxbuf->m_head != NULL) { 3706 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3707 BUS_DMASYNC_POSTREAD); 3708 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3709 m_freem(rxbuf->m_head); 3710 rxbuf->m_head = NULL; 3711 } 3712 } 3713 } 3714 3715 return (ENOBUFS); 3716} 3717 3718/********************************************************************* 3719 * 3720 * Enable receive unit. 3721 * 3722 **********************************************************************/ 3723static void 3724igb_initialize_receive_units(struct adapter *adapter) 3725{ 3726 struct rx_ring *rxr = adapter->rx_rings; 3727 struct ifnet *ifp = adapter->ifp; 3728 u32 rctl, rxcsum, psize, srrctl = 0; 3729 3730 INIT_DEBUGOUT("igb_initialize_receive_unit: begin"); 3731 3732 /* 3733 * Make sure receives are disabled while setting 3734 * up the descriptor ring 3735 */ 3736 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 3737 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl & ~E1000_RCTL_EN); 3738 3739 /* 3740 ** Set up for header split 3741 */ 3742 if (igb_rx_hdr_split) { 3743 /* Use a standard mbuf for the header */ 3744 srrctl |= IGB_HDR_BUF << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT; 3745 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3746 } else 3747 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF; 3748 3749 /* 3750 ** Set up for jumbo frames 3751 */ 3752 if (ifp->if_mtu > ETHERMTU) { 3753 rctl |= E1000_RCTL_LPE; 3754 srrctl |= 4096 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 3755 rctl |= E1000_RCTL_SZ_4096 | E1000_RCTL_BSEX; 3756 3757 /* Set maximum packet len */ 3758 psize = adapter->max_frame_size; 3759 /* are we on a vlan? */ 3760#if __FreeBSD_version >= 700000 3761 if (adapter->ifp->if_vlantrunk != NULL) 3762#else 3763 if (adapter->ifp->if_nvlans != 0) 3764#endif 3765 psize += VLAN_TAG_SIZE; 3766 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, psize); 3767 } else { 3768 rctl &= ~E1000_RCTL_LPE; 3769 srrctl |= 2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT; 3770 rctl |= E1000_RCTL_SZ_2048; 3771 } 3772 3773 /* Setup the Base and Length of the Rx Descriptor Rings */ 3774 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3775 u64 bus_addr = rxr->rxdma.dma_paddr; 3776 u32 rxdctl; 3777 3778 E1000_WRITE_REG(&adapter->hw, E1000_RDLEN(i), 3779 adapter->num_rx_desc * sizeof(struct e1000_rx_desc)); 3780 E1000_WRITE_REG(&adapter->hw, E1000_RDBAH(i), 3781 (uint32_t)(bus_addr >> 32)); 3782 E1000_WRITE_REG(&adapter->hw, E1000_RDBAL(i), 3783 (uint32_t)bus_addr); 3784 E1000_WRITE_REG(&adapter->hw, E1000_SRRCTL(i), srrctl); 3785 /* Enable this Queue */ 3786 rxdctl = E1000_READ_REG(&adapter->hw, E1000_RXDCTL(i)); 3787 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 3788 rxdctl &= 0xFFF00000; 3789 rxdctl |= IGB_RX_PTHRESH; 3790 rxdctl |= IGB_RX_HTHRESH << 8; 3791 rxdctl |= IGB_RX_WTHRESH << 16; 3792 E1000_WRITE_REG(&adapter->hw, E1000_RXDCTL(i), rxdctl); 3793 } 3794 3795 /* 3796 ** Setup for RX MultiQueue 3797 */ 3798 rxcsum = E1000_READ_REG(&adapter->hw, E1000_RXCSUM); 3799 if (adapter->num_rx_queues >1) { 3800 u32 random[10], mrqc, shift = 0; 3801 union igb_reta { 3802 u32 dword; 3803 u8 bytes[4]; 3804 } reta; 3805 3806 arc4rand(&random, sizeof(random), 0); 3807 if (adapter->hw.mac.type == e1000_82575) 3808 shift = 6; 3809 /* Warning FM follows */ 3810 for (int i = 0; i < 128; i++) { 3811 reta.bytes[i & 3] = 3812 (i % adapter->num_rx_queues) << shift; 3813 if ((i & 3) == 3) 3814 E1000_WRITE_REG(&adapter->hw, 3815 E1000_RETA(i >> 2), reta.dword); 3816 } 3817 /* Now fill in hash table */ 3818 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 3819 for (int i = 0; i < 10; i++) 3820 E1000_WRITE_REG_ARRAY(&adapter->hw, 3821 E1000_RSSRK(0), i, random[i]); 3822 3823 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 3824 E1000_MRQC_RSS_FIELD_IPV4_TCP); 3825 mrqc |= (E1000_MRQC_RSS_FIELD_IPV6 | 3826 E1000_MRQC_RSS_FIELD_IPV6_TCP); 3827 mrqc |=( E1000_MRQC_RSS_FIELD_IPV4_UDP | 3828 E1000_MRQC_RSS_FIELD_IPV6_UDP); 3829 mrqc |=( E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 3830 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 3831 3832 E1000_WRITE_REG(&adapter->hw, E1000_MRQC, mrqc); 3833 3834 /* 3835 ** NOTE: Receive Full-Packet Checksum Offload 3836 ** is mutually exclusive with Multiqueue. However 3837 ** this is not the same as TCP/IP checksums which 3838 ** still work. 3839 */ 3840 rxcsum |= E1000_RXCSUM_PCSD; 3841#if __FreeBSD_version >= 800000 3842 /* For SCTP Offload */ 3843 if ((adapter->hw.mac.type == e1000_82576) 3844 && (ifp->if_capenable & IFCAP_RXCSUM)) 3845 rxcsum |= E1000_RXCSUM_CRCOFL; 3846#endif 3847 } else { 3848 /* Non RSS setup */ 3849 if (ifp->if_capenable & IFCAP_RXCSUM) { 3850 rxcsum |= E1000_RXCSUM_IPPCSE; 3851#if __FreeBSD_version >= 800000 3852 if (adapter->hw.mac.type == e1000_82576) 3853 rxcsum |= E1000_RXCSUM_CRCOFL; 3854#endif 3855 } else 3856 rxcsum &= ~E1000_RXCSUM_TUOFL; 3857 } 3858 E1000_WRITE_REG(&adapter->hw, E1000_RXCSUM, rxcsum); 3859 3860 /* Setup the Receive Control Register */ 3861 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 3862 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_LBM_NO | 3863 E1000_RCTL_RDMTS_HALF | 3864 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT); 3865 3866 /* Make sure VLAN Filters are off */ 3867 rctl &= ~E1000_RCTL_VFE; 3868 /* Don't store bad packets */ 3869 rctl &= ~E1000_RCTL_SBP; 3870 3871 /* Enable Receives */ 3872 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 3873 3874 /* 3875 * Setup the HW Rx Head and Tail Descriptor Pointers 3876 * - needs to be after enable 3877 */ 3878 for (int i = 0; i < adapter->num_rx_queues; i++) { 3879 E1000_WRITE_REG(&adapter->hw, E1000_RDH(i), 0); 3880 E1000_WRITE_REG(&adapter->hw, E1000_RDT(i), 3881 adapter->num_rx_desc - 1); 3882 } 3883 return; 3884} 3885 3886/********************************************************************* 3887 * 3888 * Free receive rings. 3889 * 3890 **********************************************************************/ 3891static void 3892igb_free_receive_structures(struct adapter *adapter) 3893{ 3894 struct rx_ring *rxr = adapter->rx_rings; 3895 3896 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3897 struct lro_ctrl *lro = &rxr->lro; 3898 igb_free_receive_buffers(rxr); 3899 tcp_lro_free(lro); 3900 igb_dma_free(adapter, &rxr->rxdma); 3901 } 3902 3903 free(adapter->rx_rings, M_DEVBUF); 3904} 3905 3906/********************************************************************* 3907 * 3908 * Free receive ring data structures. 3909 * 3910 **********************************************************************/ 3911static void 3912igb_free_receive_buffers(struct rx_ring *rxr) 3913{ 3914 struct adapter *adapter = rxr->adapter; 3915 struct igb_rx_buffer *rx_buffer; 3916 3917 INIT_DEBUGOUT("free_receive_structures: begin"); 3918 3919 if (rxr->rx_spare_map) { 3920 bus_dmamap_destroy(rxr->rxtag, rxr->rx_spare_map); 3921 rxr->rx_spare_map = NULL; 3922 } 3923 3924 /* Cleanup any existing buffers */ 3925 if (rxr->rx_buffers != NULL) { 3926 rx_buffer = &rxr->rx_buffers[0]; 3927 for (int i = 0; i < adapter->num_rx_desc; i++, rx_buffer++) { 3928 if (rx_buffer->m_head != NULL) { 3929 bus_dmamap_sync(rxr->rxtag, rx_buffer->map, 3930 BUS_DMASYNC_POSTREAD); 3931 bus_dmamap_unload(rxr->rxtag, 3932 rx_buffer->map); 3933 m_freem(rx_buffer->m_head); 3934 rx_buffer->m_head = NULL; 3935 } else if (rx_buffer->map != NULL) 3936 bus_dmamap_unload(rxr->rxtag, 3937 rx_buffer->map); 3938 if (rx_buffer->map != NULL) { 3939 bus_dmamap_destroy(rxr->rxtag, 3940 rx_buffer->map); 3941 rx_buffer->map = NULL; 3942 } 3943 } 3944 } 3945 3946 if (rxr->rx_buffers != NULL) { 3947 free(rxr->rx_buffers, M_DEVBUF); 3948 rxr->rx_buffers = NULL; 3949 } 3950 3951 if (rxr->rxtag != NULL) { 3952 bus_dma_tag_destroy(rxr->rxtag); 3953 rxr->rxtag = NULL; 3954 } 3955} 3956/********************************************************************* 3957 * 3958 * This routine executes in interrupt context. It replenishes 3959 * the mbufs in the descriptor and sends data which has been 3960 * dma'ed into host memory to upper layer. 3961 * 3962 * We loop at most count times if count is > 0, or until done if 3963 * count < 0. 3964 * 3965 * Return TRUE if more to clean, FALSE otherwise 3966 *********************************************************************/ 3967static bool 3968igb_rxeof(struct rx_ring *rxr, int count) 3969{ 3970 struct adapter *adapter = rxr->adapter; 3971 struct ifnet *ifp; 3972 struct lro_ctrl *lro = &rxr->lro; 3973 struct lro_entry *queued; 3974 int i; 3975 u32 staterr; 3976 union e1000_adv_rx_desc *cur; 3977 3978 3979 IGB_RX_LOCK(rxr); 3980 ifp = adapter->ifp; 3981 i = rxr->next_to_check; 3982 cur = &rxr->rx_base[i]; 3983 staterr = cur->wb.upper.status_error; 3984 3985 if (!(staterr & E1000_RXD_STAT_DD)) { 3986 IGB_RX_UNLOCK(rxr); 3987 return FALSE; 3988 } 3989 3990 /* Sync the ring */ 3991 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3992 BUS_DMASYNC_POSTREAD); 3993 3994 /* Main clean loop */ 3995 while ((staterr & E1000_RXD_STAT_DD) && 3996 (count != 0) && 3997 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { 3998 struct mbuf *sendmp, *mh, *mp; 3999 u16 hlen, plen, hdr, ptype, len_adj; 4000 u8 dopayload, accept_frame, eop; 4001 4002 accept_frame = 1; 4003 hlen = plen = len_adj = 0; 4004 sendmp = mh = mp = NULL; 4005 ptype = (u16)cur->wb.lower.lo_dword.data; 4006 4007 /* Sync the buffers */ 4008 bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[i].map, 4009 BUS_DMASYNC_POSTREAD); 4010 4011 /* 4012 ** The way the hardware is configured to 4013 ** split, it will ONLY use the header buffer 4014 ** when header split is enabled, otherwise we 4015 ** get normal behavior, ie, both header and 4016 ** payload are DMA'd into the payload buffer. 4017 ** 4018 ** The fmp test is to catch the case where a 4019 ** packet spans multiple descriptors, in that 4020 ** case only the first header is valid. 4021 */ 4022 if ((igb_rx_hdr_split) && (rxr->fmp == NULL)){ 4023 hdr = le16toh(cur-> 4024 wb.lower.lo_dword.hs_rss.hdr_info); 4025 hlen = (hdr & E1000_RXDADV_HDRBUFLEN_MASK) >> 4026 E1000_RXDADV_HDRBUFLEN_SHIFT; 4027 if (hlen > IGB_HDR_BUF) 4028 hlen = IGB_HDR_BUF; 4029 plen = le16toh(cur->wb.upper.length); 4030 /* Handle the header mbuf */ 4031 mh = rxr->rx_buffers[i].m_head; 4032 mh->m_len = hlen; 4033 dopayload = IGB_CLEAN_HEADER; 4034 /* 4035 ** Get the payload length, this 4036 ** could be zero if its a small 4037 ** packet. 4038 */ 4039 if (plen) { 4040 mp = rxr->rx_buffers[i].m_pack; 4041 mp->m_len = plen; 4042 mp->m_next = NULL; 4043 mp->m_flags &= ~M_PKTHDR; 4044 mh->m_next = mp; 4045 mh->m_flags |= M_PKTHDR; 4046 dopayload = IGB_CLEAN_BOTH; 4047 rxr->rx_split_packets++; 4048 } else { /* small packets */ 4049 mh->m_flags &= ~M_PKTHDR; 4050 mh->m_next = NULL; 4051 } 4052 } else { 4053 /* 4054 ** Either no header split, or a 4055 ** secondary piece of a fragmented 4056 ** split packet. 4057 */ 4058 mh = rxr->rx_buffers[i].m_pack; 4059 mh->m_flags |= M_PKTHDR; 4060 mh->m_len = le16toh(cur->wb.upper.length); 4061 dopayload = IGB_CLEAN_PAYLOAD; 4062 } 4063 4064 if (staterr & E1000_RXD_STAT_EOP) { 4065 count--; 4066 eop = 1; 4067 /* 4068 ** Strip CRC and account for frag 4069 */ 4070 if (mp) { 4071 if (mp->m_len < ETHER_CRC_LEN) { 4072 /* a frag, how much is left? */ 4073 len_adj = ETHER_CRC_LEN - mp->m_len; 4074 mp->m_len = 0; 4075 } else 4076 mp->m_len -= ETHER_CRC_LEN; 4077 } else { /* not split */ 4078 if (mh->m_len < ETHER_CRC_LEN) { 4079 len_adj = ETHER_CRC_LEN - mh->m_len; 4080 mh->m_len = 0; 4081 } else 4082 mh->m_len -= ETHER_CRC_LEN; 4083 } 4084 } else 4085 eop = 0; 4086 4087 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) 4088 accept_frame = 0; 4089 4090 if (accept_frame) { 4091 if (igb_get_buf(rxr, i, dopayload) != 0) { 4092 ifp->if_iqdrops++; 4093 goto discard; 4094 } 4095 /* Initial frame - setup */ 4096 if (rxr->fmp == NULL) { 4097 mh->m_flags |= M_PKTHDR; 4098 mh->m_pkthdr.len = mh->m_len; 4099 rxr->fmp = mh; /* Store the first mbuf */ 4100 rxr->lmp = mh; 4101 if (mp) { /* Add payload if split */ 4102 mh->m_pkthdr.len += mp->m_len; 4103 rxr->lmp = mh->m_next; 4104 } 4105 } else { 4106 /* Chain mbuf's together */ 4107 mh->m_flags &= ~M_PKTHDR; 4108 rxr->lmp->m_next = mh; 4109 rxr->lmp = rxr->lmp->m_next; 4110 rxr->fmp->m_pkthdr.len += mh->m_len; 4111 /* Adjust for CRC frag */ 4112 if (len_adj) { 4113 rxr->lmp->m_len -= len_adj; 4114 rxr->fmp->m_pkthdr.len -= len_adj; 4115 } 4116 } 4117 4118 if (eop) { 4119 bool sctp = ((ptype & 0x40) != 0); 4120 rxr->fmp->m_pkthdr.rcvif = ifp; 4121 ifp->if_ipackets++; 4122 rxr->rx_packets++; 4123 /* capture data for AIM */ 4124 rxr->bytes += rxr->fmp->m_pkthdr.len; 4125 rxr->rx_bytes += rxr->fmp->m_pkthdr.len; 4126 4127 igb_rx_checksum(staterr, rxr->fmp, sctp); 4128 if (staterr & E1000_RXD_STAT_VP) { 4129#if __FreeBSD_version >= 700000 4130 rxr->fmp->m_pkthdr.ether_vtag = 4131 le16toh(cur->wb.upper.vlan); 4132 rxr->fmp->m_flags |= M_VLANTAG; 4133#else 4134 VLAN_INPUT_TAG_NEW(ifp, rxr->fmp, 4135 (le16toh(cur->wb.upper.vlan) & 4136 E1000_RXD_SPC_VLAN_MASK)); 4137#endif 4138 } 4139 sendmp = rxr->fmp; 4140 rxr->fmp = NULL; 4141 rxr->lmp = NULL; 4142 } 4143 } else { 4144 ifp->if_ierrors++; 4145discard: 4146 /* Reuse loaded DMA map and just update mbuf chain */ 4147 if (hlen) { 4148 mh = rxr->rx_buffers[i].m_head; 4149 mh->m_len = MHLEN; 4150 mh->m_next = NULL; 4151 } 4152 mp = rxr->rx_buffers[i].m_pack; 4153 mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz; 4154 mp->m_data = mp->m_ext.ext_buf; 4155 mp->m_next = NULL; 4156 if (adapter->max_frame_size <= 4157 (MCLBYTES - ETHER_ALIGN)) 4158 m_adj(mp, ETHER_ALIGN); 4159 if (rxr->fmp != NULL) { 4160 /* handles the whole chain */ 4161 m_freem(rxr->fmp); 4162 rxr->fmp = NULL; 4163 rxr->lmp = NULL; 4164 } 4165 sendmp = NULL; 4166 } 4167 4168 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 4169 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 4170 4171 rxr->last_cleaned = i; /* For updating tail */ 4172 4173 /* Advance our pointers to the next descriptor. */ 4174 if (++i == adapter->num_rx_desc) 4175 i = 0; 4176 4177 /* 4178 ** Note that we hold the RX lock thru 4179 ** the following call so this ring's 4180 ** next_to_check is not gonna change. 4181 */ 4182 if (sendmp != NULL) { 4183 /* Use LRO if possible */ 4184 if ((!lro->lro_cnt) || (tcp_lro_rx(lro, sendmp, 0))) 4185 /* Pass up to the stack */ 4186 (*ifp->if_input)(ifp, sendmp); 4187 } 4188 4189 /* Get the next descriptor */ 4190 cur = &rxr->rx_base[i]; 4191 staterr = cur->wb.upper.status_error; 4192 } 4193 rxr->next_to_check = i; 4194 4195 /* Advance the E1000's Receive Queue #0 "Tail Pointer". */ 4196 E1000_WRITE_REG(&adapter->hw, E1000_RDT(rxr->me), rxr->last_cleaned); 4197 4198 /* 4199 * Flush any outstanding LRO work 4200 */ 4201 while (!SLIST_EMPTY(&lro->lro_active)) { 4202 queued = SLIST_FIRST(&lro->lro_active); 4203 SLIST_REMOVE_HEAD(&lro->lro_active, next); 4204 tcp_lro_flush(lro, queued); 4205 } 4206 4207 IGB_RX_UNLOCK(rxr); 4208 4209 /* 4210 ** We still have cleaning to do? 4211 ** Schedule another interrupt if so. 4212 */ 4213 if (staterr & E1000_RXD_STAT_DD) { 4214 E1000_WRITE_REG(&adapter->hw, E1000_EICS, rxr->eims); 4215 return TRUE; 4216 } 4217 4218 return FALSE; 4219} 4220 4221 4222/********************************************************************* 4223 * 4224 * Verify that the hardware indicated that the checksum is valid. 4225 * Inform the stack about the status of checksum so that stack 4226 * doesn't spend time verifying the checksum. 4227 * 4228 *********************************************************************/ 4229static void 4230igb_rx_checksum(u32 staterr, struct mbuf *mp, bool sctp) 4231{ 4232 u16 status = (u16)staterr; 4233 u8 errors = (u8) (staterr >> 24); 4234 4235 /* Ignore Checksum bit is set */ 4236 if (status & E1000_RXD_STAT_IXSM) { 4237 mp->m_pkthdr.csum_flags = 0; 4238 return; 4239 } 4240 4241 if (status & E1000_RXD_STAT_IPCS) { 4242 /* Did it pass? */ 4243 if (!(errors & E1000_RXD_ERR_IPE)) { 4244 /* IP Checksum Good */ 4245 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 4246 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 4247 } else 4248 mp->m_pkthdr.csum_flags = 0; 4249 } 4250 4251 if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { 4252 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 4253#if __FreeBSD_version >= 800000 4254 if (sctp) /* reassign */ 4255 type = CSUM_SCTP_VALID; 4256#endif 4257 /* Did it pass? */ 4258 if (!(errors & E1000_RXD_ERR_TCPE)) { 4259 mp->m_pkthdr.csum_flags = type; 4260 if (!sctp) 4261 mp->m_pkthdr.csum_data = htons(0xffff); 4262 } 4263 } 4264 return; 4265} 4266 4267/* 4268 * This routine is run via an vlan 4269 * config EVENT 4270 */ 4271static void 4272igb_register_vlan(void *unused, struct ifnet *ifp, u16 vtag) 4273{ 4274 struct adapter *adapter = ifp->if_softc; 4275 u32 ctrl, rctl, index, vfta; 4276 4277 ctrl = E1000_READ_REG(&adapter->hw, E1000_CTRL); 4278 ctrl |= E1000_CTRL_VME; 4279 E1000_WRITE_REG(&adapter->hw, E1000_CTRL, ctrl); 4280 4281 /* Setup for Hardware Filter */ 4282 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4283 rctl |= E1000_RCTL_VFE; 4284 rctl &= ~E1000_RCTL_CFIEN; 4285 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4286 4287 /* Make entry in the hardware filter table */ 4288 index = ((vtag >> 5) & 0x7F); 4289 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index); 4290 vfta |= (1 << (vtag & 0x1F)); 4291 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta); 4292 4293 /* Update the frame size */ 4294 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4295 adapter->max_frame_size + VLAN_TAG_SIZE); 4296 4297} 4298 4299/* 4300 * This routine is run via an vlan 4301 * unconfig EVENT 4302 */ 4303static void 4304igb_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag) 4305{ 4306 struct adapter *adapter = ifp->if_softc; 4307 u32 index, vfta; 4308 4309 /* Remove entry in the hardware filter table */ 4310 index = ((vtag >> 5) & 0x7F); 4311 vfta = E1000_READ_REG_ARRAY(&adapter->hw, E1000_VFTA, index); 4312 vfta &= ~(1 << (vtag & 0x1F)); 4313 E1000_WRITE_REG_ARRAY(&adapter->hw, E1000_VFTA, index, vfta); 4314 /* Have all vlans unregistered? */ 4315 if (adapter->ifp->if_vlantrunk == NULL) { 4316 u32 rctl; 4317 /* Turn off the filter table */ 4318 rctl = E1000_READ_REG(&adapter->hw, E1000_RCTL); 4319 rctl &= ~E1000_RCTL_VFE; 4320 rctl |= E1000_RCTL_CFIEN; 4321 E1000_WRITE_REG(&adapter->hw, E1000_RCTL, rctl); 4322 /* Reset the frame size */ 4323 E1000_WRITE_REG(&adapter->hw, E1000_RLPML, 4324 adapter->max_frame_size); 4325 } 4326} 4327 4328static void 4329igb_enable_intr(struct adapter *adapter) 4330{ 4331 /* With RSS set up what to auto clear */ 4332 if (adapter->msix_mem) { 4333 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 4334 adapter->eims_mask); 4335 E1000_WRITE_REG(&adapter->hw, E1000_EIAM, 4336 adapter->eims_mask); 4337 E1000_WRITE_REG(&adapter->hw, E1000_EIMS, 4338 adapter->eims_mask); 4339 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 4340 E1000_IMS_LSC); 4341 } else { 4342 E1000_WRITE_REG(&adapter->hw, E1000_IMS, 4343 IMS_ENABLE_MASK); 4344 } 4345 E1000_WRITE_FLUSH(&adapter->hw); 4346 4347 return; 4348} 4349 4350static void 4351igb_disable_intr(struct adapter *adapter) 4352{ 4353 if (adapter->msix_mem) { 4354 E1000_WRITE_REG(&adapter->hw, E1000_EIMC, ~0); 4355 E1000_WRITE_REG(&adapter->hw, E1000_EIAC, 0); 4356 } 4357 E1000_WRITE_REG(&adapter->hw, E1000_IMC, ~0); 4358 E1000_WRITE_FLUSH(&adapter->hw); 4359 return; 4360} 4361 4362/* 4363 * Bit of a misnomer, what this really means is 4364 * to enable OS management of the system... aka 4365 * to disable special hardware management features 4366 */ 4367static void 4368igb_init_manageability(struct adapter *adapter) 4369{ 4370 if (adapter->has_manage) { 4371 int manc2h = E1000_READ_REG(&adapter->hw, E1000_MANC2H); 4372 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4373 4374 /* disable hardware interception of ARP */ 4375 manc &= ~(E1000_MANC_ARP_EN); 4376 4377 /* enable receiving management packets to the host */ 4378 manc |= E1000_MANC_EN_MNG2HOST; 4379 manc2h |= 1 << 5; /* Mng Port 623 */ 4380 manc2h |= 1 << 6; /* Mng Port 664 */ 4381 E1000_WRITE_REG(&adapter->hw, E1000_MANC2H, manc2h); 4382 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4383 } 4384} 4385 4386/* 4387 * Give control back to hardware management 4388 * controller if there is one. 4389 */ 4390static void 4391igb_release_manageability(struct adapter *adapter) 4392{ 4393 if (adapter->has_manage) { 4394 int manc = E1000_READ_REG(&adapter->hw, E1000_MANC); 4395 4396 /* re-enable hardware interception of ARP */ 4397 manc |= E1000_MANC_ARP_EN; 4398 manc &= ~E1000_MANC_EN_MNG2HOST; 4399 4400 E1000_WRITE_REG(&adapter->hw, E1000_MANC, manc); 4401 } 4402} 4403 4404/* 4405 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit. 4406 * For ASF and Pass Through versions of f/w this means that 4407 * the driver is loaded. 4408 * 4409 */ 4410static void 4411igb_get_hw_control(struct adapter *adapter) 4412{ 4413 u32 ctrl_ext; 4414 4415 /* Let firmware know the driver has taken over */ 4416 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4417 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4418 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD); 4419} 4420 4421/* 4422 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit. 4423 * For ASF and Pass Through versions of f/w this means that the 4424 * driver is no longer loaded. 4425 * 4426 */ 4427static void 4428igb_release_hw_control(struct adapter *adapter) 4429{ 4430 u32 ctrl_ext; 4431 4432 /* Let firmware taken over control of h/w */ 4433 ctrl_ext = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); 4434 E1000_WRITE_REG(&adapter->hw, E1000_CTRL_EXT, 4435 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD); 4436} 4437 4438static int 4439igb_is_valid_ether_addr(uint8_t *addr) 4440{ 4441 char zero_addr[6] = { 0, 0, 0, 0, 0, 0 }; 4442 4443 if ((addr[0] & 1) || (!bcmp(addr, zero_addr, ETHER_ADDR_LEN))) { 4444 return (FALSE); 4445 } 4446 4447 return (TRUE); 4448} 4449 4450 4451/* 4452 * Enable PCI Wake On Lan capability 4453 */ 4454void 4455igb_enable_wakeup(device_t dev) 4456{ 4457 u16 cap, status; 4458 u8 id; 4459 4460 /* First find the capabilities pointer*/ 4461 cap = pci_read_config(dev, PCIR_CAP_PTR, 2); 4462 /* Read the PM Capabilities */ 4463 id = pci_read_config(dev, cap, 1); 4464 if (id != PCIY_PMG) /* Something wrong */ 4465 return; 4466 /* OK, we have the power capabilities, so 4467 now get the status register */ 4468 cap += PCIR_POWER_STATUS; 4469 status = pci_read_config(dev, cap, 2); 4470 status |= PCIM_PSTAT_PME | PCIM_PSTAT_PMEENABLE; 4471 pci_write_config(dev, cap, status, 2); 4472 return; 4473} 4474 4475 4476/********************************************************************** 4477 * 4478 * Update the board statistics counters. 4479 * 4480 **********************************************************************/ 4481static void 4482igb_update_stats_counters(struct adapter *adapter) 4483{ 4484 struct ifnet *ifp; 4485 4486 if(adapter->hw.phy.media_type == e1000_media_type_copper || 4487 (E1000_READ_REG(&adapter->hw, E1000_STATUS) & E1000_STATUS_LU)) { 4488 adapter->stats.symerrs += E1000_READ_REG(&adapter->hw, E1000_SYMERRS); 4489 adapter->stats.sec += E1000_READ_REG(&adapter->hw, E1000_SEC); 4490 } 4491 adapter->stats.crcerrs += E1000_READ_REG(&adapter->hw, E1000_CRCERRS); 4492 adapter->stats.mpc += E1000_READ_REG(&adapter->hw, E1000_MPC); 4493 adapter->stats.scc += E1000_READ_REG(&adapter->hw, E1000_SCC); 4494 adapter->stats.ecol += E1000_READ_REG(&adapter->hw, E1000_ECOL); 4495 4496 adapter->stats.mcc += E1000_READ_REG(&adapter->hw, E1000_MCC); 4497 adapter->stats.latecol += E1000_READ_REG(&adapter->hw, E1000_LATECOL); 4498 adapter->stats.colc += E1000_READ_REG(&adapter->hw, E1000_COLC); 4499 adapter->stats.dc += E1000_READ_REG(&adapter->hw, E1000_DC); 4500 adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC); 4501 adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC); 4502 adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC); 4503 adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC); 4504 adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC); 4505 adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC); 4506 adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64); 4507 adapter->stats.prc127 += E1000_READ_REG(&adapter->hw, E1000_PRC127); 4508 adapter->stats.prc255 += E1000_READ_REG(&adapter->hw, E1000_PRC255); 4509 adapter->stats.prc511 += E1000_READ_REG(&adapter->hw, E1000_PRC511); 4510 adapter->stats.prc1023 += E1000_READ_REG(&adapter->hw, E1000_PRC1023); 4511 adapter->stats.prc1522 += E1000_READ_REG(&adapter->hw, E1000_PRC1522); 4512 adapter->stats.gprc += E1000_READ_REG(&adapter->hw, E1000_GPRC); 4513 adapter->stats.bprc += E1000_READ_REG(&adapter->hw, E1000_BPRC); 4514 adapter->stats.mprc += E1000_READ_REG(&adapter->hw, E1000_MPRC); 4515 adapter->stats.gptc += E1000_READ_REG(&adapter->hw, E1000_GPTC); 4516 4517 /* For the 64-bit byte counters the low dword must be read first. */ 4518 /* Both registers clear on the read of the high dword */ 4519 4520 adapter->stats.gorc += E1000_READ_REG(&adapter->hw, E1000_GORCH); 4521 adapter->stats.gotc += E1000_READ_REG(&adapter->hw, E1000_GOTCH); 4522 4523 adapter->stats.rnbc += E1000_READ_REG(&adapter->hw, E1000_RNBC); 4524 adapter->stats.ruc += E1000_READ_REG(&adapter->hw, E1000_RUC); 4525 adapter->stats.rfc += E1000_READ_REG(&adapter->hw, E1000_RFC); 4526 adapter->stats.roc += E1000_READ_REG(&adapter->hw, E1000_ROC); 4527 adapter->stats.rjc += E1000_READ_REG(&adapter->hw, E1000_RJC); 4528 4529 adapter->stats.tor += E1000_READ_REG(&adapter->hw, E1000_TORH); 4530 adapter->stats.tot += E1000_READ_REG(&adapter->hw, E1000_TOTH); 4531 4532 adapter->stats.tpr += E1000_READ_REG(&adapter->hw, E1000_TPR); 4533 adapter->stats.tpt += E1000_READ_REG(&adapter->hw, E1000_TPT); 4534 adapter->stats.ptc64 += E1000_READ_REG(&adapter->hw, E1000_PTC64); 4535 adapter->stats.ptc127 += E1000_READ_REG(&adapter->hw, E1000_PTC127); 4536 adapter->stats.ptc255 += E1000_READ_REG(&adapter->hw, E1000_PTC255); 4537 adapter->stats.ptc511 += E1000_READ_REG(&adapter->hw, E1000_PTC511); 4538 adapter->stats.ptc1023 += E1000_READ_REG(&adapter->hw, E1000_PTC1023); 4539 adapter->stats.ptc1522 += E1000_READ_REG(&adapter->hw, E1000_PTC1522); 4540 adapter->stats.mptc += E1000_READ_REG(&adapter->hw, E1000_MPTC); 4541 adapter->stats.bptc += E1000_READ_REG(&adapter->hw, E1000_BPTC); 4542 4543 adapter->stats.algnerrc += 4544 E1000_READ_REG(&adapter->hw, E1000_ALGNERRC); 4545 adapter->stats.rxerrc += 4546 E1000_READ_REG(&adapter->hw, E1000_RXERRC); 4547 adapter->stats.tncrs += 4548 E1000_READ_REG(&adapter->hw, E1000_TNCRS); 4549 adapter->stats.cexterr += 4550 E1000_READ_REG(&adapter->hw, E1000_CEXTERR); 4551 adapter->stats.tsctc += 4552 E1000_READ_REG(&adapter->hw, E1000_TSCTC); 4553 adapter->stats.tsctfc += 4554 E1000_READ_REG(&adapter->hw, E1000_TSCTFC); 4555 ifp = adapter->ifp; 4556 4557 ifp->if_collisions = adapter->stats.colc; 4558 4559 /* Rx Errors */ 4560 ifp->if_ierrors = adapter->dropped_pkts + adapter->stats.rxerrc + 4561 adapter->stats.crcerrs + adapter->stats.algnerrc + 4562 adapter->stats.ruc + adapter->stats.roc + 4563 adapter->stats.mpc + adapter->stats.cexterr; 4564 4565 /* Tx Errors */ 4566 ifp->if_oerrors = adapter->stats.ecol + 4567 adapter->stats.latecol + adapter->watchdog_events; 4568} 4569 4570 4571/********************************************************************** 4572 * 4573 * This routine is called only when igb_display_debug_stats is enabled. 4574 * This routine provides a way to take a look at important statistics 4575 * maintained by the driver and hardware. 4576 * 4577 **********************************************************************/ 4578static void 4579igb_print_debug_info(struct adapter *adapter) 4580{ 4581 device_t dev = adapter->dev; 4582 struct rx_ring *rxr = adapter->rx_rings; 4583 struct tx_ring *txr = adapter->tx_rings; 4584 uint8_t *hw_addr = adapter->hw.hw_addr; 4585 4586 device_printf(dev, "Adapter hardware address = %p \n", hw_addr); 4587 device_printf(dev, "CTRL = 0x%x RCTL = 0x%x \n", 4588 E1000_READ_REG(&adapter->hw, E1000_CTRL), 4589 E1000_READ_REG(&adapter->hw, E1000_RCTL)); 4590 4591#if (DEBUG_HW > 0) /* Dont output these errors normally */ 4592 device_printf(dev, "IMS = 0x%x EIMS = 0x%x \n", 4593 E1000_READ_REG(&adapter->hw, E1000_IMS), 4594 E1000_READ_REG(&adapter->hw, E1000_EIMS)); 4595#endif 4596 4597 device_printf(dev, "Packet buffer = Tx=%dk Rx=%dk \n", 4598 ((E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff0000) >> 16),\ 4599 (E1000_READ_REG(&adapter->hw, E1000_PBA) & 0xffff) ); 4600 device_printf(dev, "Flow control watermarks high = %d low = %d\n", 4601 adapter->hw.fc.high_water, 4602 adapter->hw.fc.low_water); 4603 4604 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 4605 device_printf(dev, "Queue(%d) tdh = %d, tdt = %d\n", i, 4606 E1000_READ_REG(&adapter->hw, E1000_TDH(i)), 4607 E1000_READ_REG(&adapter->hw, E1000_TDT(i))); 4608 device_printf(dev, "no descriptors avail event = %lu\n", 4609 txr->no_desc_avail); 4610 device_printf(dev, "TX(%d) MSIX IRQ Handled = %lld\n", txr->me, 4611 (long long)txr->tx_irq); 4612 device_printf(dev, "TX(%d) Packets sent = %lld\n", txr->me, 4613 (long long)txr->tx_packets); 4614 } 4615 4616 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 4617 struct lro_ctrl *lro = &rxr->lro; 4618 device_printf(dev, "Queue(%d) rdh = %d, rdt = %d\n", i, 4619 E1000_READ_REG(&adapter->hw, E1000_RDH(i)), 4620 E1000_READ_REG(&adapter->hw, E1000_RDT(i))); 4621 device_printf(dev, "RX(%d) Packets received = %lld\n", rxr->me, 4622 (long long)rxr->rx_packets); 4623 device_printf(dev, "RX(%d) Split Packets = %lld\n", rxr->me, 4624 (long long)rxr->rx_split_packets); 4625 device_printf(dev, "RX(%d) Byte count = %lld\n", rxr->me, 4626 (long long)rxr->rx_bytes); 4627 device_printf(dev, "RX(%d) MSIX IRQ Handled = %lld\n", rxr->me, 4628 (long long)rxr->rx_irq); 4629 device_printf(dev,"RX(%d) LRO Queued= %d\n", 4630 rxr->me, lro->lro_queued); 4631 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 4632 rxr->me, lro->lro_flushed); 4633 } 4634 4635 device_printf(dev, "LINK MSIX IRQ Handled = %u\n", adapter->link_irq); 4636 4637 device_printf(dev, "Mbuf defrag failed = %ld\n", 4638 adapter->mbuf_defrag_failed); 4639 device_printf(dev, "Std mbuf header failed = %ld\n", 4640 adapter->mbuf_header_failed); 4641 device_printf(dev, "Std mbuf packet failed = %ld\n", 4642 adapter->mbuf_packet_failed); 4643 device_printf(dev, "Driver dropped packets = %ld\n", 4644 adapter->dropped_pkts); 4645 device_printf(dev, "Driver tx dma failure in xmit = %ld\n", 4646 adapter->no_tx_dma_setup); 4647} 4648 4649static void 4650igb_print_hw_stats(struct adapter *adapter) 4651{ 4652 device_t dev = adapter->dev; 4653 4654 device_printf(dev, "Excessive collisions = %lld\n", 4655 (long long)adapter->stats.ecol); 4656#if (DEBUG_HW > 0) /* Dont output these errors normally */ 4657 device_printf(dev, "Symbol errors = %lld\n", 4658 (long long)adapter->stats.symerrs); 4659#endif 4660 device_printf(dev, "Sequence errors = %lld\n", 4661 (long long)adapter->stats.sec); 4662 device_printf(dev, "Defer count = %lld\n", 4663 (long long)adapter->stats.dc); 4664 device_printf(dev, "Missed Packets = %lld\n", 4665 (long long)adapter->stats.mpc); 4666 device_printf(dev, "Receive No Buffers = %lld\n", 4667 (long long)adapter->stats.rnbc); 4668 /* RLEC is inaccurate on some hardware, calculate our own. */ 4669 device_printf(dev, "Receive Length Errors = %lld\n", 4670 ((long long)adapter->stats.roc + (long long)adapter->stats.ruc)); 4671 device_printf(dev, "Receive errors = %lld\n", 4672 (long long)adapter->stats.rxerrc); 4673 device_printf(dev, "Crc errors = %lld\n", 4674 (long long)adapter->stats.crcerrs); 4675 device_printf(dev, "Alignment errors = %lld\n", 4676 (long long)adapter->stats.algnerrc); 4677 /* On 82575 these are collision counts */ 4678 device_printf(dev, "Collision/Carrier extension errors = %lld\n", 4679 (long long)adapter->stats.cexterr); 4680 device_printf(dev, "RX overruns = %ld\n", adapter->rx_overruns); 4681 device_printf(dev, "watchdog timeouts = %ld\n", 4682 adapter->watchdog_events); 4683 device_printf(dev, "XON Rcvd = %lld\n", 4684 (long long)adapter->stats.xonrxc); 4685 device_printf(dev, "XON Xmtd = %lld\n", 4686 (long long)adapter->stats.xontxc); 4687 device_printf(dev, "XOFF Rcvd = %lld\n", 4688 (long long)adapter->stats.xoffrxc); 4689 device_printf(dev, "XOFF Xmtd = %lld\n", 4690 (long long)adapter->stats.xofftxc); 4691 device_printf(dev, "Good Packets Rcvd = %lld\n", 4692 (long long)adapter->stats.gprc); 4693 device_printf(dev, "Good Packets Xmtd = %lld\n", 4694 (long long)adapter->stats.gptc); 4695 device_printf(dev, "TSO Contexts Xmtd = %lld\n", 4696 (long long)adapter->stats.tsctc); 4697 device_printf(dev, "TSO Contexts Failed = %lld\n", 4698 (long long)adapter->stats.tsctfc); 4699} 4700 4701/********************************************************************** 4702 * 4703 * This routine provides a way to dump out the adapter eeprom, 4704 * often a useful debug/service tool. This only dumps the first 4705 * 32 words, stuff that matters is in that extent. 4706 * 4707 **********************************************************************/ 4708static void 4709igb_print_nvm_info(struct adapter *adapter) 4710{ 4711 u16 eeprom_data; 4712 int i, j, row = 0; 4713 4714 /* Its a bit crude, but it gets the job done */ 4715 printf("\nInterface EEPROM Dump:\n"); 4716 printf("Offset\n0x0000 "); 4717 for (i = 0, j = 0; i < 32; i++, j++) { 4718 if (j == 8) { /* Make the offset block */ 4719 j = 0; ++row; 4720 printf("\n0x00%x0 ",row); 4721 } 4722 e1000_read_nvm(&adapter->hw, i, 1, &eeprom_data); 4723 printf("%04x ", eeprom_data); 4724 } 4725 printf("\n"); 4726} 4727 4728static int 4729igb_sysctl_debug_info(SYSCTL_HANDLER_ARGS) 4730{ 4731 struct adapter *adapter; 4732 int error; 4733 int result; 4734 4735 result = -1; 4736 error = sysctl_handle_int(oidp, &result, 0, req); 4737 4738 if (error || !req->newptr) 4739 return (error); 4740 4741 if (result == 1) { 4742 adapter = (struct adapter *)arg1; 4743 igb_print_debug_info(adapter); 4744 } 4745 /* 4746 * This value will cause a hex dump of the 4747 * first 32 16-bit words of the EEPROM to 4748 * the screen. 4749 */ 4750 if (result == 2) { 4751 adapter = (struct adapter *)arg1; 4752 igb_print_nvm_info(adapter); 4753 } 4754 4755 return (error); 4756} 4757 4758 4759static int 4760igb_sysctl_stats(SYSCTL_HANDLER_ARGS) 4761{ 4762 struct adapter *adapter; 4763 int error; 4764 int result; 4765 4766 result = -1; 4767 error = sysctl_handle_int(oidp, &result, 0, req); 4768 4769 if (error || !req->newptr) 4770 return (error); 4771 4772 if (result == 1) { 4773 adapter = (struct adapter *)arg1; 4774 igb_print_hw_stats(adapter); 4775 } 4776 4777 return (error); 4778} 4779 4780static void 4781igb_add_rx_process_limit(struct adapter *adapter, const char *name, 4782 const char *description, int *limit, int value) 4783{ 4784 *limit = value; 4785 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 4786 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 4787 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 4788} 4789 4790#ifdef IGB_TIMESYNC 4791/* 4792 * Initialize the Time Sync Feature 4793 */ 4794static int 4795igb_tsync_init(struct adapter *adapter) 4796{ 4797 device_t dev = adapter->dev; 4798 u32 tx_ctl, rx_ctl, val; 4799 4800 4801 E1000_WRITE_REG(&adapter->hw, E1000_TIMINCA, (1<<24) | 4802 20833/PICOSECS_PER_TICK); 4803 4804 adapter->last_stamp = E1000_READ_REG(&adapter->hw, E1000_SYSTIML); 4805 adapter->last_stamp |= (u64)E1000_READ_REG(&adapter->hw, 4806 E1000_SYSTIMH) << 32ULL; 4807 4808 /* Enable the TX side */ 4809 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 4810 tx_ctl |= 0x10; 4811 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl); 4812 E1000_WRITE_FLUSH(&adapter->hw); 4813 4814 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 4815 if ((tx_ctl & 0x10) == 0) { 4816 device_printf(dev, "Failed to enable TX timestamping\n"); 4817 return (ENXIO); 4818 } 4819 4820 /* Enable RX */ 4821 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 4822 rx_ctl |= 0x10; /* Enable the feature */ 4823 rx_ctl |= 0x04; /* This value turns on Ver 1 and 2 */ 4824 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl); 4825 4826 /* 4827 * Ethertype Filter Queue Filter[0][15:0] = 0x88F7 (Ethertype) 4828 * Ethertype Filter Queue Filter[0][26] = 0x1 (Enable filter) 4829 * Ethertype Filter Queue Filter[0][31] = 0x1 (Enable Timestamping) 4830 */ 4831 E1000_WRITE_REG(&adapter->hw, E1000_ETQF(0), 0x440088f7); 4832 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCFG, 0x0); 4833 4834 /* 4835 * Source Port Queue Filter Setup: 4836 * this is for UDP port filtering 4837 */ 4838 E1000_WRITE_REG(&adapter->hw, E1000_SPQF(0), TSYNC_PORT); 4839 /* Protocol = UDP, enable Timestamp, and filter on source/protocol */ 4840 val = (0x11 | (1 << 27) | (6 << 28)); 4841 E1000_WRITE_REG(&adapter->hw, E1000_FTQF(0), val); 4842 4843 E1000_WRITE_FLUSH(&adapter->hw); 4844 4845 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 4846 if ((rx_ctl & 0x10) == 0) { 4847 device_printf(dev, "Failed to enable RX timestamping\n"); 4848 return (ENXIO); 4849 } 4850 4851 device_printf(dev, "IEEE 1588 Precision Time Protocol enabled\n"); 4852 4853 return (0); 4854} 4855 4856/* 4857 * Disable the Time Sync Feature 4858 */ 4859static void 4860igb_tsync_disable(struct adapter *adapter) 4861{ 4862 u32 tx_ctl, rx_ctl; 4863 4864 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 4865 tx_ctl &= ~0x10; 4866 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCTXCTL, tx_ctl); 4867 E1000_WRITE_FLUSH(&adapter->hw); 4868 4869 /* Invalidate TX Timestamp */ 4870 E1000_READ_REG(&adapter->hw, E1000_TXSTMPH); 4871 4872 tx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCTXCTL); 4873 if (tx_ctl & 0x10) 4874 HW_DEBUGOUT("Failed to disable TX timestamping\n"); 4875 4876 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 4877 rx_ctl &= ~0x10; 4878 4879 E1000_WRITE_REG(&adapter->hw, E1000_TSYNCRXCTL, rx_ctl); 4880 E1000_WRITE_FLUSH(&adapter->hw); 4881 4882 /* Invalidate RX Timestamp */ 4883 E1000_READ_REG(&adapter->hw, E1000_RXSATRH); 4884 4885 rx_ctl = E1000_READ_REG(&adapter->hw, E1000_TSYNCRXCTL); 4886 if (rx_ctl & 0x10) 4887 HW_DEBUGOUT("Failed to disable RX timestamping\n"); 4888 4889 return; 4890} 4891 4892#endif /* IGB_TIMESYNC */ 4893