1/****************************************************************************** 2 3 Copyright (c) 2001-2008, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 --- 16 unchanged lines hidden (view full) --- 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ |
33/*$FreeBSD: head/sys/dev/ixgbe/ixgbe.c 185352 2008-11-26 23:41:18Z jfv $*/ |
34 35#ifdef HAVE_KERNEL_OPTION_HEADERS 36#include "opt_device_polling.h" 37#endif 38 |
39#include "ixgbe.h" 40 41/********************************************************************* 42 * Set this to one to display debug statistics 43 *********************************************************************/ 44int ixgbe_display_debug_stats = 0; 45 46/********************************************************************* 47 * Driver version 48 *********************************************************************/ |
49char ixgbe_driver_version[] = "1.6.2"; |
50 51/********************************************************************* 52 * PCI Device ID Table 53 * 54 * Used by probe to select devices to load on 55 * Last field stores an index into ixgbe_strings 56 * Last entry must be all 0s 57 * 58 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 59 *********************************************************************/ 60 61static ixgbe_vendor_info_t ixgbe_vendor_info_array[] = 62{ 63 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT, 0, 0, 0}, 64 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT, 0, 0, 0}, |
65 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4, 0, 0, 0}, |
66 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, 67 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598, 0, 0, 0}, 68 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT, 0, 0, 0}, |
69 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT, 0, 0, 0}, 70 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR, 0, 0, 0}, 71 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT, 0, 0, 0}, |
72 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM, 0, 0, 0}, 73 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM, 0, 0, 0}, |
74 /* required last entry */ 75 {0, 0, 0, 0, 0} 76}; 77 78/********************************************************************* 79 * Table of branding strings 80 *********************************************************************/ 81 --- 42 unchanged lines hidden (view full) --- 124static void ixgbe_free_receive_structures(struct adapter *); 125static void ixgbe_free_receive_buffers(struct rx_ring *); 126 127static void ixgbe_enable_intr(struct adapter *); 128static void ixgbe_disable_intr(struct adapter *); 129static void ixgbe_update_stats_counters(struct adapter *); 130static bool ixgbe_txeof(struct tx_ring *); 131static bool ixgbe_rxeof(struct rx_ring *, int); |
132static void ixgbe_rx_checksum(u32, struct mbuf *); |
133static void ixgbe_set_promisc(struct adapter *); 134static void ixgbe_disable_promisc(struct adapter *); 135static void ixgbe_set_multi(struct adapter *); 136static void ixgbe_print_hw_stats(struct adapter *); 137static void ixgbe_print_debug_info(struct adapter *); 138static void ixgbe_update_link_status(struct adapter *); |
139static int ixgbe_get_buf(struct rx_ring *, int, u8); |
140static int ixgbe_xmit(struct tx_ring *, struct mbuf **); 141static int ixgbe_sysctl_stats(SYSCTL_HANDLER_ARGS); 142static int ixgbe_sysctl_debug(SYSCTL_HANDLER_ARGS); 143static int ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS); 144static int ixgbe_dma_malloc(struct adapter *, bus_size_t, 145 struct ixgbe_dma_alloc *, int); 146static void ixgbe_dma_free(struct adapter *, struct ixgbe_dma_alloc *); 147static void ixgbe_add_rx_process_limit(struct adapter *, const char *, 148 const char *, int *, int); 149static boolean_t ixgbe_tx_ctx_setup(struct tx_ring *, struct mbuf *); 150static boolean_t ixgbe_tso_setup(struct tx_ring *, struct mbuf *, u32 *); |
151static void ixgbe_set_ivar(struct adapter *, u16, u8, s8); |
152static void ixgbe_configure_ivars(struct adapter *); 153static u8 * ixgbe_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 154 |
155#ifdef IXGBE_HW_VLAN_SUPPORT |
156static void ixgbe_register_vlan(void *, struct ifnet *, u16); 157static void ixgbe_unregister_vlan(void *, struct ifnet *, u16); 158#endif 159 |
160static void ixgbe_update_aim(struct rx_ring *); 161 162/* Support for pluggable optic modules */ 163static bool ixgbe_sfp_probe(struct adapter *); 164 |
165/* Legacy (single vector interrupt handler */ 166static void ixgbe_legacy_irq(void *); 167 168/* The MSI/X Interrupt handlers */ 169static void ixgbe_msix_tx(void *); 170static void ixgbe_msix_rx(void *); 171static void ixgbe_msix_link(void *); 172 173/* Legacy interrupts use deferred handlers */ 174static void ixgbe_handle_tx(void *context, int pending); 175static void ixgbe_handle_rx(void *context, int pending); 176 |
177 178/********************************************************************* 179 * FreeBSD Device Interface Entry Points 180 *********************************************************************/ 181 182static device_method_t ixgbe_methods[] = { 183 /* Device interface */ 184 DEVMETHOD(device_probe, ixgbe_probe), --- 12 unchanged lines hidden (view full) --- 197 198MODULE_DEPEND(ixgbe, pci, 1, 1, 1); 199MODULE_DEPEND(ixgbe, ether, 1, 1, 1); 200 201/* 202** TUNEABLE PARAMETERS: 203*/ 204 |
205/* 206** These parameters are used in Adaptive 207** Interrupt Moderation. The value is set 208** into EITR and controls the interrupt 209** frequency. They can be modified but 210** be careful in tuning them. 211*/ 212static int ixgbe_enable_aim = TRUE; 213TUNABLE_INT("hw.ixgbe.enable_aim", &ixgbe_enable_aim); 214static int ixgbe_low_latency = IXGBE_LOW_LATENCY; 215TUNABLE_INT("hw.ixgbe.low_latency", &ixgbe_low_latency); 216static int ixgbe_ave_latency = IXGBE_LOW_LATENCY; 217TUNABLE_INT("hw.ixgbe.ave_latency", &ixgbe_low_latency); 218static int ixgbe_bulk_latency = IXGBE_BULK_LATENCY; 219TUNABLE_INT("hw.ixgbe.bulk_latency", &ixgbe_bulk_latency); 220 |
221/* How many packets rxeof tries to clean at a time */ 222static int ixgbe_rx_process_limit = 100; 223TUNABLE_INT("hw.ixgbe.rx_process_limit", &ixgbe_rx_process_limit); 224 225/* Flow control setting, default to full */ |
226static int ixgbe_flow_control = ixgbe_fc_none; |
227TUNABLE_INT("hw.ixgbe.flow_control", &ixgbe_flow_control); 228 229/* 230 * Should the driver do LRO on the RX end 231 * this can be toggled on the fly, but the 232 * interface must be reset (down/up) for it 233 * to take effect. 234 */ |
235static int ixgbe_enable_lro = 1; |
236TUNABLE_INT("hw.ixgbe.enable_lro", &ixgbe_enable_lro); 237 238/* 239 * MSIX should be the default for best performance, 240 * but this allows it to be forced off for testing. 241 */ 242static int ixgbe_enable_msix = 1; 243TUNABLE_INT("hw.ixgbe.enable_msix", &ixgbe_enable_msix); 244 245/* |
246 * Enable RX Header Split 247 */ 248static int ixgbe_rx_hdr_split = 1; 249TUNABLE_INT("hw.ixgbe.rx_hdr_split", &ixgbe_rx_hdr_split); 250 251/* |
252 * Number of TX/RX Queues, with 0 setting 253 * it autoconfigures to the number of cpus. 254 */ 255static int ixgbe_tx_queues = 1; 256TUNABLE_INT("hw.ixgbe.tx_queues", &ixgbe_tx_queues); |
257static int ixgbe_rx_queues = 1; |
258TUNABLE_INT("hw.ixgbe.rx_queues", &ixgbe_rx_queues); 259 260/* Number of TX descriptors per ring */ 261static int ixgbe_txd = DEFAULT_TXD; 262TUNABLE_INT("hw.ixgbe.txd", &ixgbe_txd); 263 264/* Number of RX descriptors per ring */ 265static int ixgbe_rxd = DEFAULT_RXD; 266TUNABLE_INT("hw.ixgbe.rxd", &ixgbe_rxd); 267 268/* Total number of Interfaces - need for config sanity check */ 269static int ixgbe_total_ports; 270 |
271/********************************************************************* 272 * Device identification routine 273 * 274 * ixgbe_probe determines if the driver should be loaded on 275 * adapter based on PCI vendor/device id of the adapter. 276 * 277 * return 0 on success, positive on failure 278 *********************************************************************/ 279 280static int 281ixgbe_probe(device_t dev) 282{ 283 ixgbe_vendor_info_t *ent; 284 |
285 u16 pci_vendor_id = 0; 286 u16 pci_device_id = 0; 287 u16 pci_subvendor_id = 0; 288 u16 pci_subdevice_id = 0; 289 char adapter_name[256]; |
290 291 INIT_DEBUGOUT("ixgbe_probe: begin"); 292 293 pci_vendor_id = pci_get_vendor(dev); 294 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) 295 return (ENXIO); 296 297 pci_device_id = pci_get_device(dev); --- 8 unchanged lines hidden (view full) --- 306 ((pci_subvendor_id == ent->subvendor_id) || 307 (ent->subvendor_id == 0)) && 308 309 ((pci_subdevice_id == ent->subdevice_id) || 310 (ent->subdevice_id == 0))) { 311 sprintf(adapter_name, "%s, Version - %s", 312 ixgbe_strings[ent->index], 313 ixgbe_driver_version); |
314 device_set_desc_copy(dev, adapter_name); 315 return (0); 316 } 317 ent++; 318 } |
319 return (ENXIO); 320} 321 322/********************************************************************* 323 * Device initialization routine 324 * 325 * The attach entry point is called when the driver is being loaded. 326 * This routine identifies the type of hardware, allocates all resources 327 * and initializes the hardware. 328 * 329 * return 0 on success, positive on failure 330 *********************************************************************/ 331 332static int 333ixgbe_attach(device_t dev) 334{ 335 struct adapter *adapter; 336 int error = 0; |
337 u16 pci_device_id; 338 u32 ctrl_ext; |
339 340 INIT_DEBUGOUT("ixgbe_attach: begin"); 341 342 /* Allocate, clear, and link in our adapter structure */ 343 adapter = device_get_softc(dev); 344 adapter->dev = adapter->osdep.dev = dev; 345 346 /* Core Lock Init*/ 347 IXGBE_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 348 |
349 /* Keep track of number of ports and optics */ 350 pci_device_id = pci_get_device(dev); 351 switch (pci_device_id) { 352 case IXGBE_DEV_ID_82598_CX4_DUAL_PORT : 353 adapter->optics = IFM_10G_CX4; 354 ixgbe_total_ports += 2; 355 break; 356 case IXGBE_DEV_ID_82598AF_DUAL_PORT : 357 adapter->optics = IFM_10G_SR; 358 ixgbe_total_ports += 2; 359 break; 360 case IXGBE_DEV_ID_82598AF_SINGLE_PORT : 361 adapter->optics = IFM_10G_SR; 362 ixgbe_total_ports += 1; 363 break; 364 case IXGBE_DEV_ID_82598EB_XF_LR : 365 adapter->optics = IFM_10G_LR; 366 ixgbe_total_ports += 1; 367 break; 368 case IXGBE_DEV_ID_82598EB_CX4 : 369 adapter->optics = IFM_10G_CX4; 370 ixgbe_total_ports += 1; 371 break; 372 case IXGBE_DEV_ID_82598AT : 373 ixgbe_total_ports += 1; 374 case IXGBE_DEV_ID_82598_DA_DUAL_PORT : 375 ixgbe_total_ports += 2; 376 default: 377 break; 378 } 379 |
380 /* SYSCTL APIs */ 381 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 382 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 383 OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW, 384 adapter, 0, ixgbe_sysctl_stats, "I", "Statistics"); 385 386 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 387 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), --- 5 unchanged lines hidden (view full) --- 393 OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW, 394 adapter, 0, ixgbe_set_flowcntl, "I", "Flow Control"); 395 396 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 397 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 398 OID_AUTO, "enable_lro", CTLTYPE_INT|CTLFLAG_RW, 399 &ixgbe_enable_lro, 1, "Large Receive Offload"); 400 |
401 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 402 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 403 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 404 &ixgbe_enable_aim, 1, "Interrupt Moderation"); 405 406 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 407 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 408 OID_AUTO, "low_latency", CTLTYPE_INT|CTLFLAG_RW, 409 &ixgbe_low_latency, 1, "Low Latency"); 410 411 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 412 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 413 OID_AUTO, "ave_latency", CTLTYPE_INT|CTLFLAG_RW, 414 &ixgbe_ave_latency, 1, "Average Latency"); 415 416 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 417 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 418 OID_AUTO, "bulk_latency", CTLTYPE_INT|CTLFLAG_RW, 419 &ixgbe_bulk_latency, 1, "Bulk Latency"); 420 421 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 422 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 423 OID_AUTO, "hdr_split", CTLTYPE_INT|CTLFLAG_RW, 424 &ixgbe_rx_hdr_split, 1, "RX Header Split"); 425 |
426 /* Set up the timer callout */ 427 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 428 429 /* Determine hardware revision */ 430 ixgbe_identify_hardware(adapter); 431 |
432 /* Do base PCI setup - map BAR0 */ 433 if (ixgbe_allocate_pci_resources(adapter)) { 434 device_printf(dev, "Allocation of PCI resources failed\n"); 435 error = ENXIO; 436 goto err_out; 437 } 438 439 /* Do descriptor calc and sanity checks */ --- 29 unchanged lines hidden (view full) --- 469 470 /* Allocate our TX/RX Queues */ 471 if (ixgbe_allocate_queues(adapter)) { 472 error = ENOMEM; 473 goto err_out; 474 } 475 476 /* Initialize the shared code */ |
477 error = ixgbe_init_shared_code(&adapter->hw); 478 if (error == IXGBE_ERR_SFP_NOT_PRESENT) { 479 /* 480 ** No optics in this port, set up 481 ** so the timer routine will probe 482 ** for later insertion. 483 */ 484 adapter->sfp_probe = TRUE; 485 error = 0; 486 } else if (error == IXGBE_ERR_SFP_NOT_SUPPORTED) { 487 device_printf(dev,"Unsupported SFP+ module detected!\n"); 488 error = EIO; 489 goto err_late; 490 } else if (error) { |
491 device_printf(dev,"Unable to initialize the shared code\n"); 492 error = EIO; 493 goto err_late; 494 } 495 496 /* Initialize the hardware */ 497 if (ixgbe_hardware_init(adapter)) { 498 device_printf(dev,"Unable to initialize the hardware\n"); --- 14 unchanged lines hidden (view full) --- 513 /* Sysctl for limiting the amount of work done in the taskqueue */ 514 ixgbe_add_rx_process_limit(adapter, "rx_processing_limit", 515 "max number of rx packets to process", &adapter->rx_process_limit, 516 ixgbe_rx_process_limit); 517 518 /* Initialize statistics */ 519 ixgbe_update_stats_counters(adapter); 520 |
521#ifdef IXGBE_HW_VLAN_SUPPORT |
522 /* Register for VLAN events */ 523 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 524 ixgbe_register_vlan, 0, EVENTHANDLER_PRI_FIRST); 525 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 526 ixgbe_unregister_vlan, 0, EVENTHANDLER_PRI_FIRST); 527#endif |
528 |
529 /* let hardware know driver is loaded */ 530 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 531 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 532 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 533 534 INIT_DEBUGOUT("ixgbe_attach: end"); 535 return (0); 536err_late: --- 38 unchanged lines hidden (view full) --- 575 IXGBE_CORE_LOCK(adapter); 576 ixgbe_stop(adapter); 577 IXGBE_CORE_UNLOCK(adapter); 578 579 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 580 if (txr->tq) { 581 taskqueue_drain(txr->tq, &txr->tx_task); 582 taskqueue_free(txr->tq); |
583 } 584 } 585 586 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 587 if (rxr->tq) { 588 taskqueue_drain(rxr->tq, &rxr->rx_task); 589 taskqueue_free(rxr->tq); |
590 } 591 } 592 |
593 /* let hardware know driver is unloading */ 594 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); 595 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 596 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext); 597 598#ifdef IXGBE_HW_VLAN_SUPPORT |
599 /* Unregister VLAN events */ 600 if (adapter->vlan_attach != NULL) 601 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 602 if (adapter->vlan_detach != NULL) 603 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); |
604#endif |
605 |
606 ether_ifdetach(adapter->ifp); 607 callout_drain(&adapter->timer); 608 ixgbe_free_pci_resources(adapter); 609 bus_generic_detach(dev); 610 if_free(adapter->ifp); 611 612 ixgbe_free_transmit_structures(adapter); 613 ixgbe_free_receive_structures(adapter); --- 286 unchanged lines hidden (view full) --- 900 * 901 * return 0 on success, positive on failure 902 **********************************************************************/ 903#define IXGBE_MHADD_MFS_SHIFT 16 904 905static void 906ixgbe_init_locked(struct adapter *adapter) 907{ |
908 struct rx_ring *rxr = adapter->rx_rings; 909 struct tx_ring *txr = adapter->tx_rings; |
910 struct ifnet *ifp = adapter->ifp; 911 device_t dev = adapter->dev; 912 struct ixgbe_hw *hw; |
913 u32 k, txdctl, mhadd, gpie; 914 u32 rxdctl, rxctrl; |
915 916 INIT_DEBUGOUT("ixgbe_init: begin"); 917 918 hw = &adapter->hw; 919 mtx_assert(&adapter->core_mtx, MA_OWNED); 920 921 ixgbe_stop(adapter); 922 --- 4 unchanged lines hidden (view full) --- 927 adapter->hw.addr_ctrl.rar_used_count = 1; 928 929 /* Initialize the hardware */ 930 if (ixgbe_hardware_init(adapter)) { 931 device_printf(dev, "Unable to initialize the hardware\n"); 932 return; 933 } 934 |
935#ifndef IXGBE_HW_VLAN_SUPPORT |
936 if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) { |
937 u32 ctrl; 938 |
939 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); 940 ctrl |= IXGBE_VLNCTRL_VME; 941 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 942 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); 943 } 944#endif |
945 /* Prepare transmit descriptors and buffers */ 946 if (ixgbe_setup_transmit_structures(adapter)) { 947 device_printf(dev,"Could not setup transmit structures\n"); 948 ixgbe_stop(adapter); 949 return; 950 } 951 952 ixgbe_initialize_transmit_units(adapter); 953 |
954 /* TX irq moderation rate is fixed */ 955 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 956 IXGBE_WRITE_REG(&adapter->hw, 957 IXGBE_EITR(txr->msix), ixgbe_ave_latency); 958 txr->watchdog_timer = FALSE; 959 } 960 |
961 /* Setup Multicast table */ 962 ixgbe_set_multi(adapter); 963 964 /* |
965 ** Determine the correct mbuf pool 966 ** for doing jumbo/headersplit |
967 */ |
968 if (ifp->if_mtu > ETHERMTU) 969 adapter->rx_mbuf_sz = MJUMPAGESIZE; 970 else 971 adapter->rx_mbuf_sz = MCLBYTES; |
972 973 /* Prepare receive descriptors and buffers */ 974 if (ixgbe_setup_receive_structures(adapter)) { 975 device_printf(dev,"Could not setup receive structures\n"); 976 ixgbe_stop(adapter); 977 return; 978 } 979 980 /* Configure RX settings */ 981 ixgbe_initialize_receive_units(adapter); 982 |
983 /* RX moderation will be adapted over time, set default */ 984 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 985 IXGBE_WRITE_REG(&adapter->hw, 986 IXGBE_EITR(rxr->msix), ixgbe_low_latency); 987 } 988 989 /* Set Link moderation */ 990 IXGBE_WRITE_REG(&adapter->hw, 991 IXGBE_EITR(adapter->linkvec), IXGBE_LINK_ITR); 992 |
993 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); |
994 |
995 /* Enable Fan Failure Interrupt */ 996 if (adapter->hw.phy.media_type == ixgbe_media_type_copper) 997 gpie |= IXGBE_SDP1_GPIEN; |
998 |
999 if (adapter->msix) { 1000 /* Enable Enhanced MSIX mode */ 1001 gpie |= IXGBE_GPIE_MSIX_MODE; 1002 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 1003 IXGBE_GPIE_OCD; 1004 } 1005 IXGBE_WRITE_REG(&adapter->hw, IXGBE_GPIE, gpie); 1006 --- 23 unchanged lines hidden (view full) --- 1030 } 1031 1032 for (int i = 0; i < adapter->num_rx_queues; i++) { 1033 rxdctl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXDCTL(i)); 1034 /* PTHRESH set to 32 */ 1035 rxdctl |= 0x0020; 1036 rxdctl |= IXGBE_RXDCTL_ENABLE; 1037 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RXDCTL(i), rxdctl); |
1038 for (k = 0; k < 10; k++) { 1039 if (IXGBE_READ_REG(hw, IXGBE_RXDCTL(i)) & 1040 IXGBE_RXDCTL_ENABLE) 1041 break; 1042 else 1043 msec_delay(1); 1044 } 1045 wmb(); 1046 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), adapter->num_rx_desc - 1); |
1047 } 1048 |
1049 /* Enable Receive engine */ 1050 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 1051 if (adapter->hw.mac.type == ixgbe_mac_82598EB) 1052 rxctrl |= IXGBE_RXCTRL_DMBYPS; 1053 rxctrl |= IXGBE_RXCTRL_RXEN; 1054 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, rxctrl); 1055 |
1056 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1057 1058 /* Set up MSI/X routing */ |
1059 if (ixgbe_enable_msix) 1060 ixgbe_configure_ivars(adapter); |
1061 1062 ixgbe_enable_intr(adapter); 1063 1064 /* Now inform the stack we're ready */ 1065 ifp->if_drv_flags |= IFF_DRV_RUNNING; 1066 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 1067 1068 return; --- 7 unchanged lines hidden (view full) --- 1076 IXGBE_CORE_LOCK(adapter); 1077 ixgbe_init_locked(adapter); 1078 IXGBE_CORE_UNLOCK(adapter); 1079 return; 1080} 1081 1082 1083/* |
1084** MSIX Interrupt Handlers |
1085*/ 1086 1087static void 1088ixgbe_handle_rx(void *context, int pending) 1089{ 1090 struct rx_ring *rxr = context; 1091 struct adapter *adapter = rxr->adapter; |
1092 u32 loop = MAX_LOOP; 1093 bool more; |
1094 |
1095 do { 1096 more = ixgbe_rxeof(rxr, -1); 1097 } while (loop-- && more); 1098 /* Reenable this interrupt */ 1099 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims); |
1100} 1101 1102static void 1103ixgbe_handle_tx(void *context, int pending) 1104{ 1105 struct tx_ring *txr = context; 1106 struct adapter *adapter = txr->adapter; 1107 struct ifnet *ifp = adapter->ifp; |
1108 u32 loop = MAX_LOOP; 1109 bool more; |
1110 |
1111 IXGBE_TX_LOCK(txr); 1112 do { 1113 more = ixgbe_txeof(txr); 1114 } while (loop-- && more); 1115 1116 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 1117 ixgbe_start_locked(txr, ifp); 1118 1119 IXGBE_TX_UNLOCK(txr); 1120 1121 /* Reenable this interrupt */ 1122 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims); |
1123} 1124 1125 1126/********************************************************************* 1127 * 1128 * Legacy Interrupt Service routine 1129 * 1130 **********************************************************************/ 1131 1132static void 1133ixgbe_legacy_irq(void *arg) 1134{ |
1135 struct adapter *adapter = arg; |
1136 struct ixgbe_hw *hw = &adapter->hw; |
1137 struct tx_ring *txr = adapter->tx_rings; 1138 struct rx_ring *rxr = adapter->rx_rings; |
1139 u32 reg_eicr; |
1140 |
1141 1142 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1143 1144 if (reg_eicr == 0) { 1145 ixgbe_enable_intr(adapter); |
1146 return; |
1147 } |
1148 |
1149 if (ixgbe_rxeof(rxr, adapter->rx_process_limit)) |
1150 taskqueue_enqueue(rxr->tq, &rxr->rx_task); |
1151 if (ixgbe_txeof(txr)) 1152 taskqueue_enqueue(txr->tq, &txr->tx_task); |
1153 1154 /* Check for fan failure */ 1155 if ((hw->phy.media_type == ixgbe_media_type_copper) && 1156 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1157 device_printf(adapter->dev, "\nCRITICAL: FAN FAILURE!! " 1158 "REPLACE IMMEDIATELY!!\n"); |
1159 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EICR_GPI_SDP1); |
1160 } |
1161 |
1162 /* Link status change */ 1163 if (reg_eicr & IXGBE_EICR_LSC) 1164 ixgbe_update_link_status(adapter); 1165 |
1166 ixgbe_enable_intr(adapter); |
1167 return; 1168} 1169 1170 1171/********************************************************************* 1172 * 1173 * MSI TX Interrupt Service routine 1174 * 1175 **********************************************************************/ 1176 1177void 1178ixgbe_msix_tx(void *arg) 1179{ |
1180 struct tx_ring *txr = arg; 1181 struct adapter *adapter = txr->adapter; 1182 bool more; |
1183 |
1184 IXGBE_TX_LOCK(txr); |
1185 ++txr->tx_irq; 1186 more = ixgbe_txeof(txr); |
1187 IXGBE_TX_UNLOCK(txr); |
1188 if (more) 1189 taskqueue_enqueue(txr->tq, &txr->tx_task); 1190 else /* Reenable this interrupt */ 1191 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, txr->eims); |
1192 return; 1193} 1194 |
1195 |
1196/********************************************************************* 1197 * |
1198 * MSIX RX Interrupt Service routine |
1199 * 1200 **********************************************************************/ 1201 1202static void 1203ixgbe_msix_rx(void *arg) 1204{ 1205 struct rx_ring *rxr = arg; |
1206 struct adapter *adapter = rxr->adapter; 1207 bool more; |
1208 1209 ++rxr->rx_irq; |
1210 more = ixgbe_rxeof(rxr, -1); 1211 if (more) 1212 taskqueue_enqueue(rxr->tq, &rxr->rx_task); 1213 else 1214 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rxr->eims); 1215 /* Update interrupt rate */ 1216 if (ixgbe_enable_aim == TRUE) 1217 ixgbe_update_aim(rxr); |
1218 return; 1219} 1220 |
1221/* 1222** Routine to do adjust the RX EITR value based on traffic, 1223** its a simple three state model, but seems to help. 1224** 1225** Note that the three EITR values are tuneable using 1226** sysctl in real time. The feature can be effectively 1227** nullified by setting them equal. 1228*/ 1229#define BULK_THRESHOLD 10000 1230#define AVE_THRESHOLD 1600 1231 |
1232static void |
1233ixgbe_update_aim(struct rx_ring *rxr) 1234{ 1235 struct adapter *adapter = rxr->adapter; 1236 u32 olditr, newitr; 1237 1238 /* Update interrupt moderation based on traffic */ 1239 olditr = rxr->eitr_setting; 1240 newitr = olditr; 1241 1242 /* Idle, don't change setting */ 1243 if (rxr->bytes == 0) 1244 return; 1245 1246 if (olditr == ixgbe_low_latency) { 1247 if (rxr->bytes > AVE_THRESHOLD) 1248 newitr = ixgbe_ave_latency; 1249 } else if (olditr == ixgbe_ave_latency) { 1250 if (rxr->bytes < AVE_THRESHOLD) 1251 newitr = ixgbe_low_latency; 1252 else if (rxr->bytes > BULK_THRESHOLD) 1253 newitr = ixgbe_bulk_latency; 1254 } else if (olditr == ixgbe_bulk_latency) { 1255 if (rxr->bytes < BULK_THRESHOLD) 1256 newitr = ixgbe_ave_latency; 1257 } 1258 1259 if (olditr != newitr) { 1260 /* Change interrupt rate */ 1261 rxr->eitr_setting = newitr; 1262 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(rxr->me), 1263 newitr | (newitr << 16)); 1264 } 1265 1266 rxr->bytes = 0; 1267 return; 1268} 1269 1270 1271static void |
1272ixgbe_msix_link(void *arg) 1273{ 1274 struct adapter *adapter = arg; 1275 struct ixgbe_hw *hw = &adapter->hw; 1276 u32 reg_eicr; 1277 1278 ++adapter->link_irq; 1279 --- 42 unchanged lines hidden (view full) --- 1322 1323 ifmr->ifm_status |= IFM_ACTIVE; 1324 1325 switch (adapter->link_speed) { 1326 case IXGBE_LINK_SPEED_1GB_FULL: 1327 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1328 break; 1329 case IXGBE_LINK_SPEED_10GB_FULL: |
1330 ifmr->ifm_active |= adapter->optics | IFM_FDX; |
1331 break; 1332 } 1333 1334 IXGBE_CORE_UNLOCK(adapter); 1335 1336 return; 1337} 1338 --- 39 unchanged lines hidden (view full) --- 1378 * return 0 on success, positive on failure 1379 **********************************************************************/ 1380 1381static int 1382ixgbe_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1383{ 1384 struct adapter *adapter = txr->adapter; 1385 u32 olinfo_status = 0, cmd_type_len = 0; |
1386 u32 paylen = 0; |
1387 int i, j, error, nsegs; 1388 int first, last = 0; 1389 struct mbuf *m_head; 1390 bus_dma_segment_t segs[IXGBE_MAX_SCATTER]; 1391 bus_dmamap_t map; 1392 struct ixgbe_tx_buf *txbuf, *txbuf_mapped; 1393 union ixgbe_adv_tx_desc *txd = NULL; 1394 1395 m_head = *m_headp; |
1396 1397 /* Basic descriptor defines */ 1398 cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; 1399 cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; 1400 1401 if (m_head->m_flags & M_VLANTAG) 1402 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1403 --- 27 unchanged lines hidden (view full) --- 1431 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1432 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1433 1434 if (error == EFBIG) { 1435 struct mbuf *m; 1436 1437 m = m_defrag(*m_headp, M_DONTWAIT); 1438 if (m == NULL) { |
1439 adapter->mbuf_defrag_failed++; |
1440 m_freem(*m_headp); 1441 *m_headp = NULL; 1442 return (ENOBUFS); 1443 } 1444 *m_headp = m; 1445 1446 /* Try it again */ 1447 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, --- 35 unchanged lines hidden (view full) --- 1483 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1484 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1485 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1486 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1487 ++adapter->tso_tx; 1488 } else if (ixgbe_tx_ctx_setup(txr, m_head)) 1489 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1490 |
1491 /* Record payload length */ 1492 if (paylen == 0) 1493 olinfo_status |= m_head->m_pkthdr.len << 1494 IXGBE_ADVTXD_PAYLEN_SHIFT; 1495 |
1496 i = txr->next_avail_tx_desc; 1497 for (j = 0; j < nsegs; j++) { 1498 bus_size_t seglen; 1499 bus_addr_t segaddr; 1500 1501 txbuf = &txr->tx_buffers[i]; 1502 txd = &txr->tx_base[i]; 1503 seglen = segs[j].ds_len; --- 4 unchanged lines hidden (view full) --- 1508 cmd_type_len |seglen); 1509 txd->read.olinfo_status = htole32(olinfo_status); 1510 last = i; /* Next descriptor that will get completed */ 1511 1512 if (++i == adapter->num_tx_desc) 1513 i = 0; 1514 1515 txbuf->m_head = NULL; |
1516 } 1517 |
1518 txd->read.cmd_type_len |= 1519 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); |
1520 txr->tx_avail -= nsegs; 1521 txr->next_avail_tx_desc = i; 1522 1523 txbuf->m_head = m_head; 1524 txbuf->map = map; 1525 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1526 1527 /* Set the index of the descriptor that will be marked done */ 1528 txbuf = &txr->tx_buffers[first]; 1529 1530 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1531 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1532 /* 1533 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1534 * hardware that this frame is available to transmit. 1535 */ |
1536 ++txr->total_packets; |
1537 IXGBE_WRITE_REG(&adapter->hw, IXGBE_TDT(txr->me), i); |
1538 return (0); 1539 1540xmit_fail: 1541 bus_dmamap_unload(txr->txtag, txbuf->map); 1542 return (error); 1543 1544} 1545 --- 111 unchanged lines hidden (view full) --- 1657static void 1658ixgbe_local_timer(void *arg) 1659{ 1660 struct adapter *adapter = arg; 1661 struct ifnet *ifp = adapter->ifp; 1662 1663 mtx_assert(&adapter->core_mtx, MA_OWNED); 1664 |
1665 /* Check for pluggable optics */ 1666 if (adapter->sfp_probe) 1667 if (!ixgbe_sfp_probe(adapter)) 1668 goto out; /* Nothing to do */ 1669 |
1670 ixgbe_update_link_status(adapter); 1671 ixgbe_update_stats_counters(adapter); 1672 if (ixgbe_display_debug_stats && ifp->if_drv_flags & IFF_DRV_RUNNING) { 1673 ixgbe_print_hw_stats(adapter); 1674 } 1675 /* |
1676 * Each tick we check the watchdog |
1677 * to protect against hardware hangs. 1678 */ 1679 ixgbe_watchdog(adapter); 1680 |
1681out: 1682 /* Trigger an RX interrupt on all queues */ 1683 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, adapter->rx_mask); 1684 |
1685 callout_reset(&adapter->timer, hz, ixgbe_local_timer, adapter); 1686} 1687 1688static void 1689ixgbe_update_link_status(struct adapter *adapter) 1690{ 1691 boolean_t link_up = FALSE; 1692 struct ifnet *ifp = adapter->ifp; --- 170 unchanged lines hidden (view full) --- 1863 ixgbe_msix_tx, txr, &adapter->tag[vector]); 1864 if (error) { 1865 adapter->res[vector] = NULL; 1866 device_printf(dev, "Failed to register TX handler"); 1867 return (error); 1868 } 1869 txr->msix = vector; 1870 txr->eims = IXGBE_IVAR_TX_QUEUE(vector); |
1871 TASK_INIT(&txr->tx_task, 0, ixgbe_handle_tx, txr); 1872 txr->tq = taskqueue_create_fast("ixgbe_txq", M_NOWAIT, 1873 taskqueue_thread_enqueue, &txr->tq); 1874 taskqueue_start_threads(&txr->tq, 1, PI_NET, "%s txq", 1875 device_get_nameunit(adapter->dev)); |
1876 } 1877 1878 /* RX setup */ 1879 for (int i = 0; i < adapter->num_rx_queues; i++, vector++, rxr++) { 1880 adapter->res[vector] = bus_alloc_resource_any(dev, 1881 SYS_RES_IRQ, &adapter->rid[vector], 1882 RF_SHAREABLE | RF_ACTIVE); 1883 if (!adapter->res[vector]) { --- 8 unchanged lines hidden (view full) --- 1892 rxr, &adapter->tag[vector]); 1893 if (error) { 1894 adapter->res[vector] = NULL; 1895 device_printf(dev, "Failed to register RX handler"); 1896 return (error); 1897 } 1898 rxr->msix = vector; 1899 rxr->eims = IXGBE_IVAR_RX_QUEUE(vector); |
1900 /* used in local timer */ 1901 adapter->rx_mask |= rxr->eims; 1902 TASK_INIT(&rxr->rx_task, 0, ixgbe_handle_rx, rxr); 1903 rxr->tq = taskqueue_create_fast("ixgbe_rxq", M_NOWAIT, 1904 taskqueue_thread_enqueue, &rxr->tq); 1905 taskqueue_start_threads(&rxr->tq, 1, PI_NET, "%s rxq", 1906 device_get_nameunit(adapter->dev)); |
1907 } 1908 1909 /* Now for Link changes */ 1910 adapter->res[vector] = bus_alloc_resource_any(dev, 1911 SYS_RES_IRQ, &adapter->rid[vector], RF_SHAREABLE | RF_ACTIVE); 1912 if (!adapter->res[vector]) { 1913 device_printf(dev,"Unable to allocate" 1914 " bus resource: Link interrupt [%d]\n", adapter->rid[vector]); --- 18 unchanged lines hidden (view full) --- 1933 * Setup Either MSI/X or MSI 1934 */ 1935static int 1936ixgbe_setup_msix(struct adapter *adapter) 1937{ 1938 device_t dev = adapter->dev; 1939 int rid, want, queues, msgs; 1940 |
1941 /* Override by tuneable */ 1942 if (ixgbe_enable_msix == 0) 1943 goto msi; 1944 |
1945 /* First try MSI/X */ |
1946 rid = PCIR_BAR(MSIX_82598_BAR); |
1947 adapter->msix_mem = bus_alloc_resource_any(dev, 1948 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1949 if (!adapter->msix_mem) { |
1950 rid += 4; /* 82599 maps in higher BAR */ 1951 adapter->msix_mem = bus_alloc_resource_any(dev, 1952 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1953 } 1954 if (!adapter->msix_mem) { |
1955 /* May not be enabled */ 1956 device_printf(adapter->dev, 1957 "Unable to map MSIX table \n"); 1958 goto msi; 1959 } 1960 1961 msgs = pci_msix_count(dev); 1962 if (msgs == 0) { /* system has msix disabled */ 1963 bus_release_resource(dev, SYS_RES_MEMORY, |
1964 rid, adapter->msix_mem); |
1965 adapter->msix_mem = NULL; 1966 goto msi; 1967 } 1968 1969 /* Figure out a reasonable auto config value */ 1970 queues = (mp_ncpus > ((msgs-1)/2)) ? (msgs-1)/2 : mp_ncpus; 1971 1972 if (ixgbe_tx_queues == 0) --- 63 unchanged lines hidden (view full) --- 2036 2037 adapter->hw.back = &adapter->osdep; 2038 return (0); 2039} 2040 2041static void 2042ixgbe_free_pci_resources(struct adapter * adapter) 2043{ |
2044 device_t dev = adapter->dev; 2045 int rid; |
2046 2047 /* 2048 * Legacy has this set to 0, but we need 2049 * to run this once, so reset it. 2050 */ 2051 if (adapter->msix == 0) 2052 adapter->msix = 1; 2053 |
2054 rid = PCIR_BAR(MSIX_82598_BAR); 2055 |
2056 /* 2057 * First release all the interrupt resources: 2058 * notice that since these are just kept 2059 * in an array we can do the same logic 2060 * whether its MSIX or just legacy. 2061 */ 2062 for (int i = 0; i < adapter->msix; i++) { 2063 if (adapter->tag[i] != NULL) { --- 7 unchanged lines hidden (view full) --- 2071 } 2072 } 2073 2074 if (adapter->msix) 2075 pci_release_msi(dev); 2076 2077 if (adapter->msix_mem != NULL) 2078 bus_release_resource(dev, SYS_RES_MEMORY, |
2079 rid, adapter->msix_mem); |
2080 2081 if (adapter->pci_mem != NULL) 2082 bus_release_resource(dev, SYS_RES_MEMORY, 2083 PCIR_BAR(0), adapter->pci_mem); 2084 2085 return; 2086} 2087 --- 18 unchanged lines hidden (view full) --- 2106 2107 /* Make sure we have a good EEPROM before we read from it */ 2108 if (ixgbe_validate_eeprom_checksum(&adapter->hw, &csum) < 0) { 2109 device_printf(dev,"The EEPROM Checksum Is Not Valid\n"); 2110 return (EIO); 2111 } 2112 2113 /* Get Hardware Flow Control setting */ |
2114 adapter->hw.fc.requested_mode = ixgbe_fc_full; |
2115 adapter->hw.fc.pause_time = IXGBE_FC_PAUSE; 2116 adapter->hw.fc.low_water = IXGBE_FC_LO; 2117 adapter->hw.fc.high_water = IXGBE_FC_HI; 2118 adapter->hw.fc.send_xon = TRUE; 2119 2120 if (ixgbe_init_hw(&adapter->hw)) { 2121 device_printf(dev,"Hardware Initialization Failed"); 2122 return (EIO); --- 40 unchanged lines hidden (view full) --- 2163 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 2164 2165 ifp->if_capabilities |= (IFCAP_HWCSUM | IFCAP_TSO4); 2166 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 2167 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 2168 2169 ifp->if_capenable = ifp->if_capabilities; 2170 |
2171 if (hw->device_id == IXGBE_DEV_ID_82598AT) |
2172 ixgbe_setup_link_speed(hw, (IXGBE_LINK_SPEED_10GB_FULL | 2173 IXGBE_LINK_SPEED_1GB_FULL), TRUE, TRUE); 2174 else 2175 ixgbe_setup_link_speed(hw, IXGBE_LINK_SPEED_10GB_FULL, 2176 TRUE, FALSE); 2177 2178 /* 2179 * Specify the media types supported by this adapter and register 2180 * callbacks to update media and link information 2181 */ 2182 ifmedia_init(&adapter->media, IFM_IMASK, ixgbe_media_change, 2183 ixgbe_media_status); |
2184 ifmedia_add(&adapter->media, IFM_ETHER | adapter->optics | |
2185 IFM_FDX, 0, NULL); |
2186 if (hw->device_id == IXGBE_DEV_ID_82598AT) { |
2187 ifmedia_add(&adapter->media, 2188 IFM_ETHER | IFM_1000_T | IFM_FDX, 0, NULL); 2189 ifmedia_add(&adapter->media, 2190 IFM_ETHER | IFM_1000_T, 0, NULL); 2191 } 2192 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2193 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2194 --- 84 unchanged lines hidden (view full) --- 2279 **********************************************************************/ 2280static int 2281ixgbe_allocate_queues(struct adapter *adapter) 2282{ 2283 device_t dev = adapter->dev; 2284 struct tx_ring *txr; 2285 struct rx_ring *rxr; 2286 int rsize, tsize, error = IXGBE_SUCCESS; |
2287 int txconf = 0, rxconf = 0; 2288 2289 /* First allocate the TX ring struct memory */ 2290 if (!(adapter->tx_rings = 2291 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2292 adapter->num_tx_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2293 device_printf(dev, "Unable to allocate TX ring memory\n"); 2294 error = ENOMEM; --- 22 unchanged lines hidden (view full) --- 2317 */ 2318 for (int i = 0; i < adapter->num_tx_queues; i++, txconf++) { 2319 /* Set up some basics */ 2320 txr = &adapter->tx_rings[i]; 2321 txr->adapter = adapter; 2322 txr->me = i; 2323 2324 /* Initialize the TX side lock */ |
2325 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", |
2326 device_get_nameunit(dev), txr->me); |
2327 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); |
2328 2329 if (ixgbe_dma_malloc(adapter, tsize, 2330 &txr->txdma, BUS_DMA_NOWAIT)) { 2331 device_printf(dev, 2332 "Unable to allocate TX Descriptor memory\n"); 2333 error = ENOMEM; 2334 goto err_tx_desc; 2335 } --- 16 unchanged lines hidden (view full) --- 2352 rsize = roundup2(adapter->num_rx_desc * 2353 sizeof(union ixgbe_adv_rx_desc), 4096); 2354 for (int i = 0; i < adapter->num_rx_queues; i++, rxconf++) { 2355 rxr = &adapter->rx_rings[i]; 2356 /* Set up some basics */ 2357 rxr->adapter = adapter; 2358 rxr->me = i; 2359 |
2360 /* Initialize the RX side lock */ 2361 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", |
2362 device_get_nameunit(dev), rxr->me); |
2363 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); |
2364 2365 if (ixgbe_dma_malloc(adapter, rsize, 2366 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2367 device_printf(dev, 2368 "Unable to allocate RxDescriptor memory\n"); 2369 error = ENOMEM; 2370 goto err_rx_desc; 2371 } --- 365 unchanged lines hidden (view full) --- 2737 } 2738 2739 /* Now copy bits into descriptor */ 2740 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2741 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2742 TXD->seqnum_seed = htole32(0); 2743 TXD->mss_l4len_idx = htole32(0); 2744 |
2745 tx_buffer->m_head = NULL; 2746 2747 /* We've consumed the first desc, adjust counters */ 2748 if (++ctxd == adapter->num_tx_desc) 2749 ctxd = 0; 2750 txr->next_avail_tx_desc = ctxd; 2751 --txr->tx_avail; 2752 --- 77 unchanged lines hidden (view full) --- 2830 /* MSS L4LEN IDX */ 2831 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2832 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2833 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2834 2835 TXD->seqnum_seed = htole32(0); 2836 tx_buffer->m_head = NULL; 2837 |
2838 if (++ctxd == adapter->num_tx_desc) 2839 ctxd = 0; 2840 2841 txr->tx_avail--; 2842 txr->next_avail_tx_desc = ctxd; 2843 return TRUE; 2844} 2845 --- 105 unchanged lines hidden (view full) --- 2951} 2952 2953/********************************************************************* 2954 * 2955 * Get a buffer from system mbuf buffer pool. 2956 * 2957 **********************************************************************/ 2958static int |
2959ixgbe_get_buf(struct rx_ring *rxr, int i, u8 clean) |
2960{ |
2961 struct adapter *adapter = rxr->adapter; 2962 bus_dma_segment_t seg[2]; |
2963 struct ixgbe_rx_buf *rxbuf; |
2964 struct mbuf *mh, *mp; 2965 bus_dmamap_t map; 2966 int nsegs, error; 2967 int merr = 0; |
2968 |
2969 |
2970 rxbuf = &rxr->rx_buffers[i]; |
2971 |
2972 /* First get our header and payload mbuf */ 2973 if (clean & IXGBE_CLEAN_HDR) { 2974 mh = m_gethdr(M_DONTWAIT, MT_DATA); 2975 if (mh == NULL) 2976 goto remap; 2977 } else /* reuse */ 2978 mh = rxr->rx_buffers[i].m_head; |
2979 |
2980 mh->m_len = MHLEN; 2981 mh->m_flags |= M_PKTHDR; 2982 2983 if (clean & IXGBE_CLEAN_PKT) { 2984 mp = m_getjcl(M_DONTWAIT, MT_DATA, 2985 M_PKTHDR, adapter->rx_mbuf_sz); 2986 if (mp == NULL) 2987 goto remap; 2988 mp->m_len = adapter->rx_mbuf_sz; 2989 mp->m_flags &= ~M_PKTHDR; 2990 } else { /* reusing */ 2991 mp = rxr->rx_buffers[i].m_pack; 2992 mp->m_len = adapter->rx_mbuf_sz; 2993 mp->m_flags &= ~M_PKTHDR; 2994 } |
2995 /* |
2996 ** Need to create a chain for the following 2997 ** dmamap call at this point. 2998 */ 2999 mh->m_next = mp; 3000 mh->m_pkthdr.len = mh->m_len + mp->m_len; 3001 3002 /* Get the memory mapping */ 3003 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3004 rxr->spare_map, mh, seg, &nsegs, BUS_DMA_NOWAIT); 3005 if (error != 0) { 3006 printf("GET BUF: dmamap load failure - %d\n", error); 3007 m_free(mh); |
3008 return (error); 3009 } 3010 |
3011 /* Unload old mapping and update buffer struct */ |
3012 if (rxbuf->m_head != NULL) |
3013 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3014 map = rxbuf->map; 3015 rxbuf->map = rxr->spare_map; 3016 rxr->spare_map = map; 3017 rxbuf->m_head = mh; 3018 rxbuf->m_pack = mp; 3019 bus_dmamap_sync(rxr->rxtag, 3020 rxbuf->map, BUS_DMASYNC_PREREAD); |
3021 |
3022 /* Update descriptor */ 3023 rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr); 3024 rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr); |
3025 |
3026 return (0); |
3027 |
3028 /* 3029 ** If we get here, we have an mbuf resource 3030 ** issue, so we discard the incoming packet 3031 ** and attempt to reuse existing mbufs next 3032 ** pass thru the ring, but to do so we must 3033 ** fix up the descriptor which had the address 3034 ** clobbered with writeback info. 3035 */ 3036remap: 3037 adapter->mbuf_header_failed++; 3038 merr = ENOBUFS; 3039 /* Is there a reusable buffer? */ 3040 mh = rxr->rx_buffers[i].m_head; 3041 if (mh == NULL) /* Nope, init error */ 3042 return (merr); 3043 mp = rxr->rx_buffers[i].m_pack; 3044 if (mp == NULL) /* Nope, init error */ 3045 return (merr); 3046 /* Get our old mapping */ 3047 rxbuf = &rxr->rx_buffers[i]; 3048 error = bus_dmamap_load_mbuf_sg(rxr->rxtag, 3049 rxbuf->map, mh, seg, &nsegs, BUS_DMA_NOWAIT); 3050 if (error != 0) { 3051 /* We really have a problem */ 3052 m_free(mh); 3053 return (error); |
3054 } |
3055 /* Now fix the descriptor as needed */ 3056 rxr->rx_base[i].read.hdr_addr = htole64(seg[0].ds_addr); 3057 rxr->rx_base[i].read.pkt_addr = htole64(seg[1].ds_addr); |
3058 |
3059 return (merr); |
3060} 3061 |
3062 |
3063/********************************************************************* 3064 * 3065 * Allocate memory for rx_buffer structures. Since we use one 3066 * rx_buffer per received packet, the maximum number of rx_buffer's 3067 * that we'll need is equal to the number of receive descriptors 3068 * that we've allocated. 3069 * 3070 **********************************************************************/ --- 9 unchanged lines hidden (view full) --- 3080 if (!(rxr->rx_buffers = 3081 (struct ixgbe_rx_buf *) malloc(bsize, 3082 M_DEVBUF, M_NOWAIT | M_ZERO))) { 3083 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 3084 error = ENOMEM; 3085 goto fail; 3086 } 3087 |
3088 /* 3089 ** The tag is made to accomodate the largest buffer size 3090 ** with packet split (hence the two segments, even though 3091 ** it may not always use this. 3092 */ |
3093 if ((error = bus_dma_tag_create(NULL, /* parent */ 3094 PAGE_SIZE, 0, /* alignment, bounds */ 3095 BUS_SPACE_MAXADDR, /* lowaddr */ 3096 BUS_SPACE_MAXADDR, /* highaddr */ 3097 NULL, NULL, /* filter, filterarg */ |
3098 MJUM16BYTES, /* maxsize */ 3099 2, /* nsegments */ |
3100 MJUMPAGESIZE, /* maxsegsize */ 3101 0, /* flags */ 3102 NULL, /* lockfunc */ 3103 NULL, /* lockfuncarg */ |
3104 &rxr->rxtag))) { 3105 device_printf(dev, "Unable to create RX DMA tag\n"); |
3106 goto fail; 3107 } 3108 |
3109 /* Create the spare map (used by getbuf) */ 3110 error = bus_dmamap_create(rxr->rxtag, BUS_DMA_NOWAIT, 3111 &rxr->spare_map); |
3112 if (error) { 3113 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 3114 __func__, error); 3115 goto fail; 3116 } 3117 3118 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 3119 rxbuf = &rxr->rx_buffers[i]; |
3120 error = bus_dmamap_create(rxr->rxtag, 3121 BUS_DMA_NOWAIT, &rxbuf->map); |
3122 if (error) { |
3123 device_printf(dev, "Unable to create RX DMA map\n"); |
3124 goto fail; 3125 } |
3126 } 3127 3128 return (0); 3129 3130fail: 3131 /* Frees all, but can handle partial completion */ 3132 ixgbe_free_receive_structures(adapter); 3133 return (error); --- 6 unchanged lines hidden (view full) --- 3140 **********************************************************************/ 3141static int 3142ixgbe_setup_receive_ring(struct rx_ring *rxr) 3143{ 3144 struct adapter *adapter; 3145 device_t dev; 3146 struct ixgbe_rx_buf *rxbuf; 3147 struct lro_ctrl *lro = &rxr->lro; |
3148 int j, rsize; |
3149 3150 adapter = rxr->adapter; 3151 dev = adapter->dev; |
3152 |
3153 /* Clear the ring contents */ |
3154 rsize = roundup2(adapter->num_rx_desc * 3155 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); |
3156 bzero((void *)rxr->rx_base, rsize); 3157 3158 /* |
3159 ** Free current RX buffer structs and their mbufs |
3160 */ 3161 for (int i = 0; i < adapter->num_rx_desc; i++) { 3162 rxbuf = &rxr->rx_buffers[i]; |
3163 if (rxbuf->m_head != NULL) { |
3164 bus_dmamap_sync(rxr->rxtag, rxbuf->map, |
3165 BUS_DMASYNC_POSTREAD); |
3166 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3167 if (rxbuf->m_head) { 3168 rxbuf->m_head->m_next = rxbuf->m_pack; 3169 m_freem(rxbuf->m_head); 3170 } |
3171 rxbuf->m_head = NULL; |
3172 rxbuf->m_pack = NULL; |
3173 } 3174 } 3175 |
3176 /* Now refresh the mbufs */ |
3177 for (j = 0; j < adapter->num_rx_desc; j++) { |
3178 if (ixgbe_get_buf(rxr, j, IXGBE_CLEAN_ALL) == ENOBUFS) { |
3179 rxr->rx_buffers[j].m_head = NULL; |
3180 rxr->rx_buffers[j].m_pack = NULL; 3181 rxr->rx_base[j].read.hdr_addr = 0; |
3182 rxr->rx_base[j].read.pkt_addr = 0; |
3183 goto fail; 3184 } 3185 } 3186 3187 /* Setup our descriptor indices */ 3188 rxr->next_to_check = 0; 3189 rxr->last_cleaned = 0; 3190 3191 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3192 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3193 3194 /* Now set up the LRO interface */ 3195 if (ixgbe_enable_lro) { 3196 int err = tcp_lro_init(lro); 3197 if (err) { |
3198 INIT_DEBUGOUT("LRO Initialization failed!\n"); |
3199 goto fail; 3200 } |
3201 INIT_DEBUGOUT("RX LRO Initialized\n"); |
3202 lro->ifp = adapter->ifp; 3203 } 3204 |
3205 return (0); |
3206 |
3207fail: 3208 /* |
3209 * We need to clean up any buffers allocated 3210 * so far, 'j' is the failing index. |
3211 */ |
3212 for (int i = 0; i < j; i++) { 3213 rxbuf = &rxr->rx_buffers[i]; |
3214 if (rxbuf->m_head != NULL) { |
3215 bus_dmamap_sync(rxr->rxtag, rxbuf->map, |
3216 BUS_DMASYNC_POSTREAD); |
3217 bus_dmamap_unload(rxr->rxtag, rxbuf->map); |
3218 m_freem(rxbuf->m_head); 3219 rxbuf->m_head = NULL; 3220 } 3221 } 3222 return (ENOBUFS); 3223} 3224 3225/********************************************************************* 3226 * 3227 * Initialize all receive rings. 3228 * 3229 **********************************************************************/ 3230static int 3231ixgbe_setup_receive_structures(struct adapter *adapter) 3232{ 3233 struct rx_ring *rxr = adapter->rx_rings; |
3234 int j; |
3235 |
3236 for (j = 0; j < adapter->num_rx_queues; j++, rxr++) |
3237 if (ixgbe_setup_receive_ring(rxr)) 3238 goto fail; 3239 3240 return (0); 3241fail: 3242 /* 3243 * Free RX buffers allocated so far, we will only handle 3244 * the rings that completed, the failing case will have |
3245 * cleaned up for itself. 'j' failed, so its the terminus. |
3246 */ |
3247 for (int i = 0; i < j; ++i) { 3248 rxr = &adapter->rx_rings[i]; 3249 for (int n = 0; n < adapter->num_rx_desc; n++) { |
3250 struct ixgbe_rx_buf *rxbuf; |
3251 rxbuf = &rxr->rx_buffers[n]; |
3252 if (rxbuf->m_head != NULL) { |
3253 bus_dmamap_sync(rxr->rxtag, rxbuf->map, |
3254 BUS_DMASYNC_POSTREAD); |
3255 bus_dmamap_unload(rxr->rxtag, rxbuf->map); |
3256 m_freem(rxbuf->m_head); 3257 rxbuf->m_head = NULL; 3258 } 3259 } 3260 } 3261 3262 return (ENOBUFS); 3263} 3264 3265/********************************************************************* 3266 * |
3267 * Setup receive registers and features. |
3268 * 3269 **********************************************************************/ |
3270#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3271 |
3272static void 3273ixgbe_initialize_receive_units(struct adapter *adapter) 3274{ 3275 struct rx_ring *rxr = adapter->rx_rings; |
3276 struct ixgbe_hw *hw = &adapter->hw; |
3277 struct ifnet *ifp = adapter->ifp; 3278 u32 rxctrl, fctrl, srrctl, rxcsum; |
3279 u32 reta, mrqc = 0, hlreg, random[10]; |
3280 3281 3282 /* 3283 * Make sure receives are disabled while 3284 * setting up the descriptor ring 3285 */ |
3286 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 3287 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, |
3288 rxctrl & ~IXGBE_RXCTRL_RXEN); 3289 3290 /* Enable broadcasts */ |
3291 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); |
3292 fctrl |= IXGBE_FCTRL_BAM; |
3293 fctrl |= IXGBE_FCTRL_DPF; 3294 fctrl |= IXGBE_FCTRL_PMCF; 3295 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); |
3296 |
3297 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(0)); |
3298 srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3299 srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; |
3300 3301 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3302 /* Set for Jumbo Frames? */ 3303 if (ifp->if_mtu > ETHERMTU) { 3304 hlreg |= IXGBE_HLREG0_JUMBOEN; |
3305 srrctl |= 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; |
3306 } else { 3307 hlreg &= ~IXGBE_HLREG0_JUMBOEN; |
3308 srrctl |= 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; |
3309 } 3310 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); |
3311 |
3312 if (ixgbe_rx_hdr_split) { 3313 /* Use a standard mbuf for the header */ 3314 srrctl |= ((IXGBE_RX_HDR << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3315 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3316 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3317 } else 3318 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; |
3319 |
3320 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(0), srrctl); |
3321 3322 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 3323 u64 rdba = rxr->rxdma.dma_paddr; 3324 /* Setup the Base and Length of the Rx Descriptor Ring */ |
3325 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), |
3326 (rdba & 0x00000000ffffffffULL)); |
3327 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 3328 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), |
3329 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3330 3331 /* Setup the HW Rx Head and Tail Descriptor Pointers */ |
3332 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 3333 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); |
3334 } 3335 |
3336 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); |
3337 |
3338 /* Setup RSS */ |
3339 if (adapter->num_rx_queues > 1) { |
3340 int i, j; 3341 reta = 0; 3342 |
3343 /* set up random bits */ 3344 arc4rand(&random, sizeof(random), 0); 3345 |
3346 /* Set up the redirection table */ |
3347 for (i = 0, j = 0; i < 128; i++, j++) { 3348 if (j == adapter->num_rx_queues) j = 0; 3349 reta = (reta << 8) | (j * 0x11); 3350 if ((i & 3) == 3) 3351 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 3352 } |
3353 3354 /* Now fill our hash function seeds */ 3355 for (int i = 0; i < 10; i++) |
3356 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random[i]); |
3357 |
3358 /* Perform hash on these packet types */ 3359 mrqc |= IXGBE_MRQC_RSSEN 3360 | IXGBE_MRQC_RSS_FIELD_IPV4 3361 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 3362 | IXGBE_MRQC_RSS_FIELD_IPV4_UDP 3363 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 3364 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 3365 | IXGBE_MRQC_RSS_FIELD_IPV6 3366 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 3367 | IXGBE_MRQC_RSS_FIELD_IPV6_UDP 3368 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 3369 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); |
3370 3371 /* RSS and RX IPP Checksum are mutually exclusive */ 3372 rxcsum |= IXGBE_RXCSUM_PCSD; 3373 } 3374 3375 if (ifp->if_capenable & IFCAP_RXCSUM) 3376 rxcsum |= IXGBE_RXCSUM_PCSD; 3377 3378 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3379 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3380 |
3381 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); |
3382 |
3383 return; 3384} 3385 3386/********************************************************************* 3387 * 3388 * Free all receive rings. 3389 * 3390 **********************************************************************/ --- 25 unchanged lines hidden (view full) --- 3416 struct adapter *adapter = NULL; 3417 struct ixgbe_rx_buf *rxbuf = NULL; 3418 3419 INIT_DEBUGOUT("free_receive_buffers: begin"); 3420 adapter = rxr->adapter; 3421 if (rxr->rx_buffers != NULL) { 3422 rxbuf = &rxr->rx_buffers[0]; 3423 for (int i = 0; i < adapter->num_rx_desc; i++) { |
3424 if (rxbuf->map != NULL) { |
3425 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 3426 BUS_DMASYNC_POSTREAD); 3427 bus_dmamap_unload(rxr->rxtag, rxbuf->map); 3428 bus_dmamap_destroy(rxr->rxtag, rxbuf->map); |
3429 } 3430 if (rxbuf->m_head != NULL) { 3431 m_freem(rxbuf->m_head); 3432 } 3433 rxbuf->m_head = NULL; 3434 ++rxbuf; 3435 } 3436 } 3437 if (rxr->rx_buffers != NULL) { 3438 free(rxr->rx_buffers, M_DEVBUF); 3439 rxr->rx_buffers = NULL; 3440 } |
3441 if (rxr->rxtag != NULL) { 3442 bus_dma_tag_destroy(rxr->rxtag); 3443 rxr->rxtag = NULL; |
3444 } 3445 return; 3446} 3447 3448/********************************************************************* 3449 * 3450 * This routine executes in interrupt context. It replenishes 3451 * the mbufs in the descriptor and sends data which has been 3452 * dma'ed into host memory to upper layer. 3453 * 3454 * We loop at most count times if count is > 0, or until done if 3455 * count < 0. 3456 * |
3457 * Return TRUE for more work, FALSE for all clean. |
3458 *********************************************************************/ 3459static bool 3460ixgbe_rxeof(struct rx_ring *rxr, int count) 3461{ 3462 struct adapter *adapter = rxr->adapter; 3463 struct ifnet *ifp = adapter->ifp; 3464 struct lro_ctrl *lro = &rxr->lro; 3465 struct lro_entry *queued; |
3466 int i; 3467 u32 staterr; |
3468 union ixgbe_adv_rx_desc *cur; 3469 3470 3471 IXGBE_RX_LOCK(rxr); 3472 i = rxr->next_to_check; 3473 cur = &rxr->rx_base[i]; 3474 staterr = cur->wb.upper.status_error; 3475 3476 if (!(staterr & IXGBE_RXD_STAT_DD)) { 3477 IXGBE_RX_UNLOCK(rxr); 3478 return FALSE; 3479 } 3480 |
3481 /* Sync the ring */ 3482 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3483 BUS_DMASYNC_POSTREAD); 3484 |
3485 while ((staterr & IXGBE_RXD_STAT_DD) && (count != 0) && 3486 (ifp->if_drv_flags & IFF_DRV_RUNNING)) { |
3487 struct mbuf *sendmp, *mh, *mp; 3488 u16 hlen, plen, hdr; 3489 u8 dopayload, accept_frame, eop; |
3490 |
3491 |
3492 accept_frame = 1; |
3493 hlen = plen = 0; 3494 sendmp = mh = mp = NULL; 3495 3496 /* Sync the buffers */ 3497 bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[i].map, 3498 BUS_DMASYNC_POSTREAD); 3499 3500 /* 3501 ** The way the hardware is configured to 3502 ** split, it will ONLY use the header buffer 3503 ** when header split is enabled, otherwise we 3504 ** get normal behavior, ie, both header and 3505 ** payload are DMA'd into the payload buffer. 3506 ** 3507 ** The fmp test is to catch the case where a 3508 ** packet spans multiple descriptors, in that 3509 ** case only the first header is valid. 3510 */ 3511 if ((ixgbe_rx_hdr_split) && (rxr->fmp == NULL)){ 3512 hdr = le16toh(cur-> 3513 wb.lower.lo_dword.hs_rss.hdr_info); 3514 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3515 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3516 if (hlen > IXGBE_RX_HDR) 3517 hlen = IXGBE_RX_HDR; 3518 plen = le16toh(cur->wb.upper.length); 3519 /* Handle the header mbuf */ 3520 mh = rxr->rx_buffers[i].m_head; 3521 mh->m_len = hlen; 3522 dopayload = IXGBE_CLEAN_HDR; 3523 /* 3524 ** Get the payload length, this 3525 ** could be zero if its a small 3526 ** packet. 3527 */ 3528 if (plen) { 3529 mp = rxr->rx_buffers[i].m_pack; 3530 mp->m_len = plen; 3531 mp->m_next = NULL; 3532 mp->m_flags &= ~M_PKTHDR; 3533 mh->m_next = mp; 3534 mh->m_flags |= M_PKTHDR; 3535 dopayload = IXGBE_CLEAN_ALL; 3536 rxr->rx_split_packets++; 3537 } else { /* small packets */ 3538 mh->m_flags &= ~M_PKTHDR; 3539 mh->m_next = NULL; 3540 } 3541 } else { 3542 /* 3543 ** Either no header split, or a 3544 ** secondary piece of a fragmented 3545 ** split packet. 3546 */ 3547 mh = rxr->rx_buffers[i].m_pack; 3548 mh->m_flags |= M_PKTHDR; 3549 mh->m_len = le16toh(cur->wb.upper.length); 3550 dopayload = IXGBE_CLEAN_PKT; 3551 } 3552 |
3553 if (staterr & IXGBE_RXD_STAT_EOP) { 3554 count--; 3555 eop = 1; |
3556 } else |
3557 eop = 0; |
3558 3559 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) 3560 accept_frame = 0; 3561 3562 if (accept_frame) { |
3563 if (ixgbe_get_buf(rxr, i, dopayload) != 0) { |
3564 ifp->if_iqdrops++; 3565 goto discard; 3566 } |
3567 /* Initial frame - setup */ |
3568 if (rxr->fmp == NULL) { |
3569 mh->m_flags |= M_PKTHDR; 3570 mh->m_pkthdr.len = mh->m_len; 3571 rxr->fmp = mh; /* Store the first mbuf */ 3572 rxr->lmp = mh; 3573 if (mp) { /* Add payload if split */ 3574 mh->m_pkthdr.len += mp->m_len; 3575 rxr->lmp = mh->m_next; 3576 } |
3577 } else { 3578 /* Chain mbuf's together */ |
3579 mh->m_flags &= ~M_PKTHDR; 3580 rxr->lmp->m_next = mh; |
3581 rxr->lmp = rxr->lmp->m_next; |
3582 rxr->fmp->m_pkthdr.len += mh->m_len; |
3583 } 3584 3585 if (eop) { 3586 rxr->fmp->m_pkthdr.rcvif = ifp; 3587 ifp->if_ipackets++; |
3588 rxr->rx_packets++; 3589 /* capture data for AIM */ 3590 rxr->bytes += rxr->fmp->m_pkthdr.len; 3591 rxr->rx_bytes += rxr->bytes; 3592 if (ifp->if_capenable & IFCAP_RXCSUM) 3593 ixgbe_rx_checksum(staterr, rxr->fmp); 3594 else 3595 rxr->fmp->m_pkthdr.csum_flags = 0; |
3596 if (staterr & IXGBE_RXD_STAT_VP) { |
3597 rxr->fmp->m_pkthdr.ether_vtag = |
3598 le16toh(cur->wb.upper.vlan); 3599 rxr->fmp->m_flags |= M_VLANTAG; |
3600 } |
3601 sendmp = rxr->fmp; |
3602 rxr->fmp = NULL; 3603 rxr->lmp = NULL; 3604 } 3605 } else { 3606 ifp->if_ierrors++; 3607discard: 3608 /* Reuse loaded DMA map and just update mbuf chain */ |
3609 if (hlen) { 3610 mh = rxr->rx_buffers[i].m_head; 3611 mh->m_len = MHLEN; 3612 mh->m_next = NULL; 3613 } 3614 mp = rxr->rx_buffers[i].m_pack; 3615 mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz; |
3616 mp->m_data = mp->m_ext.ext_buf; 3617 mp->m_next = NULL; |
3618 if (adapter->max_frame_size <= 3619 (MCLBYTES - ETHER_ALIGN)) |
3620 m_adj(mp, ETHER_ALIGN); 3621 if (rxr->fmp != NULL) { |
3622 /* handles the whole chain */ |
3623 m_freem(rxr->fmp); 3624 rxr->fmp = NULL; 3625 rxr->lmp = NULL; 3626 } |
3627 sendmp = NULL; |
3628 } |
3629 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3630 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3631 3632 rxr->last_cleaned = i; /* for updating tail */ 3633 3634 if (++i == adapter->num_rx_desc) 3635 i = 0; 3636 |
3637 /* 3638 ** Now send up to the stack, 3639 ** note the the value of next_to_check 3640 ** is safe because we keep the RX lock 3641 ** thru this call. 3642 */ 3643 if (sendmp != NULL) { |
3644 /* Use LRO if possible */ |
3645 if ((!lro->lro_cnt) || (tcp_lro_rx(lro, sendmp, 0))) 3646 (*ifp->if_input)(ifp, sendmp); |
3647 } |
3648 |
3649 /* Get next descriptor */ 3650 cur = &rxr->rx_base[i]; 3651 staterr = cur->wb.upper.status_error; 3652 } 3653 rxr->next_to_check = i; 3654 3655 /* Advance the IXGB's Receive Queue "Tail Pointer" */ 3656 IXGBE_WRITE_REG(&adapter->hw, IXGBE_RDT(rxr->me), rxr->last_cleaned); |
3657 3658 /* |
3659 * Flush any outstanding LRO work 3660 */ 3661 while (!SLIST_EMPTY(&lro->lro_active)) { |
3662 queued = SLIST_FIRST(&lro->lro_active); 3663 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3664 tcp_lro_flush(lro, queued); 3665 } 3666 |
3667 IXGBE_RX_UNLOCK(rxr); |
3668 |
3669 /* 3670 ** Leaving with more to clean? 3671 ** then schedule another interrupt. 3672 */ 3673 if (staterr & IXGBE_RXD_STAT_DD) { 3674 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, rxr->eims); 3675 return TRUE; 3676 } 3677 3678 return FALSE; |
3679} 3680 3681/********************************************************************* 3682 * 3683 * Verify that the hardware indicated that the checksum is valid. 3684 * Inform the stack about the status of checksum so that stack 3685 * doesn't spend time verifying the checksum. 3686 * 3687 *********************************************************************/ 3688static void |
3689ixgbe_rx_checksum(u32 staterr, struct mbuf * mp) |
3690{ |
3691 u16 status = (u16) staterr; 3692 u8 errors = (u8) (staterr >> 24); 3693 |
3694 if (status & IXGBE_RXD_STAT_IPCS) { 3695 /* Did it pass? */ 3696 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3697 /* IP Checksum Good */ 3698 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 3699 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 3700 3701 } else --- 5 unchanged lines hidden (view full) --- 3707 mp->m_pkthdr.csum_flags |= 3708 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3709 mp->m_pkthdr.csum_data = htons(0xffff); 3710 } 3711 } 3712 return; 3713} 3714 |
3715 3716#ifdef IXGBE_HW_VLAN_SUPPORT |
3717/* 3718 * This routine is run via an vlan 3719 * config EVENT 3720 */ 3721static void 3722ixgbe_register_vlan(void *unused, struct ifnet *ifp, u16 vtag) 3723{ 3724 struct adapter *adapter = ifp->if_softc; |
3725 u32 ctrl, rctl, index, vfta; |
3726 3727 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); 3728 ctrl |= IXGBE_VLNCTRL_VME | IXGBE_VLNCTRL_VFE; 3729 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3730 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); 3731 3732 /* Make entry in the hardware filter table */ 3733 ixgbe_set_vfta(&adapter->hw, vtag, 0, TRUE); 3734} 3735 3736/* 3737 * This routine is run via an vlan 3738 * unconfig EVENT 3739 */ 3740static void 3741ixgbe_unregister_vlan(void *unused, struct ifnet *ifp, u16 vtag) 3742{ 3743 struct adapter *adapter = ifp->if_softc; |
3744 u32 index, vfta; |
3745 3746 /* Remove entry in the hardware filter table */ 3747 ixgbe_set_vfta(&adapter->hw, vtag, 0, FALSE); 3748 3749 /* Have all vlans unregistered? */ 3750 if (adapter->ifp->if_vlantrunk == NULL) { 3751 u32 ctrl; 3752 /* Turn off the filter table */ 3753 ctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_VLNCTRL); 3754 ctrl &= ~IXGBE_VLNCTRL_VME; 3755 ctrl &= ~IXGBE_VLNCTRL_VFE; 3756 ctrl |= IXGBE_VLNCTRL_CFIEN; 3757 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VLNCTRL, ctrl); 3758 } 3759} |
3760#endif |
3761 3762static void 3763ixgbe_enable_intr(struct adapter *adapter) 3764{ 3765 struct ixgbe_hw *hw = &adapter->hw; 3766 u32 mask = IXGBE_EIMS_ENABLE_MASK; 3767 3768 /* Enable Fan Failure detection */ 3769 if (hw->phy.media_type == ixgbe_media_type_copper) 3770 mask |= IXGBE_EIMS_GPI_SDP1; |
3771 3772 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3773 |
3774 /* With RSS we use auto clear */ 3775 if (adapter->msix_mem) { 3776 /* Dont autoclear Link */ 3777 mask &= ~IXGBE_EIMS_OTHER; 3778 mask &= ~IXGBE_EIMS_LSC; |
3779 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); |
3780 } 3781 |
3782 IXGBE_WRITE_FLUSH(hw); 3783 3784 return; 3785} 3786 3787static void 3788ixgbe_disable_intr(struct adapter *adapter) 3789{ --- 10 unchanged lines hidden (view full) --- 3800 u16 value; 3801 3802 value = pci_read_config(((struct ixgbe_osdep *)hw->back)->dev, 3803 reg, 2); 3804 3805 return (value); 3806} 3807 |
3808/* 3809** Setup the correct IVAR register for a particular MSIX interrupt 3810** (yes this is all very magic and confusing :) 3811** - entry is the register array entry 3812** - vector is the MSIX vector for this queue 3813** - type is RX/TX/MISC 3814*/ |
3815static void |
3816ixgbe_set_ivar(struct adapter *adapter, u16 entry, u8 vector, s8 type) |
3817{ |
3818 struct ixgbe_hw *hw = &adapter->hw; |
3819 u32 ivar, index; 3820 3821 vector |= IXGBE_IVAR_ALLOC_VAL; |
3822 3823 switch (hw->mac.type) { 3824 3825 case ixgbe_mac_82598EB: 3826 if (type == -1) 3827 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3828 else 3829 entry += (type * 64); 3830 index = (entry >> 2) & 0x1F; 3831 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3832 ivar &= ~(0xFF << (8 * (entry & 0x3))); 3833 ivar |= (vector << (8 * (entry & 0x3))); 3834 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); 3835 break; 3836 3837 default: 3838 break; 3839 } |
3840} 3841 3842static void 3843ixgbe_configure_ivars(struct adapter *adapter) 3844{ 3845 struct tx_ring *txr = adapter->tx_rings; 3846 struct rx_ring *rxr = adapter->rx_rings; 3847 |
3848 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) 3849 ixgbe_set_ivar(adapter, i, rxr->msix, 0); |
3850 |
3851 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) 3852 ixgbe_set_ivar(adapter, i, txr->msix, 1); |
3853 3854 /* For the Link interrupt */ |
3855 ixgbe_set_ivar(adapter, 1, adapter->linkvec, -1); |
3856} 3857 |
3858/* 3859** ixgbe_sfp_probe - called in the local timer to 3860** determine if a port had optics inserted. 3861*/ 3862static bool ixgbe_sfp_probe(struct adapter *adapter) 3863{ 3864 struct ixgbe_hw *hw = &adapter->hw; 3865 device_t dev = adapter->dev; 3866 bool result = FALSE; 3867 3868 if ((hw->phy.type == ixgbe_phy_nl) && 3869 (hw->phy.sfp_type == ixgbe_sfp_type_not_present)) { 3870 s32 ret = hw->phy.ops.identify_sfp(hw); 3871 if (ret) 3872 goto out; 3873 ret = hw->phy.ops.reset(hw); 3874 if (ret == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3875 device_printf(dev,"Unsupported SFP+ module detected!"); 3876 printf(" Reload driver with supported module.\n"); 3877 adapter->sfp_probe = FALSE; 3878 goto out; 3879 } else 3880 device_printf(dev,"SFP+ module detected!\n"); 3881 /* We now have supported optics */ 3882 adapter->sfp_probe = FALSE; 3883 result = TRUE; 3884 } 3885out: 3886 return (result); 3887} 3888 3889 |
3890/********************************************************************** 3891 * 3892 * Update the board statistics counters. 3893 * 3894 **********************************************************************/ 3895static void 3896ixgbe_update_stats_counters(struct adapter *adapter) 3897{ --- 32 unchanged lines hidden (view full) --- 3930 adapter->stats.prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64); 3931 adapter->stats.prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127); 3932 adapter->stats.prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255); 3933 adapter->stats.prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511); 3934 adapter->stats.prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023); 3935 adapter->stats.prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522); 3936 adapter->stats.rlec += IXGBE_READ_REG(hw, IXGBE_RLEC); 3937 |
3938 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC); 3939 adapter->stats.lxontxc += lxon; 3940 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC); 3941 adapter->stats.lxofftxc += lxoff; 3942 total = lxon + lxoff; 3943 3944 adapter->stats.gptc += IXGBE_READ_REG(hw, IXGBE_GPTC); 3945 adapter->stats.mptc += IXGBE_READ_REG(hw, IXGBE_MPTC); --- 38 unchanged lines hidden (view full) --- 3984 **********************************************************************/ 3985static void 3986ixgbe_print_hw_stats(struct adapter * adapter) 3987{ 3988 device_t dev = adapter->dev; 3989 3990 3991 device_printf(dev,"Std Mbuf Failed = %lu\n", |
3992 adapter->mbuf_defrag_failed); |
3993 device_printf(dev,"Missed Packets = %llu\n", 3994 (long long)adapter->stats.mpc[0]); 3995 device_printf(dev,"Receive length errors = %llu\n", 3996 ((long long)adapter->stats.roc + 3997 (long long)adapter->stats.ruc)); 3998 device_printf(dev,"Crc errors = %llu\n", 3999 (long long)adapter->stats.crcerrs); 4000 device_printf(dev,"Driver dropped packets = %lu\n", --- 40 unchanged lines hidden (view full) --- 4041 device_printf(dev,"Error Byte Count = %u \n", 4042 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 4043 4044 for (int i = 0; i < adapter->num_rx_queues; i++, rxr++) { 4045 struct lro_ctrl *lro = &rxr->lro; 4046 device_printf(dev,"Queue[%d]: rdh = %d, hw rdt = %d\n", 4047 i, IXGBE_READ_REG(hw, IXGBE_RDH(i)), 4048 IXGBE_READ_REG(hw, IXGBE_RDT(i))); |
4049 device_printf(dev,"RX(%d) Packets Received: %lld\n", 4050 rxr->me, (long long)rxr->rx_packets); 4051 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 4052 rxr->me, (long long)rxr->rx_split_packets); |
4053 device_printf(dev,"RX(%d) Bytes Received: %lu\n", |
4054 rxr->me, (long)rxr->rx_bytes); |
4055 device_printf(dev,"RX(%d) IRQ Handled: %lu\n", 4056 rxr->me, (long)rxr->rx_irq); 4057 device_printf(dev,"RX(%d) LRO Queued= %d\n", 4058 rxr->me, lro->lro_queued); 4059 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 4060 rxr->me, lro->lro_flushed); 4061 } 4062 4063 for (int i = 0; i < adapter->num_tx_queues; i++, txr++) { 4064 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", i, 4065 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 4066 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 4067 device_printf(dev,"TX(%d) Packets Sent: %lu\n", |
4068 txr->me, (long)txr->total_packets); |
4069 device_printf(dev,"TX(%d) IRQ Handled: %lu\n", 4070 txr->me, (long)txr->tx_irq); 4071 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 4072 txr->me, (long)txr->no_tx_desc_avail); 4073 } 4074 4075 device_printf(dev,"Link IRQ Handled: %lu\n", 4076 (long)adapter->link_irq); --- 58 unchanged lines hidden (view full) --- 4135 if (error) 4136 return (error); 4137 4138 adapter = (struct adapter *) arg1; 4139 switch (ixgbe_flow_control) { 4140 case ixgbe_fc_rx_pause: 4141 case ixgbe_fc_tx_pause: 4142 case ixgbe_fc_full: |
4143 adapter->hw.fc.requested_mode = ixgbe_flow_control; |
4144 break; 4145 case ixgbe_fc_none: 4146 default: |
4147 adapter->hw.fc.requested_mode = ixgbe_fc_none; |
4148 } 4149 4150 ixgbe_setup_fc(&adapter->hw, 0); 4151 return error; 4152} 4153 4154static void 4155ixgbe_add_rx_process_limit(struct adapter *adapter, const char *name, 4156 const char *description, int *limit, int value) 4157{ 4158 *limit = value; 4159 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 4160 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 4161 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 4162} |