ixv.c revision 219902
1/****************************************************************************** 2 3 Copyright (c) 2001-2010, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ 33/*$FreeBSD: head/sys/dev/ixgbe/ixv.c 219902 2011-03-23 13:10:15Z jhb $*/ 34 35#ifdef HAVE_KERNEL_OPTION_HEADERS 36#include "opt_device_polling.h" 37#endif 38 39#include "ixv.h" 40 41/********************************************************************* 42 * Driver version 43 *********************************************************************/ 44char ixv_driver_version[] = "1.0.0"; 45 46/********************************************************************* 47 * PCI Device ID Table 48 * 49 * Used by probe to select devices to load on 50 * Last field stores an index into ixv_strings 51 * Last entry must be all 0s 52 * 53 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 54 *********************************************************************/ 55 56static ixv_vendor_info_t ixv_vendor_info_array[] = 57{ 58 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 59 /* required last entry */ 60 {0, 0, 0, 0, 0} 61}; 62 63/********************************************************************* 64 * Table of branding strings 65 *********************************************************************/ 66 67static char *ixv_strings[] = { 68 "Intel(R) PRO/10GbE Virtual Function Network Driver" 69}; 70 71/********************************************************************* 72 * Function prototypes 73 *********************************************************************/ 74static int ixv_probe(device_t); 75static int ixv_attach(device_t); 76static int ixv_detach(device_t); 77static int ixv_shutdown(device_t); 78#if __FreeBSD_version < 800000 79static void ixv_start(struct ifnet *); 80static void ixv_start_locked(struct tx_ring *, struct ifnet *); 81#else 82static int ixv_mq_start(struct ifnet *, struct mbuf *); 83static int ixv_mq_start_locked(struct ifnet *, 84 struct tx_ring *, struct mbuf *); 85static void ixv_qflush(struct ifnet *); 86#endif 87static int ixv_ioctl(struct ifnet *, u_long, caddr_t); 88static void ixv_init(void *); 89static void ixv_init_locked(struct adapter *); 90static void ixv_stop(void *); 91static void ixv_media_status(struct ifnet *, struct ifmediareq *); 92static int ixv_media_change(struct ifnet *); 93static void ixv_identify_hardware(struct adapter *); 94static int ixv_allocate_pci_resources(struct adapter *); 95static int ixv_allocate_msix(struct adapter *); 96static int ixv_allocate_queues(struct adapter *); 97static int ixv_setup_msix(struct adapter *); 98static void ixv_free_pci_resources(struct adapter *); 99static void ixv_local_timer(void *); 100static void ixv_setup_interface(device_t, struct adapter *); 101static void ixv_config_link(struct adapter *); 102 103static int ixv_allocate_transmit_buffers(struct tx_ring *); 104static int ixv_setup_transmit_structures(struct adapter *); 105static void ixv_setup_transmit_ring(struct tx_ring *); 106static void ixv_initialize_transmit_units(struct adapter *); 107static void ixv_free_transmit_structures(struct adapter *); 108static void ixv_free_transmit_buffers(struct tx_ring *); 109 110static int ixv_allocate_receive_buffers(struct rx_ring *); 111static int ixv_setup_receive_structures(struct adapter *); 112static int ixv_setup_receive_ring(struct rx_ring *); 113static void ixv_initialize_receive_units(struct adapter *); 114static void ixv_free_receive_structures(struct adapter *); 115static void ixv_free_receive_buffers(struct rx_ring *); 116 117static void ixv_enable_intr(struct adapter *); 118static void ixv_disable_intr(struct adapter *); 119static bool ixv_txeof(struct tx_ring *); 120static bool ixv_rxeof(struct ix_queue *, int); 121static void ixv_rx_checksum(u32, struct mbuf *, u32); 122static void ixv_set_multi(struct adapter *); 123static void ixv_update_link_status(struct adapter *); 124static void ixv_refresh_mbufs(struct rx_ring *, int); 125static int ixv_xmit(struct tx_ring *, struct mbuf **); 126static int ixv_sysctl_stats(SYSCTL_HANDLER_ARGS); 127static int ixv_sysctl_debug(SYSCTL_HANDLER_ARGS); 128static int ixv_set_flowcntl(SYSCTL_HANDLER_ARGS); 129static int ixv_dma_malloc(struct adapter *, bus_size_t, 130 struct ixv_dma_alloc *, int); 131static void ixv_dma_free(struct adapter *, struct ixv_dma_alloc *); 132static void ixv_add_rx_process_limit(struct adapter *, const char *, 133 const char *, int *, int); 134static bool ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *); 135static bool ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 136static void ixv_set_ivar(struct adapter *, u8, u8, s8); 137static void ixv_configure_ivars(struct adapter *); 138static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 139 140static void ixv_setup_vlan_support(struct adapter *); 141static void ixv_register_vlan(void *, struct ifnet *, u16); 142static void ixv_unregister_vlan(void *, struct ifnet *, u16); 143 144static void ixv_save_stats(struct adapter *); 145static void ixv_init_stats(struct adapter *); 146static void ixv_update_stats(struct adapter *); 147 148static __inline void ixv_rx_discard(struct rx_ring *, int); 149static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *, 150 struct mbuf *, u32); 151 152/* The MSI/X Interrupt handlers */ 153static void ixv_msix_que(void *); 154static void ixv_msix_mbx(void *); 155 156/* Deferred interrupt tasklets */ 157static void ixv_handle_que(void *, int); 158static void ixv_handle_mbx(void *, int); 159 160/********************************************************************* 161 * FreeBSD Device Interface Entry Points 162 *********************************************************************/ 163 164static device_method_t ixv_methods[] = { 165 /* Device interface */ 166 DEVMETHOD(device_probe, ixv_probe), 167 DEVMETHOD(device_attach, ixv_attach), 168 DEVMETHOD(device_detach, ixv_detach), 169 DEVMETHOD(device_shutdown, ixv_shutdown), 170 {0, 0} 171}; 172 173static driver_t ixv_driver = { 174 "ix", ixv_methods, sizeof(struct adapter), 175}; 176 177extern devclass_t ixgbe_devclass; 178DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0); 179MODULE_DEPEND(ixv, pci, 1, 1, 1); 180MODULE_DEPEND(ixv, ether, 1, 1, 1); 181 182/* 183** TUNEABLE PARAMETERS: 184*/ 185 186/* 187** AIM: Adaptive Interrupt Moderation 188** which means that the interrupt rate 189** is varied over time based on the 190** traffic for that interrupt vector 191*/ 192static int ixv_enable_aim = FALSE; 193TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 194 195/* How many packets rxeof tries to clean at a time */ 196static int ixv_rx_process_limit = 128; 197TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 198 199/* Flow control setting, default to full */ 200static int ixv_flow_control = ixgbe_fc_full; 201TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control); 202 203/* 204 * Header split: this causes the hardware to DMA 205 * the header into a seperate mbuf from the payload, 206 * it can be a performance win in some workloads, but 207 * in others it actually hurts, its off by default. 208 */ 209static bool ixv_header_split = FALSE; 210TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split); 211 212/* 213** Number of TX descriptors per ring, 214** setting higher than RX as this seems 215** the better performing choice. 216*/ 217static int ixv_txd = DEFAULT_TXD; 218TUNABLE_INT("hw.ixv.txd", &ixv_txd); 219 220/* Number of RX descriptors per ring */ 221static int ixv_rxd = DEFAULT_RXD; 222TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 223 224/* 225** Shadow VFTA table, this is needed because 226** the real filter table gets cleared during 227** a soft reset and we need to repopulate it. 228*/ 229static u32 ixv_shadow_vfta[VFTA_SIZE]; 230 231/********************************************************************* 232 * Device identification routine 233 * 234 * ixv_probe determines if the driver should be loaded on 235 * adapter based on PCI vendor/device id of the adapter. 236 * 237 * return 0 on success, positive on failure 238 *********************************************************************/ 239 240static int 241ixv_probe(device_t dev) 242{ 243 ixv_vendor_info_t *ent; 244 245 u16 pci_vendor_id = 0; 246 u16 pci_device_id = 0; 247 u16 pci_subvendor_id = 0; 248 u16 pci_subdevice_id = 0; 249 char adapter_name[256]; 250 251 252 pci_vendor_id = pci_get_vendor(dev); 253 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) 254 return (ENXIO); 255 256 pci_device_id = pci_get_device(dev); 257 pci_subvendor_id = pci_get_subvendor(dev); 258 pci_subdevice_id = pci_get_subdevice(dev); 259 260 ent = ixv_vendor_info_array; 261 while (ent->vendor_id != 0) { 262 if ((pci_vendor_id == ent->vendor_id) && 263 (pci_device_id == ent->device_id) && 264 265 ((pci_subvendor_id == ent->subvendor_id) || 266 (ent->subvendor_id == 0)) && 267 268 ((pci_subdevice_id == ent->subdevice_id) || 269 (ent->subdevice_id == 0))) { 270 sprintf(adapter_name, "%s, Version - %s", 271 ixv_strings[ent->index], 272 ixv_driver_version); 273 device_set_desc_copy(dev, adapter_name); 274 return (0); 275 } 276 ent++; 277 } 278 return (ENXIO); 279} 280 281/********************************************************************* 282 * Device initialization routine 283 * 284 * The attach entry point is called when the driver is being loaded. 285 * This routine identifies the type of hardware, allocates all resources 286 * and initializes the hardware. 287 * 288 * return 0 on success, positive on failure 289 *********************************************************************/ 290 291static int 292ixv_attach(device_t dev) 293{ 294 struct adapter *adapter; 295 struct ixgbe_hw *hw; 296 int error = 0; 297 298 INIT_DEBUGOUT("ixv_attach: begin"); 299 300 /* Allocate, clear, and link in our adapter structure */ 301 adapter = device_get_softc(dev); 302 adapter->dev = adapter->osdep.dev = dev; 303 hw = &adapter->hw; 304 305 /* Core Lock Init*/ 306 IXV_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 307 308 /* SYSCTL APIs */ 309 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 310 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 311 OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW, 312 adapter, 0, ixv_sysctl_stats, "I", "Statistics"); 313 314 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 315 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 316 OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW, 317 adapter, 0, ixv_sysctl_debug, "I", "Debug Info"); 318 319 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 320 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 321 OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW, 322 adapter, 0, ixv_set_flowcntl, "I", "Flow Control"); 323 324 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 325 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 326 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 327 &ixv_enable_aim, 1, "Interrupt Moderation"); 328 329 /* Set up the timer callout */ 330 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 331 332 /* Determine hardware revision */ 333 ixv_identify_hardware(adapter); 334 335 /* Do base PCI setup - map BAR0 */ 336 if (ixv_allocate_pci_resources(adapter)) { 337 device_printf(dev, "Allocation of PCI resources failed\n"); 338 error = ENXIO; 339 goto err_out; 340 } 341 342 /* Do descriptor calc and sanity checks */ 343 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 344 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 345 device_printf(dev, "TXD config issue, using default!\n"); 346 adapter->num_tx_desc = DEFAULT_TXD; 347 } else 348 adapter->num_tx_desc = ixv_txd; 349 350 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 351 ixv_rxd < MIN_TXD || ixv_rxd > MAX_TXD) { 352 device_printf(dev, "RXD config issue, using default!\n"); 353 adapter->num_rx_desc = DEFAULT_RXD; 354 } else 355 adapter->num_rx_desc = ixv_rxd; 356 357 /* Allocate our TX/RX Queues */ 358 if (ixv_allocate_queues(adapter)) { 359 error = ENOMEM; 360 goto err_out; 361 } 362 363 /* 364 ** Initialize the shared code: its 365 ** at this point the mac type is set. 366 */ 367 error = ixgbe_init_shared_code(hw); 368 if (error) { 369 device_printf(dev,"Shared Code Initialization Failure\n"); 370 error = EIO; 371 goto err_late; 372 } 373 374 /* Setup the mailbox */ 375 ixgbe_init_mbx_params_vf(hw); 376 377 ixgbe_reset_hw(hw); 378 379 /* Get Hardware Flow Control setting */ 380 hw->fc.requested_mode = ixgbe_fc_full; 381 hw->fc.pause_time = IXV_FC_PAUSE; 382 hw->fc.low_water = IXV_FC_LO; 383 hw->fc.high_water = IXV_FC_HI; 384 hw->fc.send_xon = TRUE; 385 386 error = ixgbe_init_hw(hw); 387 if (error) { 388 device_printf(dev,"Hardware Initialization Failure\n"); 389 error = EIO; 390 goto err_late; 391 } 392 393 error = ixv_allocate_msix(adapter); 394 if (error) 395 goto err_late; 396 397 /* Setup OS specific network interface */ 398 ixv_setup_interface(dev, adapter); 399 400 /* Sysctl for limiting the amount of work done in the taskqueue */ 401 ixv_add_rx_process_limit(adapter, "rx_processing_limit", 402 "max number of rx packets to process", &adapter->rx_process_limit, 403 ixv_rx_process_limit); 404 405 /* Do the stats setup */ 406 ixv_save_stats(adapter); 407 ixv_init_stats(adapter); 408 409 /* Register for VLAN events */ 410 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 411 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 412 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 413 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 414 415 INIT_DEBUGOUT("ixv_attach: end"); 416 return (0); 417 418err_late: 419 ixv_free_transmit_structures(adapter); 420 ixv_free_receive_structures(adapter); 421err_out: 422 ixv_free_pci_resources(adapter); 423 return (error); 424 425} 426 427/********************************************************************* 428 * Device removal routine 429 * 430 * The detach entry point is called when the driver is being removed. 431 * This routine stops the adapter and deallocates all the resources 432 * that were allocated for driver operation. 433 * 434 * return 0 on success, positive on failure 435 *********************************************************************/ 436 437static int 438ixv_detach(device_t dev) 439{ 440 struct adapter *adapter = device_get_softc(dev); 441 struct ix_queue *que = adapter->queues; 442 443 INIT_DEBUGOUT("ixv_detach: begin"); 444 445 /* Make sure VLANS are not using driver */ 446 if (adapter->ifp->if_vlantrunk != NULL) { 447 device_printf(dev,"Vlan in use, detach first\n"); 448 return (EBUSY); 449 } 450 451 IXV_CORE_LOCK(adapter); 452 ixv_stop(adapter); 453 IXV_CORE_UNLOCK(adapter); 454 455 for (int i = 0; i < adapter->num_queues; i++, que++) { 456 if (que->tq) { 457 taskqueue_drain(que->tq, &que->que_task); 458 taskqueue_free(que->tq); 459 } 460 } 461 462 /* Drain the Link queue */ 463 if (adapter->tq) { 464 taskqueue_drain(adapter->tq, &adapter->mbx_task); 465 taskqueue_free(adapter->tq); 466 } 467 468 /* Unregister VLAN events */ 469 if (adapter->vlan_attach != NULL) 470 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 471 if (adapter->vlan_detach != NULL) 472 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 473 474 ether_ifdetach(adapter->ifp); 475 callout_drain(&adapter->timer); 476 ixv_free_pci_resources(adapter); 477 bus_generic_detach(dev); 478 if_free(adapter->ifp); 479 480 ixv_free_transmit_structures(adapter); 481 ixv_free_receive_structures(adapter); 482 483 IXV_CORE_LOCK_DESTROY(adapter); 484 return (0); 485} 486 487/********************************************************************* 488 * 489 * Shutdown entry point 490 * 491 **********************************************************************/ 492static int 493ixv_shutdown(device_t dev) 494{ 495 struct adapter *adapter = device_get_softc(dev); 496 IXV_CORE_LOCK(adapter); 497 ixv_stop(adapter); 498 IXV_CORE_UNLOCK(adapter); 499 return (0); 500} 501 502#if __FreeBSD_version < 800000 503/********************************************************************* 504 * Transmit entry point 505 * 506 * ixv_start is called by the stack to initiate a transmit. 507 * The driver will remain in this routine as long as there are 508 * packets to transmit and transmit resources are available. 509 * In case resources are not available stack is notified and 510 * the packet is requeued. 511 **********************************************************************/ 512static void 513ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp) 514{ 515 struct mbuf *m_head; 516 struct adapter *adapter = txr->adapter; 517 518 IXV_TX_LOCK_ASSERT(txr); 519 520 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 521 IFF_DRV_RUNNING) 522 return; 523 if (!adapter->link_active) 524 return; 525 526 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 527 528 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 529 if (m_head == NULL) 530 break; 531 532 if (ixv_xmit(txr, &m_head)) { 533 if (m_head == NULL) 534 break; 535 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 536 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 537 break; 538 } 539 /* Send a copy of the frame to the BPF listener */ 540 ETHER_BPF_MTAP(ifp, m_head); 541 542 /* Set watchdog on */ 543 txr->watchdog_check = TRUE; 544 txr->watchdog_time = ticks; 545 546 } 547 return; 548} 549 550/* 551 * Legacy TX start - called by the stack, this 552 * always uses the first tx ring, and should 553 * not be used with multiqueue tx enabled. 554 */ 555static void 556ixv_start(struct ifnet *ifp) 557{ 558 struct adapter *adapter = ifp->if_softc; 559 struct tx_ring *txr = adapter->tx_rings; 560 561 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 562 IXV_TX_LOCK(txr); 563 ixv_start_locked(txr, ifp); 564 IXV_TX_UNLOCK(txr); 565 } 566 return; 567} 568 569#else 570 571/* 572** Multiqueue Transmit driver 573** 574*/ 575static int 576ixv_mq_start(struct ifnet *ifp, struct mbuf *m) 577{ 578 struct adapter *adapter = ifp->if_softc; 579 struct ix_queue *que; 580 struct tx_ring *txr; 581 int i = 0, err = 0; 582 583 /* Which queue to use */ 584 if ((m->m_flags & M_FLOWID) != 0) 585 i = m->m_pkthdr.flowid % adapter->num_queues; 586 587 txr = &adapter->tx_rings[i]; 588 que = &adapter->queues[i]; 589 590 if (IXV_TX_TRYLOCK(txr)) { 591 err = ixv_mq_start_locked(ifp, txr, m); 592 IXV_TX_UNLOCK(txr); 593 } else { 594 err = drbr_enqueue(ifp, txr->br, m); 595 taskqueue_enqueue(que->tq, &que->que_task); 596 } 597 598 return (err); 599} 600 601static int 602ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 603{ 604 struct adapter *adapter = txr->adapter; 605 struct mbuf *next; 606 int enqueued, err = 0; 607 608 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 609 IFF_DRV_RUNNING || adapter->link_active == 0) { 610 if (m != NULL) 611 err = drbr_enqueue(ifp, txr->br, m); 612 return (err); 613 } 614 615 /* Do a clean if descriptors are low */ 616 if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD) 617 ixv_txeof(txr); 618 619 enqueued = 0; 620 if (m == NULL) { 621 next = drbr_dequeue(ifp, txr->br); 622 } else if (drbr_needs_enqueue(ifp, txr->br)) { 623 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 624 return (err); 625 next = drbr_dequeue(ifp, txr->br); 626 } else 627 next = m; 628 629 /* Process the queue */ 630 while (next != NULL) { 631 if ((err = ixv_xmit(txr, &next)) != 0) { 632 if (next != NULL) 633 err = drbr_enqueue(ifp, txr->br, next); 634 break; 635 } 636 enqueued++; 637 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 638 /* Send a copy of the frame to the BPF listener */ 639 ETHER_BPF_MTAP(ifp, next); 640 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 641 break; 642 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) { 643 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 644 break; 645 } 646 next = drbr_dequeue(ifp, txr->br); 647 } 648 649 if (enqueued > 0) { 650 /* Set watchdog on */ 651 txr->watchdog_check = TRUE; 652 txr->watchdog_time = ticks; 653 } 654 655 return (err); 656} 657 658/* 659** Flush all ring buffers 660*/ 661static void 662ixv_qflush(struct ifnet *ifp) 663{ 664 struct adapter *adapter = ifp->if_softc; 665 struct tx_ring *txr = adapter->tx_rings; 666 struct mbuf *m; 667 668 for (int i = 0; i < adapter->num_queues; i++, txr++) { 669 IXV_TX_LOCK(txr); 670 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 671 m_freem(m); 672 IXV_TX_UNLOCK(txr); 673 } 674 if_qflush(ifp); 675} 676 677#endif 678 679/********************************************************************* 680 * Ioctl entry point 681 * 682 * ixv_ioctl is called when the user wants to configure the 683 * interface. 684 * 685 * return 0 on success, positive on failure 686 **********************************************************************/ 687 688static int 689ixv_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 690{ 691 struct adapter *adapter = ifp->if_softc; 692 struct ifreq *ifr = (struct ifreq *) data; 693 int error = 0; 694 695 switch (command) { 696 697 case SIOCSIFMTU: 698 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 699 if (ifr->ifr_mtu > IXV_MAX_FRAME_SIZE - ETHER_HDR_LEN) { 700 error = EINVAL; 701 } else { 702 IXV_CORE_LOCK(adapter); 703 ifp->if_mtu = ifr->ifr_mtu; 704 adapter->max_frame_size = 705 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 706 ixv_init_locked(adapter); 707 IXV_CORE_UNLOCK(adapter); 708 } 709 break; 710 case SIOCSIFFLAGS: 711 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 712 IXV_CORE_LOCK(adapter); 713 if (ifp->if_flags & IFF_UP) { 714 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 715 ixv_init_locked(adapter); 716 } else 717 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 718 ixv_stop(adapter); 719 adapter->if_flags = ifp->if_flags; 720 IXV_CORE_UNLOCK(adapter); 721 break; 722 case SIOCADDMULTI: 723 case SIOCDELMULTI: 724 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 725 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 726 IXV_CORE_LOCK(adapter); 727 ixv_disable_intr(adapter); 728 ixv_set_multi(adapter); 729 ixv_enable_intr(adapter); 730 IXV_CORE_UNLOCK(adapter); 731 } 732 break; 733 case SIOCSIFMEDIA: 734 case SIOCGIFMEDIA: 735 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 736 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 737 break; 738 case SIOCSIFCAP: 739 { 740 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 741 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 742 if (mask & IFCAP_HWCSUM) 743 ifp->if_capenable ^= IFCAP_HWCSUM; 744 if (mask & IFCAP_TSO4) 745 ifp->if_capenable ^= IFCAP_TSO4; 746 if (mask & IFCAP_LRO) 747 ifp->if_capenable ^= IFCAP_LRO; 748 if (mask & IFCAP_VLAN_HWTAGGING) 749 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 750 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 751 IXV_CORE_LOCK(adapter); 752 ixv_init_locked(adapter); 753 IXV_CORE_UNLOCK(adapter); 754 } 755 VLAN_CAPABILITIES(ifp); 756 break; 757 } 758 759 default: 760 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 761 error = ether_ioctl(ifp, command, data); 762 break; 763 } 764 765 return (error); 766} 767 768/********************************************************************* 769 * Init entry point 770 * 771 * This routine is used in two ways. It is used by the stack as 772 * init entry point in network interface structure. It is also used 773 * by the driver as a hw/sw initialization routine to get to a 774 * consistent state. 775 * 776 * return 0 on success, positive on failure 777 **********************************************************************/ 778#define IXGBE_MHADD_MFS_SHIFT 16 779 780static void 781ixv_init_locked(struct adapter *adapter) 782{ 783 struct ifnet *ifp = adapter->ifp; 784 device_t dev = adapter->dev; 785 struct ixgbe_hw *hw = &adapter->hw; 786 u32 mhadd, gpie; 787 788 INIT_DEBUGOUT("ixv_init: begin"); 789 mtx_assert(&adapter->core_mtx, MA_OWNED); 790 hw->adapter_stopped = FALSE; 791 ixgbe_stop_adapter(hw); 792 callout_stop(&adapter->timer); 793 794 /* reprogram the RAR[0] in case user changed it. */ 795 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 796 797 /* Get the latest mac address, User can use a LAA */ 798 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr, 799 IXGBE_ETH_LENGTH_OF_ADDRESS); 800 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 801 hw->addr_ctrl.rar_used_count = 1; 802 803 /* Prepare transmit descriptors and buffers */ 804 if (ixv_setup_transmit_structures(adapter)) { 805 device_printf(dev,"Could not setup transmit structures\n"); 806 ixv_stop(adapter); 807 return; 808 } 809 810 ixgbe_reset_hw(hw); 811 ixv_initialize_transmit_units(adapter); 812 813 /* Setup Multicast table */ 814 ixv_set_multi(adapter); 815 816 /* 817 ** Determine the correct mbuf pool 818 ** for doing jumbo/headersplit 819 */ 820 if (ifp->if_mtu > ETHERMTU) 821 adapter->rx_mbuf_sz = MJUMPAGESIZE; 822 else 823 adapter->rx_mbuf_sz = MCLBYTES; 824 825 /* Prepare receive descriptors and buffers */ 826 if (ixv_setup_receive_structures(adapter)) { 827 device_printf(dev,"Could not setup receive structures\n"); 828 ixv_stop(adapter); 829 return; 830 } 831 832 /* Configure RX settings */ 833 ixv_initialize_receive_units(adapter); 834 835 /* Enable Enhanced MSIX mode */ 836 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 837 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME; 838 gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; 839 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 840 841 /* Set the various hardware offload abilities */ 842 ifp->if_hwassist = 0; 843 if (ifp->if_capenable & IFCAP_TSO4) 844 ifp->if_hwassist |= CSUM_TSO; 845 if (ifp->if_capenable & IFCAP_TXCSUM) { 846 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 847#if __FreeBSD_version >= 800000 848 ifp->if_hwassist |= CSUM_SCTP; 849#endif 850 } 851 852 /* Set MTU size */ 853 if (ifp->if_mtu > ETHERMTU) { 854 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 855 mhadd &= ~IXGBE_MHADD_MFS_MASK; 856 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 857 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 858 } 859 860 /* Set up VLAN offload and filter */ 861 ixv_setup_vlan_support(adapter); 862 863 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 864 865 /* Set up MSI/X routing */ 866 ixv_configure_ivars(adapter); 867 868 /* Set up auto-mask */ 869 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 870 871 /* Set moderation on the Link interrupt */ 872 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR); 873 874 /* Stats init */ 875 ixv_init_stats(adapter); 876 877 /* Config/Enable Link */ 878 ixv_config_link(adapter); 879 880 /* And now turn on interrupts */ 881 ixv_enable_intr(adapter); 882 883 /* Now inform the stack we're ready */ 884 ifp->if_drv_flags |= IFF_DRV_RUNNING; 885 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 886 887 return; 888} 889 890static void 891ixv_init(void *arg) 892{ 893 struct adapter *adapter = arg; 894 895 IXV_CORE_LOCK(adapter); 896 ixv_init_locked(adapter); 897 IXV_CORE_UNLOCK(adapter); 898 return; 899} 900 901 902/* 903** 904** MSIX Interrupt Handlers and Tasklets 905** 906*/ 907 908static inline void 909ixv_enable_queue(struct adapter *adapter, u32 vector) 910{ 911 struct ixgbe_hw *hw = &adapter->hw; 912 u32 queue = 1 << vector; 913 u32 mask; 914 915 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 916 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 917} 918 919static inline void 920ixv_disable_queue(struct adapter *adapter, u32 vector) 921{ 922 struct ixgbe_hw *hw = &adapter->hw; 923 u64 queue = (u64)(1 << vector); 924 u32 mask; 925 926 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 927 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 928} 929 930static inline void 931ixv_rearm_queues(struct adapter *adapter, u64 queues) 932{ 933 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 934 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 935} 936 937 938static void 939ixv_handle_que(void *context, int pending) 940{ 941 struct ix_queue *que = context; 942 struct adapter *adapter = que->adapter; 943 struct tx_ring *txr = que->txr; 944 struct ifnet *ifp = adapter->ifp; 945 bool more; 946 947 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 948 more = ixv_rxeof(que, adapter->rx_process_limit); 949 IXV_TX_LOCK(txr); 950 ixv_txeof(txr); 951#if __FreeBSD_version >= 800000 952 if (!drbr_empty(ifp, txr->br)) 953 ixv_mq_start_locked(ifp, txr, NULL); 954#else 955 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 956 ixv_start_locked(txr, ifp); 957#endif 958 IXV_TX_UNLOCK(txr); 959 if (more) { 960 taskqueue_enqueue(que->tq, &que->que_task); 961 return; 962 } 963 } 964 965 /* Reenable this interrupt */ 966 ixv_enable_queue(adapter, que->msix); 967 return; 968} 969 970/********************************************************************* 971 * 972 * MSI Queue Interrupt Service routine 973 * 974 **********************************************************************/ 975void 976ixv_msix_que(void *arg) 977{ 978 struct ix_queue *que = arg; 979 struct adapter *adapter = que->adapter; 980 struct tx_ring *txr = que->txr; 981 struct rx_ring *rxr = que->rxr; 982 bool more_tx, more_rx; 983 u32 newitr = 0; 984 985 ixv_disable_queue(adapter, que->msix); 986 ++que->irqs; 987 988 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 989 990 IXV_TX_LOCK(txr); 991 more_tx = ixv_txeof(txr); 992 IXV_TX_UNLOCK(txr); 993 994 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 995 996 /* Do AIM now? */ 997 998 if (ixv_enable_aim == FALSE) 999 goto no_calc; 1000 /* 1001 ** Do Adaptive Interrupt Moderation: 1002 ** - Write out last calculated setting 1003 ** - Calculate based on average size over 1004 ** the last interval. 1005 */ 1006 if (que->eitr_setting) 1007 IXGBE_WRITE_REG(&adapter->hw, 1008 IXGBE_VTEITR(que->msix), 1009 que->eitr_setting); 1010 1011 que->eitr_setting = 0; 1012 1013 /* Idle, do nothing */ 1014 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1015 goto no_calc; 1016 1017 if ((txr->bytes) && (txr->packets)) 1018 newitr = txr->bytes/txr->packets; 1019 if ((rxr->bytes) && (rxr->packets)) 1020 newitr = max(newitr, 1021 (rxr->bytes / rxr->packets)); 1022 newitr += 24; /* account for hardware frame, crc */ 1023 1024 /* set an upper boundary */ 1025 newitr = min(newitr, 3000); 1026 1027 /* Be nice to the mid range */ 1028 if ((newitr > 300) && (newitr < 1200)) 1029 newitr = (newitr / 3); 1030 else 1031 newitr = (newitr / 2); 1032 1033 newitr |= newitr << 16; 1034 1035 /* save for next interrupt */ 1036 que->eitr_setting = newitr; 1037 1038 /* Reset state */ 1039 txr->bytes = 0; 1040 txr->packets = 0; 1041 rxr->bytes = 0; 1042 rxr->packets = 0; 1043 1044no_calc: 1045 if (more_tx || more_rx) 1046 taskqueue_enqueue(que->tq, &que->que_task); 1047 else /* Reenable this interrupt */ 1048 ixv_enable_queue(adapter, que->msix); 1049 return; 1050} 1051 1052static void 1053ixv_msix_mbx(void *arg) 1054{ 1055 struct adapter *adapter = arg; 1056 struct ixgbe_hw *hw = &adapter->hw; 1057 u32 reg; 1058 1059 ++adapter->mbx_irq; 1060 1061 /* First get the cause */ 1062 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 1063 /* Clear interrupt with write */ 1064 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 1065 1066 /* Link status change */ 1067 if (reg & IXGBE_EICR_LSC) 1068 taskqueue_enqueue(adapter->tq, &adapter->mbx_task); 1069 1070 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 1071 return; 1072} 1073 1074/********************************************************************* 1075 * 1076 * Media Ioctl callback 1077 * 1078 * This routine is called whenever the user queries the status of 1079 * the interface using ifconfig. 1080 * 1081 **********************************************************************/ 1082static void 1083ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1084{ 1085 struct adapter *adapter = ifp->if_softc; 1086 1087 INIT_DEBUGOUT("ixv_media_status: begin"); 1088 IXV_CORE_LOCK(adapter); 1089 ixv_update_link_status(adapter); 1090 1091 ifmr->ifm_status = IFM_AVALID; 1092 ifmr->ifm_active = IFM_ETHER; 1093 1094 if (!adapter->link_active) { 1095 IXV_CORE_UNLOCK(adapter); 1096 return; 1097 } 1098 1099 ifmr->ifm_status |= IFM_ACTIVE; 1100 1101 switch (adapter->link_speed) { 1102 case IXGBE_LINK_SPEED_1GB_FULL: 1103 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1104 break; 1105 case IXGBE_LINK_SPEED_10GB_FULL: 1106 ifmr->ifm_active |= IFM_FDX; 1107 break; 1108 } 1109 1110 IXV_CORE_UNLOCK(adapter); 1111 1112 return; 1113} 1114 1115/********************************************************************* 1116 * 1117 * Media Ioctl callback 1118 * 1119 * This routine is called when the user changes speed/duplex using 1120 * media/mediopt option with ifconfig. 1121 * 1122 **********************************************************************/ 1123static int 1124ixv_media_change(struct ifnet * ifp) 1125{ 1126 struct adapter *adapter = ifp->if_softc; 1127 struct ifmedia *ifm = &adapter->media; 1128 1129 INIT_DEBUGOUT("ixv_media_change: begin"); 1130 1131 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1132 return (EINVAL); 1133 1134 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1135 case IFM_AUTO: 1136 break; 1137 default: 1138 device_printf(adapter->dev, "Only auto media type\n"); 1139 return (EINVAL); 1140 } 1141 1142 return (0); 1143} 1144 1145/********************************************************************* 1146 * 1147 * This routine maps the mbufs to tx descriptors, allowing the 1148 * TX engine to transmit the packets. 1149 * - return 0 on success, positive on failure 1150 * 1151 **********************************************************************/ 1152 1153static int 1154ixv_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1155{ 1156 struct adapter *adapter = txr->adapter; 1157 u32 olinfo_status = 0, cmd_type_len; 1158 u32 paylen = 0; 1159 int i, j, error, nsegs; 1160 int first, last = 0; 1161 struct mbuf *m_head; 1162 bus_dma_segment_t segs[32]; 1163 bus_dmamap_t map; 1164 struct ixv_tx_buf *txbuf, *txbuf_mapped; 1165 union ixgbe_adv_tx_desc *txd = NULL; 1166 1167 m_head = *m_headp; 1168 1169 /* Basic descriptor defines */ 1170 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1171 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1172 1173 if (m_head->m_flags & M_VLANTAG) 1174 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1175 1176 /* 1177 * Important to capture the first descriptor 1178 * used because it will contain the index of 1179 * the one we tell the hardware to report back 1180 */ 1181 first = txr->next_avail_desc; 1182 txbuf = &txr->tx_buffers[first]; 1183 txbuf_mapped = txbuf; 1184 map = txbuf->map; 1185 1186 /* 1187 * Map the packet for DMA. 1188 */ 1189 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1190 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1191 1192 if (error == EFBIG) { 1193 struct mbuf *m; 1194 1195 m = m_defrag(*m_headp, M_DONTWAIT); 1196 if (m == NULL) { 1197 adapter->mbuf_defrag_failed++; 1198 m_freem(*m_headp); 1199 *m_headp = NULL; 1200 return (ENOBUFS); 1201 } 1202 *m_headp = m; 1203 1204 /* Try it again */ 1205 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1206 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1207 1208 if (error == ENOMEM) { 1209 adapter->no_tx_dma_setup++; 1210 return (error); 1211 } else if (error != 0) { 1212 adapter->no_tx_dma_setup++; 1213 m_freem(*m_headp); 1214 *m_headp = NULL; 1215 return (error); 1216 } 1217 } else if (error == ENOMEM) { 1218 adapter->no_tx_dma_setup++; 1219 return (error); 1220 } else if (error != 0) { 1221 adapter->no_tx_dma_setup++; 1222 m_freem(*m_headp); 1223 *m_headp = NULL; 1224 return (error); 1225 } 1226 1227 /* Make certain there are enough descriptors */ 1228 if (nsegs > txr->tx_avail - 2) { 1229 txr->no_desc_avail++; 1230 error = ENOBUFS; 1231 goto xmit_fail; 1232 } 1233 m_head = *m_headp; 1234 1235 /* 1236 ** Set up the appropriate offload context 1237 ** this becomes the first descriptor of 1238 ** a packet. 1239 */ 1240 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1241 if (ixv_tso_setup(txr, m_head, &paylen)) { 1242 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1243 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1244 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1245 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1246 ++adapter->tso_tx; 1247 } else 1248 return (ENXIO); 1249 } else if (ixv_tx_ctx_setup(txr, m_head)) 1250 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1251 1252 /* Record payload length */ 1253 if (paylen == 0) 1254 olinfo_status |= m_head->m_pkthdr.len << 1255 IXGBE_ADVTXD_PAYLEN_SHIFT; 1256 1257 i = txr->next_avail_desc; 1258 for (j = 0; j < nsegs; j++) { 1259 bus_size_t seglen; 1260 bus_addr_t segaddr; 1261 1262 txbuf = &txr->tx_buffers[i]; 1263 txd = &txr->tx_base[i]; 1264 seglen = segs[j].ds_len; 1265 segaddr = htole64(segs[j].ds_addr); 1266 1267 txd->read.buffer_addr = segaddr; 1268 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1269 cmd_type_len |seglen); 1270 txd->read.olinfo_status = htole32(olinfo_status); 1271 last = i; /* descriptor that will get completion IRQ */ 1272 1273 if (++i == adapter->num_tx_desc) 1274 i = 0; 1275 1276 txbuf->m_head = NULL; 1277 txbuf->eop_index = -1; 1278 } 1279 1280 txd->read.cmd_type_len |= 1281 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1282 txr->tx_avail -= nsegs; 1283 txr->next_avail_desc = i; 1284 1285 txbuf->m_head = m_head; 1286 txbuf->map = map; 1287 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1288 1289 /* Set the index of the descriptor that will be marked done */ 1290 txbuf = &txr->tx_buffers[first]; 1291 txbuf->eop_index = last; 1292 1293 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1294 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1295 /* 1296 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1297 * hardware that this frame is available to transmit. 1298 */ 1299 ++txr->total_packets; 1300 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i); 1301 1302 return (0); 1303 1304xmit_fail: 1305 bus_dmamap_unload(txr->txtag, txbuf->map); 1306 return (error); 1307 1308} 1309 1310 1311/********************************************************************* 1312 * Multicast Update 1313 * 1314 * This routine is called whenever multicast address list is updated. 1315 * 1316 **********************************************************************/ 1317#define IXGBE_RAR_ENTRIES 16 1318 1319static void 1320ixv_set_multi(struct adapter *adapter) 1321{ 1322 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1323 u8 *update_ptr; 1324 struct ifmultiaddr *ifma; 1325 int mcnt = 0; 1326 struct ifnet *ifp = adapter->ifp; 1327 1328 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1329 1330#if __FreeBSD_version < 800000 1331 IF_ADDR_LOCK(ifp); 1332#else 1333 if_maddr_rlock(ifp); 1334#endif 1335 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1336 if (ifma->ifma_addr->sa_family != AF_LINK) 1337 continue; 1338 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), 1339 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1340 IXGBE_ETH_LENGTH_OF_ADDRESS); 1341 mcnt++; 1342 } 1343#if __FreeBSD_version < 800000 1344 IF_ADDR_UNLOCK(ifp); 1345#else 1346 if_maddr_runlock(ifp); 1347#endif 1348 1349 update_ptr = mta; 1350 1351 ixgbe_update_mc_addr_list(&adapter->hw, 1352 update_ptr, mcnt, ixv_mc_array_itr); 1353 1354 return; 1355} 1356 1357/* 1358 * This is an iterator function now needed by the multicast 1359 * shared code. It simply feeds the shared code routine the 1360 * addresses in the array of ixv_set_multi() one by one. 1361 */ 1362static u8 * 1363ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1364{ 1365 u8 *addr = *update_ptr; 1366 u8 *newptr; 1367 *vmdq = 0; 1368 1369 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1370 *update_ptr = newptr; 1371 return addr; 1372} 1373 1374/********************************************************************* 1375 * Timer routine 1376 * 1377 * This routine checks for link status,updates statistics, 1378 * and runs the watchdog check. 1379 * 1380 **********************************************************************/ 1381 1382static void 1383ixv_local_timer(void *arg) 1384{ 1385 struct adapter *adapter = arg; 1386 device_t dev = adapter->dev; 1387 struct tx_ring *txr = adapter->tx_rings; 1388 int i; 1389 1390 mtx_assert(&adapter->core_mtx, MA_OWNED); 1391 1392 ixv_update_link_status(adapter); 1393 1394 /* Stats Update */ 1395 ixv_update_stats(adapter); 1396 1397 /* 1398 * If the interface has been paused 1399 * then don't do the watchdog check 1400 */ 1401 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 1402 goto out; 1403 /* 1404 ** Check for time since any descriptor was cleaned 1405 */ 1406 for (i = 0; i < adapter->num_queues; i++, txr++) { 1407 IXV_TX_LOCK(txr); 1408 if (txr->watchdog_check == FALSE) { 1409 IXV_TX_UNLOCK(txr); 1410 continue; 1411 } 1412 if ((ticks - txr->watchdog_time) > IXV_WATCHDOG) 1413 goto hung; 1414 IXV_TX_UNLOCK(txr); 1415 } 1416out: 1417 ixv_rearm_queues(adapter, adapter->que_mask); 1418 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1419 return; 1420 1421hung: 1422 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1423 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1424 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)), 1425 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i))); 1426 device_printf(dev,"TX(%d) desc avail = %d," 1427 "Next TX to Clean = %d\n", 1428 txr->me, txr->tx_avail, txr->next_to_clean); 1429 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1430 adapter->watchdog_events++; 1431 IXV_TX_UNLOCK(txr); 1432 ixv_init_locked(adapter); 1433} 1434 1435/* 1436** Note: this routine updates the OS on the link state 1437** the real check of the hardware only happens with 1438** a link interrupt. 1439*/ 1440static void 1441ixv_update_link_status(struct adapter *adapter) 1442{ 1443 struct ifnet *ifp = adapter->ifp; 1444 struct tx_ring *txr = adapter->tx_rings; 1445 device_t dev = adapter->dev; 1446 1447 1448 if (adapter->link_up){ 1449 if (adapter->link_active == FALSE) { 1450 if (bootverbose) 1451 device_printf(dev,"Link is up %d Gbps %s \n", 1452 ((adapter->link_speed == 128)? 10:1), 1453 "Full Duplex"); 1454 adapter->link_active = TRUE; 1455 if_link_state_change(ifp, LINK_STATE_UP); 1456 } 1457 } else { /* Link down */ 1458 if (adapter->link_active == TRUE) { 1459 if (bootverbose) 1460 device_printf(dev,"Link is Down\n"); 1461 if_link_state_change(ifp, LINK_STATE_DOWN); 1462 adapter->link_active = FALSE; 1463 for (int i = 0; i < adapter->num_queues; 1464 i++, txr++) 1465 txr->watchdog_check = FALSE; 1466 } 1467 } 1468 1469 return; 1470} 1471 1472 1473/********************************************************************* 1474 * 1475 * This routine disables all traffic on the adapter by issuing a 1476 * global reset on the MAC and deallocates TX/RX buffers. 1477 * 1478 **********************************************************************/ 1479 1480static void 1481ixv_stop(void *arg) 1482{ 1483 struct ifnet *ifp; 1484 struct adapter *adapter = arg; 1485 struct ixgbe_hw *hw = &adapter->hw; 1486 ifp = adapter->ifp; 1487 1488 mtx_assert(&adapter->core_mtx, MA_OWNED); 1489 1490 INIT_DEBUGOUT("ixv_stop: begin\n"); 1491 ixv_disable_intr(adapter); 1492 1493 /* Tell the stack that the interface is no longer active */ 1494 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1495 1496 ixgbe_reset_hw(hw); 1497 adapter->hw.adapter_stopped = FALSE; 1498 ixgbe_stop_adapter(hw); 1499 callout_stop(&adapter->timer); 1500 1501 /* reprogram the RAR[0] in case user changed it. */ 1502 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1503 1504 return; 1505} 1506 1507 1508/********************************************************************* 1509 * 1510 * Determine hardware revision. 1511 * 1512 **********************************************************************/ 1513static void 1514ixv_identify_hardware(struct adapter *adapter) 1515{ 1516 device_t dev = adapter->dev; 1517 u16 pci_cmd_word; 1518 1519 /* 1520 ** Make sure BUSMASTER is set, on a VM under 1521 ** KVM it may not be and will break things. 1522 */ 1523 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 1524 if (!((pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 1525 (pci_cmd_word & PCIM_CMD_MEMEN))) { 1526 INIT_DEBUGOUT("Memory Access and/or Bus Master " 1527 "bits were not set!\n"); 1528 pci_cmd_word |= (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 1529 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); 1530 } 1531 1532 /* Save off the information about this board */ 1533 adapter->hw.vendor_id = pci_get_vendor(dev); 1534 adapter->hw.device_id = pci_get_device(dev); 1535 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1536 adapter->hw.subsystem_vendor_id = 1537 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1538 adapter->hw.subsystem_device_id = 1539 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1540 1541 return; 1542} 1543 1544/********************************************************************* 1545 * 1546 * Setup MSIX Interrupt resources and handlers 1547 * 1548 **********************************************************************/ 1549static int 1550ixv_allocate_msix(struct adapter *adapter) 1551{ 1552 device_t dev = adapter->dev; 1553 struct ix_queue *que = adapter->queues; 1554 int error, rid, vector = 0; 1555 1556 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 1557 rid = vector + 1; 1558 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 1559 RF_SHAREABLE | RF_ACTIVE); 1560 if (que->res == NULL) { 1561 device_printf(dev,"Unable to allocate" 1562 " bus resource: que interrupt [%d]\n", vector); 1563 return (ENXIO); 1564 } 1565 /* Set the handler function */ 1566 error = bus_setup_intr(dev, que->res, 1567 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1568 ixv_msix_que, que, &que->tag); 1569 if (error) { 1570 que->res = NULL; 1571 device_printf(dev, "Failed to register QUE handler"); 1572 return (error); 1573 } 1574#if __FreeBSD_version >= 800504 1575 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 1576#endif 1577 que->msix = vector; 1578 adapter->que_mask |= (u64)(1 << que->msix); 1579 /* 1580 ** Bind the msix vector, and thus the 1581 ** ring to the corresponding cpu. 1582 */ 1583 if (adapter->num_queues > 1) 1584 bus_bind_intr(dev, que->res, i); 1585 1586 TASK_INIT(&que->que_task, 0, ixv_handle_que, que); 1587 que->tq = taskqueue_create_fast("ixv_que", M_NOWAIT, 1588 taskqueue_thread_enqueue, &que->tq); 1589 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 1590 device_get_nameunit(adapter->dev)); 1591 } 1592 1593 /* and Mailbox */ 1594 rid = vector + 1; 1595 adapter->res = bus_alloc_resource_any(dev, 1596 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 1597 if (!adapter->res) { 1598 device_printf(dev,"Unable to allocate" 1599 " bus resource: MBX interrupt [%d]\n", rid); 1600 return (ENXIO); 1601 } 1602 /* Set the mbx handler function */ 1603 error = bus_setup_intr(dev, adapter->res, 1604 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1605 ixv_msix_mbx, adapter, &adapter->tag); 1606 if (error) { 1607 adapter->res = NULL; 1608 device_printf(dev, "Failed to register LINK handler"); 1609 return (error); 1610 } 1611#if __FreeBSD_version >= 800504 1612 bus_describe_intr(dev, adapter->res, adapter->tag, "mbx"); 1613#endif 1614 adapter->mbxvec = vector; 1615 /* Tasklets for Mailbox */ 1616 TASK_INIT(&adapter->mbx_task, 0, ixv_handle_mbx, adapter); 1617 adapter->tq = taskqueue_create_fast("ixv_mbx", M_NOWAIT, 1618 taskqueue_thread_enqueue, &adapter->tq); 1619 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s mbxq", 1620 device_get_nameunit(adapter->dev)); 1621 /* 1622 ** Due to a broken design QEMU will fail to properly 1623 ** enable the guest for MSIX unless the vectors in 1624 ** the table are all set up, so we must rewrite the 1625 ** ENABLE in the MSIX control register again at this 1626 ** point to cause it to successfully initialize us. 1627 */ 1628 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 1629 int msix_ctrl; 1630 pci_find_cap(dev, PCIY_MSIX, &rid); 1631 rid += PCIR_MSIX_CTRL; 1632 msix_ctrl = pci_read_config(dev, rid, 2); 1633 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 1634 pci_write_config(dev, rid, msix_ctrl, 2); 1635 } 1636 1637 return (0); 1638} 1639 1640/* 1641 * Setup MSIX resources, note that the VF 1642 * device MUST use MSIX, there is no fallback. 1643 */ 1644static int 1645ixv_setup_msix(struct adapter *adapter) 1646{ 1647 device_t dev = adapter->dev; 1648 int rid, vectors, want = 2; 1649 1650 1651 /* First try MSI/X */ 1652 rid = PCIR_BAR(3); 1653 adapter->msix_mem = bus_alloc_resource_any(dev, 1654 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1655 if (!adapter->msix_mem) { 1656 device_printf(adapter->dev, 1657 "Unable to map MSIX table \n"); 1658 goto out; 1659 } 1660 1661 vectors = pci_msix_count(dev); 1662 if (vectors < 2) { 1663 bus_release_resource(dev, SYS_RES_MEMORY, 1664 rid, adapter->msix_mem); 1665 adapter->msix_mem = NULL; 1666 goto out; 1667 } 1668 1669 /* 1670 ** Want two vectors: one for a queue, 1671 ** plus an additional for mailbox. 1672 */ 1673 if (pci_alloc_msix(dev, &want) == 0) { 1674 device_printf(adapter->dev, 1675 "Using MSIX interrupts with %d vectors\n", want); 1676 return (want); 1677 } 1678out: 1679 device_printf(adapter->dev,"MSIX config error\n"); 1680 return (ENXIO); 1681} 1682 1683 1684static int 1685ixv_allocate_pci_resources(struct adapter *adapter) 1686{ 1687 int rid; 1688 device_t dev = adapter->dev; 1689 1690 rid = PCIR_BAR(0); 1691 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1692 &rid, RF_ACTIVE); 1693 1694 if (!(adapter->pci_mem)) { 1695 device_printf(dev,"Unable to allocate bus resource: memory\n"); 1696 return (ENXIO); 1697 } 1698 1699 adapter->osdep.mem_bus_space_tag = 1700 rman_get_bustag(adapter->pci_mem); 1701 adapter->osdep.mem_bus_space_handle = 1702 rman_get_bushandle(adapter->pci_mem); 1703 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle; 1704 1705 adapter->num_queues = 1; 1706 adapter->hw.back = &adapter->osdep; 1707 1708 /* 1709 ** Now setup MSI/X, should 1710 ** return us the number of 1711 ** configured vectors. 1712 */ 1713 adapter->msix = ixv_setup_msix(adapter); 1714 if (adapter->msix == ENXIO) 1715 return (ENXIO); 1716 else 1717 return (0); 1718} 1719 1720static void 1721ixv_free_pci_resources(struct adapter * adapter) 1722{ 1723 struct ix_queue *que = adapter->queues; 1724 device_t dev = adapter->dev; 1725 int rid, memrid; 1726 1727 memrid = PCIR_BAR(MSIX_BAR); 1728 1729 /* 1730 ** There is a slight possibility of a failure mode 1731 ** in attach that will result in entering this function 1732 ** before interrupt resources have been initialized, and 1733 ** in that case we do not want to execute the loops below 1734 ** We can detect this reliably by the state of the adapter 1735 ** res pointer. 1736 */ 1737 if (adapter->res == NULL) 1738 goto mem; 1739 1740 /* 1741 ** Release all msix queue resources: 1742 */ 1743 for (int i = 0; i < adapter->num_queues; i++, que++) { 1744 rid = que->msix + 1; 1745 if (que->tag != NULL) { 1746 bus_teardown_intr(dev, que->res, que->tag); 1747 que->tag = NULL; 1748 } 1749 if (que->res != NULL) 1750 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 1751 } 1752 1753 1754 /* Clean the Legacy or Link interrupt last */ 1755 if (adapter->mbxvec) /* we are doing MSIX */ 1756 rid = adapter->mbxvec + 1; 1757 else 1758 (adapter->msix != 0) ? (rid = 1):(rid = 0); 1759 1760 if (adapter->tag != NULL) { 1761 bus_teardown_intr(dev, adapter->res, adapter->tag); 1762 adapter->tag = NULL; 1763 } 1764 if (adapter->res != NULL) 1765 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 1766 1767mem: 1768 if (adapter->msix) 1769 pci_release_msi(dev); 1770 1771 if (adapter->msix_mem != NULL) 1772 bus_release_resource(dev, SYS_RES_MEMORY, 1773 memrid, adapter->msix_mem); 1774 1775 if (adapter->pci_mem != NULL) 1776 bus_release_resource(dev, SYS_RES_MEMORY, 1777 PCIR_BAR(0), adapter->pci_mem); 1778 1779 return; 1780} 1781 1782/********************************************************************* 1783 * 1784 * Setup networking device structure and register an interface. 1785 * 1786 **********************************************************************/ 1787static void 1788ixv_setup_interface(device_t dev, struct adapter *adapter) 1789{ 1790 struct ifnet *ifp; 1791 1792 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1793 1794 ifp = adapter->ifp = if_alloc(IFT_ETHER); 1795 if (ifp == NULL) 1796 panic("%s: can not if_alloc()\n", device_get_nameunit(dev)); 1797 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1798 ifp->if_mtu = ETHERMTU; 1799 ifp->if_baudrate = 1000000000; 1800 ifp->if_init = ixv_init; 1801 ifp->if_softc = adapter; 1802 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1803 ifp->if_ioctl = ixv_ioctl; 1804#if __FreeBSD_version >= 800000 1805 ifp->if_transmit = ixv_mq_start; 1806 ifp->if_qflush = ixv_qflush; 1807#else 1808 ifp->if_start = ixv_start; 1809#endif 1810 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 1811 1812 ether_ifattach(ifp, adapter->hw.mac.addr); 1813 1814 adapter->max_frame_size = 1815 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1816 1817 /* 1818 * Tell the upper layer(s) we support long frames. 1819 */ 1820 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1821 1822 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM; 1823 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU; 1824 ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_LRO; 1825 1826 ifp->if_capenable = ifp->if_capabilities; 1827 1828 /* 1829 * Specify the media types supported by this adapter and register 1830 * callbacks to update media and link information 1831 */ 1832 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1833 ixv_media_status); 1834 ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL); 1835 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1836 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1837 1838 return; 1839} 1840 1841static void 1842ixv_config_link(struct adapter *adapter) 1843{ 1844 struct ixgbe_hw *hw = &adapter->hw; 1845 u32 autoneg, err = 0; 1846 bool negotiate = TRUE; 1847 1848 if (hw->mac.ops.check_link) 1849 err = hw->mac.ops.check_link(hw, &autoneg, 1850 &adapter->link_up, FALSE); 1851 if (err) 1852 goto out; 1853 1854 if (hw->mac.ops.setup_link) 1855 err = hw->mac.ops.setup_link(hw, autoneg, 1856 negotiate, adapter->link_up); 1857out: 1858 return; 1859} 1860 1861/******************************************************************** 1862 * Manage DMA'able memory. 1863 *******************************************************************/ 1864static void 1865ixv_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) 1866{ 1867 if (error) 1868 return; 1869 *(bus_addr_t *) arg = segs->ds_addr; 1870 return; 1871} 1872 1873static int 1874ixv_dma_malloc(struct adapter *adapter, bus_size_t size, 1875 struct ixv_dma_alloc *dma, int mapflags) 1876{ 1877 device_t dev = adapter->dev; 1878 int r; 1879 1880 r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 1881 DBA_ALIGN, 0, /* alignment, bounds */ 1882 BUS_SPACE_MAXADDR, /* lowaddr */ 1883 BUS_SPACE_MAXADDR, /* highaddr */ 1884 NULL, NULL, /* filter, filterarg */ 1885 size, /* maxsize */ 1886 1, /* nsegments */ 1887 size, /* maxsegsize */ 1888 BUS_DMA_ALLOCNOW, /* flags */ 1889 NULL, /* lockfunc */ 1890 NULL, /* lockfuncarg */ 1891 &dma->dma_tag); 1892 if (r != 0) { 1893 device_printf(dev,"ixv_dma_malloc: bus_dma_tag_create failed; " 1894 "error %u\n", r); 1895 goto fail_0; 1896 } 1897 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 1898 BUS_DMA_NOWAIT, &dma->dma_map); 1899 if (r != 0) { 1900 device_printf(dev,"ixv_dma_malloc: bus_dmamem_alloc failed; " 1901 "error %u\n", r); 1902 goto fail_1; 1903 } 1904 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1905 size, 1906 ixv_dmamap_cb, 1907 &dma->dma_paddr, 1908 mapflags | BUS_DMA_NOWAIT); 1909 if (r != 0) { 1910 device_printf(dev,"ixv_dma_malloc: bus_dmamap_load failed; " 1911 "error %u\n", r); 1912 goto fail_2; 1913 } 1914 dma->dma_size = size; 1915 return (0); 1916fail_2: 1917 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1918fail_1: 1919 bus_dma_tag_destroy(dma->dma_tag); 1920fail_0: 1921 dma->dma_map = NULL; 1922 dma->dma_tag = NULL; 1923 return (r); 1924} 1925 1926static void 1927ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma) 1928{ 1929 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 1930 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1931 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1932 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1933 bus_dma_tag_destroy(dma->dma_tag); 1934} 1935 1936 1937/********************************************************************* 1938 * 1939 * Allocate memory for the transmit and receive rings, and then 1940 * the descriptors associated with each, called only once at attach. 1941 * 1942 **********************************************************************/ 1943static int 1944ixv_allocate_queues(struct adapter *adapter) 1945{ 1946 device_t dev = adapter->dev; 1947 struct ix_queue *que; 1948 struct tx_ring *txr; 1949 struct rx_ring *rxr; 1950 int rsize, tsize, error = 0; 1951 int txconf = 0, rxconf = 0; 1952 1953 /* First allocate the top level queue structs */ 1954 if (!(adapter->queues = 1955 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 1956 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1957 device_printf(dev, "Unable to allocate queue memory\n"); 1958 error = ENOMEM; 1959 goto fail; 1960 } 1961 1962 /* First allocate the TX ring struct memory */ 1963 if (!(adapter->tx_rings = 1964 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 1965 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1966 device_printf(dev, "Unable to allocate TX ring memory\n"); 1967 error = ENOMEM; 1968 goto tx_fail; 1969 } 1970 1971 /* Next allocate the RX */ 1972 if (!(adapter->rx_rings = 1973 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 1974 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1975 device_printf(dev, "Unable to allocate RX ring memory\n"); 1976 error = ENOMEM; 1977 goto rx_fail; 1978 } 1979 1980 /* For the ring itself */ 1981 tsize = roundup2(adapter->num_tx_desc * 1982 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 1983 1984 /* 1985 * Now set up the TX queues, txconf is needed to handle the 1986 * possibility that things fail midcourse and we need to 1987 * undo memory gracefully 1988 */ 1989 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 1990 /* Set up some basics */ 1991 txr = &adapter->tx_rings[i]; 1992 txr->adapter = adapter; 1993 txr->me = i; 1994 1995 /* Initialize the TX side lock */ 1996 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 1997 device_get_nameunit(dev), txr->me); 1998 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 1999 2000 if (ixv_dma_malloc(adapter, tsize, 2001 &txr->txdma, BUS_DMA_NOWAIT)) { 2002 device_printf(dev, 2003 "Unable to allocate TX Descriptor memory\n"); 2004 error = ENOMEM; 2005 goto err_tx_desc; 2006 } 2007 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2008 bzero((void *)txr->tx_base, tsize); 2009 2010 /* Now allocate transmit buffers for the ring */ 2011 if (ixv_allocate_transmit_buffers(txr)) { 2012 device_printf(dev, 2013 "Critical Failure setting up transmit buffers\n"); 2014 error = ENOMEM; 2015 goto err_tx_desc; 2016 } 2017#if __FreeBSD_version >= 800000 2018 /* Allocate a buf ring */ 2019 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF, 2020 M_WAITOK, &txr->tx_mtx); 2021 if (txr->br == NULL) { 2022 device_printf(dev, 2023 "Critical Failure setting up buf ring\n"); 2024 error = ENOMEM; 2025 goto err_tx_desc; 2026 } 2027#endif 2028 } 2029 2030 /* 2031 * Next the RX queues... 2032 */ 2033 rsize = roundup2(adapter->num_rx_desc * 2034 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2035 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2036 rxr = &adapter->rx_rings[i]; 2037 /* Set up some basics */ 2038 rxr->adapter = adapter; 2039 rxr->me = i; 2040 2041 /* Initialize the RX side lock */ 2042 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2043 device_get_nameunit(dev), rxr->me); 2044 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2045 2046 if (ixv_dma_malloc(adapter, rsize, 2047 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2048 device_printf(dev, 2049 "Unable to allocate RxDescriptor memory\n"); 2050 error = ENOMEM; 2051 goto err_rx_desc; 2052 } 2053 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2054 bzero((void *)rxr->rx_base, rsize); 2055 2056 /* Allocate receive buffers for the ring*/ 2057 if (ixv_allocate_receive_buffers(rxr)) { 2058 device_printf(dev, 2059 "Critical Failure setting up receive buffers\n"); 2060 error = ENOMEM; 2061 goto err_rx_desc; 2062 } 2063 } 2064 2065 /* 2066 ** Finally set up the queue holding structs 2067 */ 2068 for (int i = 0; i < adapter->num_queues; i++) { 2069 que = &adapter->queues[i]; 2070 que->adapter = adapter; 2071 que->txr = &adapter->tx_rings[i]; 2072 que->rxr = &adapter->rx_rings[i]; 2073 } 2074 2075 return (0); 2076 2077err_rx_desc: 2078 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2079 ixv_dma_free(adapter, &rxr->rxdma); 2080err_tx_desc: 2081 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2082 ixv_dma_free(adapter, &txr->txdma); 2083 free(adapter->rx_rings, M_DEVBUF); 2084rx_fail: 2085 free(adapter->tx_rings, M_DEVBUF); 2086tx_fail: 2087 free(adapter->queues, M_DEVBUF); 2088fail: 2089 return (error); 2090} 2091 2092 2093/********************************************************************* 2094 * 2095 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2096 * the information needed to transmit a packet on the wire. This is 2097 * called only once at attach, setup is done every reset. 2098 * 2099 **********************************************************************/ 2100static int 2101ixv_allocate_transmit_buffers(struct tx_ring *txr) 2102{ 2103 struct adapter *adapter = txr->adapter; 2104 device_t dev = adapter->dev; 2105 struct ixv_tx_buf *txbuf; 2106 int error, i; 2107 2108 /* 2109 * Setup DMA descriptor areas. 2110 */ 2111 if ((error = bus_dma_tag_create(NULL, /* parent */ 2112 1, 0, /* alignment, bounds */ 2113 BUS_SPACE_MAXADDR, /* lowaddr */ 2114 BUS_SPACE_MAXADDR, /* highaddr */ 2115 NULL, NULL, /* filter, filterarg */ 2116 IXV_TSO_SIZE, /* maxsize */ 2117 32, /* nsegments */ 2118 PAGE_SIZE, /* maxsegsize */ 2119 0, /* flags */ 2120 NULL, /* lockfunc */ 2121 NULL, /* lockfuncarg */ 2122 &txr->txtag))) { 2123 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2124 goto fail; 2125 } 2126 2127 if (!(txr->tx_buffers = 2128 (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) * 2129 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2130 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2131 error = ENOMEM; 2132 goto fail; 2133 } 2134 2135 /* Create the descriptor buffer dma maps */ 2136 txbuf = txr->tx_buffers; 2137 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2138 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2139 if (error != 0) { 2140 device_printf(dev, "Unable to create TX DMA map\n"); 2141 goto fail; 2142 } 2143 } 2144 2145 return 0; 2146fail: 2147 /* We free all, it handles case where we are in the middle */ 2148 ixv_free_transmit_structures(adapter); 2149 return (error); 2150} 2151 2152/********************************************************************* 2153 * 2154 * Initialize a transmit ring. 2155 * 2156 **********************************************************************/ 2157static void 2158ixv_setup_transmit_ring(struct tx_ring *txr) 2159{ 2160 struct adapter *adapter = txr->adapter; 2161 struct ixv_tx_buf *txbuf; 2162 int i; 2163 2164 /* Clear the old ring contents */ 2165 IXV_TX_LOCK(txr); 2166 bzero((void *)txr->tx_base, 2167 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2168 /* Reset indices */ 2169 txr->next_avail_desc = 0; 2170 txr->next_to_clean = 0; 2171 2172 /* Free any existing tx buffers. */ 2173 txbuf = txr->tx_buffers; 2174 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2175 if (txbuf->m_head != NULL) { 2176 bus_dmamap_sync(txr->txtag, txbuf->map, 2177 BUS_DMASYNC_POSTWRITE); 2178 bus_dmamap_unload(txr->txtag, txbuf->map); 2179 m_freem(txbuf->m_head); 2180 txbuf->m_head = NULL; 2181 } 2182 /* Clear the EOP index */ 2183 txbuf->eop_index = -1; 2184 } 2185 2186 /* Set number of descriptors available */ 2187 txr->tx_avail = adapter->num_tx_desc; 2188 2189 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2190 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2191 IXV_TX_UNLOCK(txr); 2192} 2193 2194/********************************************************************* 2195 * 2196 * Initialize all transmit rings. 2197 * 2198 **********************************************************************/ 2199static int 2200ixv_setup_transmit_structures(struct adapter *adapter) 2201{ 2202 struct tx_ring *txr = adapter->tx_rings; 2203 2204 for (int i = 0; i < adapter->num_queues; i++, txr++) 2205 ixv_setup_transmit_ring(txr); 2206 2207 return (0); 2208} 2209 2210/********************************************************************* 2211 * 2212 * Enable transmit unit. 2213 * 2214 **********************************************************************/ 2215static void 2216ixv_initialize_transmit_units(struct adapter *adapter) 2217{ 2218 struct tx_ring *txr = adapter->tx_rings; 2219 struct ixgbe_hw *hw = &adapter->hw; 2220 2221 2222 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2223 u64 tdba = txr->txdma.dma_paddr; 2224 u32 txctrl, txdctl; 2225 2226 /* Set WTHRESH to 8, burst writeback */ 2227 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2228 txdctl |= (8 << 16); 2229 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2230 /* Now enable */ 2231 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2232 txdctl |= IXGBE_TXDCTL_ENABLE; 2233 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2234 2235 /* Set the HW Tx Head and Tail indices */ 2236 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0); 2237 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0); 2238 2239 /* Setup Transmit Descriptor Cmd Settings */ 2240 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2241 txr->watchdog_check = FALSE; 2242 2243 /* Set Ring parameters */ 2244 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 2245 (tdba & 0x00000000ffffffffULL)); 2246 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32)); 2247 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 2248 adapter->num_tx_desc * 2249 sizeof(struct ixgbe_legacy_tx_desc)); 2250 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i)); 2251 txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 2252 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl); 2253 break; 2254 } 2255 2256 return; 2257} 2258 2259/********************************************************************* 2260 * 2261 * Free all transmit rings. 2262 * 2263 **********************************************************************/ 2264static void 2265ixv_free_transmit_structures(struct adapter *adapter) 2266{ 2267 struct tx_ring *txr = adapter->tx_rings; 2268 2269 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2270 IXV_TX_LOCK(txr); 2271 ixv_free_transmit_buffers(txr); 2272 ixv_dma_free(adapter, &txr->txdma); 2273 IXV_TX_UNLOCK(txr); 2274 IXV_TX_LOCK_DESTROY(txr); 2275 } 2276 free(adapter->tx_rings, M_DEVBUF); 2277} 2278 2279/********************************************************************* 2280 * 2281 * Free transmit ring related data structures. 2282 * 2283 **********************************************************************/ 2284static void 2285ixv_free_transmit_buffers(struct tx_ring *txr) 2286{ 2287 struct adapter *adapter = txr->adapter; 2288 struct ixv_tx_buf *tx_buffer; 2289 int i; 2290 2291 INIT_DEBUGOUT("free_transmit_ring: begin"); 2292 2293 if (txr->tx_buffers == NULL) 2294 return; 2295 2296 tx_buffer = txr->tx_buffers; 2297 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2298 if (tx_buffer->m_head != NULL) { 2299 bus_dmamap_sync(txr->txtag, tx_buffer->map, 2300 BUS_DMASYNC_POSTWRITE); 2301 bus_dmamap_unload(txr->txtag, 2302 tx_buffer->map); 2303 m_freem(tx_buffer->m_head); 2304 tx_buffer->m_head = NULL; 2305 if (tx_buffer->map != NULL) { 2306 bus_dmamap_destroy(txr->txtag, 2307 tx_buffer->map); 2308 tx_buffer->map = NULL; 2309 } 2310 } else if (tx_buffer->map != NULL) { 2311 bus_dmamap_unload(txr->txtag, 2312 tx_buffer->map); 2313 bus_dmamap_destroy(txr->txtag, 2314 tx_buffer->map); 2315 tx_buffer->map = NULL; 2316 } 2317 } 2318#if __FreeBSD_version >= 800000 2319 if (txr->br != NULL) 2320 buf_ring_free(txr->br, M_DEVBUF); 2321#endif 2322 if (txr->tx_buffers != NULL) { 2323 free(txr->tx_buffers, M_DEVBUF); 2324 txr->tx_buffers = NULL; 2325 } 2326 if (txr->txtag != NULL) { 2327 bus_dma_tag_destroy(txr->txtag); 2328 txr->txtag = NULL; 2329 } 2330 return; 2331} 2332 2333/********************************************************************* 2334 * 2335 * Advanced Context Descriptor setup for VLAN or CSUM 2336 * 2337 **********************************************************************/ 2338 2339static boolean_t 2340ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2341{ 2342 struct adapter *adapter = txr->adapter; 2343 struct ixgbe_adv_tx_context_desc *TXD; 2344 struct ixv_tx_buf *tx_buffer; 2345 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2346 struct ether_vlan_header *eh; 2347 struct ip *ip; 2348 struct ip6_hdr *ip6; 2349 int ehdrlen, ip_hlen = 0; 2350 u16 etype; 2351 u8 ipproto = 0; 2352 bool offload = TRUE; 2353 int ctxd = txr->next_avail_desc; 2354 u16 vtag = 0; 2355 2356 2357 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2358 offload = FALSE; 2359 2360 2361 tx_buffer = &txr->tx_buffers[ctxd]; 2362 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2363 2364 /* 2365 ** In advanced descriptors the vlan tag must 2366 ** be placed into the descriptor itself. 2367 */ 2368 if (mp->m_flags & M_VLANTAG) { 2369 vtag = htole16(mp->m_pkthdr.ether_vtag); 2370 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2371 } else if (offload == FALSE) 2372 return FALSE; 2373 2374 /* 2375 * Determine where frame payload starts. 2376 * Jump over vlan headers if already present, 2377 * helpful for QinQ too. 2378 */ 2379 eh = mtod(mp, struct ether_vlan_header *); 2380 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2381 etype = ntohs(eh->evl_proto); 2382 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2383 } else { 2384 etype = ntohs(eh->evl_encap_proto); 2385 ehdrlen = ETHER_HDR_LEN; 2386 } 2387 2388 /* Set the ether header length */ 2389 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2390 2391 switch (etype) { 2392 case ETHERTYPE_IP: 2393 ip = (struct ip *)(mp->m_data + ehdrlen); 2394 ip_hlen = ip->ip_hl << 2; 2395 if (mp->m_len < ehdrlen + ip_hlen) 2396 return (FALSE); 2397 ipproto = ip->ip_p; 2398 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2399 break; 2400 case ETHERTYPE_IPV6: 2401 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2402 ip_hlen = sizeof(struct ip6_hdr); 2403 if (mp->m_len < ehdrlen + ip_hlen) 2404 return (FALSE); 2405 ipproto = ip6->ip6_nxt; 2406 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2407 break; 2408 default: 2409 offload = FALSE; 2410 break; 2411 } 2412 2413 vlan_macip_lens |= ip_hlen; 2414 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2415 2416 switch (ipproto) { 2417 case IPPROTO_TCP: 2418 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2419 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2420 break; 2421 2422 case IPPROTO_UDP: 2423 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2424 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2425 break; 2426 2427#if __FreeBSD_version >= 800000 2428 case IPPROTO_SCTP: 2429 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 2430 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 2431 break; 2432#endif 2433 default: 2434 offload = FALSE; 2435 break; 2436 } 2437 2438 /* Now copy bits into descriptor */ 2439 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2440 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2441 TXD->seqnum_seed = htole32(0); 2442 TXD->mss_l4len_idx = htole32(0); 2443 2444 tx_buffer->m_head = NULL; 2445 tx_buffer->eop_index = -1; 2446 2447 /* We've consumed the first desc, adjust counters */ 2448 if (++ctxd == adapter->num_tx_desc) 2449 ctxd = 0; 2450 txr->next_avail_desc = ctxd; 2451 --txr->tx_avail; 2452 2453 return (offload); 2454} 2455 2456/********************************************************************** 2457 * 2458 * Setup work for hardware segmentation offload (TSO) on 2459 * adapters using advanced tx descriptors 2460 * 2461 **********************************************************************/ 2462static boolean_t 2463ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2464{ 2465 struct adapter *adapter = txr->adapter; 2466 struct ixgbe_adv_tx_context_desc *TXD; 2467 struct ixv_tx_buf *tx_buffer; 2468 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2469 u32 mss_l4len_idx = 0; 2470 u16 vtag = 0; 2471 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2472 struct ether_vlan_header *eh; 2473 struct ip *ip; 2474 struct tcphdr *th; 2475 2476 2477 /* 2478 * Determine where frame payload starts. 2479 * Jump over vlan headers if already present 2480 */ 2481 eh = mtod(mp, struct ether_vlan_header *); 2482 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2483 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2484 else 2485 ehdrlen = ETHER_HDR_LEN; 2486 2487 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2488 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2489 return FALSE; 2490 2491 ctxd = txr->next_avail_desc; 2492 tx_buffer = &txr->tx_buffers[ctxd]; 2493 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2494 2495 ip = (struct ip *)(mp->m_data + ehdrlen); 2496 if (ip->ip_p != IPPROTO_TCP) 2497 return FALSE; /* 0 */ 2498 ip->ip_sum = 0; 2499 ip_hlen = ip->ip_hl << 2; 2500 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2501 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2502 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2503 tcp_hlen = th->th_off << 2; 2504 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2505 2506 /* This is used in the transmit desc in encap */ 2507 *paylen = mp->m_pkthdr.len - hdrlen; 2508 2509 /* VLAN MACLEN IPLEN */ 2510 if (mp->m_flags & M_VLANTAG) { 2511 vtag = htole16(mp->m_pkthdr.ether_vtag); 2512 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2513 } 2514 2515 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2516 vlan_macip_lens |= ip_hlen; 2517 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2518 2519 /* ADV DTYPE TUCMD */ 2520 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2521 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2522 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2523 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2524 2525 2526 /* MSS L4LEN IDX */ 2527 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2528 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2529 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2530 2531 TXD->seqnum_seed = htole32(0); 2532 tx_buffer->m_head = NULL; 2533 tx_buffer->eop_index = -1; 2534 2535 if (++ctxd == adapter->num_tx_desc) 2536 ctxd = 0; 2537 2538 txr->tx_avail--; 2539 txr->next_avail_desc = ctxd; 2540 return TRUE; 2541} 2542 2543 2544/********************************************************************** 2545 * 2546 * Examine each tx_buffer in the used queue. If the hardware is done 2547 * processing the packet then free associated resources. The 2548 * tx_buffer is put back on the free queue. 2549 * 2550 **********************************************************************/ 2551static boolean_t 2552ixv_txeof(struct tx_ring *txr) 2553{ 2554 struct adapter *adapter = txr->adapter; 2555 struct ifnet *ifp = adapter->ifp; 2556 u32 first, last, done; 2557 struct ixv_tx_buf *tx_buffer; 2558 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2559 2560 mtx_assert(&txr->tx_mtx, MA_OWNED); 2561 2562 if (txr->tx_avail == adapter->num_tx_desc) 2563 return FALSE; 2564 2565 first = txr->next_to_clean; 2566 tx_buffer = &txr->tx_buffers[first]; 2567 /* For cleanup we just use legacy struct */ 2568 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2569 last = tx_buffer->eop_index; 2570 if (last == -1) 2571 return FALSE; 2572 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2573 2574 /* 2575 ** Get the index of the first descriptor 2576 ** BEYOND the EOP and call that 'done'. 2577 ** I do this so the comparison in the 2578 ** inner while loop below can be simple 2579 */ 2580 if (++last == adapter->num_tx_desc) last = 0; 2581 done = last; 2582 2583 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2584 BUS_DMASYNC_POSTREAD); 2585 /* 2586 ** Only the EOP descriptor of a packet now has the DD 2587 ** bit set, this is what we look for... 2588 */ 2589 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2590 /* We clean the range of the packet */ 2591 while (first != done) { 2592 tx_desc->upper.data = 0; 2593 tx_desc->lower.data = 0; 2594 tx_desc->buffer_addr = 0; 2595 ++txr->tx_avail; 2596 2597 if (tx_buffer->m_head) { 2598 bus_dmamap_sync(txr->txtag, 2599 tx_buffer->map, 2600 BUS_DMASYNC_POSTWRITE); 2601 bus_dmamap_unload(txr->txtag, 2602 tx_buffer->map); 2603 m_freem(tx_buffer->m_head); 2604 tx_buffer->m_head = NULL; 2605 tx_buffer->map = NULL; 2606 } 2607 tx_buffer->eop_index = -1; 2608 txr->watchdog_time = ticks; 2609 2610 if (++first == adapter->num_tx_desc) 2611 first = 0; 2612 2613 tx_buffer = &txr->tx_buffers[first]; 2614 tx_desc = 2615 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2616 } 2617 ++ifp->if_opackets; 2618 /* See if there is more work now */ 2619 last = tx_buffer->eop_index; 2620 if (last != -1) { 2621 eop_desc = 2622 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2623 /* Get next done point */ 2624 if (++last == adapter->num_tx_desc) last = 0; 2625 done = last; 2626 } else 2627 break; 2628 } 2629 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2630 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2631 2632 txr->next_to_clean = first; 2633 2634 /* 2635 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that 2636 * it is OK to send packets. If there are no pending descriptors, 2637 * clear the timeout. Otherwise, if some descriptors have been freed, 2638 * restart the timeout. 2639 */ 2640 if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) { 2641 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 2642 if (txr->tx_avail == adapter->num_tx_desc) { 2643 txr->watchdog_check = FALSE; 2644 return FALSE; 2645 } 2646 } 2647 2648 return TRUE; 2649} 2650 2651/********************************************************************* 2652 * 2653 * Refresh mbuf buffers for RX descriptor rings 2654 * - now keeps its own state so discards due to resource 2655 * exhaustion are unnecessary, if an mbuf cannot be obtained 2656 * it just returns, keeping its placeholder, thus it can simply 2657 * be recalled to try again. 2658 * 2659 **********************************************************************/ 2660static void 2661ixv_refresh_mbufs(struct rx_ring *rxr, int limit) 2662{ 2663 struct adapter *adapter = rxr->adapter; 2664 bus_dma_segment_t hseg[1]; 2665 bus_dma_segment_t pseg[1]; 2666 struct ixv_rx_buf *rxbuf; 2667 struct mbuf *mh, *mp; 2668 int i, nsegs, error, cleaned; 2669 2670 i = rxr->next_to_refresh; 2671 cleaned = -1; /* Signify no completions */ 2672 while (i != limit) { 2673 rxbuf = &rxr->rx_buffers[i]; 2674 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 2675 mh = m_gethdr(M_DONTWAIT, MT_DATA); 2676 if (mh == NULL) 2677 goto update; 2678 mh->m_pkthdr.len = mh->m_len = MHLEN; 2679 mh->m_len = MHLEN; 2680 mh->m_flags |= M_PKTHDR; 2681 m_adj(mh, ETHER_ALIGN); 2682 /* Get the memory mapping */ 2683 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2684 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); 2685 if (error != 0) { 2686 printf("GET BUF: dmamap load" 2687 " failure - %d\n", error); 2688 m_free(mh); 2689 goto update; 2690 } 2691 rxbuf->m_head = mh; 2692 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2693 BUS_DMASYNC_PREREAD); 2694 rxr->rx_base[i].read.hdr_addr = 2695 htole64(hseg[0].ds_addr); 2696 } 2697 2698 if (rxbuf->m_pack == NULL) { 2699 mp = m_getjcl(M_DONTWAIT, MT_DATA, 2700 M_PKTHDR, adapter->rx_mbuf_sz); 2701 if (mp == NULL) 2702 goto update; 2703 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2704 /* Get the memory mapping */ 2705 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2706 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); 2707 if (error != 0) { 2708 printf("GET BUF: dmamap load" 2709 " failure - %d\n", error); 2710 m_free(mp); 2711 goto update; 2712 } 2713 rxbuf->m_pack = mp; 2714 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2715 BUS_DMASYNC_PREREAD); 2716 rxr->rx_base[i].read.pkt_addr = 2717 htole64(pseg[0].ds_addr); 2718 } 2719 2720 cleaned = i; 2721 /* Calculate next index */ 2722 if (++i == adapter->num_rx_desc) 2723 i = 0; 2724 /* This is the work marker for refresh */ 2725 rxr->next_to_refresh = i; 2726 } 2727update: 2728 if (cleaned != -1) /* If we refreshed some, bump tail */ 2729 IXGBE_WRITE_REG(&adapter->hw, 2730 IXGBE_VFRDT(rxr->me), cleaned); 2731 return; 2732} 2733 2734/********************************************************************* 2735 * 2736 * Allocate memory for rx_buffer structures. Since we use one 2737 * rx_buffer per received packet, the maximum number of rx_buffer's 2738 * that we'll need is equal to the number of receive descriptors 2739 * that we've allocated. 2740 * 2741 **********************************************************************/ 2742static int 2743ixv_allocate_receive_buffers(struct rx_ring *rxr) 2744{ 2745 struct adapter *adapter = rxr->adapter; 2746 device_t dev = adapter->dev; 2747 struct ixv_rx_buf *rxbuf; 2748 int i, bsize, error; 2749 2750 bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc; 2751 if (!(rxr->rx_buffers = 2752 (struct ixv_rx_buf *) malloc(bsize, 2753 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2754 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 2755 error = ENOMEM; 2756 goto fail; 2757 } 2758 2759 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2760 1, 0, /* alignment, bounds */ 2761 BUS_SPACE_MAXADDR, /* lowaddr */ 2762 BUS_SPACE_MAXADDR, /* highaddr */ 2763 NULL, NULL, /* filter, filterarg */ 2764 MSIZE, /* maxsize */ 2765 1, /* nsegments */ 2766 MSIZE, /* maxsegsize */ 2767 0, /* flags */ 2768 NULL, /* lockfunc */ 2769 NULL, /* lockfuncarg */ 2770 &rxr->htag))) { 2771 device_printf(dev, "Unable to create RX DMA tag\n"); 2772 goto fail; 2773 } 2774 2775 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2776 1, 0, /* alignment, bounds */ 2777 BUS_SPACE_MAXADDR, /* lowaddr */ 2778 BUS_SPACE_MAXADDR, /* highaddr */ 2779 NULL, NULL, /* filter, filterarg */ 2780 MJUMPAGESIZE, /* maxsize */ 2781 1, /* nsegments */ 2782 MJUMPAGESIZE, /* maxsegsize */ 2783 0, /* flags */ 2784 NULL, /* lockfunc */ 2785 NULL, /* lockfuncarg */ 2786 &rxr->ptag))) { 2787 device_printf(dev, "Unable to create RX DMA tag\n"); 2788 goto fail; 2789 } 2790 2791 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 2792 rxbuf = &rxr->rx_buffers[i]; 2793 error = bus_dmamap_create(rxr->htag, 2794 BUS_DMA_NOWAIT, &rxbuf->hmap); 2795 if (error) { 2796 device_printf(dev, "Unable to create RX head map\n"); 2797 goto fail; 2798 } 2799 error = bus_dmamap_create(rxr->ptag, 2800 BUS_DMA_NOWAIT, &rxbuf->pmap); 2801 if (error) { 2802 device_printf(dev, "Unable to create RX pkt map\n"); 2803 goto fail; 2804 } 2805 } 2806 2807 return (0); 2808 2809fail: 2810 /* Frees all, but can handle partial completion */ 2811 ixv_free_receive_structures(adapter); 2812 return (error); 2813} 2814 2815static void 2816ixv_free_receive_ring(struct rx_ring *rxr) 2817{ 2818 struct adapter *adapter; 2819 struct ixv_rx_buf *rxbuf; 2820 int i; 2821 2822 adapter = rxr->adapter; 2823 for (i = 0; i < adapter->num_rx_desc; i++) { 2824 rxbuf = &rxr->rx_buffers[i]; 2825 if (rxbuf->m_head != NULL) { 2826 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2827 BUS_DMASYNC_POSTREAD); 2828 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 2829 rxbuf->m_head->m_flags |= M_PKTHDR; 2830 m_freem(rxbuf->m_head); 2831 } 2832 if (rxbuf->m_pack != NULL) { 2833 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2834 BUS_DMASYNC_POSTREAD); 2835 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 2836 rxbuf->m_pack->m_flags |= M_PKTHDR; 2837 m_freem(rxbuf->m_pack); 2838 } 2839 rxbuf->m_head = NULL; 2840 rxbuf->m_pack = NULL; 2841 } 2842} 2843 2844 2845/********************************************************************* 2846 * 2847 * Initialize a receive ring and its buffers. 2848 * 2849 **********************************************************************/ 2850static int 2851ixv_setup_receive_ring(struct rx_ring *rxr) 2852{ 2853 struct adapter *adapter; 2854 struct ifnet *ifp; 2855 device_t dev; 2856 struct ixv_rx_buf *rxbuf; 2857 bus_dma_segment_t pseg[1], hseg[1]; 2858 struct lro_ctrl *lro = &rxr->lro; 2859 int rsize, nsegs, error = 0; 2860 2861 adapter = rxr->adapter; 2862 ifp = adapter->ifp; 2863 dev = adapter->dev; 2864 2865 /* Clear the ring contents */ 2866 IXV_RX_LOCK(rxr); 2867 rsize = roundup2(adapter->num_rx_desc * 2868 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2869 bzero((void *)rxr->rx_base, rsize); 2870 2871 /* Free current RX buffer structs and their mbufs */ 2872 ixv_free_receive_ring(rxr); 2873 2874 /* Configure header split? */ 2875 if (ixv_header_split) 2876 rxr->hdr_split = TRUE; 2877 2878 /* Now replenish the mbufs */ 2879 for (int j = 0; j != adapter->num_rx_desc; ++j) { 2880 struct mbuf *mh, *mp; 2881 2882 rxbuf = &rxr->rx_buffers[j]; 2883 /* 2884 ** Dont allocate mbufs if not 2885 ** doing header split, its wasteful 2886 */ 2887 if (rxr->hdr_split == FALSE) 2888 goto skip_head; 2889 2890 /* First the header */ 2891 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA); 2892 if (rxbuf->m_head == NULL) { 2893 error = ENOBUFS; 2894 goto fail; 2895 } 2896 m_adj(rxbuf->m_head, ETHER_ALIGN); 2897 mh = rxbuf->m_head; 2898 mh->m_len = mh->m_pkthdr.len = MHLEN; 2899 mh->m_flags |= M_PKTHDR; 2900 /* Get the memory mapping */ 2901 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2902 rxbuf->hmap, rxbuf->m_head, hseg, 2903 &nsegs, BUS_DMA_NOWAIT); 2904 if (error != 0) /* Nothing elegant to do here */ 2905 goto fail; 2906 bus_dmamap_sync(rxr->htag, 2907 rxbuf->hmap, BUS_DMASYNC_PREREAD); 2908 /* Update descriptor */ 2909 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 2910 2911skip_head: 2912 /* Now the payload cluster */ 2913 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, 2914 M_PKTHDR, adapter->rx_mbuf_sz); 2915 if (rxbuf->m_pack == NULL) { 2916 error = ENOBUFS; 2917 goto fail; 2918 } 2919 mp = rxbuf->m_pack; 2920 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2921 /* Get the memory mapping */ 2922 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2923 rxbuf->pmap, mp, pseg, 2924 &nsegs, BUS_DMA_NOWAIT); 2925 if (error != 0) 2926 goto fail; 2927 bus_dmamap_sync(rxr->ptag, 2928 rxbuf->pmap, BUS_DMASYNC_PREREAD); 2929 /* Update descriptor */ 2930 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 2931 } 2932 2933 2934 /* Setup our descriptor indices */ 2935 rxr->next_to_check = 0; 2936 rxr->next_to_refresh = 0; 2937 rxr->lro_enabled = FALSE; 2938 rxr->rx_split_packets = 0; 2939 rxr->rx_bytes = 0; 2940 2941 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2942 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2943 2944 /* 2945 ** Now set up the LRO interface: 2946 */ 2947 if (ifp->if_capenable & IFCAP_LRO) { 2948 int err = tcp_lro_init(lro); 2949 if (err) { 2950 device_printf(dev, "LRO Initialization failed!\n"); 2951 goto fail; 2952 } 2953 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 2954 rxr->lro_enabled = TRUE; 2955 lro->ifp = adapter->ifp; 2956 } 2957 2958 IXV_RX_UNLOCK(rxr); 2959 return (0); 2960 2961fail: 2962 ixv_free_receive_ring(rxr); 2963 IXV_RX_UNLOCK(rxr); 2964 return (error); 2965} 2966 2967/********************************************************************* 2968 * 2969 * Initialize all receive rings. 2970 * 2971 **********************************************************************/ 2972static int 2973ixv_setup_receive_structures(struct adapter *adapter) 2974{ 2975 struct rx_ring *rxr = adapter->rx_rings; 2976 int j; 2977 2978 for (j = 0; j < adapter->num_queues; j++, rxr++) 2979 if (ixv_setup_receive_ring(rxr)) 2980 goto fail; 2981 2982 return (0); 2983fail: 2984 /* 2985 * Free RX buffers allocated so far, we will only handle 2986 * the rings that completed, the failing case will have 2987 * cleaned up for itself. 'j' failed, so its the terminus. 2988 */ 2989 for (int i = 0; i < j; ++i) { 2990 rxr = &adapter->rx_rings[i]; 2991 ixv_free_receive_ring(rxr); 2992 } 2993 2994 return (ENOBUFS); 2995} 2996 2997/********************************************************************* 2998 * 2999 * Setup receive registers and features. 3000 * 3001 **********************************************************************/ 3002#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3003 3004static void 3005ixv_initialize_receive_units(struct adapter *adapter) 3006{ 3007 struct rx_ring *rxr = adapter->rx_rings; 3008 struct ixgbe_hw *hw = &adapter->hw; 3009 struct ifnet *ifp = adapter->ifp; 3010 u32 bufsz, fctrl, rxcsum, hlreg; 3011 3012 3013 /* Enable broadcasts */ 3014 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3015 fctrl |= IXGBE_FCTRL_BAM; 3016 fctrl |= IXGBE_FCTRL_DPF; 3017 fctrl |= IXGBE_FCTRL_PMCF; 3018 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3019 3020 /* Set for Jumbo Frames? */ 3021 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3022 if (ifp->if_mtu > ETHERMTU) { 3023 hlreg |= IXGBE_HLREG0_JUMBOEN; 3024 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3025 } else { 3026 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 3027 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3028 } 3029 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 3030 3031 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3032 u64 rdba = rxr->rxdma.dma_paddr; 3033 u32 reg, rxdctl; 3034 3035 /* Do the queue enabling first */ 3036 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3037 rxdctl |= IXGBE_RXDCTL_ENABLE; 3038 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 3039 for (int k = 0; k < 10; k++) { 3040 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 3041 IXGBE_RXDCTL_ENABLE) 3042 break; 3043 else 3044 msec_delay(1); 3045 } 3046 wmb(); 3047 3048 /* Setup the Base and Length of the Rx Descriptor Ring */ 3049 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 3050 (rdba & 0x00000000ffffffffULL)); 3051 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 3052 (rdba >> 32)); 3053 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 3054 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3055 3056 /* Set up the SRRCTL register */ 3057 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i)); 3058 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3059 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 3060 reg |= bufsz; 3061 if (rxr->hdr_split) { 3062 /* Use a standard mbuf for the header */ 3063 reg |= ((IXV_RX_HDR << 3064 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3065 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3066 reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3067 } else 3068 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3069 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg); 3070 3071 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 3072 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 3073 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 3074 adapter->num_rx_desc - 1); 3075 } 3076 3077 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3078 3079 if (ifp->if_capenable & IFCAP_RXCSUM) 3080 rxcsum |= IXGBE_RXCSUM_PCSD; 3081 3082 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3083 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3084 3085 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3086 3087 return; 3088} 3089 3090/********************************************************************* 3091 * 3092 * Free all receive rings. 3093 * 3094 **********************************************************************/ 3095static void 3096ixv_free_receive_structures(struct adapter *adapter) 3097{ 3098 struct rx_ring *rxr = adapter->rx_rings; 3099 3100 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3101 struct lro_ctrl *lro = &rxr->lro; 3102 ixv_free_receive_buffers(rxr); 3103 /* Free LRO memory */ 3104 tcp_lro_free(lro); 3105 /* Free the ring memory as well */ 3106 ixv_dma_free(adapter, &rxr->rxdma); 3107 } 3108 3109 free(adapter->rx_rings, M_DEVBUF); 3110} 3111 3112 3113/********************************************************************* 3114 * 3115 * Free receive ring data structures 3116 * 3117 **********************************************************************/ 3118static void 3119ixv_free_receive_buffers(struct rx_ring *rxr) 3120{ 3121 struct adapter *adapter = rxr->adapter; 3122 struct ixv_rx_buf *rxbuf; 3123 3124 INIT_DEBUGOUT("free_receive_structures: begin"); 3125 3126 /* Cleanup any existing buffers */ 3127 if (rxr->rx_buffers != NULL) { 3128 for (int i = 0; i < adapter->num_rx_desc; i++) { 3129 rxbuf = &rxr->rx_buffers[i]; 3130 if (rxbuf->m_head != NULL) { 3131 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3132 BUS_DMASYNC_POSTREAD); 3133 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 3134 rxbuf->m_head->m_flags |= M_PKTHDR; 3135 m_freem(rxbuf->m_head); 3136 } 3137 if (rxbuf->m_pack != NULL) { 3138 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3139 BUS_DMASYNC_POSTREAD); 3140 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 3141 rxbuf->m_pack->m_flags |= M_PKTHDR; 3142 m_freem(rxbuf->m_pack); 3143 } 3144 rxbuf->m_head = NULL; 3145 rxbuf->m_pack = NULL; 3146 if (rxbuf->hmap != NULL) { 3147 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 3148 rxbuf->hmap = NULL; 3149 } 3150 if (rxbuf->pmap != NULL) { 3151 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 3152 rxbuf->pmap = NULL; 3153 } 3154 } 3155 if (rxr->rx_buffers != NULL) { 3156 free(rxr->rx_buffers, M_DEVBUF); 3157 rxr->rx_buffers = NULL; 3158 } 3159 } 3160 3161 if (rxr->htag != NULL) { 3162 bus_dma_tag_destroy(rxr->htag); 3163 rxr->htag = NULL; 3164 } 3165 if (rxr->ptag != NULL) { 3166 bus_dma_tag_destroy(rxr->ptag); 3167 rxr->ptag = NULL; 3168 } 3169 3170 return; 3171} 3172 3173static __inline void 3174ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 3175{ 3176 3177 /* 3178 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 3179 * should be computed by hardware. Also it should not have VLAN tag in 3180 * ethernet header. 3181 */ 3182 if (rxr->lro_enabled && 3183 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 3184 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3185 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 3186 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 3187 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 3188 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 3189 /* 3190 * Send to the stack if: 3191 ** - LRO not enabled, or 3192 ** - no LRO resources, or 3193 ** - lro enqueue fails 3194 */ 3195 if (rxr->lro.lro_cnt != 0) 3196 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 3197 return; 3198 } 3199 (*ifp->if_input)(ifp, m); 3200} 3201 3202static __inline void 3203ixv_rx_discard(struct rx_ring *rxr, int i) 3204{ 3205 struct adapter *adapter = rxr->adapter; 3206 struct ixv_rx_buf *rbuf; 3207 struct mbuf *mh, *mp; 3208 3209 rbuf = &rxr->rx_buffers[i]; 3210 if (rbuf->fmp != NULL) /* Partial chain ? */ 3211 m_freem(rbuf->fmp); 3212 3213 mh = rbuf->m_head; 3214 mp = rbuf->m_pack; 3215 3216 /* Reuse loaded DMA map and just update mbuf chain */ 3217 mh->m_len = MHLEN; 3218 mh->m_flags |= M_PKTHDR; 3219 mh->m_next = NULL; 3220 3221 mp->m_len = mp->m_pkthdr.len = adapter->rx_mbuf_sz; 3222 mp->m_data = mp->m_ext.ext_buf; 3223 mp->m_next = NULL; 3224 return; 3225} 3226 3227 3228/********************************************************************* 3229 * 3230 * This routine executes in interrupt context. It replenishes 3231 * the mbufs in the descriptor and sends data which has been 3232 * dma'ed into host memory to upper layer. 3233 * 3234 * We loop at most count times if count is > 0, or until done if 3235 * count < 0. 3236 * 3237 * Return TRUE for more work, FALSE for all clean. 3238 *********************************************************************/ 3239static bool 3240ixv_rxeof(struct ix_queue *que, int count) 3241{ 3242 struct adapter *adapter = que->adapter; 3243 struct rx_ring *rxr = que->rxr; 3244 struct ifnet *ifp = adapter->ifp; 3245 struct lro_ctrl *lro = &rxr->lro; 3246 struct lro_entry *queued; 3247 int i, nextp, processed = 0; 3248 u32 staterr = 0; 3249 union ixgbe_adv_rx_desc *cur; 3250 struct ixv_rx_buf *rbuf, *nbuf; 3251 3252 IXV_RX_LOCK(rxr); 3253 3254 for (i = rxr->next_to_check; count != 0;) { 3255 struct mbuf *sendmp, *mh, *mp; 3256 u32 rsc, ptype; 3257 u16 hlen, plen, hdr, vtag; 3258 bool eop; 3259 3260 /* Sync the ring. */ 3261 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3262 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3263 3264 cur = &rxr->rx_base[i]; 3265 staterr = le32toh(cur->wb.upper.status_error); 3266 3267 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 3268 break; 3269 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 3270 break; 3271 3272 count--; 3273 sendmp = NULL; 3274 nbuf = NULL; 3275 rsc = 0; 3276 cur->wb.upper.status_error = 0; 3277 rbuf = &rxr->rx_buffers[i]; 3278 mh = rbuf->m_head; 3279 mp = rbuf->m_pack; 3280 3281 plen = le16toh(cur->wb.upper.length); 3282 ptype = le32toh(cur->wb.lower.lo_dword.data) & 3283 IXGBE_RXDADV_PKTTYPE_MASK; 3284 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 3285 vtag = le16toh(cur->wb.upper.vlan); 3286 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3287 3288 /* Make sure all parts of a bad packet are discarded */ 3289 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 3290 (rxr->discard)) { 3291 ifp->if_ierrors++; 3292 rxr->rx_discarded++; 3293 if (!eop) 3294 rxr->discard = TRUE; 3295 else 3296 rxr->discard = FALSE; 3297 ixv_rx_discard(rxr, i); 3298 goto next_desc; 3299 } 3300 3301 if (!eop) { 3302 nextp = i + 1; 3303 if (nextp == adapter->num_rx_desc) 3304 nextp = 0; 3305 nbuf = &rxr->rx_buffers[nextp]; 3306 prefetch(nbuf); 3307 } 3308 /* 3309 ** The header mbuf is ONLY used when header 3310 ** split is enabled, otherwise we get normal 3311 ** behavior, ie, both header and payload 3312 ** are DMA'd into the payload buffer. 3313 ** 3314 ** Rather than using the fmp/lmp global pointers 3315 ** we now keep the head of a packet chain in the 3316 ** buffer struct and pass this along from one 3317 ** descriptor to the next, until we get EOP. 3318 */ 3319 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 3320 /* This must be an initial descriptor */ 3321 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3322 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3323 if (hlen > IXV_RX_HDR) 3324 hlen = IXV_RX_HDR; 3325 mh->m_len = hlen; 3326 mh->m_flags |= M_PKTHDR; 3327 mh->m_next = NULL; 3328 mh->m_pkthdr.len = mh->m_len; 3329 /* Null buf pointer so it is refreshed */ 3330 rbuf->m_head = NULL; 3331 /* 3332 ** Check the payload length, this 3333 ** could be zero if its a small 3334 ** packet. 3335 */ 3336 if (plen > 0) { 3337 mp->m_len = plen; 3338 mp->m_next = NULL; 3339 mp->m_flags &= ~M_PKTHDR; 3340 mh->m_next = mp; 3341 mh->m_pkthdr.len += mp->m_len; 3342 /* Null buf pointer so it is refreshed */ 3343 rbuf->m_pack = NULL; 3344 rxr->rx_split_packets++; 3345 } 3346 /* 3347 ** Now create the forward 3348 ** chain so when complete 3349 ** we wont have to. 3350 */ 3351 if (eop == 0) { 3352 /* stash the chain head */ 3353 nbuf->fmp = mh; 3354 /* Make forward chain */ 3355 if (plen) 3356 mp->m_next = nbuf->m_pack; 3357 else 3358 mh->m_next = nbuf->m_pack; 3359 } else { 3360 /* Singlet, prepare to send */ 3361 sendmp = mh; 3362 if (staterr & IXGBE_RXD_STAT_VP) { 3363 sendmp->m_pkthdr.ether_vtag = vtag; 3364 sendmp->m_flags |= M_VLANTAG; 3365 } 3366 } 3367 } else { 3368 /* 3369 ** Either no header split, or a 3370 ** secondary piece of a fragmented 3371 ** split packet. 3372 */ 3373 mp->m_len = plen; 3374 /* 3375 ** See if there is a stored head 3376 ** that determines what we are 3377 */ 3378 sendmp = rbuf->fmp; 3379 rbuf->m_pack = rbuf->fmp = NULL; 3380 3381 if (sendmp != NULL) /* secondary frag */ 3382 sendmp->m_pkthdr.len += mp->m_len; 3383 else { 3384 /* first desc of a non-ps chain */ 3385 sendmp = mp; 3386 sendmp->m_flags |= M_PKTHDR; 3387 sendmp->m_pkthdr.len = mp->m_len; 3388 if (staterr & IXGBE_RXD_STAT_VP) { 3389 sendmp->m_pkthdr.ether_vtag = vtag; 3390 sendmp->m_flags |= M_VLANTAG; 3391 } 3392 } 3393 /* Pass the head pointer on */ 3394 if (eop == 0) { 3395 nbuf->fmp = sendmp; 3396 sendmp = NULL; 3397 mp->m_next = nbuf->m_pack; 3398 } 3399 } 3400 ++processed; 3401 /* Sending this frame? */ 3402 if (eop) { 3403 sendmp->m_pkthdr.rcvif = ifp; 3404 ifp->if_ipackets++; 3405 rxr->rx_packets++; 3406 /* capture data for AIM */ 3407 rxr->bytes += sendmp->m_pkthdr.len; 3408 rxr->rx_bytes += sendmp->m_pkthdr.len; 3409 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 3410 ixv_rx_checksum(staterr, sendmp, ptype); 3411#if __FreeBSD_version >= 800000 3412 sendmp->m_pkthdr.flowid = que->msix; 3413 sendmp->m_flags |= M_FLOWID; 3414#endif 3415 } 3416next_desc: 3417 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3418 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3419 3420 /* Advance our pointers to the next descriptor. */ 3421 if (++i == adapter->num_rx_desc) 3422 i = 0; 3423 3424 /* Now send to the stack or do LRO */ 3425 if (sendmp != NULL) 3426 ixv_rx_input(rxr, ifp, sendmp, ptype); 3427 3428 /* Every 8 descriptors we go to refresh mbufs */ 3429 if (processed == 8) { 3430 ixv_refresh_mbufs(rxr, i); 3431 processed = 0; 3432 } 3433 } 3434 3435 /* Refresh any remaining buf structs */ 3436 if (processed != 0) { 3437 ixv_refresh_mbufs(rxr, i); 3438 processed = 0; 3439 } 3440 3441 rxr->next_to_check = i; 3442 3443 /* 3444 * Flush any outstanding LRO work 3445 */ 3446 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 3447 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3448 tcp_lro_flush(lro, queued); 3449 } 3450 3451 IXV_RX_UNLOCK(rxr); 3452 3453 /* 3454 ** We still have cleaning to do? 3455 ** Schedule another interrupt if so. 3456 */ 3457 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 3458 ixv_rearm_queues(adapter, (u64)(1 << que->msix)); 3459 return (TRUE); 3460 } 3461 3462 return (FALSE); 3463} 3464 3465 3466/********************************************************************* 3467 * 3468 * Verify that the hardware indicated that the checksum is valid. 3469 * Inform the stack about the status of checksum so that stack 3470 * doesn't spend time verifying the checksum. 3471 * 3472 *********************************************************************/ 3473static void 3474ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype) 3475{ 3476 u16 status = (u16) staterr; 3477 u8 errors = (u8) (staterr >> 24); 3478 bool sctp = FALSE; 3479 3480 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3481 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 3482 sctp = TRUE; 3483 3484 if (status & IXGBE_RXD_STAT_IPCS) { 3485 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3486 /* IP Checksum Good */ 3487 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 3488 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 3489 3490 } else 3491 mp->m_pkthdr.csum_flags = 0; 3492 } 3493 if (status & IXGBE_RXD_STAT_L4CS) { 3494 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3495#if __FreeBSD_version >= 800000 3496 if (sctp) 3497 type = CSUM_SCTP_VALID; 3498#endif 3499 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3500 mp->m_pkthdr.csum_flags |= type; 3501 if (!sctp) 3502 mp->m_pkthdr.csum_data = htons(0xffff); 3503 } 3504 } 3505 return; 3506} 3507 3508static void 3509ixv_setup_vlan_support(struct adapter *adapter) 3510{ 3511 struct ixgbe_hw *hw = &adapter->hw; 3512 u32 ctrl, vid, vfta, retry; 3513 3514 3515 /* 3516 ** We get here thru init_locked, meaning 3517 ** a soft reset, this has already cleared 3518 ** the VFTA and other state, so if there 3519 ** have been no vlan's registered do nothing. 3520 */ 3521 if (adapter->num_vlans == 0) 3522 return; 3523 3524 /* Enable the queues */ 3525 for (int i = 0; i < adapter->num_queues; i++) { 3526 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3527 ctrl |= IXGBE_RXDCTL_VME; 3528 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 3529 } 3530 3531 /* 3532 ** A soft reset zero's out the VFTA, so 3533 ** we need to repopulate it now. 3534 */ 3535 for (int i = 0; i < VFTA_SIZE; i++) { 3536 if (ixv_shadow_vfta[i] == 0) 3537 continue; 3538 vfta = ixv_shadow_vfta[i]; 3539 /* 3540 ** Reconstruct the vlan id's 3541 ** based on the bits set in each 3542 ** of the array ints. 3543 */ 3544 for ( int j = 0; j < 32; j++) { 3545 retry = 0; 3546 if ((vfta & (1 << j)) == 0) 3547 continue; 3548 vid = (i * 32) + j; 3549 /* Call the shared code mailbox routine */ 3550 while (ixgbe_set_vfta(hw, vid, 0, TRUE)) { 3551 if (++retry > 5) 3552 break; 3553 } 3554 } 3555 } 3556} 3557 3558/* 3559** This routine is run via an vlan config EVENT, 3560** it enables us to use the HW Filter table since 3561** we can get the vlan id. This just creates the 3562** entry in the soft version of the VFTA, init will 3563** repopulate the real table. 3564*/ 3565static void 3566ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3567{ 3568 struct adapter *adapter = ifp->if_softc; 3569 u16 index, bit; 3570 3571 if (ifp->if_softc != arg) /* Not our event */ 3572 return; 3573 3574 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3575 return; 3576 3577 index = (vtag >> 5) & 0x7F; 3578 bit = vtag & 0x1F; 3579 ixv_shadow_vfta[index] |= (1 << bit); 3580 ++adapter->num_vlans; 3581 /* Re-init to load the changes */ 3582 ixv_init(adapter); 3583} 3584 3585/* 3586** This routine is run via an vlan 3587** unconfig EVENT, remove our entry 3588** in the soft vfta. 3589*/ 3590static void 3591ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3592{ 3593 struct adapter *adapter = ifp->if_softc; 3594 u16 index, bit; 3595 3596 if (ifp->if_softc != arg) 3597 return; 3598 3599 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3600 return; 3601 3602 index = (vtag >> 5) & 0x7F; 3603 bit = vtag & 0x1F; 3604 ixv_shadow_vfta[index] &= ~(1 << bit); 3605 --adapter->num_vlans; 3606 /* Re-init to load the changes */ 3607 ixv_init(adapter); 3608} 3609 3610static void 3611ixv_enable_intr(struct adapter *adapter) 3612{ 3613 struct ixgbe_hw *hw = &adapter->hw; 3614 struct ix_queue *que = adapter->queues; 3615 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3616 3617 3618 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 3619 3620 mask = IXGBE_EIMS_ENABLE_MASK; 3621 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 3622 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 3623 3624 for (int i = 0; i < adapter->num_queues; i++, que++) 3625 ixv_enable_queue(adapter, que->msix); 3626 3627 IXGBE_WRITE_FLUSH(hw); 3628 3629 return; 3630} 3631 3632static void 3633ixv_disable_intr(struct adapter *adapter) 3634{ 3635 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 3636 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0); 3637 IXGBE_WRITE_FLUSH(&adapter->hw); 3638 return; 3639} 3640 3641/* 3642** Setup the correct IVAR register for a particular MSIX interrupt 3643** - entry is the register array entry 3644** - vector is the MSIX vector for this queue 3645** - type is RX/TX/MISC 3646*/ 3647static void 3648ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 3649{ 3650 struct ixgbe_hw *hw = &adapter->hw; 3651 u32 ivar, index; 3652 3653 vector |= IXGBE_IVAR_ALLOC_VAL; 3654 3655 if (type == -1) { /* MISC IVAR */ 3656 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 3657 ivar &= ~0xFF; 3658 ivar |= vector; 3659 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 3660 } else { /* RX/TX IVARS */ 3661 index = (16 * (entry & 1)) + (8 * type); 3662 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 3663 ivar &= ~(0xFF << index); 3664 ivar |= (vector << index); 3665 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 3666 } 3667} 3668 3669static void 3670ixv_configure_ivars(struct adapter *adapter) 3671{ 3672 struct ix_queue *que = adapter->queues; 3673 3674 for (int i = 0; i < adapter->num_queues; i++, que++) { 3675 /* First the RX queue entry */ 3676 ixv_set_ivar(adapter, i, que->msix, 0); 3677 /* ... and the TX */ 3678 ixv_set_ivar(adapter, i, que->msix, 1); 3679 /* Set an initial value in EITR */ 3680 IXGBE_WRITE_REG(&adapter->hw, 3681 IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT); 3682 } 3683 3684 /* For the Link interrupt */ 3685 ixv_set_ivar(adapter, 1, adapter->mbxvec, -1); 3686} 3687 3688 3689/* 3690** Tasklet handler for MSIX MBX interrupts 3691** - do outside interrupt since it might sleep 3692*/ 3693static void 3694ixv_handle_mbx(void *context, int pending) 3695{ 3696 struct adapter *adapter = context; 3697 3698 ixgbe_check_link(&adapter->hw, 3699 &adapter->link_speed, &adapter->link_up, 0); 3700 ixv_update_link_status(adapter); 3701} 3702 3703/* 3704** The VF stats registers never have a truely virgin 3705** starting point, so this routine tries to make an 3706** artificial one, marking ground zero on attach as 3707** it were. 3708*/ 3709static void 3710ixv_save_stats(struct adapter *adapter) 3711{ 3712 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 3713 adapter->stats.saved_reset_vfgprc += 3714 adapter->stats.vfgprc - adapter->stats.base_vfgprc; 3715 adapter->stats.saved_reset_vfgptc += 3716 adapter->stats.vfgptc - adapter->stats.base_vfgptc; 3717 adapter->stats.saved_reset_vfgorc += 3718 adapter->stats.vfgorc - adapter->stats.base_vfgorc; 3719 adapter->stats.saved_reset_vfgotc += 3720 adapter->stats.vfgotc - adapter->stats.base_vfgotc; 3721 adapter->stats.saved_reset_vfmprc += 3722 adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3723 } 3724} 3725 3726static void 3727ixv_init_stats(struct adapter *adapter) 3728{ 3729 struct ixgbe_hw *hw = &adapter->hw; 3730 3731 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 3732 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 3733 adapter->stats.last_vfgorc |= 3734 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 3735 3736 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 3737 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 3738 adapter->stats.last_vfgotc |= 3739 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 3740 3741 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 3742 3743 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 3744 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 3745 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 3746 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 3747 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 3748} 3749 3750#define UPDATE_STAT_32(reg, last, count) \ 3751{ \ 3752 u32 current = IXGBE_READ_REG(hw, reg); \ 3753 if (current < last) \ 3754 count += 0x100000000LL; \ 3755 last = current; \ 3756 count &= 0xFFFFFFFF00000000LL; \ 3757 count |= current; \ 3758} 3759 3760#define UPDATE_STAT_36(lsb, msb, last, count) \ 3761{ \ 3762 u64 cur_lsb = IXGBE_READ_REG(hw, lsb); \ 3763 u64 cur_msb = IXGBE_READ_REG(hw, msb); \ 3764 u64 current = ((cur_msb << 32) | cur_lsb); \ 3765 if (current < last) \ 3766 count += 0x1000000000LL; \ 3767 last = current; \ 3768 count &= 0xFFFFFFF000000000LL; \ 3769 count |= current; \ 3770} 3771 3772/* 3773** ixv_update_stats - Update the board statistics counters. 3774*/ 3775void 3776ixv_update_stats(struct adapter *adapter) 3777{ 3778 struct ixgbe_hw *hw = &adapter->hw; 3779 3780 UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3781 adapter->stats.vfgprc); 3782 UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3783 adapter->stats.vfgptc); 3784 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3785 adapter->stats.last_vfgorc, adapter->stats.vfgorc); 3786 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3787 adapter->stats.last_vfgotc, adapter->stats.vfgotc); 3788 UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3789 adapter->stats.vfmprc); 3790} 3791 3792/********************************************************************** 3793 * 3794 * This routine is called only when ixgbe_display_debug_stats is enabled. 3795 * This routine provides a way to take a look at important statistics 3796 * maintained by the driver and hardware. 3797 * 3798 **********************************************************************/ 3799static void 3800ixv_print_hw_stats(struct adapter * adapter) 3801{ 3802 device_t dev = adapter->dev; 3803 3804 device_printf(dev,"Std Mbuf Failed = %lu\n", 3805 adapter->mbuf_defrag_failed); 3806 device_printf(dev,"Driver dropped packets = %lu\n", 3807 adapter->dropped_pkts); 3808 device_printf(dev, "watchdog timeouts = %ld\n", 3809 adapter->watchdog_events); 3810 3811 device_printf(dev,"Good Packets Rcvd = %llu\n", 3812 (long long)adapter->stats.vfgprc); 3813 device_printf(dev,"Good Packets Xmtd = %llu\n", 3814 (long long)adapter->stats.vfgptc); 3815 device_printf(dev,"TSO Transmissions = %lu\n", 3816 adapter->tso_tx); 3817 3818} 3819 3820/********************************************************************** 3821 * 3822 * This routine is called only when em_display_debug_stats is enabled. 3823 * This routine provides a way to take a look at important statistics 3824 * maintained by the driver and hardware. 3825 * 3826 **********************************************************************/ 3827static void 3828ixv_print_debug_info(struct adapter *adapter) 3829{ 3830 device_t dev = adapter->dev; 3831 struct ixgbe_hw *hw = &adapter->hw; 3832 struct ix_queue *que = adapter->queues; 3833 struct rx_ring *rxr; 3834 struct tx_ring *txr; 3835 struct lro_ctrl *lro; 3836 3837 device_printf(dev,"Error Byte Count = %u \n", 3838 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 3839 3840 for (int i = 0; i < adapter->num_queues; i++, que++) { 3841 txr = que->txr; 3842 rxr = que->rxr; 3843 lro = &rxr->lro; 3844 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n", 3845 que->msix, (long)que->irqs); 3846 device_printf(dev,"RX(%d) Packets Received: %lld\n", 3847 rxr->me, (long long)rxr->rx_packets); 3848 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 3849 rxr->me, (long long)rxr->rx_split_packets); 3850 device_printf(dev,"RX(%d) Bytes Received: %lu\n", 3851 rxr->me, (long)rxr->rx_bytes); 3852 device_printf(dev,"RX(%d) LRO Queued= %d\n", 3853 rxr->me, lro->lro_queued); 3854 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 3855 rxr->me, lro->lro_flushed); 3856 device_printf(dev,"TX(%d) Packets Sent: %lu\n", 3857 txr->me, (long)txr->total_packets); 3858 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 3859 txr->me, (long)txr->no_desc_avail); 3860 } 3861 3862 device_printf(dev,"MBX IRQ Handled: %lu\n", 3863 (long)adapter->mbx_irq); 3864 return; 3865} 3866 3867static int 3868ixv_sysctl_stats(SYSCTL_HANDLER_ARGS) 3869{ 3870 int error; 3871 int result; 3872 struct adapter *adapter; 3873 3874 result = -1; 3875 error = sysctl_handle_int(oidp, &result, 0, req); 3876 3877 if (error || !req->newptr) 3878 return (error); 3879 3880 if (result == 1) { 3881 adapter = (struct adapter *) arg1; 3882 ixv_print_hw_stats(adapter); 3883 } 3884 return error; 3885} 3886 3887static int 3888ixv_sysctl_debug(SYSCTL_HANDLER_ARGS) 3889{ 3890 int error, result; 3891 struct adapter *adapter; 3892 3893 result = -1; 3894 error = sysctl_handle_int(oidp, &result, 0, req); 3895 3896 if (error || !req->newptr) 3897 return (error); 3898 3899 if (result == 1) { 3900 adapter = (struct adapter *) arg1; 3901 ixv_print_debug_info(adapter); 3902 } 3903 return error; 3904} 3905 3906/* 3907** Set flow control using sysctl: 3908** Flow control values: 3909** 0 - off 3910** 1 - rx pause 3911** 2 - tx pause 3912** 3 - full 3913*/ 3914static int 3915ixv_set_flowcntl(SYSCTL_HANDLER_ARGS) 3916{ 3917 int error; 3918 struct adapter *adapter; 3919 3920 error = sysctl_handle_int(oidp, &ixv_flow_control, 0, req); 3921 3922 if (error) 3923 return (error); 3924 3925 adapter = (struct adapter *) arg1; 3926 switch (ixv_flow_control) { 3927 case ixgbe_fc_rx_pause: 3928 case ixgbe_fc_tx_pause: 3929 case ixgbe_fc_full: 3930 adapter->hw.fc.requested_mode = ixv_flow_control; 3931 break; 3932 case ixgbe_fc_none: 3933 default: 3934 adapter->hw.fc.requested_mode = ixgbe_fc_none; 3935 } 3936 3937 ixgbe_fc_enable(&adapter->hw, 0); 3938 return error; 3939} 3940 3941static void 3942ixv_add_rx_process_limit(struct adapter *adapter, const char *name, 3943 const char *description, int *limit, int value) 3944{ 3945 *limit = value; 3946 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 3947 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 3948 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 3949} 3950 3951