ixv.c revision 248287
1/****************************************************************************** 2 3 Copyright (c) 2001-2013, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ 33/*$FreeBSD: stable/9/sys/dev/ixgbe/ixv.c 248287 2013-03-14 21:39:39Z jfv $*/ 34 35#ifdef HAVE_KERNEL_OPTION_HEADERS 36#include "opt_inet.h" 37#include "opt_inet6.h" 38#endif 39 40#include "ixv.h" 41 42/********************************************************************* 43 * Driver version 44 *********************************************************************/ 45char ixv_driver_version[] = "1.1.4"; 46 47/********************************************************************* 48 * PCI Device ID Table 49 * 50 * Used by probe to select devices to load on 51 * Last field stores an index into ixv_strings 52 * Last entry must be all 0s 53 * 54 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 55 *********************************************************************/ 56 57static ixv_vendor_info_t ixv_vendor_info_array[] = 58{ 59 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 60 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 61 /* required last entry */ 62 {0, 0, 0, 0, 0} 63}; 64 65/********************************************************************* 66 * Table of branding strings 67 *********************************************************************/ 68 69static char *ixv_strings[] = { 70 "Intel(R) PRO/10GbE Virtual Function Network Driver" 71}; 72 73/********************************************************************* 74 * Function prototypes 75 *********************************************************************/ 76static int ixv_probe(device_t); 77static int ixv_attach(device_t); 78static int ixv_detach(device_t); 79static int ixv_shutdown(device_t); 80#if __FreeBSD_version < 800000 81static void ixv_start(struct ifnet *); 82static void ixv_start_locked(struct tx_ring *, struct ifnet *); 83#else 84static int ixv_mq_start(struct ifnet *, struct mbuf *); 85static int ixv_mq_start_locked(struct ifnet *, 86 struct tx_ring *, struct mbuf *); 87static void ixv_qflush(struct ifnet *); 88#endif 89static int ixv_ioctl(struct ifnet *, u_long, caddr_t); 90static void ixv_init(void *); 91static void ixv_init_locked(struct adapter *); 92static void ixv_stop(void *); 93static void ixv_media_status(struct ifnet *, struct ifmediareq *); 94static int ixv_media_change(struct ifnet *); 95static void ixv_identify_hardware(struct adapter *); 96static int ixv_allocate_pci_resources(struct adapter *); 97static int ixv_allocate_msix(struct adapter *); 98static int ixv_allocate_queues(struct adapter *); 99static int ixv_setup_msix(struct adapter *); 100static void ixv_free_pci_resources(struct adapter *); 101static void ixv_local_timer(void *); 102static void ixv_setup_interface(device_t, struct adapter *); 103static void ixv_config_link(struct adapter *); 104 105static int ixv_allocate_transmit_buffers(struct tx_ring *); 106static int ixv_setup_transmit_structures(struct adapter *); 107static void ixv_setup_transmit_ring(struct tx_ring *); 108static void ixv_initialize_transmit_units(struct adapter *); 109static void ixv_free_transmit_structures(struct adapter *); 110static void ixv_free_transmit_buffers(struct tx_ring *); 111 112static int ixv_allocate_receive_buffers(struct rx_ring *); 113static int ixv_setup_receive_structures(struct adapter *); 114static int ixv_setup_receive_ring(struct rx_ring *); 115static void ixv_initialize_receive_units(struct adapter *); 116static void ixv_free_receive_structures(struct adapter *); 117static void ixv_free_receive_buffers(struct rx_ring *); 118 119static void ixv_enable_intr(struct adapter *); 120static void ixv_disable_intr(struct adapter *); 121static bool ixv_txeof(struct tx_ring *); 122static bool ixv_rxeof(struct ix_queue *, int); 123static void ixv_rx_checksum(u32, struct mbuf *, u32); 124static void ixv_set_multi(struct adapter *); 125static void ixv_update_link_status(struct adapter *); 126static void ixv_refresh_mbufs(struct rx_ring *, int); 127static int ixv_xmit(struct tx_ring *, struct mbuf **); 128static int ixv_sysctl_stats(SYSCTL_HANDLER_ARGS); 129static int ixv_sysctl_debug(SYSCTL_HANDLER_ARGS); 130static int ixv_set_flowcntl(SYSCTL_HANDLER_ARGS); 131static int ixv_dma_malloc(struct adapter *, bus_size_t, 132 struct ixv_dma_alloc *, int); 133static void ixv_dma_free(struct adapter *, struct ixv_dma_alloc *); 134static void ixv_add_rx_process_limit(struct adapter *, const char *, 135 const char *, int *, int); 136static bool ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *); 137static bool ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 138static void ixv_set_ivar(struct adapter *, u8, u8, s8); 139static void ixv_configure_ivars(struct adapter *); 140static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 141 142static void ixv_setup_vlan_support(struct adapter *); 143static void ixv_register_vlan(void *, struct ifnet *, u16); 144static void ixv_unregister_vlan(void *, struct ifnet *, u16); 145 146static void ixv_save_stats(struct adapter *); 147static void ixv_init_stats(struct adapter *); 148static void ixv_update_stats(struct adapter *); 149 150static __inline void ixv_rx_discard(struct rx_ring *, int); 151static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *, 152 struct mbuf *, u32); 153 154/* The MSI/X Interrupt handlers */ 155static void ixv_msix_que(void *); 156static void ixv_msix_mbx(void *); 157 158/* Deferred interrupt tasklets */ 159static void ixv_handle_que(void *, int); 160static void ixv_handle_mbx(void *, int); 161 162/********************************************************************* 163 * FreeBSD Device Interface Entry Points 164 *********************************************************************/ 165 166static device_method_t ixv_methods[] = { 167 /* Device interface */ 168 DEVMETHOD(device_probe, ixv_probe), 169 DEVMETHOD(device_attach, ixv_attach), 170 DEVMETHOD(device_detach, ixv_detach), 171 DEVMETHOD(device_shutdown, ixv_shutdown), 172 DEVMETHOD_END 173}; 174 175static driver_t ixv_driver = { 176 "ix", ixv_methods, sizeof(struct adapter), 177}; 178 179extern devclass_t ixgbe_devclass; 180DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0); 181MODULE_DEPEND(ixv, pci, 1, 1, 1); 182MODULE_DEPEND(ixv, ether, 1, 1, 1); 183 184/* 185** TUNEABLE PARAMETERS: 186*/ 187 188/* 189** AIM: Adaptive Interrupt Moderation 190** which means that the interrupt rate 191** is varied over time based on the 192** traffic for that interrupt vector 193*/ 194static int ixv_enable_aim = FALSE; 195TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 196 197/* How many packets rxeof tries to clean at a time */ 198static int ixv_rx_process_limit = 128; 199TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 200 201/* Flow control setting, default to full */ 202static int ixv_flow_control = ixgbe_fc_full; 203TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control); 204 205/* 206 * Header split: this causes the hardware to DMA 207 * the header into a seperate mbuf from the payload, 208 * it can be a performance win in some workloads, but 209 * in others it actually hurts, its off by default. 210 */ 211static int ixv_header_split = FALSE; 212TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split); 213 214/* 215** Number of TX descriptors per ring, 216** setting higher than RX as this seems 217** the better performing choice. 218*/ 219static int ixv_txd = DEFAULT_TXD; 220TUNABLE_INT("hw.ixv.txd", &ixv_txd); 221 222/* Number of RX descriptors per ring */ 223static int ixv_rxd = DEFAULT_RXD; 224TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 225 226/* 227** Shadow VFTA table, this is needed because 228** the real filter table gets cleared during 229** a soft reset and we need to repopulate it. 230*/ 231static u32 ixv_shadow_vfta[VFTA_SIZE]; 232 233/********************************************************************* 234 * Device identification routine 235 * 236 * ixv_probe determines if the driver should be loaded on 237 * adapter based on PCI vendor/device id of the adapter. 238 * 239 * return BUS_PROBE_DEFAULT on success, positive on failure 240 *********************************************************************/ 241 242static int 243ixv_probe(device_t dev) 244{ 245 ixv_vendor_info_t *ent; 246 247 u16 pci_vendor_id = 0; 248 u16 pci_device_id = 0; 249 u16 pci_subvendor_id = 0; 250 u16 pci_subdevice_id = 0; 251 char adapter_name[256]; 252 253 254 pci_vendor_id = pci_get_vendor(dev); 255 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) 256 return (ENXIO); 257 258 pci_device_id = pci_get_device(dev); 259 pci_subvendor_id = pci_get_subvendor(dev); 260 pci_subdevice_id = pci_get_subdevice(dev); 261 262 ent = ixv_vendor_info_array; 263 while (ent->vendor_id != 0) { 264 if ((pci_vendor_id == ent->vendor_id) && 265 (pci_device_id == ent->device_id) && 266 267 ((pci_subvendor_id == ent->subvendor_id) || 268 (ent->subvendor_id == 0)) && 269 270 ((pci_subdevice_id == ent->subdevice_id) || 271 (ent->subdevice_id == 0))) { 272 sprintf(adapter_name, "%s, Version - %s", 273 ixv_strings[ent->index], 274 ixv_driver_version); 275 device_set_desc_copy(dev, adapter_name); 276 return (BUS_PROBE_DEFAULT); 277 } 278 ent++; 279 } 280 return (ENXIO); 281} 282 283/********************************************************************* 284 * Device initialization routine 285 * 286 * The attach entry point is called when the driver is being loaded. 287 * This routine identifies the type of hardware, allocates all resources 288 * and initializes the hardware. 289 * 290 * return 0 on success, positive on failure 291 *********************************************************************/ 292 293static int 294ixv_attach(device_t dev) 295{ 296 struct adapter *adapter; 297 struct ixgbe_hw *hw; 298 int error = 0; 299 300 INIT_DEBUGOUT("ixv_attach: begin"); 301 302 /* Allocate, clear, and link in our adapter structure */ 303 adapter = device_get_softc(dev); 304 adapter->dev = adapter->osdep.dev = dev; 305 hw = &adapter->hw; 306 307 /* Core Lock Init*/ 308 IXV_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 309 310 /* SYSCTL APIs */ 311 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 312 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 313 OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW, 314 adapter, 0, ixv_sysctl_stats, "I", "Statistics"); 315 316 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 317 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 318 OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW, 319 adapter, 0, ixv_sysctl_debug, "I", "Debug Info"); 320 321 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 322 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 323 OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW, 324 adapter, 0, ixv_set_flowcntl, "I", "Flow Control"); 325 326 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 327 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 328 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 329 &ixv_enable_aim, 1, "Interrupt Moderation"); 330 331 /* Set up the timer callout */ 332 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 333 334 /* Determine hardware revision */ 335 ixv_identify_hardware(adapter); 336 337 /* Do base PCI setup - map BAR0 */ 338 if (ixv_allocate_pci_resources(adapter)) { 339 device_printf(dev, "Allocation of PCI resources failed\n"); 340 error = ENXIO; 341 goto err_out; 342 } 343 344 /* Do descriptor calc and sanity checks */ 345 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 346 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 347 device_printf(dev, "TXD config issue, using default!\n"); 348 adapter->num_tx_desc = DEFAULT_TXD; 349 } else 350 adapter->num_tx_desc = ixv_txd; 351 352 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 353 ixv_rxd < MIN_TXD || ixv_rxd > MAX_TXD) { 354 device_printf(dev, "RXD config issue, using default!\n"); 355 adapter->num_rx_desc = DEFAULT_RXD; 356 } else 357 adapter->num_rx_desc = ixv_rxd; 358 359 /* Allocate our TX/RX Queues */ 360 if (ixv_allocate_queues(adapter)) { 361 error = ENOMEM; 362 goto err_out; 363 } 364 365 /* 366 ** Initialize the shared code: its 367 ** at this point the mac type is set. 368 */ 369 error = ixgbe_init_shared_code(hw); 370 if (error) { 371 device_printf(dev,"Shared Code Initialization Failure\n"); 372 error = EIO; 373 goto err_late; 374 } 375 376 /* Setup the mailbox */ 377 ixgbe_init_mbx_params_vf(hw); 378 379 ixgbe_reset_hw(hw); 380 381 /* Get Hardware Flow Control setting */ 382 hw->fc.requested_mode = ixgbe_fc_full; 383 hw->fc.pause_time = IXV_FC_PAUSE; 384 hw->fc.low_water[0] = IXV_FC_LO; 385 hw->fc.high_water[0] = IXV_FC_HI; 386 hw->fc.send_xon = TRUE; 387 388 error = ixgbe_init_hw(hw); 389 if (error) { 390 device_printf(dev,"Hardware Initialization Failure\n"); 391 error = EIO; 392 goto err_late; 393 } 394 395 error = ixv_allocate_msix(adapter); 396 if (error) 397 goto err_late; 398 399 /* Setup OS specific network interface */ 400 ixv_setup_interface(dev, adapter); 401 402 /* Sysctl for limiting the amount of work done in the taskqueue */ 403 ixv_add_rx_process_limit(adapter, "rx_processing_limit", 404 "max number of rx packets to process", &adapter->rx_process_limit, 405 ixv_rx_process_limit); 406 407 /* Do the stats setup */ 408 ixv_save_stats(adapter); 409 ixv_init_stats(adapter); 410 411 /* Register for VLAN events */ 412 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 413 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 414 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 415 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 416 417 INIT_DEBUGOUT("ixv_attach: end"); 418 return (0); 419 420err_late: 421 ixv_free_transmit_structures(adapter); 422 ixv_free_receive_structures(adapter); 423err_out: 424 ixv_free_pci_resources(adapter); 425 return (error); 426 427} 428 429/********************************************************************* 430 * Device removal routine 431 * 432 * The detach entry point is called when the driver is being removed. 433 * This routine stops the adapter and deallocates all the resources 434 * that were allocated for driver operation. 435 * 436 * return 0 on success, positive on failure 437 *********************************************************************/ 438 439static int 440ixv_detach(device_t dev) 441{ 442 struct adapter *adapter = device_get_softc(dev); 443 struct ix_queue *que = adapter->queues; 444 445 INIT_DEBUGOUT("ixv_detach: begin"); 446 447 /* Make sure VLANS are not using driver */ 448 if (adapter->ifp->if_vlantrunk != NULL) { 449 device_printf(dev,"Vlan in use, detach first\n"); 450 return (EBUSY); 451 } 452 453 IXV_CORE_LOCK(adapter); 454 ixv_stop(adapter); 455 IXV_CORE_UNLOCK(adapter); 456 457 for (int i = 0; i < adapter->num_queues; i++, que++) { 458 if (que->tq) { 459 taskqueue_drain(que->tq, &que->que_task); 460 taskqueue_free(que->tq); 461 } 462 } 463 464 /* Drain the Link queue */ 465 if (adapter->tq) { 466 taskqueue_drain(adapter->tq, &adapter->mbx_task); 467 taskqueue_free(adapter->tq); 468 } 469 470 /* Unregister VLAN events */ 471 if (adapter->vlan_attach != NULL) 472 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 473 if (adapter->vlan_detach != NULL) 474 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 475 476 ether_ifdetach(adapter->ifp); 477 callout_drain(&adapter->timer); 478 ixv_free_pci_resources(adapter); 479 bus_generic_detach(dev); 480 if_free(adapter->ifp); 481 482 ixv_free_transmit_structures(adapter); 483 ixv_free_receive_structures(adapter); 484 485 IXV_CORE_LOCK_DESTROY(adapter); 486 return (0); 487} 488 489/********************************************************************* 490 * 491 * Shutdown entry point 492 * 493 **********************************************************************/ 494static int 495ixv_shutdown(device_t dev) 496{ 497 struct adapter *adapter = device_get_softc(dev); 498 IXV_CORE_LOCK(adapter); 499 ixv_stop(adapter); 500 IXV_CORE_UNLOCK(adapter); 501 return (0); 502} 503 504#if __FreeBSD_version < 800000 505/********************************************************************* 506 * Transmit entry point 507 * 508 * ixv_start is called by the stack to initiate a transmit. 509 * The driver will remain in this routine as long as there are 510 * packets to transmit and transmit resources are available. 511 * In case resources are not available stack is notified and 512 * the packet is requeued. 513 **********************************************************************/ 514static void 515ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp) 516{ 517 struct mbuf *m_head; 518 struct adapter *adapter = txr->adapter; 519 520 IXV_TX_LOCK_ASSERT(txr); 521 522 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 523 IFF_DRV_RUNNING) 524 return; 525 if (!adapter->link_active) 526 return; 527 528 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 529 530 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 531 if (m_head == NULL) 532 break; 533 534 if (ixv_xmit(txr, &m_head)) { 535 if (m_head == NULL) 536 break; 537 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 538 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 539 break; 540 } 541 /* Send a copy of the frame to the BPF listener */ 542 ETHER_BPF_MTAP(ifp, m_head); 543 544 /* Set watchdog on */ 545 txr->watchdog_check = TRUE; 546 txr->watchdog_time = ticks; 547 548 } 549 return; 550} 551 552/* 553 * Legacy TX start - called by the stack, this 554 * always uses the first tx ring, and should 555 * not be used with multiqueue tx enabled. 556 */ 557static void 558ixv_start(struct ifnet *ifp) 559{ 560 struct adapter *adapter = ifp->if_softc; 561 struct tx_ring *txr = adapter->tx_rings; 562 563 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 564 IXV_TX_LOCK(txr); 565 ixv_start_locked(txr, ifp); 566 IXV_TX_UNLOCK(txr); 567 } 568 return; 569} 570 571#else 572 573/* 574** Multiqueue Transmit driver 575** 576*/ 577static int 578ixv_mq_start(struct ifnet *ifp, struct mbuf *m) 579{ 580 struct adapter *adapter = ifp->if_softc; 581 struct ix_queue *que; 582 struct tx_ring *txr; 583 int i = 0, err = 0; 584 585 /* Which queue to use */ 586 if ((m->m_flags & M_FLOWID) != 0) 587 i = m->m_pkthdr.flowid % adapter->num_queues; 588 589 txr = &adapter->tx_rings[i]; 590 que = &adapter->queues[i]; 591 592 if (IXV_TX_TRYLOCK(txr)) { 593 err = ixv_mq_start_locked(ifp, txr, m); 594 IXV_TX_UNLOCK(txr); 595 } else { 596 err = drbr_enqueue(ifp, txr->br, m); 597 taskqueue_enqueue(que->tq, &que->que_task); 598 } 599 600 return (err); 601} 602 603static int 604ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 605{ 606 struct adapter *adapter = txr->adapter; 607 struct mbuf *next; 608 int enqueued, err = 0; 609 610 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 611 IFF_DRV_RUNNING || adapter->link_active == 0) { 612 if (m != NULL) 613 err = drbr_enqueue(ifp, txr->br, m); 614 return (err); 615 } 616 617 /* Do a clean if descriptors are low */ 618 if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD) 619 ixv_txeof(txr); 620 621 enqueued = 0; 622 if (m != NULL) { 623 err = drbr_enqueue(ifp, txr->br, m); 624 if (err) { 625 return (err); 626 } 627 } 628 /* Process the queue */ 629 while ((next = drbr_peek(ifp, txr->br)) != NULL) { 630 if ((err = ixv_xmit(txr, &next)) != 0) { 631 if (next == NULL) { 632 drbr_advance(ifp, txr->br); 633 } else { 634 drbr_putback(ifp, txr->br, next); 635 } 636 break; 637 } 638 drbr_advance(ifp, txr->br); 639 enqueued++; 640 ifp->if_obytes += next->m_pkthdr.len; 641 if (next->m_flags & M_MCAST) 642 ifp->if_omcasts++; 643 /* Send a copy of the frame to the BPF listener */ 644 ETHER_BPF_MTAP(ifp, next); 645 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 646 break; 647 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) { 648 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 649 break; 650 } 651 } 652 653 if (enqueued > 0) { 654 /* Set watchdog on */ 655 txr->watchdog_check = TRUE; 656 txr->watchdog_time = ticks; 657 } 658 659 return (err); 660} 661 662/* 663** Flush all ring buffers 664*/ 665static void 666ixv_qflush(struct ifnet *ifp) 667{ 668 struct adapter *adapter = ifp->if_softc; 669 struct tx_ring *txr = adapter->tx_rings; 670 struct mbuf *m; 671 672 for (int i = 0; i < adapter->num_queues; i++, txr++) { 673 IXV_TX_LOCK(txr); 674 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 675 m_freem(m); 676 IXV_TX_UNLOCK(txr); 677 } 678 if_qflush(ifp); 679} 680 681#endif 682 683/********************************************************************* 684 * Ioctl entry point 685 * 686 * ixv_ioctl is called when the user wants to configure the 687 * interface. 688 * 689 * return 0 on success, positive on failure 690 **********************************************************************/ 691 692static int 693ixv_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 694{ 695 struct adapter *adapter = ifp->if_softc; 696 struct ifreq *ifr = (struct ifreq *) data; 697#if defined(INET) || defined(INET6) 698 struct ifaddr *ifa = (struct ifaddr *) data; 699 bool avoid_reset = FALSE; 700#endif 701 int error = 0; 702 703 switch (command) { 704 705 case SIOCSIFADDR: 706#ifdef INET 707 if (ifa->ifa_addr->sa_family == AF_INET) 708 avoid_reset = TRUE; 709#endif 710#ifdef INET6 711 if (ifa->ifa_addr->sa_family == AF_INET6) 712 avoid_reset = TRUE; 713#endif 714#if defined(INET) || defined(INET6) 715 /* 716 ** Calling init results in link renegotiation, 717 ** so we avoid doing it when possible. 718 */ 719 if (avoid_reset) { 720 ifp->if_flags |= IFF_UP; 721 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 722 ixv_init(adapter); 723 if (!(ifp->if_flags & IFF_NOARP)) 724 arp_ifinit(ifp, ifa); 725 } else 726 error = ether_ioctl(ifp, command, data); 727 break; 728#endif 729 case SIOCSIFMTU: 730 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 731 if (ifr->ifr_mtu > IXV_MAX_FRAME_SIZE - ETHER_HDR_LEN) { 732 error = EINVAL; 733 } else { 734 IXV_CORE_LOCK(adapter); 735 ifp->if_mtu = ifr->ifr_mtu; 736 adapter->max_frame_size = 737 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 738 ixv_init_locked(adapter); 739 IXV_CORE_UNLOCK(adapter); 740 } 741 break; 742 case SIOCSIFFLAGS: 743 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 744 IXV_CORE_LOCK(adapter); 745 if (ifp->if_flags & IFF_UP) { 746 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 747 ixv_init_locked(adapter); 748 } else 749 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 750 ixv_stop(adapter); 751 adapter->if_flags = ifp->if_flags; 752 IXV_CORE_UNLOCK(adapter); 753 break; 754 case SIOCADDMULTI: 755 case SIOCDELMULTI: 756 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 757 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 758 IXV_CORE_LOCK(adapter); 759 ixv_disable_intr(adapter); 760 ixv_set_multi(adapter); 761 ixv_enable_intr(adapter); 762 IXV_CORE_UNLOCK(adapter); 763 } 764 break; 765 case SIOCSIFMEDIA: 766 case SIOCGIFMEDIA: 767 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 768 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 769 break; 770 case SIOCSIFCAP: 771 { 772 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 773 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 774 if (mask & IFCAP_HWCSUM) 775 ifp->if_capenable ^= IFCAP_HWCSUM; 776 if (mask & IFCAP_TSO4) 777 ifp->if_capenable ^= IFCAP_TSO4; 778 if (mask & IFCAP_LRO) 779 ifp->if_capenable ^= IFCAP_LRO; 780 if (mask & IFCAP_VLAN_HWTAGGING) 781 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 782 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 783 IXV_CORE_LOCK(adapter); 784 ixv_init_locked(adapter); 785 IXV_CORE_UNLOCK(adapter); 786 } 787 VLAN_CAPABILITIES(ifp); 788 break; 789 } 790 791 default: 792 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 793 error = ether_ioctl(ifp, command, data); 794 break; 795 } 796 797 return (error); 798} 799 800/********************************************************************* 801 * Init entry point 802 * 803 * This routine is used in two ways. It is used by the stack as 804 * init entry point in network interface structure. It is also used 805 * by the driver as a hw/sw initialization routine to get to a 806 * consistent state. 807 * 808 * return 0 on success, positive on failure 809 **********************************************************************/ 810#define IXGBE_MHADD_MFS_SHIFT 16 811 812static void 813ixv_init_locked(struct adapter *adapter) 814{ 815 struct ifnet *ifp = adapter->ifp; 816 device_t dev = adapter->dev; 817 struct ixgbe_hw *hw = &adapter->hw; 818 u32 mhadd, gpie; 819 820 INIT_DEBUGOUT("ixv_init: begin"); 821 mtx_assert(&adapter->core_mtx, MA_OWNED); 822 hw->adapter_stopped = FALSE; 823 ixgbe_stop_adapter(hw); 824 callout_stop(&adapter->timer); 825 826 /* reprogram the RAR[0] in case user changed it. */ 827 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 828 829 /* Get the latest mac address, User can use a LAA */ 830 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr, 831 IXGBE_ETH_LENGTH_OF_ADDRESS); 832 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 833 hw->addr_ctrl.rar_used_count = 1; 834 835 /* Prepare transmit descriptors and buffers */ 836 if (ixv_setup_transmit_structures(adapter)) { 837 device_printf(dev,"Could not setup transmit structures\n"); 838 ixv_stop(adapter); 839 return; 840 } 841 842 ixgbe_reset_hw(hw); 843 ixv_initialize_transmit_units(adapter); 844 845 /* Setup Multicast table */ 846 ixv_set_multi(adapter); 847 848 /* 849 ** Determine the correct mbuf pool 850 ** for doing jumbo/headersplit 851 */ 852 if (ifp->if_mtu > ETHERMTU) 853 adapter->rx_mbuf_sz = MJUMPAGESIZE; 854 else 855 adapter->rx_mbuf_sz = MCLBYTES; 856 857 /* Prepare receive descriptors and buffers */ 858 if (ixv_setup_receive_structures(adapter)) { 859 device_printf(dev,"Could not setup receive structures\n"); 860 ixv_stop(adapter); 861 return; 862 } 863 864 /* Configure RX settings */ 865 ixv_initialize_receive_units(adapter); 866 867 /* Enable Enhanced MSIX mode */ 868 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 869 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME; 870 gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; 871 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 872 873 /* Set the various hardware offload abilities */ 874 ifp->if_hwassist = 0; 875 if (ifp->if_capenable & IFCAP_TSO4) 876 ifp->if_hwassist |= CSUM_TSO; 877 if (ifp->if_capenable & IFCAP_TXCSUM) { 878 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 879#if __FreeBSD_version >= 800000 880 ifp->if_hwassist |= CSUM_SCTP; 881#endif 882 } 883 884 /* Set MTU size */ 885 if (ifp->if_mtu > ETHERMTU) { 886 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 887 mhadd &= ~IXGBE_MHADD_MFS_MASK; 888 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 889 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 890 } 891 892 /* Set up VLAN offload and filter */ 893 ixv_setup_vlan_support(adapter); 894 895 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 896 897 /* Set up MSI/X routing */ 898 ixv_configure_ivars(adapter); 899 900 /* Set up auto-mask */ 901 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 902 903 /* Set moderation on the Link interrupt */ 904 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR); 905 906 /* Stats init */ 907 ixv_init_stats(adapter); 908 909 /* Config/Enable Link */ 910 ixv_config_link(adapter); 911 912 /* And now turn on interrupts */ 913 ixv_enable_intr(adapter); 914 915 /* Now inform the stack we're ready */ 916 ifp->if_drv_flags |= IFF_DRV_RUNNING; 917 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 918 919 return; 920} 921 922static void 923ixv_init(void *arg) 924{ 925 struct adapter *adapter = arg; 926 927 IXV_CORE_LOCK(adapter); 928 ixv_init_locked(adapter); 929 IXV_CORE_UNLOCK(adapter); 930 return; 931} 932 933 934/* 935** 936** MSIX Interrupt Handlers and Tasklets 937** 938*/ 939 940static inline void 941ixv_enable_queue(struct adapter *adapter, u32 vector) 942{ 943 struct ixgbe_hw *hw = &adapter->hw; 944 u32 queue = 1 << vector; 945 u32 mask; 946 947 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 948 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 949} 950 951static inline void 952ixv_disable_queue(struct adapter *adapter, u32 vector) 953{ 954 struct ixgbe_hw *hw = &adapter->hw; 955 u64 queue = (u64)(1 << vector); 956 u32 mask; 957 958 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 959 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 960} 961 962static inline void 963ixv_rearm_queues(struct adapter *adapter, u64 queues) 964{ 965 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 966 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 967} 968 969 970static void 971ixv_handle_que(void *context, int pending) 972{ 973 struct ix_queue *que = context; 974 struct adapter *adapter = que->adapter; 975 struct tx_ring *txr = que->txr; 976 struct ifnet *ifp = adapter->ifp; 977 bool more; 978 979 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 980 more = ixv_rxeof(que, adapter->rx_process_limit); 981 IXV_TX_LOCK(txr); 982 ixv_txeof(txr); 983#if __FreeBSD_version >= 800000 984 if (!drbr_empty(ifp, txr->br)) 985 ixv_mq_start_locked(ifp, txr, NULL); 986#else 987 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 988 ixv_start_locked(txr, ifp); 989#endif 990 IXV_TX_UNLOCK(txr); 991 if (more) { 992 taskqueue_enqueue(que->tq, &que->que_task); 993 return; 994 } 995 } 996 997 /* Reenable this interrupt */ 998 ixv_enable_queue(adapter, que->msix); 999 return; 1000} 1001 1002/********************************************************************* 1003 * 1004 * MSI Queue Interrupt Service routine 1005 * 1006 **********************************************************************/ 1007void 1008ixv_msix_que(void *arg) 1009{ 1010 struct ix_queue *que = arg; 1011 struct adapter *adapter = que->adapter; 1012 struct tx_ring *txr = que->txr; 1013 struct rx_ring *rxr = que->rxr; 1014 bool more_tx, more_rx; 1015 u32 newitr = 0; 1016 1017 ixv_disable_queue(adapter, que->msix); 1018 ++que->irqs; 1019 1020 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1021 1022 IXV_TX_LOCK(txr); 1023 more_tx = ixv_txeof(txr); 1024 /* 1025 ** Make certain that if the stack 1026 ** has anything queued the task gets 1027 ** scheduled to handle it. 1028 */ 1029#if __FreeBSD_version < 800000 1030 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd)) 1031#else 1032 if (!drbr_empty(adapter->ifp, txr->br)) 1033#endif 1034 more_tx = 1; 1035 IXV_TX_UNLOCK(txr); 1036 1037 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1038 1039 /* Do AIM now? */ 1040 1041 if (ixv_enable_aim == FALSE) 1042 goto no_calc; 1043 /* 1044 ** Do Adaptive Interrupt Moderation: 1045 ** - Write out last calculated setting 1046 ** - Calculate based on average size over 1047 ** the last interval. 1048 */ 1049 if (que->eitr_setting) 1050 IXGBE_WRITE_REG(&adapter->hw, 1051 IXGBE_VTEITR(que->msix), 1052 que->eitr_setting); 1053 1054 que->eitr_setting = 0; 1055 1056 /* Idle, do nothing */ 1057 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1058 goto no_calc; 1059 1060 if ((txr->bytes) && (txr->packets)) 1061 newitr = txr->bytes/txr->packets; 1062 if ((rxr->bytes) && (rxr->packets)) 1063 newitr = max(newitr, 1064 (rxr->bytes / rxr->packets)); 1065 newitr += 24; /* account for hardware frame, crc */ 1066 1067 /* set an upper boundary */ 1068 newitr = min(newitr, 3000); 1069 1070 /* Be nice to the mid range */ 1071 if ((newitr > 300) && (newitr < 1200)) 1072 newitr = (newitr / 3); 1073 else 1074 newitr = (newitr / 2); 1075 1076 newitr |= newitr << 16; 1077 1078 /* save for next interrupt */ 1079 que->eitr_setting = newitr; 1080 1081 /* Reset state */ 1082 txr->bytes = 0; 1083 txr->packets = 0; 1084 rxr->bytes = 0; 1085 rxr->packets = 0; 1086 1087no_calc: 1088 if (more_tx || more_rx) 1089 taskqueue_enqueue(que->tq, &que->que_task); 1090 else /* Reenable this interrupt */ 1091 ixv_enable_queue(adapter, que->msix); 1092 return; 1093} 1094 1095static void 1096ixv_msix_mbx(void *arg) 1097{ 1098 struct adapter *adapter = arg; 1099 struct ixgbe_hw *hw = &adapter->hw; 1100 u32 reg; 1101 1102 ++adapter->mbx_irq; 1103 1104 /* First get the cause */ 1105 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 1106 /* Clear interrupt with write */ 1107 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 1108 1109 /* Link status change */ 1110 if (reg & IXGBE_EICR_LSC) 1111 taskqueue_enqueue(adapter->tq, &adapter->mbx_task); 1112 1113 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 1114 return; 1115} 1116 1117/********************************************************************* 1118 * 1119 * Media Ioctl callback 1120 * 1121 * This routine is called whenever the user queries the status of 1122 * the interface using ifconfig. 1123 * 1124 **********************************************************************/ 1125static void 1126ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1127{ 1128 struct adapter *adapter = ifp->if_softc; 1129 1130 INIT_DEBUGOUT("ixv_media_status: begin"); 1131 IXV_CORE_LOCK(adapter); 1132 ixv_update_link_status(adapter); 1133 1134 ifmr->ifm_status = IFM_AVALID; 1135 ifmr->ifm_active = IFM_ETHER; 1136 1137 if (!adapter->link_active) { 1138 IXV_CORE_UNLOCK(adapter); 1139 return; 1140 } 1141 1142 ifmr->ifm_status |= IFM_ACTIVE; 1143 1144 switch (adapter->link_speed) { 1145 case IXGBE_LINK_SPEED_1GB_FULL: 1146 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1147 break; 1148 case IXGBE_LINK_SPEED_10GB_FULL: 1149 ifmr->ifm_active |= IFM_FDX; 1150 break; 1151 } 1152 1153 IXV_CORE_UNLOCK(adapter); 1154 1155 return; 1156} 1157 1158/********************************************************************* 1159 * 1160 * Media Ioctl callback 1161 * 1162 * This routine is called when the user changes speed/duplex using 1163 * media/mediopt option with ifconfig. 1164 * 1165 **********************************************************************/ 1166static int 1167ixv_media_change(struct ifnet * ifp) 1168{ 1169 struct adapter *adapter = ifp->if_softc; 1170 struct ifmedia *ifm = &adapter->media; 1171 1172 INIT_DEBUGOUT("ixv_media_change: begin"); 1173 1174 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1175 return (EINVAL); 1176 1177 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1178 case IFM_AUTO: 1179 break; 1180 default: 1181 device_printf(adapter->dev, "Only auto media type\n"); 1182 return (EINVAL); 1183 } 1184 1185 return (0); 1186} 1187 1188/********************************************************************* 1189 * 1190 * This routine maps the mbufs to tx descriptors, allowing the 1191 * TX engine to transmit the packets. 1192 * - return 0 on success, positive on failure 1193 * 1194 **********************************************************************/ 1195 1196static int 1197ixv_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1198{ 1199 struct adapter *adapter = txr->adapter; 1200 u32 olinfo_status = 0, cmd_type_len; 1201 u32 paylen = 0; 1202 int i, j, error, nsegs; 1203 int first, last = 0; 1204 struct mbuf *m_head; 1205 bus_dma_segment_t segs[32]; 1206 bus_dmamap_t map; 1207 struct ixv_tx_buf *txbuf, *txbuf_mapped; 1208 union ixgbe_adv_tx_desc *txd = NULL; 1209 1210 m_head = *m_headp; 1211 1212 /* Basic descriptor defines */ 1213 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1214 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1215 1216 if (m_head->m_flags & M_VLANTAG) 1217 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1218 1219 /* 1220 * Important to capture the first descriptor 1221 * used because it will contain the index of 1222 * the one we tell the hardware to report back 1223 */ 1224 first = txr->next_avail_desc; 1225 txbuf = &txr->tx_buffers[first]; 1226 txbuf_mapped = txbuf; 1227 map = txbuf->map; 1228 1229 /* 1230 * Map the packet for DMA. 1231 */ 1232 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1233 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1234 1235 if (error == EFBIG) { 1236 struct mbuf *m; 1237 1238 m = m_defrag(*m_headp, M_NOWAIT); 1239 if (m == NULL) { 1240 adapter->mbuf_defrag_failed++; 1241 m_freem(*m_headp); 1242 *m_headp = NULL; 1243 return (ENOBUFS); 1244 } 1245 *m_headp = m; 1246 1247 /* Try it again */ 1248 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1249 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1250 1251 if (error == ENOMEM) { 1252 adapter->no_tx_dma_setup++; 1253 return (error); 1254 } else if (error != 0) { 1255 adapter->no_tx_dma_setup++; 1256 m_freem(*m_headp); 1257 *m_headp = NULL; 1258 return (error); 1259 } 1260 } else if (error == ENOMEM) { 1261 adapter->no_tx_dma_setup++; 1262 return (error); 1263 } else if (error != 0) { 1264 adapter->no_tx_dma_setup++; 1265 m_freem(*m_headp); 1266 *m_headp = NULL; 1267 return (error); 1268 } 1269 1270 /* Make certain there are enough descriptors */ 1271 if (nsegs > txr->tx_avail - 2) { 1272 txr->no_desc_avail++; 1273 error = ENOBUFS; 1274 goto xmit_fail; 1275 } 1276 m_head = *m_headp; 1277 1278 /* 1279 ** Set up the appropriate offload context 1280 ** this becomes the first descriptor of 1281 ** a packet. 1282 */ 1283 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1284 if (ixv_tso_setup(txr, m_head, &paylen)) { 1285 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1286 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1287 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1288 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1289 ++adapter->tso_tx; 1290 } else 1291 return (ENXIO); 1292 } else if (ixv_tx_ctx_setup(txr, m_head)) 1293 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1294 1295 /* Record payload length */ 1296 if (paylen == 0) 1297 olinfo_status |= m_head->m_pkthdr.len << 1298 IXGBE_ADVTXD_PAYLEN_SHIFT; 1299 1300 i = txr->next_avail_desc; 1301 for (j = 0; j < nsegs; j++) { 1302 bus_size_t seglen; 1303 bus_addr_t segaddr; 1304 1305 txbuf = &txr->tx_buffers[i]; 1306 txd = &txr->tx_base[i]; 1307 seglen = segs[j].ds_len; 1308 segaddr = htole64(segs[j].ds_addr); 1309 1310 txd->read.buffer_addr = segaddr; 1311 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1312 cmd_type_len |seglen); 1313 txd->read.olinfo_status = htole32(olinfo_status); 1314 last = i; /* descriptor that will get completion IRQ */ 1315 1316 if (++i == adapter->num_tx_desc) 1317 i = 0; 1318 1319 txbuf->m_head = NULL; 1320 txbuf->eop_index = -1; 1321 } 1322 1323 txd->read.cmd_type_len |= 1324 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1325 txr->tx_avail -= nsegs; 1326 txr->next_avail_desc = i; 1327 1328 txbuf->m_head = m_head; 1329 txr->tx_buffers[first].map = txbuf->map; 1330 txbuf->map = map; 1331 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1332 1333 /* Set the index of the descriptor that will be marked done */ 1334 txbuf = &txr->tx_buffers[first]; 1335 txbuf->eop_index = last; 1336 1337 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1338 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1339 /* 1340 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1341 * hardware that this frame is available to transmit. 1342 */ 1343 ++txr->total_packets; 1344 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i); 1345 1346 return (0); 1347 1348xmit_fail: 1349 bus_dmamap_unload(txr->txtag, txbuf->map); 1350 return (error); 1351 1352} 1353 1354 1355/********************************************************************* 1356 * Multicast Update 1357 * 1358 * This routine is called whenever multicast address list is updated. 1359 * 1360 **********************************************************************/ 1361#define IXGBE_RAR_ENTRIES 16 1362 1363static void 1364ixv_set_multi(struct adapter *adapter) 1365{ 1366 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1367 u8 *update_ptr; 1368 struct ifmultiaddr *ifma; 1369 int mcnt = 0; 1370 struct ifnet *ifp = adapter->ifp; 1371 1372 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1373 1374#if __FreeBSD_version < 800000 1375 IF_ADDR_LOCK(ifp); 1376#else 1377 if_maddr_rlock(ifp); 1378#endif 1379 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1380 if (ifma->ifma_addr->sa_family != AF_LINK) 1381 continue; 1382 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), 1383 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1384 IXGBE_ETH_LENGTH_OF_ADDRESS); 1385 mcnt++; 1386 } 1387#if __FreeBSD_version < 800000 1388 IF_ADDR_UNLOCK(ifp); 1389#else 1390 if_maddr_runlock(ifp); 1391#endif 1392 1393 update_ptr = mta; 1394 1395 ixgbe_update_mc_addr_list(&adapter->hw, 1396 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1397 1398 return; 1399} 1400 1401/* 1402 * This is an iterator function now needed by the multicast 1403 * shared code. It simply feeds the shared code routine the 1404 * addresses in the array of ixv_set_multi() one by one. 1405 */ 1406static u8 * 1407ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1408{ 1409 u8 *addr = *update_ptr; 1410 u8 *newptr; 1411 *vmdq = 0; 1412 1413 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1414 *update_ptr = newptr; 1415 return addr; 1416} 1417 1418/********************************************************************* 1419 * Timer routine 1420 * 1421 * This routine checks for link status,updates statistics, 1422 * and runs the watchdog check. 1423 * 1424 **********************************************************************/ 1425 1426static void 1427ixv_local_timer(void *arg) 1428{ 1429 struct adapter *adapter = arg; 1430 device_t dev = adapter->dev; 1431 struct tx_ring *txr = adapter->tx_rings; 1432 int i; 1433 1434 mtx_assert(&adapter->core_mtx, MA_OWNED); 1435 1436 ixv_update_link_status(adapter); 1437 1438 /* Stats Update */ 1439 ixv_update_stats(adapter); 1440 1441 /* 1442 * If the interface has been paused 1443 * then don't do the watchdog check 1444 */ 1445 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 1446 goto out; 1447 /* 1448 ** Check for time since any descriptor was cleaned 1449 */ 1450 for (i = 0; i < adapter->num_queues; i++, txr++) { 1451 IXV_TX_LOCK(txr); 1452 if (txr->watchdog_check == FALSE) { 1453 IXV_TX_UNLOCK(txr); 1454 continue; 1455 } 1456 if ((ticks - txr->watchdog_time) > IXV_WATCHDOG) 1457 goto hung; 1458 IXV_TX_UNLOCK(txr); 1459 } 1460out: 1461 ixv_rearm_queues(adapter, adapter->que_mask); 1462 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1463 return; 1464 1465hung: 1466 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1467 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1468 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)), 1469 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i))); 1470 device_printf(dev,"TX(%d) desc avail = %d," 1471 "Next TX to Clean = %d\n", 1472 txr->me, txr->tx_avail, txr->next_to_clean); 1473 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1474 adapter->watchdog_events++; 1475 IXV_TX_UNLOCK(txr); 1476 ixv_init_locked(adapter); 1477} 1478 1479/* 1480** Note: this routine updates the OS on the link state 1481** the real check of the hardware only happens with 1482** a link interrupt. 1483*/ 1484static void 1485ixv_update_link_status(struct adapter *adapter) 1486{ 1487 struct ifnet *ifp = adapter->ifp; 1488 struct tx_ring *txr = adapter->tx_rings; 1489 device_t dev = adapter->dev; 1490 1491 1492 if (adapter->link_up){ 1493 if (adapter->link_active == FALSE) { 1494 if (bootverbose) 1495 device_printf(dev,"Link is up %d Gbps %s \n", 1496 ((adapter->link_speed == 128)? 10:1), 1497 "Full Duplex"); 1498 adapter->link_active = TRUE; 1499 if_link_state_change(ifp, LINK_STATE_UP); 1500 } 1501 } else { /* Link down */ 1502 if (adapter->link_active == TRUE) { 1503 if (bootverbose) 1504 device_printf(dev,"Link is Down\n"); 1505 if_link_state_change(ifp, LINK_STATE_DOWN); 1506 adapter->link_active = FALSE; 1507 for (int i = 0; i < adapter->num_queues; 1508 i++, txr++) 1509 txr->watchdog_check = FALSE; 1510 } 1511 } 1512 1513 return; 1514} 1515 1516 1517/********************************************************************* 1518 * 1519 * This routine disables all traffic on the adapter by issuing a 1520 * global reset on the MAC and deallocates TX/RX buffers. 1521 * 1522 **********************************************************************/ 1523 1524static void 1525ixv_stop(void *arg) 1526{ 1527 struct ifnet *ifp; 1528 struct adapter *adapter = arg; 1529 struct ixgbe_hw *hw = &adapter->hw; 1530 ifp = adapter->ifp; 1531 1532 mtx_assert(&adapter->core_mtx, MA_OWNED); 1533 1534 INIT_DEBUGOUT("ixv_stop: begin\n"); 1535 ixv_disable_intr(adapter); 1536 1537 /* Tell the stack that the interface is no longer active */ 1538 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1539 1540 ixgbe_reset_hw(hw); 1541 adapter->hw.adapter_stopped = FALSE; 1542 ixgbe_stop_adapter(hw); 1543 callout_stop(&adapter->timer); 1544 1545 /* reprogram the RAR[0] in case user changed it. */ 1546 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1547 1548 return; 1549} 1550 1551 1552/********************************************************************* 1553 * 1554 * Determine hardware revision. 1555 * 1556 **********************************************************************/ 1557static void 1558ixv_identify_hardware(struct adapter *adapter) 1559{ 1560 device_t dev = adapter->dev; 1561 u16 pci_cmd_word; 1562 1563 /* 1564 ** Make sure BUSMASTER is set, on a VM under 1565 ** KVM it may not be and will break things. 1566 */ 1567 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 1568 if (!((pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 1569 (pci_cmd_word & PCIM_CMD_MEMEN))) { 1570 INIT_DEBUGOUT("Memory Access and/or Bus Master " 1571 "bits were not set!\n"); 1572 pci_cmd_word |= (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 1573 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); 1574 } 1575 1576 /* Save off the information about this board */ 1577 adapter->hw.vendor_id = pci_get_vendor(dev); 1578 adapter->hw.device_id = pci_get_device(dev); 1579 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1580 adapter->hw.subsystem_vendor_id = 1581 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1582 adapter->hw.subsystem_device_id = 1583 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1584 1585 return; 1586} 1587 1588/********************************************************************* 1589 * 1590 * Setup MSIX Interrupt resources and handlers 1591 * 1592 **********************************************************************/ 1593static int 1594ixv_allocate_msix(struct adapter *adapter) 1595{ 1596 device_t dev = adapter->dev; 1597 struct ix_queue *que = adapter->queues; 1598 int error, rid, vector = 0; 1599 1600 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 1601 rid = vector + 1; 1602 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 1603 RF_SHAREABLE | RF_ACTIVE); 1604 if (que->res == NULL) { 1605 device_printf(dev,"Unable to allocate" 1606 " bus resource: que interrupt [%d]\n", vector); 1607 return (ENXIO); 1608 } 1609 /* Set the handler function */ 1610 error = bus_setup_intr(dev, que->res, 1611 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1612 ixv_msix_que, que, &que->tag); 1613 if (error) { 1614 que->res = NULL; 1615 device_printf(dev, "Failed to register QUE handler"); 1616 return (error); 1617 } 1618#if __FreeBSD_version >= 800504 1619 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 1620#endif 1621 que->msix = vector; 1622 adapter->que_mask |= (u64)(1 << que->msix); 1623 /* 1624 ** Bind the msix vector, and thus the 1625 ** ring to the corresponding cpu. 1626 */ 1627 if (adapter->num_queues > 1) 1628 bus_bind_intr(dev, que->res, i); 1629 1630 TASK_INIT(&que->que_task, 0, ixv_handle_que, que); 1631 que->tq = taskqueue_create_fast("ixv_que", M_NOWAIT, 1632 taskqueue_thread_enqueue, &que->tq); 1633 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 1634 device_get_nameunit(adapter->dev)); 1635 } 1636 1637 /* and Mailbox */ 1638 rid = vector + 1; 1639 adapter->res = bus_alloc_resource_any(dev, 1640 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 1641 if (!adapter->res) { 1642 device_printf(dev,"Unable to allocate" 1643 " bus resource: MBX interrupt [%d]\n", rid); 1644 return (ENXIO); 1645 } 1646 /* Set the mbx handler function */ 1647 error = bus_setup_intr(dev, adapter->res, 1648 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1649 ixv_msix_mbx, adapter, &adapter->tag); 1650 if (error) { 1651 adapter->res = NULL; 1652 device_printf(dev, "Failed to register LINK handler"); 1653 return (error); 1654 } 1655#if __FreeBSD_version >= 800504 1656 bus_describe_intr(dev, adapter->res, adapter->tag, "mbx"); 1657#endif 1658 adapter->mbxvec = vector; 1659 /* Tasklets for Mailbox */ 1660 TASK_INIT(&adapter->mbx_task, 0, ixv_handle_mbx, adapter); 1661 adapter->tq = taskqueue_create_fast("ixv_mbx", M_NOWAIT, 1662 taskqueue_thread_enqueue, &adapter->tq); 1663 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s mbxq", 1664 device_get_nameunit(adapter->dev)); 1665 /* 1666 ** Due to a broken design QEMU will fail to properly 1667 ** enable the guest for MSIX unless the vectors in 1668 ** the table are all set up, so we must rewrite the 1669 ** ENABLE in the MSIX control register again at this 1670 ** point to cause it to successfully initialize us. 1671 */ 1672 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 1673 int msix_ctrl; 1674 pci_find_cap(dev, PCIY_MSIX, &rid); 1675 rid += PCIR_MSIX_CTRL; 1676 msix_ctrl = pci_read_config(dev, rid, 2); 1677 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 1678 pci_write_config(dev, rid, msix_ctrl, 2); 1679 } 1680 1681 return (0); 1682} 1683 1684/* 1685 * Setup MSIX resources, note that the VF 1686 * device MUST use MSIX, there is no fallback. 1687 */ 1688static int 1689ixv_setup_msix(struct adapter *adapter) 1690{ 1691 device_t dev = adapter->dev; 1692 int rid, vectors, want = 2; 1693 1694 1695 /* First try MSI/X */ 1696 rid = PCIR_BAR(3); 1697 adapter->msix_mem = bus_alloc_resource_any(dev, 1698 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1699 if (!adapter->msix_mem) { 1700 device_printf(adapter->dev, 1701 "Unable to map MSIX table \n"); 1702 goto out; 1703 } 1704 1705 vectors = pci_msix_count(dev); 1706 if (vectors < 2) { 1707 bus_release_resource(dev, SYS_RES_MEMORY, 1708 rid, adapter->msix_mem); 1709 adapter->msix_mem = NULL; 1710 goto out; 1711 } 1712 1713 /* 1714 ** Want two vectors: one for a queue, 1715 ** plus an additional for mailbox. 1716 */ 1717 if (pci_alloc_msix(dev, &want) == 0) { 1718 device_printf(adapter->dev, 1719 "Using MSIX interrupts with %d vectors\n", want); 1720 return (want); 1721 } 1722out: 1723 device_printf(adapter->dev,"MSIX config error\n"); 1724 return (ENXIO); 1725} 1726 1727 1728static int 1729ixv_allocate_pci_resources(struct adapter *adapter) 1730{ 1731 int rid; 1732 device_t dev = adapter->dev; 1733 1734 rid = PCIR_BAR(0); 1735 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1736 &rid, RF_ACTIVE); 1737 1738 if (!(adapter->pci_mem)) { 1739 device_printf(dev,"Unable to allocate bus resource: memory\n"); 1740 return (ENXIO); 1741 } 1742 1743 adapter->osdep.mem_bus_space_tag = 1744 rman_get_bustag(adapter->pci_mem); 1745 adapter->osdep.mem_bus_space_handle = 1746 rman_get_bushandle(adapter->pci_mem); 1747 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle; 1748 1749 adapter->num_queues = 1; 1750 adapter->hw.back = &adapter->osdep; 1751 1752 /* 1753 ** Now setup MSI/X, should 1754 ** return us the number of 1755 ** configured vectors. 1756 */ 1757 adapter->msix = ixv_setup_msix(adapter); 1758 if (adapter->msix == ENXIO) 1759 return (ENXIO); 1760 else 1761 return (0); 1762} 1763 1764static void 1765ixv_free_pci_resources(struct adapter * adapter) 1766{ 1767 struct ix_queue *que = adapter->queues; 1768 device_t dev = adapter->dev; 1769 int rid, memrid; 1770 1771 memrid = PCIR_BAR(MSIX_BAR); 1772 1773 /* 1774 ** There is a slight possibility of a failure mode 1775 ** in attach that will result in entering this function 1776 ** before interrupt resources have been initialized, and 1777 ** in that case we do not want to execute the loops below 1778 ** We can detect this reliably by the state of the adapter 1779 ** res pointer. 1780 */ 1781 if (adapter->res == NULL) 1782 goto mem; 1783 1784 /* 1785 ** Release all msix queue resources: 1786 */ 1787 for (int i = 0; i < adapter->num_queues; i++, que++) { 1788 rid = que->msix + 1; 1789 if (que->tag != NULL) { 1790 bus_teardown_intr(dev, que->res, que->tag); 1791 que->tag = NULL; 1792 } 1793 if (que->res != NULL) 1794 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 1795 } 1796 1797 1798 /* Clean the Legacy or Link interrupt last */ 1799 if (adapter->mbxvec) /* we are doing MSIX */ 1800 rid = adapter->mbxvec + 1; 1801 else 1802 (adapter->msix != 0) ? (rid = 1):(rid = 0); 1803 1804 if (adapter->tag != NULL) { 1805 bus_teardown_intr(dev, adapter->res, adapter->tag); 1806 adapter->tag = NULL; 1807 } 1808 if (adapter->res != NULL) 1809 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 1810 1811mem: 1812 if (adapter->msix) 1813 pci_release_msi(dev); 1814 1815 if (adapter->msix_mem != NULL) 1816 bus_release_resource(dev, SYS_RES_MEMORY, 1817 memrid, adapter->msix_mem); 1818 1819 if (adapter->pci_mem != NULL) 1820 bus_release_resource(dev, SYS_RES_MEMORY, 1821 PCIR_BAR(0), adapter->pci_mem); 1822 1823 return; 1824} 1825 1826/********************************************************************* 1827 * 1828 * Setup networking device structure and register an interface. 1829 * 1830 **********************************************************************/ 1831static void 1832ixv_setup_interface(device_t dev, struct adapter *adapter) 1833{ 1834 struct ifnet *ifp; 1835 1836 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1837 1838 ifp = adapter->ifp = if_alloc(IFT_ETHER); 1839 if (ifp == NULL) 1840 panic("%s: can not if_alloc()\n", device_get_nameunit(dev)); 1841 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1842 ifp->if_baudrate = 1000000000; 1843 ifp->if_init = ixv_init; 1844 ifp->if_softc = adapter; 1845 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1846 ifp->if_ioctl = ixv_ioctl; 1847#if __FreeBSD_version >= 800000 1848 ifp->if_transmit = ixv_mq_start; 1849 ifp->if_qflush = ixv_qflush; 1850#else 1851 ifp->if_start = ixv_start; 1852#endif 1853 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 1854 1855 ether_ifattach(ifp, adapter->hw.mac.addr); 1856 1857 adapter->max_frame_size = 1858 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1859 1860 /* 1861 * Tell the upper layer(s) we support long frames. 1862 */ 1863 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1864 1865 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM; 1866 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 1867 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING 1868 | IFCAP_VLAN_HWTSO 1869 | IFCAP_VLAN_MTU; 1870 ifp->if_capenable = ifp->if_capabilities; 1871 1872 /* Don't enable LRO by default */ 1873 ifp->if_capabilities |= IFCAP_LRO; 1874 1875 /* 1876 * Specify the media types supported by this adapter and register 1877 * callbacks to update media and link information 1878 */ 1879 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1880 ixv_media_status); 1881 ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL); 1882 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1883 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1884 1885 return; 1886} 1887 1888static void 1889ixv_config_link(struct adapter *adapter) 1890{ 1891 struct ixgbe_hw *hw = &adapter->hw; 1892 u32 autoneg, err = 0; 1893 1894 if (hw->mac.ops.check_link) 1895 err = hw->mac.ops.check_link(hw, &autoneg, 1896 &adapter->link_up, FALSE); 1897 if (err) 1898 goto out; 1899 1900 if (hw->mac.ops.setup_link) 1901 err = hw->mac.ops.setup_link(hw, 1902 autoneg, adapter->link_up); 1903out: 1904 return; 1905} 1906 1907/******************************************************************** 1908 * Manage DMA'able memory. 1909 *******************************************************************/ 1910static void 1911ixv_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) 1912{ 1913 if (error) 1914 return; 1915 *(bus_addr_t *) arg = segs->ds_addr; 1916 return; 1917} 1918 1919static int 1920ixv_dma_malloc(struct adapter *adapter, bus_size_t size, 1921 struct ixv_dma_alloc *dma, int mapflags) 1922{ 1923 device_t dev = adapter->dev; 1924 int r; 1925 1926 r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 1927 DBA_ALIGN, 0, /* alignment, bounds */ 1928 BUS_SPACE_MAXADDR, /* lowaddr */ 1929 BUS_SPACE_MAXADDR, /* highaddr */ 1930 NULL, NULL, /* filter, filterarg */ 1931 size, /* maxsize */ 1932 1, /* nsegments */ 1933 size, /* maxsegsize */ 1934 BUS_DMA_ALLOCNOW, /* flags */ 1935 NULL, /* lockfunc */ 1936 NULL, /* lockfuncarg */ 1937 &dma->dma_tag); 1938 if (r != 0) { 1939 device_printf(dev,"ixv_dma_malloc: bus_dma_tag_create failed; " 1940 "error %u\n", r); 1941 goto fail_0; 1942 } 1943 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 1944 BUS_DMA_NOWAIT, &dma->dma_map); 1945 if (r != 0) { 1946 device_printf(dev,"ixv_dma_malloc: bus_dmamem_alloc failed; " 1947 "error %u\n", r); 1948 goto fail_1; 1949 } 1950 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1951 size, 1952 ixv_dmamap_cb, 1953 &dma->dma_paddr, 1954 mapflags | BUS_DMA_NOWAIT); 1955 if (r != 0) { 1956 device_printf(dev,"ixv_dma_malloc: bus_dmamap_load failed; " 1957 "error %u\n", r); 1958 goto fail_2; 1959 } 1960 dma->dma_size = size; 1961 return (0); 1962fail_2: 1963 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1964fail_1: 1965 bus_dma_tag_destroy(dma->dma_tag); 1966fail_0: 1967 dma->dma_map = NULL; 1968 dma->dma_tag = NULL; 1969 return (r); 1970} 1971 1972static void 1973ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma) 1974{ 1975 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 1976 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1977 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1978 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1979 bus_dma_tag_destroy(dma->dma_tag); 1980} 1981 1982 1983/********************************************************************* 1984 * 1985 * Allocate memory for the transmit and receive rings, and then 1986 * the descriptors associated with each, called only once at attach. 1987 * 1988 **********************************************************************/ 1989static int 1990ixv_allocate_queues(struct adapter *adapter) 1991{ 1992 device_t dev = adapter->dev; 1993 struct ix_queue *que; 1994 struct tx_ring *txr; 1995 struct rx_ring *rxr; 1996 int rsize, tsize, error = 0; 1997 int txconf = 0, rxconf = 0; 1998 1999 /* First allocate the top level queue structs */ 2000 if (!(adapter->queues = 2001 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2002 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2003 device_printf(dev, "Unable to allocate queue memory\n"); 2004 error = ENOMEM; 2005 goto fail; 2006 } 2007 2008 /* First allocate the TX ring struct memory */ 2009 if (!(adapter->tx_rings = 2010 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2011 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2012 device_printf(dev, "Unable to allocate TX ring memory\n"); 2013 error = ENOMEM; 2014 goto tx_fail; 2015 } 2016 2017 /* Next allocate the RX */ 2018 if (!(adapter->rx_rings = 2019 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2020 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2021 device_printf(dev, "Unable to allocate RX ring memory\n"); 2022 error = ENOMEM; 2023 goto rx_fail; 2024 } 2025 2026 /* For the ring itself */ 2027 tsize = roundup2(adapter->num_tx_desc * 2028 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2029 2030 /* 2031 * Now set up the TX queues, txconf is needed to handle the 2032 * possibility that things fail midcourse and we need to 2033 * undo memory gracefully 2034 */ 2035 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2036 /* Set up some basics */ 2037 txr = &adapter->tx_rings[i]; 2038 txr->adapter = adapter; 2039 txr->me = i; 2040 2041 /* Initialize the TX side lock */ 2042 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2043 device_get_nameunit(dev), txr->me); 2044 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2045 2046 if (ixv_dma_malloc(adapter, tsize, 2047 &txr->txdma, BUS_DMA_NOWAIT)) { 2048 device_printf(dev, 2049 "Unable to allocate TX Descriptor memory\n"); 2050 error = ENOMEM; 2051 goto err_tx_desc; 2052 } 2053 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2054 bzero((void *)txr->tx_base, tsize); 2055 2056 /* Now allocate transmit buffers for the ring */ 2057 if (ixv_allocate_transmit_buffers(txr)) { 2058 device_printf(dev, 2059 "Critical Failure setting up transmit buffers\n"); 2060 error = ENOMEM; 2061 goto err_tx_desc; 2062 } 2063#if __FreeBSD_version >= 800000 2064 /* Allocate a buf ring */ 2065 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF, 2066 M_WAITOK, &txr->tx_mtx); 2067 if (txr->br == NULL) { 2068 device_printf(dev, 2069 "Critical Failure setting up buf ring\n"); 2070 error = ENOMEM; 2071 goto err_tx_desc; 2072 } 2073#endif 2074 } 2075 2076 /* 2077 * Next the RX queues... 2078 */ 2079 rsize = roundup2(adapter->num_rx_desc * 2080 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2081 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2082 rxr = &adapter->rx_rings[i]; 2083 /* Set up some basics */ 2084 rxr->adapter = adapter; 2085 rxr->me = i; 2086 2087 /* Initialize the RX side lock */ 2088 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2089 device_get_nameunit(dev), rxr->me); 2090 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2091 2092 if (ixv_dma_malloc(adapter, rsize, 2093 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2094 device_printf(dev, 2095 "Unable to allocate RxDescriptor memory\n"); 2096 error = ENOMEM; 2097 goto err_rx_desc; 2098 } 2099 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2100 bzero((void *)rxr->rx_base, rsize); 2101 2102 /* Allocate receive buffers for the ring*/ 2103 if (ixv_allocate_receive_buffers(rxr)) { 2104 device_printf(dev, 2105 "Critical Failure setting up receive buffers\n"); 2106 error = ENOMEM; 2107 goto err_rx_desc; 2108 } 2109 } 2110 2111 /* 2112 ** Finally set up the queue holding structs 2113 */ 2114 for (int i = 0; i < adapter->num_queues; i++) { 2115 que = &adapter->queues[i]; 2116 que->adapter = adapter; 2117 que->txr = &adapter->tx_rings[i]; 2118 que->rxr = &adapter->rx_rings[i]; 2119 } 2120 2121 return (0); 2122 2123err_rx_desc: 2124 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2125 ixv_dma_free(adapter, &rxr->rxdma); 2126err_tx_desc: 2127 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2128 ixv_dma_free(adapter, &txr->txdma); 2129 free(adapter->rx_rings, M_DEVBUF); 2130rx_fail: 2131 free(adapter->tx_rings, M_DEVBUF); 2132tx_fail: 2133 free(adapter->queues, M_DEVBUF); 2134fail: 2135 return (error); 2136} 2137 2138 2139/********************************************************************* 2140 * 2141 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2142 * the information needed to transmit a packet on the wire. This is 2143 * called only once at attach, setup is done every reset. 2144 * 2145 **********************************************************************/ 2146static int 2147ixv_allocate_transmit_buffers(struct tx_ring *txr) 2148{ 2149 struct adapter *adapter = txr->adapter; 2150 device_t dev = adapter->dev; 2151 struct ixv_tx_buf *txbuf; 2152 int error, i; 2153 2154 /* 2155 * Setup DMA descriptor areas. 2156 */ 2157 if ((error = bus_dma_tag_create( 2158 bus_get_dma_tag(adapter->dev), /* parent */ 2159 1, 0, /* alignment, bounds */ 2160 BUS_SPACE_MAXADDR, /* lowaddr */ 2161 BUS_SPACE_MAXADDR, /* highaddr */ 2162 NULL, NULL, /* filter, filterarg */ 2163 IXV_TSO_SIZE, /* maxsize */ 2164 32, /* nsegments */ 2165 PAGE_SIZE, /* maxsegsize */ 2166 0, /* flags */ 2167 NULL, /* lockfunc */ 2168 NULL, /* lockfuncarg */ 2169 &txr->txtag))) { 2170 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2171 goto fail; 2172 } 2173 2174 if (!(txr->tx_buffers = 2175 (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) * 2176 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2177 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2178 error = ENOMEM; 2179 goto fail; 2180 } 2181 2182 /* Create the descriptor buffer dma maps */ 2183 txbuf = txr->tx_buffers; 2184 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2185 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2186 if (error != 0) { 2187 device_printf(dev, "Unable to create TX DMA map\n"); 2188 goto fail; 2189 } 2190 } 2191 2192 return 0; 2193fail: 2194 /* We free all, it handles case where we are in the middle */ 2195 ixv_free_transmit_structures(adapter); 2196 return (error); 2197} 2198 2199/********************************************************************* 2200 * 2201 * Initialize a transmit ring. 2202 * 2203 **********************************************************************/ 2204static void 2205ixv_setup_transmit_ring(struct tx_ring *txr) 2206{ 2207 struct adapter *adapter = txr->adapter; 2208 struct ixv_tx_buf *txbuf; 2209 int i; 2210 2211 /* Clear the old ring contents */ 2212 IXV_TX_LOCK(txr); 2213 bzero((void *)txr->tx_base, 2214 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2215 /* Reset indices */ 2216 txr->next_avail_desc = 0; 2217 txr->next_to_clean = 0; 2218 2219 /* Free any existing tx buffers. */ 2220 txbuf = txr->tx_buffers; 2221 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2222 if (txbuf->m_head != NULL) { 2223 bus_dmamap_sync(txr->txtag, txbuf->map, 2224 BUS_DMASYNC_POSTWRITE); 2225 bus_dmamap_unload(txr->txtag, txbuf->map); 2226 m_freem(txbuf->m_head); 2227 txbuf->m_head = NULL; 2228 } 2229 /* Clear the EOP index */ 2230 txbuf->eop_index = -1; 2231 } 2232 2233 /* Set number of descriptors available */ 2234 txr->tx_avail = adapter->num_tx_desc; 2235 2236 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2237 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2238 IXV_TX_UNLOCK(txr); 2239} 2240 2241/********************************************************************* 2242 * 2243 * Initialize all transmit rings. 2244 * 2245 **********************************************************************/ 2246static int 2247ixv_setup_transmit_structures(struct adapter *adapter) 2248{ 2249 struct tx_ring *txr = adapter->tx_rings; 2250 2251 for (int i = 0; i < adapter->num_queues; i++, txr++) 2252 ixv_setup_transmit_ring(txr); 2253 2254 return (0); 2255} 2256 2257/********************************************************************* 2258 * 2259 * Enable transmit unit. 2260 * 2261 **********************************************************************/ 2262static void 2263ixv_initialize_transmit_units(struct adapter *adapter) 2264{ 2265 struct tx_ring *txr = adapter->tx_rings; 2266 struct ixgbe_hw *hw = &adapter->hw; 2267 2268 2269 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2270 u64 tdba = txr->txdma.dma_paddr; 2271 u32 txctrl, txdctl; 2272 2273 /* Set WTHRESH to 8, burst writeback */ 2274 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2275 txdctl |= (8 << 16); 2276 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2277 /* Now enable */ 2278 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2279 txdctl |= IXGBE_TXDCTL_ENABLE; 2280 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2281 2282 /* Set the HW Tx Head and Tail indices */ 2283 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0); 2284 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0); 2285 2286 /* Setup Transmit Descriptor Cmd Settings */ 2287 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2288 txr->watchdog_check = FALSE; 2289 2290 /* Set Ring parameters */ 2291 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 2292 (tdba & 0x00000000ffffffffULL)); 2293 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32)); 2294 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 2295 adapter->num_tx_desc * 2296 sizeof(struct ixgbe_legacy_tx_desc)); 2297 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i)); 2298 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2299 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl); 2300 break; 2301 } 2302 2303 return; 2304} 2305 2306/********************************************************************* 2307 * 2308 * Free all transmit rings. 2309 * 2310 **********************************************************************/ 2311static void 2312ixv_free_transmit_structures(struct adapter *adapter) 2313{ 2314 struct tx_ring *txr = adapter->tx_rings; 2315 2316 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2317 IXV_TX_LOCK(txr); 2318 ixv_free_transmit_buffers(txr); 2319 ixv_dma_free(adapter, &txr->txdma); 2320 IXV_TX_UNLOCK(txr); 2321 IXV_TX_LOCK_DESTROY(txr); 2322 } 2323 free(adapter->tx_rings, M_DEVBUF); 2324} 2325 2326/********************************************************************* 2327 * 2328 * Free transmit ring related data structures. 2329 * 2330 **********************************************************************/ 2331static void 2332ixv_free_transmit_buffers(struct tx_ring *txr) 2333{ 2334 struct adapter *adapter = txr->adapter; 2335 struct ixv_tx_buf *tx_buffer; 2336 int i; 2337 2338 INIT_DEBUGOUT("free_transmit_ring: begin"); 2339 2340 if (txr->tx_buffers == NULL) 2341 return; 2342 2343 tx_buffer = txr->tx_buffers; 2344 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2345 if (tx_buffer->m_head != NULL) { 2346 bus_dmamap_sync(txr->txtag, tx_buffer->map, 2347 BUS_DMASYNC_POSTWRITE); 2348 bus_dmamap_unload(txr->txtag, 2349 tx_buffer->map); 2350 m_freem(tx_buffer->m_head); 2351 tx_buffer->m_head = NULL; 2352 if (tx_buffer->map != NULL) { 2353 bus_dmamap_destroy(txr->txtag, 2354 tx_buffer->map); 2355 tx_buffer->map = NULL; 2356 } 2357 } else if (tx_buffer->map != NULL) { 2358 bus_dmamap_unload(txr->txtag, 2359 tx_buffer->map); 2360 bus_dmamap_destroy(txr->txtag, 2361 tx_buffer->map); 2362 tx_buffer->map = NULL; 2363 } 2364 } 2365#if __FreeBSD_version >= 800000 2366 if (txr->br != NULL) 2367 buf_ring_free(txr->br, M_DEVBUF); 2368#endif 2369 if (txr->tx_buffers != NULL) { 2370 free(txr->tx_buffers, M_DEVBUF); 2371 txr->tx_buffers = NULL; 2372 } 2373 if (txr->txtag != NULL) { 2374 bus_dma_tag_destroy(txr->txtag); 2375 txr->txtag = NULL; 2376 } 2377 return; 2378} 2379 2380/********************************************************************* 2381 * 2382 * Advanced Context Descriptor setup for VLAN or CSUM 2383 * 2384 **********************************************************************/ 2385 2386static bool 2387ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2388{ 2389 struct adapter *adapter = txr->adapter; 2390 struct ixgbe_adv_tx_context_desc *TXD; 2391 struct ixv_tx_buf *tx_buffer; 2392 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2393 struct ether_vlan_header *eh; 2394 struct ip *ip; 2395 struct ip6_hdr *ip6; 2396 int ehdrlen, ip_hlen = 0; 2397 u16 etype; 2398 u8 ipproto = 0; 2399 bool offload = TRUE; 2400 int ctxd = txr->next_avail_desc; 2401 u16 vtag = 0; 2402 2403 2404 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2405 offload = FALSE; 2406 2407 2408 tx_buffer = &txr->tx_buffers[ctxd]; 2409 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2410 2411 /* 2412 ** In advanced descriptors the vlan tag must 2413 ** be placed into the descriptor itself. 2414 */ 2415 if (mp->m_flags & M_VLANTAG) { 2416 vtag = htole16(mp->m_pkthdr.ether_vtag); 2417 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2418 } else if (offload == FALSE) 2419 return FALSE; 2420 2421 /* 2422 * Determine where frame payload starts. 2423 * Jump over vlan headers if already present, 2424 * helpful for QinQ too. 2425 */ 2426 eh = mtod(mp, struct ether_vlan_header *); 2427 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2428 etype = ntohs(eh->evl_proto); 2429 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2430 } else { 2431 etype = ntohs(eh->evl_encap_proto); 2432 ehdrlen = ETHER_HDR_LEN; 2433 } 2434 2435 /* Set the ether header length */ 2436 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2437 2438 switch (etype) { 2439 case ETHERTYPE_IP: 2440 ip = (struct ip *)(mp->m_data + ehdrlen); 2441 ip_hlen = ip->ip_hl << 2; 2442 if (mp->m_len < ehdrlen + ip_hlen) 2443 return (FALSE); 2444 ipproto = ip->ip_p; 2445 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2446 break; 2447 case ETHERTYPE_IPV6: 2448 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2449 ip_hlen = sizeof(struct ip6_hdr); 2450 if (mp->m_len < ehdrlen + ip_hlen) 2451 return (FALSE); 2452 ipproto = ip6->ip6_nxt; 2453 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2454 break; 2455 default: 2456 offload = FALSE; 2457 break; 2458 } 2459 2460 vlan_macip_lens |= ip_hlen; 2461 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2462 2463 switch (ipproto) { 2464 case IPPROTO_TCP: 2465 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2466 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2467 break; 2468 2469 case IPPROTO_UDP: 2470 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2471 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2472 break; 2473 2474#if __FreeBSD_version >= 800000 2475 case IPPROTO_SCTP: 2476 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 2477 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 2478 break; 2479#endif 2480 default: 2481 offload = FALSE; 2482 break; 2483 } 2484 2485 /* Now copy bits into descriptor */ 2486 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2487 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2488 TXD->seqnum_seed = htole32(0); 2489 TXD->mss_l4len_idx = htole32(0); 2490 2491 tx_buffer->m_head = NULL; 2492 tx_buffer->eop_index = -1; 2493 2494 /* We've consumed the first desc, adjust counters */ 2495 if (++ctxd == adapter->num_tx_desc) 2496 ctxd = 0; 2497 txr->next_avail_desc = ctxd; 2498 --txr->tx_avail; 2499 2500 return (offload); 2501} 2502 2503/********************************************************************** 2504 * 2505 * Setup work for hardware segmentation offload (TSO) on 2506 * adapters using advanced tx descriptors 2507 * 2508 **********************************************************************/ 2509static bool 2510ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2511{ 2512 struct adapter *adapter = txr->adapter; 2513 struct ixgbe_adv_tx_context_desc *TXD; 2514 struct ixv_tx_buf *tx_buffer; 2515 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2516 u32 mss_l4len_idx = 0; 2517 u16 vtag = 0; 2518 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2519 struct ether_vlan_header *eh; 2520 struct ip *ip; 2521 struct tcphdr *th; 2522 2523 2524 /* 2525 * Determine where frame payload starts. 2526 * Jump over vlan headers if already present 2527 */ 2528 eh = mtod(mp, struct ether_vlan_header *); 2529 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2530 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2531 else 2532 ehdrlen = ETHER_HDR_LEN; 2533 2534 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2535 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2536 return FALSE; 2537 2538 ctxd = txr->next_avail_desc; 2539 tx_buffer = &txr->tx_buffers[ctxd]; 2540 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2541 2542 ip = (struct ip *)(mp->m_data + ehdrlen); 2543 if (ip->ip_p != IPPROTO_TCP) 2544 return FALSE; /* 0 */ 2545 ip->ip_sum = 0; 2546 ip_hlen = ip->ip_hl << 2; 2547 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2548 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2549 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2550 tcp_hlen = th->th_off << 2; 2551 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2552 2553 /* This is used in the transmit desc in encap */ 2554 *paylen = mp->m_pkthdr.len - hdrlen; 2555 2556 /* VLAN MACLEN IPLEN */ 2557 if (mp->m_flags & M_VLANTAG) { 2558 vtag = htole16(mp->m_pkthdr.ether_vtag); 2559 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2560 } 2561 2562 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2563 vlan_macip_lens |= ip_hlen; 2564 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2565 2566 /* ADV DTYPE TUCMD */ 2567 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2568 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2569 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2570 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2571 2572 2573 /* MSS L4LEN IDX */ 2574 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2575 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2576 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2577 2578 TXD->seqnum_seed = htole32(0); 2579 tx_buffer->m_head = NULL; 2580 tx_buffer->eop_index = -1; 2581 2582 if (++ctxd == adapter->num_tx_desc) 2583 ctxd = 0; 2584 2585 txr->tx_avail--; 2586 txr->next_avail_desc = ctxd; 2587 return TRUE; 2588} 2589 2590 2591/********************************************************************** 2592 * 2593 * Examine each tx_buffer in the used queue. If the hardware is done 2594 * processing the packet then free associated resources. The 2595 * tx_buffer is put back on the free queue. 2596 * 2597 **********************************************************************/ 2598static bool 2599ixv_txeof(struct tx_ring *txr) 2600{ 2601 struct adapter *adapter = txr->adapter; 2602 struct ifnet *ifp = adapter->ifp; 2603 u32 first, last, done; 2604 struct ixv_tx_buf *tx_buffer; 2605 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2606 2607 mtx_assert(&txr->tx_mtx, MA_OWNED); 2608 2609 if (txr->tx_avail == adapter->num_tx_desc) 2610 return FALSE; 2611 2612 first = txr->next_to_clean; 2613 tx_buffer = &txr->tx_buffers[first]; 2614 /* For cleanup we just use legacy struct */ 2615 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2616 last = tx_buffer->eop_index; 2617 if (last == -1) 2618 return FALSE; 2619 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2620 2621 /* 2622 ** Get the index of the first descriptor 2623 ** BEYOND the EOP and call that 'done'. 2624 ** I do this so the comparison in the 2625 ** inner while loop below can be simple 2626 */ 2627 if (++last == adapter->num_tx_desc) last = 0; 2628 done = last; 2629 2630 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2631 BUS_DMASYNC_POSTREAD); 2632 /* 2633 ** Only the EOP descriptor of a packet now has the DD 2634 ** bit set, this is what we look for... 2635 */ 2636 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2637 /* We clean the range of the packet */ 2638 while (first != done) { 2639 tx_desc->upper.data = 0; 2640 tx_desc->lower.data = 0; 2641 tx_desc->buffer_addr = 0; 2642 ++txr->tx_avail; 2643 2644 if (tx_buffer->m_head) { 2645 bus_dmamap_sync(txr->txtag, 2646 tx_buffer->map, 2647 BUS_DMASYNC_POSTWRITE); 2648 bus_dmamap_unload(txr->txtag, 2649 tx_buffer->map); 2650 m_freem(tx_buffer->m_head); 2651 tx_buffer->m_head = NULL; 2652 tx_buffer->map = NULL; 2653 } 2654 tx_buffer->eop_index = -1; 2655 txr->watchdog_time = ticks; 2656 2657 if (++first == adapter->num_tx_desc) 2658 first = 0; 2659 2660 tx_buffer = &txr->tx_buffers[first]; 2661 tx_desc = 2662 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2663 } 2664 ++ifp->if_opackets; 2665 /* See if there is more work now */ 2666 last = tx_buffer->eop_index; 2667 if (last != -1) { 2668 eop_desc = 2669 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2670 /* Get next done point */ 2671 if (++last == adapter->num_tx_desc) last = 0; 2672 done = last; 2673 } else 2674 break; 2675 } 2676 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2677 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2678 2679 txr->next_to_clean = first; 2680 2681 /* 2682 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that 2683 * it is OK to send packets. If there are no pending descriptors, 2684 * clear the timeout. Otherwise, if some descriptors have been freed, 2685 * restart the timeout. 2686 */ 2687 if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) { 2688 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 2689 if (txr->tx_avail == adapter->num_tx_desc) { 2690 txr->watchdog_check = FALSE; 2691 return FALSE; 2692 } 2693 } 2694 2695 return TRUE; 2696} 2697 2698/********************************************************************* 2699 * 2700 * Refresh mbuf buffers for RX descriptor rings 2701 * - now keeps its own state so discards due to resource 2702 * exhaustion are unnecessary, if an mbuf cannot be obtained 2703 * it just returns, keeping its placeholder, thus it can simply 2704 * be recalled to try again. 2705 * 2706 **********************************************************************/ 2707static void 2708ixv_refresh_mbufs(struct rx_ring *rxr, int limit) 2709{ 2710 struct adapter *adapter = rxr->adapter; 2711 bus_dma_segment_t hseg[1]; 2712 bus_dma_segment_t pseg[1]; 2713 struct ixv_rx_buf *rxbuf; 2714 struct mbuf *mh, *mp; 2715 int i, j, nsegs, error; 2716 bool refreshed = FALSE; 2717 2718 i = j = rxr->next_to_refresh; 2719 /* Get the control variable, one beyond refresh point */ 2720 if (++j == adapter->num_rx_desc) 2721 j = 0; 2722 while (j != limit) { 2723 rxbuf = &rxr->rx_buffers[i]; 2724 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 2725 mh = m_gethdr(M_NOWAIT, MT_DATA); 2726 if (mh == NULL) 2727 goto update; 2728 mh->m_pkthdr.len = mh->m_len = MHLEN; 2729 mh->m_len = MHLEN; 2730 mh->m_flags |= M_PKTHDR; 2731 m_adj(mh, ETHER_ALIGN); 2732 /* Get the memory mapping */ 2733 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2734 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); 2735 if (error != 0) { 2736 printf("GET BUF: dmamap load" 2737 " failure - %d\n", error); 2738 m_free(mh); 2739 goto update; 2740 } 2741 rxbuf->m_head = mh; 2742 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2743 BUS_DMASYNC_PREREAD); 2744 rxr->rx_base[i].read.hdr_addr = 2745 htole64(hseg[0].ds_addr); 2746 } 2747 2748 if (rxbuf->m_pack == NULL) { 2749 mp = m_getjcl(M_NOWAIT, MT_DATA, 2750 M_PKTHDR, adapter->rx_mbuf_sz); 2751 if (mp == NULL) 2752 goto update; 2753 } else 2754 mp = rxbuf->m_pack; 2755 2756 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2757 /* Get the memory mapping */ 2758 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2759 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); 2760 if (error != 0) { 2761 printf("GET BUF: dmamap load" 2762 " failure - %d\n", error); 2763 m_free(mp); 2764 rxbuf->m_pack = NULL; 2765 goto update; 2766 } 2767 rxbuf->m_pack = mp; 2768 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2769 BUS_DMASYNC_PREREAD); 2770 rxr->rx_base[i].read.pkt_addr = 2771 htole64(pseg[0].ds_addr); 2772 2773 refreshed = TRUE; 2774 rxr->next_to_refresh = i = j; 2775 /* Calculate next index */ 2776 if (++j == adapter->num_rx_desc) 2777 j = 0; 2778 } 2779update: 2780 if (refreshed) /* update tail index */ 2781 IXGBE_WRITE_REG(&adapter->hw, 2782 IXGBE_VFRDT(rxr->me), rxr->next_to_refresh); 2783 return; 2784} 2785 2786/********************************************************************* 2787 * 2788 * Allocate memory for rx_buffer structures. Since we use one 2789 * rx_buffer per received packet, the maximum number of rx_buffer's 2790 * that we'll need is equal to the number of receive descriptors 2791 * that we've allocated. 2792 * 2793 **********************************************************************/ 2794static int 2795ixv_allocate_receive_buffers(struct rx_ring *rxr) 2796{ 2797 struct adapter *adapter = rxr->adapter; 2798 device_t dev = adapter->dev; 2799 struct ixv_rx_buf *rxbuf; 2800 int i, bsize, error; 2801 2802 bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc; 2803 if (!(rxr->rx_buffers = 2804 (struct ixv_rx_buf *) malloc(bsize, 2805 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2806 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 2807 error = ENOMEM; 2808 goto fail; 2809 } 2810 2811 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2812 1, 0, /* alignment, bounds */ 2813 BUS_SPACE_MAXADDR, /* lowaddr */ 2814 BUS_SPACE_MAXADDR, /* highaddr */ 2815 NULL, NULL, /* filter, filterarg */ 2816 MSIZE, /* maxsize */ 2817 1, /* nsegments */ 2818 MSIZE, /* maxsegsize */ 2819 0, /* flags */ 2820 NULL, /* lockfunc */ 2821 NULL, /* lockfuncarg */ 2822 &rxr->htag))) { 2823 device_printf(dev, "Unable to create RX DMA tag\n"); 2824 goto fail; 2825 } 2826 2827 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2828 1, 0, /* alignment, bounds */ 2829 BUS_SPACE_MAXADDR, /* lowaddr */ 2830 BUS_SPACE_MAXADDR, /* highaddr */ 2831 NULL, NULL, /* filter, filterarg */ 2832 MJUMPAGESIZE, /* maxsize */ 2833 1, /* nsegments */ 2834 MJUMPAGESIZE, /* maxsegsize */ 2835 0, /* flags */ 2836 NULL, /* lockfunc */ 2837 NULL, /* lockfuncarg */ 2838 &rxr->ptag))) { 2839 device_printf(dev, "Unable to create RX DMA tag\n"); 2840 goto fail; 2841 } 2842 2843 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 2844 rxbuf = &rxr->rx_buffers[i]; 2845 error = bus_dmamap_create(rxr->htag, 2846 BUS_DMA_NOWAIT, &rxbuf->hmap); 2847 if (error) { 2848 device_printf(dev, "Unable to create RX head map\n"); 2849 goto fail; 2850 } 2851 error = bus_dmamap_create(rxr->ptag, 2852 BUS_DMA_NOWAIT, &rxbuf->pmap); 2853 if (error) { 2854 device_printf(dev, "Unable to create RX pkt map\n"); 2855 goto fail; 2856 } 2857 } 2858 2859 return (0); 2860 2861fail: 2862 /* Frees all, but can handle partial completion */ 2863 ixv_free_receive_structures(adapter); 2864 return (error); 2865} 2866 2867static void 2868ixv_free_receive_ring(struct rx_ring *rxr) 2869{ 2870 struct adapter *adapter; 2871 struct ixv_rx_buf *rxbuf; 2872 int i; 2873 2874 adapter = rxr->adapter; 2875 for (i = 0; i < adapter->num_rx_desc; i++) { 2876 rxbuf = &rxr->rx_buffers[i]; 2877 if (rxbuf->m_head != NULL) { 2878 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2879 BUS_DMASYNC_POSTREAD); 2880 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 2881 rxbuf->m_head->m_flags |= M_PKTHDR; 2882 m_freem(rxbuf->m_head); 2883 } 2884 if (rxbuf->m_pack != NULL) { 2885 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2886 BUS_DMASYNC_POSTREAD); 2887 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 2888 rxbuf->m_pack->m_flags |= M_PKTHDR; 2889 m_freem(rxbuf->m_pack); 2890 } 2891 rxbuf->m_head = NULL; 2892 rxbuf->m_pack = NULL; 2893 } 2894} 2895 2896 2897/********************************************************************* 2898 * 2899 * Initialize a receive ring and its buffers. 2900 * 2901 **********************************************************************/ 2902static int 2903ixv_setup_receive_ring(struct rx_ring *rxr) 2904{ 2905 struct adapter *adapter; 2906 struct ifnet *ifp; 2907 device_t dev; 2908 struct ixv_rx_buf *rxbuf; 2909 bus_dma_segment_t pseg[1], hseg[1]; 2910 struct lro_ctrl *lro = &rxr->lro; 2911 int rsize, nsegs, error = 0; 2912 2913 adapter = rxr->adapter; 2914 ifp = adapter->ifp; 2915 dev = adapter->dev; 2916 2917 /* Clear the ring contents */ 2918 IXV_RX_LOCK(rxr); 2919 rsize = roundup2(adapter->num_rx_desc * 2920 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2921 bzero((void *)rxr->rx_base, rsize); 2922 2923 /* Free current RX buffer structs and their mbufs */ 2924 ixv_free_receive_ring(rxr); 2925 2926 /* Configure header split? */ 2927 if (ixv_header_split) 2928 rxr->hdr_split = TRUE; 2929 2930 /* Now replenish the mbufs */ 2931 for (int j = 0; j != adapter->num_rx_desc; ++j) { 2932 struct mbuf *mh, *mp; 2933 2934 rxbuf = &rxr->rx_buffers[j]; 2935 /* 2936 ** Dont allocate mbufs if not 2937 ** doing header split, its wasteful 2938 */ 2939 if (rxr->hdr_split == FALSE) 2940 goto skip_head; 2941 2942 /* First the header */ 2943 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA); 2944 if (rxbuf->m_head == NULL) { 2945 error = ENOBUFS; 2946 goto fail; 2947 } 2948 m_adj(rxbuf->m_head, ETHER_ALIGN); 2949 mh = rxbuf->m_head; 2950 mh->m_len = mh->m_pkthdr.len = MHLEN; 2951 mh->m_flags |= M_PKTHDR; 2952 /* Get the memory mapping */ 2953 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2954 rxbuf->hmap, rxbuf->m_head, hseg, 2955 &nsegs, BUS_DMA_NOWAIT); 2956 if (error != 0) /* Nothing elegant to do here */ 2957 goto fail; 2958 bus_dmamap_sync(rxr->htag, 2959 rxbuf->hmap, BUS_DMASYNC_PREREAD); 2960 /* Update descriptor */ 2961 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 2962 2963skip_head: 2964 /* Now the payload cluster */ 2965 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, 2966 M_PKTHDR, adapter->rx_mbuf_sz); 2967 if (rxbuf->m_pack == NULL) { 2968 error = ENOBUFS; 2969 goto fail; 2970 } 2971 mp = rxbuf->m_pack; 2972 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2973 /* Get the memory mapping */ 2974 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2975 rxbuf->pmap, mp, pseg, 2976 &nsegs, BUS_DMA_NOWAIT); 2977 if (error != 0) 2978 goto fail; 2979 bus_dmamap_sync(rxr->ptag, 2980 rxbuf->pmap, BUS_DMASYNC_PREREAD); 2981 /* Update descriptor */ 2982 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 2983 } 2984 2985 2986 /* Setup our descriptor indices */ 2987 rxr->next_to_check = 0; 2988 rxr->next_to_refresh = 0; 2989 rxr->lro_enabled = FALSE; 2990 rxr->rx_split_packets = 0; 2991 rxr->rx_bytes = 0; 2992 rxr->discard = FALSE; 2993 2994 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2995 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2996 2997 /* 2998 ** Now set up the LRO interface: 2999 */ 3000 if (ifp->if_capenable & IFCAP_LRO) { 3001 int err = tcp_lro_init(lro); 3002 if (err) { 3003 device_printf(dev, "LRO Initialization failed!\n"); 3004 goto fail; 3005 } 3006 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 3007 rxr->lro_enabled = TRUE; 3008 lro->ifp = adapter->ifp; 3009 } 3010 3011 IXV_RX_UNLOCK(rxr); 3012 return (0); 3013 3014fail: 3015 ixv_free_receive_ring(rxr); 3016 IXV_RX_UNLOCK(rxr); 3017 return (error); 3018} 3019 3020/********************************************************************* 3021 * 3022 * Initialize all receive rings. 3023 * 3024 **********************************************************************/ 3025static int 3026ixv_setup_receive_structures(struct adapter *adapter) 3027{ 3028 struct rx_ring *rxr = adapter->rx_rings; 3029 int j; 3030 3031 for (j = 0; j < adapter->num_queues; j++, rxr++) 3032 if (ixv_setup_receive_ring(rxr)) 3033 goto fail; 3034 3035 return (0); 3036fail: 3037 /* 3038 * Free RX buffers allocated so far, we will only handle 3039 * the rings that completed, the failing case will have 3040 * cleaned up for itself. 'j' failed, so its the terminus. 3041 */ 3042 for (int i = 0; i < j; ++i) { 3043 rxr = &adapter->rx_rings[i]; 3044 ixv_free_receive_ring(rxr); 3045 } 3046 3047 return (ENOBUFS); 3048} 3049 3050/********************************************************************* 3051 * 3052 * Setup receive registers and features. 3053 * 3054 **********************************************************************/ 3055#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3056 3057static void 3058ixv_initialize_receive_units(struct adapter *adapter) 3059{ 3060 struct rx_ring *rxr = adapter->rx_rings; 3061 struct ixgbe_hw *hw = &adapter->hw; 3062 struct ifnet *ifp = adapter->ifp; 3063 u32 bufsz, fctrl, rxcsum, hlreg; 3064 3065 3066 /* Enable broadcasts */ 3067 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3068 fctrl |= IXGBE_FCTRL_BAM; 3069 fctrl |= IXGBE_FCTRL_DPF; 3070 fctrl |= IXGBE_FCTRL_PMCF; 3071 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3072 3073 /* Set for Jumbo Frames? */ 3074 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3075 if (ifp->if_mtu > ETHERMTU) { 3076 hlreg |= IXGBE_HLREG0_JUMBOEN; 3077 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3078 } else { 3079 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 3080 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3081 } 3082 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 3083 3084 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3085 u64 rdba = rxr->rxdma.dma_paddr; 3086 u32 reg, rxdctl; 3087 3088 /* Do the queue enabling first */ 3089 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3090 rxdctl |= IXGBE_RXDCTL_ENABLE; 3091 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 3092 for (int k = 0; k < 10; k++) { 3093 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 3094 IXGBE_RXDCTL_ENABLE) 3095 break; 3096 else 3097 msec_delay(1); 3098 } 3099 wmb(); 3100 3101 /* Setup the Base and Length of the Rx Descriptor Ring */ 3102 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 3103 (rdba & 0x00000000ffffffffULL)); 3104 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 3105 (rdba >> 32)); 3106 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 3107 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3108 3109 /* Set up the SRRCTL register */ 3110 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i)); 3111 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3112 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 3113 reg |= bufsz; 3114 if (rxr->hdr_split) { 3115 /* Use a standard mbuf for the header */ 3116 reg |= ((IXV_RX_HDR << 3117 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3118 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3119 reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3120 } else 3121 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3122 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg); 3123 3124 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 3125 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 3126 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 3127 adapter->num_rx_desc - 1); 3128 } 3129 3130 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3131 3132 if (ifp->if_capenable & IFCAP_RXCSUM) 3133 rxcsum |= IXGBE_RXCSUM_PCSD; 3134 3135 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3136 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3137 3138 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3139 3140 return; 3141} 3142 3143/********************************************************************* 3144 * 3145 * Free all receive rings. 3146 * 3147 **********************************************************************/ 3148static void 3149ixv_free_receive_structures(struct adapter *adapter) 3150{ 3151 struct rx_ring *rxr = adapter->rx_rings; 3152 3153 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3154 struct lro_ctrl *lro = &rxr->lro; 3155 ixv_free_receive_buffers(rxr); 3156 /* Free LRO memory */ 3157 tcp_lro_free(lro); 3158 /* Free the ring memory as well */ 3159 ixv_dma_free(adapter, &rxr->rxdma); 3160 } 3161 3162 free(adapter->rx_rings, M_DEVBUF); 3163} 3164 3165 3166/********************************************************************* 3167 * 3168 * Free receive ring data structures 3169 * 3170 **********************************************************************/ 3171static void 3172ixv_free_receive_buffers(struct rx_ring *rxr) 3173{ 3174 struct adapter *adapter = rxr->adapter; 3175 struct ixv_rx_buf *rxbuf; 3176 3177 INIT_DEBUGOUT("free_receive_structures: begin"); 3178 3179 /* Cleanup any existing buffers */ 3180 if (rxr->rx_buffers != NULL) { 3181 for (int i = 0; i < adapter->num_rx_desc; i++) { 3182 rxbuf = &rxr->rx_buffers[i]; 3183 if (rxbuf->m_head != NULL) { 3184 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3185 BUS_DMASYNC_POSTREAD); 3186 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 3187 rxbuf->m_head->m_flags |= M_PKTHDR; 3188 m_freem(rxbuf->m_head); 3189 } 3190 if (rxbuf->m_pack != NULL) { 3191 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3192 BUS_DMASYNC_POSTREAD); 3193 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 3194 rxbuf->m_pack->m_flags |= M_PKTHDR; 3195 m_freem(rxbuf->m_pack); 3196 } 3197 rxbuf->m_head = NULL; 3198 rxbuf->m_pack = NULL; 3199 if (rxbuf->hmap != NULL) { 3200 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 3201 rxbuf->hmap = NULL; 3202 } 3203 if (rxbuf->pmap != NULL) { 3204 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 3205 rxbuf->pmap = NULL; 3206 } 3207 } 3208 if (rxr->rx_buffers != NULL) { 3209 free(rxr->rx_buffers, M_DEVBUF); 3210 rxr->rx_buffers = NULL; 3211 } 3212 } 3213 3214 if (rxr->htag != NULL) { 3215 bus_dma_tag_destroy(rxr->htag); 3216 rxr->htag = NULL; 3217 } 3218 if (rxr->ptag != NULL) { 3219 bus_dma_tag_destroy(rxr->ptag); 3220 rxr->ptag = NULL; 3221 } 3222 3223 return; 3224} 3225 3226static __inline void 3227ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 3228{ 3229 3230 /* 3231 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 3232 * should be computed by hardware. Also it should not have VLAN tag in 3233 * ethernet header. 3234 */ 3235 if (rxr->lro_enabled && 3236 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 3237 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3238 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 3239 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 3240 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 3241 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 3242 /* 3243 * Send to the stack if: 3244 ** - LRO not enabled, or 3245 ** - no LRO resources, or 3246 ** - lro enqueue fails 3247 */ 3248 if (rxr->lro.lro_cnt != 0) 3249 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 3250 return; 3251 } 3252 IXV_RX_UNLOCK(rxr); 3253 (*ifp->if_input)(ifp, m); 3254 IXV_RX_LOCK(rxr); 3255} 3256 3257static __inline void 3258ixv_rx_discard(struct rx_ring *rxr, int i) 3259{ 3260 struct ixv_rx_buf *rbuf; 3261 3262 rbuf = &rxr->rx_buffers[i]; 3263 3264 if (rbuf->fmp != NULL) {/* Partial chain ? */ 3265 rbuf->fmp->m_flags |= M_PKTHDR; 3266 m_freem(rbuf->fmp); 3267 rbuf->fmp = NULL; 3268 } 3269 3270 /* 3271 ** With advanced descriptors the writeback 3272 ** clobbers the buffer addrs, so its easier 3273 ** to just free the existing mbufs and take 3274 ** the normal refresh path to get new buffers 3275 ** and mapping. 3276 */ 3277 if (rbuf->m_head) { 3278 m_free(rbuf->m_head); 3279 rbuf->m_head = NULL; 3280 } 3281 3282 if (rbuf->m_pack) { 3283 m_free(rbuf->m_pack); 3284 rbuf->m_pack = NULL; 3285 } 3286 3287 return; 3288} 3289 3290 3291/********************************************************************* 3292 * 3293 * This routine executes in interrupt context. It replenishes 3294 * the mbufs in the descriptor and sends data which has been 3295 * dma'ed into host memory to upper layer. 3296 * 3297 * We loop at most count times if count is > 0, or until done if 3298 * count < 0. 3299 * 3300 * Return TRUE for more work, FALSE for all clean. 3301 *********************************************************************/ 3302static bool 3303ixv_rxeof(struct ix_queue *que, int count) 3304{ 3305 struct adapter *adapter = que->adapter; 3306 struct rx_ring *rxr = que->rxr; 3307 struct ifnet *ifp = adapter->ifp; 3308 struct lro_ctrl *lro = &rxr->lro; 3309 struct lro_entry *queued; 3310 int i, nextp, processed = 0; 3311 u32 staterr = 0; 3312 union ixgbe_adv_rx_desc *cur; 3313 struct ixv_rx_buf *rbuf, *nbuf; 3314 3315 IXV_RX_LOCK(rxr); 3316 3317 for (i = rxr->next_to_check; count != 0;) { 3318 struct mbuf *sendmp, *mh, *mp; 3319 u32 rsc, ptype; 3320 u16 hlen, plen, hdr, vtag; 3321 bool eop; 3322 3323 /* Sync the ring. */ 3324 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3325 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3326 3327 cur = &rxr->rx_base[i]; 3328 staterr = le32toh(cur->wb.upper.status_error); 3329 3330 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 3331 break; 3332 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 3333 break; 3334 3335 count--; 3336 sendmp = NULL; 3337 nbuf = NULL; 3338 rsc = 0; 3339 cur->wb.upper.status_error = 0; 3340 rbuf = &rxr->rx_buffers[i]; 3341 mh = rbuf->m_head; 3342 mp = rbuf->m_pack; 3343 3344 plen = le16toh(cur->wb.upper.length); 3345 ptype = le32toh(cur->wb.lower.lo_dword.data) & 3346 IXGBE_RXDADV_PKTTYPE_MASK; 3347 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 3348 vtag = le16toh(cur->wb.upper.vlan); 3349 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3350 3351 /* Make sure all parts of a bad packet are discarded */ 3352 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 3353 (rxr->discard)) { 3354 ifp->if_ierrors++; 3355 rxr->rx_discarded++; 3356 if (!eop) 3357 rxr->discard = TRUE; 3358 else 3359 rxr->discard = FALSE; 3360 ixv_rx_discard(rxr, i); 3361 goto next_desc; 3362 } 3363 3364 if (!eop) { 3365 nextp = i + 1; 3366 if (nextp == adapter->num_rx_desc) 3367 nextp = 0; 3368 nbuf = &rxr->rx_buffers[nextp]; 3369 prefetch(nbuf); 3370 } 3371 /* 3372 ** The header mbuf is ONLY used when header 3373 ** split is enabled, otherwise we get normal 3374 ** behavior, ie, both header and payload 3375 ** are DMA'd into the payload buffer. 3376 ** 3377 ** Rather than using the fmp/lmp global pointers 3378 ** we now keep the head of a packet chain in the 3379 ** buffer struct and pass this along from one 3380 ** descriptor to the next, until we get EOP. 3381 */ 3382 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 3383 /* This must be an initial descriptor */ 3384 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3385 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3386 if (hlen > IXV_RX_HDR) 3387 hlen = IXV_RX_HDR; 3388 mh->m_len = hlen; 3389 mh->m_flags |= M_PKTHDR; 3390 mh->m_next = NULL; 3391 mh->m_pkthdr.len = mh->m_len; 3392 /* Null buf pointer so it is refreshed */ 3393 rbuf->m_head = NULL; 3394 /* 3395 ** Check the payload length, this 3396 ** could be zero if its a small 3397 ** packet. 3398 */ 3399 if (plen > 0) { 3400 mp->m_len = plen; 3401 mp->m_next = NULL; 3402 mp->m_flags &= ~M_PKTHDR; 3403 mh->m_next = mp; 3404 mh->m_pkthdr.len += mp->m_len; 3405 /* Null buf pointer so it is refreshed */ 3406 rbuf->m_pack = NULL; 3407 rxr->rx_split_packets++; 3408 } 3409 /* 3410 ** Now create the forward 3411 ** chain so when complete 3412 ** we wont have to. 3413 */ 3414 if (eop == 0) { 3415 /* stash the chain head */ 3416 nbuf->fmp = mh; 3417 /* Make forward chain */ 3418 if (plen) 3419 mp->m_next = nbuf->m_pack; 3420 else 3421 mh->m_next = nbuf->m_pack; 3422 } else { 3423 /* Singlet, prepare to send */ 3424 sendmp = mh; 3425 if ((adapter->num_vlans) && 3426 (staterr & IXGBE_RXD_STAT_VP)) { 3427 sendmp->m_pkthdr.ether_vtag = vtag; 3428 sendmp->m_flags |= M_VLANTAG; 3429 } 3430 } 3431 } else { 3432 /* 3433 ** Either no header split, or a 3434 ** secondary piece of a fragmented 3435 ** split packet. 3436 */ 3437 mp->m_len = plen; 3438 /* 3439 ** See if there is a stored head 3440 ** that determines what we are 3441 */ 3442 sendmp = rbuf->fmp; 3443 rbuf->m_pack = rbuf->fmp = NULL; 3444 3445 if (sendmp != NULL) /* secondary frag */ 3446 sendmp->m_pkthdr.len += mp->m_len; 3447 else { 3448 /* first desc of a non-ps chain */ 3449 sendmp = mp; 3450 sendmp->m_flags |= M_PKTHDR; 3451 sendmp->m_pkthdr.len = mp->m_len; 3452 if (staterr & IXGBE_RXD_STAT_VP) { 3453 sendmp->m_pkthdr.ether_vtag = vtag; 3454 sendmp->m_flags |= M_VLANTAG; 3455 } 3456 } 3457 /* Pass the head pointer on */ 3458 if (eop == 0) { 3459 nbuf->fmp = sendmp; 3460 sendmp = NULL; 3461 mp->m_next = nbuf->m_pack; 3462 } 3463 } 3464 ++processed; 3465 /* Sending this frame? */ 3466 if (eop) { 3467 sendmp->m_pkthdr.rcvif = ifp; 3468 ifp->if_ipackets++; 3469 rxr->rx_packets++; 3470 /* capture data for AIM */ 3471 rxr->bytes += sendmp->m_pkthdr.len; 3472 rxr->rx_bytes += sendmp->m_pkthdr.len; 3473 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 3474 ixv_rx_checksum(staterr, sendmp, ptype); 3475#if __FreeBSD_version >= 800000 3476 sendmp->m_pkthdr.flowid = que->msix; 3477 sendmp->m_flags |= M_FLOWID; 3478#endif 3479 } 3480next_desc: 3481 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3482 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3483 3484 /* Advance our pointers to the next descriptor. */ 3485 if (++i == adapter->num_rx_desc) 3486 i = 0; 3487 3488 /* Now send to the stack or do LRO */ 3489 if (sendmp != NULL) 3490 ixv_rx_input(rxr, ifp, sendmp, ptype); 3491 3492 /* Every 8 descriptors we go to refresh mbufs */ 3493 if (processed == 8) { 3494 ixv_refresh_mbufs(rxr, i); 3495 processed = 0; 3496 } 3497 } 3498 3499 /* Refresh any remaining buf structs */ 3500 if (ixv_rx_unrefreshed(rxr)) 3501 ixv_refresh_mbufs(rxr, i); 3502 3503 rxr->next_to_check = i; 3504 3505 /* 3506 * Flush any outstanding LRO work 3507 */ 3508 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 3509 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3510 tcp_lro_flush(lro, queued); 3511 } 3512 3513 IXV_RX_UNLOCK(rxr); 3514 3515 /* 3516 ** We still have cleaning to do? 3517 ** Schedule another interrupt if so. 3518 */ 3519 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 3520 ixv_rearm_queues(adapter, (u64)(1 << que->msix)); 3521 return (TRUE); 3522 } 3523 3524 return (FALSE); 3525} 3526 3527 3528/********************************************************************* 3529 * 3530 * Verify that the hardware indicated that the checksum is valid. 3531 * Inform the stack about the status of checksum so that stack 3532 * doesn't spend time verifying the checksum. 3533 * 3534 *********************************************************************/ 3535static void 3536ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype) 3537{ 3538 u16 status = (u16) staterr; 3539 u8 errors = (u8) (staterr >> 24); 3540 bool sctp = FALSE; 3541 3542 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3543 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 3544 sctp = TRUE; 3545 3546 if (status & IXGBE_RXD_STAT_IPCS) { 3547 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3548 /* IP Checksum Good */ 3549 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 3550 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 3551 3552 } else 3553 mp->m_pkthdr.csum_flags = 0; 3554 } 3555 if (status & IXGBE_RXD_STAT_L4CS) { 3556 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3557#if __FreeBSD_version >= 800000 3558 if (sctp) 3559 type = CSUM_SCTP_VALID; 3560#endif 3561 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3562 mp->m_pkthdr.csum_flags |= type; 3563 if (!sctp) 3564 mp->m_pkthdr.csum_data = htons(0xffff); 3565 } 3566 } 3567 return; 3568} 3569 3570static void 3571ixv_setup_vlan_support(struct adapter *adapter) 3572{ 3573 struct ixgbe_hw *hw = &adapter->hw; 3574 u32 ctrl, vid, vfta, retry; 3575 3576 3577 /* 3578 ** We get here thru init_locked, meaning 3579 ** a soft reset, this has already cleared 3580 ** the VFTA and other state, so if there 3581 ** have been no vlan's registered do nothing. 3582 */ 3583 if (adapter->num_vlans == 0) 3584 return; 3585 3586 /* Enable the queues */ 3587 for (int i = 0; i < adapter->num_queues; i++) { 3588 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3589 ctrl |= IXGBE_RXDCTL_VME; 3590 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 3591 } 3592 3593 /* 3594 ** A soft reset zero's out the VFTA, so 3595 ** we need to repopulate it now. 3596 */ 3597 for (int i = 0; i < VFTA_SIZE; i++) { 3598 if (ixv_shadow_vfta[i] == 0) 3599 continue; 3600 vfta = ixv_shadow_vfta[i]; 3601 /* 3602 ** Reconstruct the vlan id's 3603 ** based on the bits set in each 3604 ** of the array ints. 3605 */ 3606 for ( int j = 0; j < 32; j++) { 3607 retry = 0; 3608 if ((vfta & (1 << j)) == 0) 3609 continue; 3610 vid = (i * 32) + j; 3611 /* Call the shared code mailbox routine */ 3612 while (ixgbe_set_vfta(hw, vid, 0, TRUE)) { 3613 if (++retry > 5) 3614 break; 3615 } 3616 } 3617 } 3618} 3619 3620/* 3621** This routine is run via an vlan config EVENT, 3622** it enables us to use the HW Filter table since 3623** we can get the vlan id. This just creates the 3624** entry in the soft version of the VFTA, init will 3625** repopulate the real table. 3626*/ 3627static void 3628ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3629{ 3630 struct adapter *adapter = ifp->if_softc; 3631 u16 index, bit; 3632 3633 if (ifp->if_softc != arg) /* Not our event */ 3634 return; 3635 3636 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3637 return; 3638 3639 IXV_CORE_LOCK(adapter); 3640 index = (vtag >> 5) & 0x7F; 3641 bit = vtag & 0x1F; 3642 ixv_shadow_vfta[index] |= (1 << bit); 3643 ++adapter->num_vlans; 3644 /* Re-init to load the changes */ 3645 ixv_init_locked(adapter); 3646 IXV_CORE_UNLOCK(adapter); 3647} 3648 3649/* 3650** This routine is run via an vlan 3651** unconfig EVENT, remove our entry 3652** in the soft vfta. 3653*/ 3654static void 3655ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3656{ 3657 struct adapter *adapter = ifp->if_softc; 3658 u16 index, bit; 3659 3660 if (ifp->if_softc != arg) 3661 return; 3662 3663 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3664 return; 3665 3666 IXV_CORE_LOCK(adapter); 3667 index = (vtag >> 5) & 0x7F; 3668 bit = vtag & 0x1F; 3669 ixv_shadow_vfta[index] &= ~(1 << bit); 3670 --adapter->num_vlans; 3671 /* Re-init to load the changes */ 3672 ixv_init_locked(adapter); 3673 IXV_CORE_UNLOCK(adapter); 3674} 3675 3676static void 3677ixv_enable_intr(struct adapter *adapter) 3678{ 3679 struct ixgbe_hw *hw = &adapter->hw; 3680 struct ix_queue *que = adapter->queues; 3681 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3682 3683 3684 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 3685 3686 mask = IXGBE_EIMS_ENABLE_MASK; 3687 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 3688 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 3689 3690 for (int i = 0; i < adapter->num_queues; i++, que++) 3691 ixv_enable_queue(adapter, que->msix); 3692 3693 IXGBE_WRITE_FLUSH(hw); 3694 3695 return; 3696} 3697 3698static void 3699ixv_disable_intr(struct adapter *adapter) 3700{ 3701 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 3702 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0); 3703 IXGBE_WRITE_FLUSH(&adapter->hw); 3704 return; 3705} 3706 3707/* 3708** Setup the correct IVAR register for a particular MSIX interrupt 3709** - entry is the register array entry 3710** - vector is the MSIX vector for this queue 3711** - type is RX/TX/MISC 3712*/ 3713static void 3714ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 3715{ 3716 struct ixgbe_hw *hw = &adapter->hw; 3717 u32 ivar, index; 3718 3719 vector |= IXGBE_IVAR_ALLOC_VAL; 3720 3721 if (type == -1) { /* MISC IVAR */ 3722 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 3723 ivar &= ~0xFF; 3724 ivar |= vector; 3725 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 3726 } else { /* RX/TX IVARS */ 3727 index = (16 * (entry & 1)) + (8 * type); 3728 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 3729 ivar &= ~(0xFF << index); 3730 ivar |= (vector << index); 3731 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 3732 } 3733} 3734 3735static void 3736ixv_configure_ivars(struct adapter *adapter) 3737{ 3738 struct ix_queue *que = adapter->queues; 3739 3740 for (int i = 0; i < adapter->num_queues; i++, que++) { 3741 /* First the RX queue entry */ 3742 ixv_set_ivar(adapter, i, que->msix, 0); 3743 /* ... and the TX */ 3744 ixv_set_ivar(adapter, i, que->msix, 1); 3745 /* Set an initial value in EITR */ 3746 IXGBE_WRITE_REG(&adapter->hw, 3747 IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT); 3748 } 3749 3750 /* For the Link interrupt */ 3751 ixv_set_ivar(adapter, 1, adapter->mbxvec, -1); 3752} 3753 3754 3755/* 3756** Tasklet handler for MSIX MBX interrupts 3757** - do outside interrupt since it might sleep 3758*/ 3759static void 3760ixv_handle_mbx(void *context, int pending) 3761{ 3762 struct adapter *adapter = context; 3763 3764 ixgbe_check_link(&adapter->hw, 3765 &adapter->link_speed, &adapter->link_up, 0); 3766 ixv_update_link_status(adapter); 3767} 3768 3769/* 3770** The VF stats registers never have a truely virgin 3771** starting point, so this routine tries to make an 3772** artificial one, marking ground zero on attach as 3773** it were. 3774*/ 3775static void 3776ixv_save_stats(struct adapter *adapter) 3777{ 3778 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 3779 adapter->stats.saved_reset_vfgprc += 3780 adapter->stats.vfgprc - adapter->stats.base_vfgprc; 3781 adapter->stats.saved_reset_vfgptc += 3782 adapter->stats.vfgptc - adapter->stats.base_vfgptc; 3783 adapter->stats.saved_reset_vfgorc += 3784 adapter->stats.vfgorc - adapter->stats.base_vfgorc; 3785 adapter->stats.saved_reset_vfgotc += 3786 adapter->stats.vfgotc - adapter->stats.base_vfgotc; 3787 adapter->stats.saved_reset_vfmprc += 3788 adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3789 } 3790} 3791 3792static void 3793ixv_init_stats(struct adapter *adapter) 3794{ 3795 struct ixgbe_hw *hw = &adapter->hw; 3796 3797 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 3798 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 3799 adapter->stats.last_vfgorc |= 3800 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 3801 3802 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 3803 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 3804 adapter->stats.last_vfgotc |= 3805 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 3806 3807 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 3808 3809 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 3810 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 3811 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 3812 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 3813 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 3814} 3815 3816#define UPDATE_STAT_32(reg, last, count) \ 3817{ \ 3818 u32 current = IXGBE_READ_REG(hw, reg); \ 3819 if (current < last) \ 3820 count += 0x100000000LL; \ 3821 last = current; \ 3822 count &= 0xFFFFFFFF00000000LL; \ 3823 count |= current; \ 3824} 3825 3826#define UPDATE_STAT_36(lsb, msb, last, count) \ 3827{ \ 3828 u64 cur_lsb = IXGBE_READ_REG(hw, lsb); \ 3829 u64 cur_msb = IXGBE_READ_REG(hw, msb); \ 3830 u64 current = ((cur_msb << 32) | cur_lsb); \ 3831 if (current < last) \ 3832 count += 0x1000000000LL; \ 3833 last = current; \ 3834 count &= 0xFFFFFFF000000000LL; \ 3835 count |= current; \ 3836} 3837 3838/* 3839** ixv_update_stats - Update the board statistics counters. 3840*/ 3841void 3842ixv_update_stats(struct adapter *adapter) 3843{ 3844 struct ixgbe_hw *hw = &adapter->hw; 3845 3846 UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3847 adapter->stats.vfgprc); 3848 UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3849 adapter->stats.vfgptc); 3850 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3851 adapter->stats.last_vfgorc, adapter->stats.vfgorc); 3852 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3853 adapter->stats.last_vfgotc, adapter->stats.vfgotc); 3854 UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3855 adapter->stats.vfmprc); 3856} 3857 3858/********************************************************************** 3859 * 3860 * This routine is called only when ixgbe_display_debug_stats is enabled. 3861 * This routine provides a way to take a look at important statistics 3862 * maintained by the driver and hardware. 3863 * 3864 **********************************************************************/ 3865static void 3866ixv_print_hw_stats(struct adapter * adapter) 3867{ 3868 device_t dev = adapter->dev; 3869 3870 device_printf(dev,"Std Mbuf Failed = %lu\n", 3871 adapter->mbuf_defrag_failed); 3872 device_printf(dev,"Driver dropped packets = %lu\n", 3873 adapter->dropped_pkts); 3874 device_printf(dev, "watchdog timeouts = %ld\n", 3875 adapter->watchdog_events); 3876 3877 device_printf(dev,"Good Packets Rcvd = %llu\n", 3878 (long long)adapter->stats.vfgprc); 3879 device_printf(dev,"Good Packets Xmtd = %llu\n", 3880 (long long)adapter->stats.vfgptc); 3881 device_printf(dev,"TSO Transmissions = %lu\n", 3882 adapter->tso_tx); 3883 3884} 3885 3886/********************************************************************** 3887 * 3888 * This routine is called only when em_display_debug_stats is enabled. 3889 * This routine provides a way to take a look at important statistics 3890 * maintained by the driver and hardware. 3891 * 3892 **********************************************************************/ 3893static void 3894ixv_print_debug_info(struct adapter *adapter) 3895{ 3896 device_t dev = adapter->dev; 3897 struct ixgbe_hw *hw = &adapter->hw; 3898 struct ix_queue *que = adapter->queues; 3899 struct rx_ring *rxr; 3900 struct tx_ring *txr; 3901 struct lro_ctrl *lro; 3902 3903 device_printf(dev,"Error Byte Count = %u \n", 3904 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 3905 3906 for (int i = 0; i < adapter->num_queues; i++, que++) { 3907 txr = que->txr; 3908 rxr = que->rxr; 3909 lro = &rxr->lro; 3910 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n", 3911 que->msix, (long)que->irqs); 3912 device_printf(dev,"RX(%d) Packets Received: %lld\n", 3913 rxr->me, (long long)rxr->rx_packets); 3914 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 3915 rxr->me, (long long)rxr->rx_split_packets); 3916 device_printf(dev,"RX(%d) Bytes Received: %lu\n", 3917 rxr->me, (long)rxr->rx_bytes); 3918 device_printf(dev,"RX(%d) LRO Queued= %d\n", 3919 rxr->me, lro->lro_queued); 3920 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 3921 rxr->me, lro->lro_flushed); 3922 device_printf(dev,"TX(%d) Packets Sent: %lu\n", 3923 txr->me, (long)txr->total_packets); 3924 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 3925 txr->me, (long)txr->no_desc_avail); 3926 } 3927 3928 device_printf(dev,"MBX IRQ Handled: %lu\n", 3929 (long)adapter->mbx_irq); 3930 return; 3931} 3932 3933static int 3934ixv_sysctl_stats(SYSCTL_HANDLER_ARGS) 3935{ 3936 int error; 3937 int result; 3938 struct adapter *adapter; 3939 3940 result = -1; 3941 error = sysctl_handle_int(oidp, &result, 0, req); 3942 3943 if (error || !req->newptr) 3944 return (error); 3945 3946 if (result == 1) { 3947 adapter = (struct adapter *) arg1; 3948 ixv_print_hw_stats(adapter); 3949 } 3950 return error; 3951} 3952 3953static int 3954ixv_sysctl_debug(SYSCTL_HANDLER_ARGS) 3955{ 3956 int error, result; 3957 struct adapter *adapter; 3958 3959 result = -1; 3960 error = sysctl_handle_int(oidp, &result, 0, req); 3961 3962 if (error || !req->newptr) 3963 return (error); 3964 3965 if (result == 1) { 3966 adapter = (struct adapter *) arg1; 3967 ixv_print_debug_info(adapter); 3968 } 3969 return error; 3970} 3971 3972/* 3973** Set flow control using sysctl: 3974** Flow control values: 3975** 0 - off 3976** 1 - rx pause 3977** 2 - tx pause 3978** 3 - full 3979*/ 3980static int 3981ixv_set_flowcntl(SYSCTL_HANDLER_ARGS) 3982{ 3983 int error; 3984 struct adapter *adapter; 3985 3986 error = sysctl_handle_int(oidp, &ixv_flow_control, 0, req); 3987 3988 if (error) 3989 return (error); 3990 3991 adapter = (struct adapter *) arg1; 3992 switch (ixv_flow_control) { 3993 case ixgbe_fc_rx_pause: 3994 case ixgbe_fc_tx_pause: 3995 case ixgbe_fc_full: 3996 adapter->hw.fc.requested_mode = ixv_flow_control; 3997 break; 3998 case ixgbe_fc_none: 3999 default: 4000 adapter->hw.fc.requested_mode = ixgbe_fc_none; 4001 } 4002 4003 ixgbe_fc_enable(&adapter->hw); 4004 return error; 4005} 4006 4007static void 4008ixv_add_rx_process_limit(struct adapter *adapter, const char *name, 4009 const char *description, int *limit, int value) 4010{ 4011 *limit = value; 4012 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 4013 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 4014 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 4015} 4016 4017