1/****************************************************************************** 2 3 Copyright (c) 2001-2013, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ 33/*$FreeBSD$*/ 34 35#include "opt_inet.h" 36#include "opt_inet6.h" 37#include "ixv.h" 38 39/********************************************************************* 40 * Driver version 41 *********************************************************************/ 42char ixv_driver_version[] = "1.1.4"; 43 44/********************************************************************* 45 * PCI Device ID Table 46 * 47 * Used by probe to select devices to load on 48 * Last field stores an index into ixv_strings 49 * Last entry must be all 0s 50 * 51 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 52 *********************************************************************/ 53 54static ixv_vendor_info_t ixv_vendor_info_array[] = 55{ 56 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 57 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 58 /* required last entry */ 59 {0, 0, 0, 0, 0} 60}; 61 62/********************************************************************* 63 * Table of branding strings 64 *********************************************************************/ 65 66static char *ixv_strings[] = { 67 "Intel(R) PRO/10GbE Virtual Function Network Driver" 68}; 69 70/********************************************************************* 71 * Function prototypes 72 *********************************************************************/ 73static int ixv_probe(device_t); 74static int ixv_attach(device_t); 75static int ixv_detach(device_t); 76static int ixv_shutdown(device_t); 77#if __FreeBSD_version < 800000 78static void ixv_start(struct ifnet *); 79static void ixv_start_locked(struct tx_ring *, struct ifnet *); 80#else 81static int ixv_mq_start(struct ifnet *, struct mbuf *); 82static int ixv_mq_start_locked(struct ifnet *, 83 struct tx_ring *, struct mbuf *); 84static void ixv_qflush(struct ifnet *); 85#endif 86static int ixv_ioctl(struct ifnet *, u_long, caddr_t); 87static void ixv_init(void *); 88static void ixv_init_locked(struct adapter *); 89static void ixv_stop(void *); 90static void ixv_media_status(struct ifnet *, struct ifmediareq *); 91static int ixv_media_change(struct ifnet *); 92static void ixv_identify_hardware(struct adapter *); 93static int ixv_allocate_pci_resources(struct adapter *); 94static int ixv_allocate_msix(struct adapter *); 95static int ixv_allocate_queues(struct adapter *); 96static int ixv_setup_msix(struct adapter *); 97static void ixv_free_pci_resources(struct adapter *); 98static void ixv_local_timer(void *); 99static void ixv_setup_interface(device_t, struct adapter *); 100static void ixv_config_link(struct adapter *); 101 102static int ixv_allocate_transmit_buffers(struct tx_ring *); 103static int ixv_setup_transmit_structures(struct adapter *); 104static void ixv_setup_transmit_ring(struct tx_ring *); 105static void ixv_initialize_transmit_units(struct adapter *); 106static void ixv_free_transmit_structures(struct adapter *); 107static void ixv_free_transmit_buffers(struct tx_ring *); 108 109static int ixv_allocate_receive_buffers(struct rx_ring *); 110static int ixv_setup_receive_structures(struct adapter *); 111static int ixv_setup_receive_ring(struct rx_ring *); 112static void ixv_initialize_receive_units(struct adapter *); 113static void ixv_free_receive_structures(struct adapter *); 114static void ixv_free_receive_buffers(struct rx_ring *); 115 116static void ixv_enable_intr(struct adapter *); 117static void ixv_disable_intr(struct adapter *); 118static bool ixv_txeof(struct tx_ring *); 119static bool ixv_rxeof(struct ix_queue *, int); 120static void ixv_rx_checksum(u32, struct mbuf *, u32); 121static void ixv_set_multi(struct adapter *); 122static void ixv_update_link_status(struct adapter *); 123static void ixv_refresh_mbufs(struct rx_ring *, int); 124static int ixv_xmit(struct tx_ring *, struct mbuf **); 125static int ixv_sysctl_stats(SYSCTL_HANDLER_ARGS); 126static int ixv_sysctl_debug(SYSCTL_HANDLER_ARGS); 127static int ixv_set_flowcntl(SYSCTL_HANDLER_ARGS); 128static int ixv_dma_malloc(struct adapter *, bus_size_t, 129 struct ixv_dma_alloc *, int); 130static void ixv_dma_free(struct adapter *, struct ixv_dma_alloc *); 131static void ixv_add_rx_process_limit(struct adapter *, const char *, 132 const char *, int *, int); 133static bool ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *); 134static bool ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 135static void ixv_set_ivar(struct adapter *, u8, u8, s8); 136static void ixv_configure_ivars(struct adapter *); 137static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 138 139static void ixv_setup_vlan_support(struct adapter *); 140static void ixv_register_vlan(void *, struct ifnet *, u16); 141static void ixv_unregister_vlan(void *, struct ifnet *, u16); 142 143static void ixv_save_stats(struct adapter *); 144static void ixv_init_stats(struct adapter *); 145static void ixv_update_stats(struct adapter *); 146 147static __inline void ixv_rx_discard(struct rx_ring *, int); 148static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *, 149 struct mbuf *, u32); 150 151/* The MSI/X Interrupt handlers */ 152static void ixv_msix_que(void *); 153static void ixv_msix_mbx(void *); 154 155/* Deferred interrupt tasklets */ 156static void ixv_handle_que(void *, int); 157static void ixv_handle_mbx(void *, int); 158 159/********************************************************************* 160 * FreeBSD Device Interface Entry Points 161 *********************************************************************/ 162 163static device_method_t ixv_methods[] = { 164 /* Device interface */ 165 DEVMETHOD(device_probe, ixv_probe), 166 DEVMETHOD(device_attach, ixv_attach), 167 DEVMETHOD(device_detach, ixv_detach), 168 DEVMETHOD(device_shutdown, ixv_shutdown), 169 DEVMETHOD_END 170}; 171 172static driver_t ixv_driver = { 173 "ix", ixv_methods, sizeof(struct adapter), 174}; 175 176extern devclass_t ixgbe_devclass; 177DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0); 178MODULE_DEPEND(ixv, pci, 1, 1, 1); 179MODULE_DEPEND(ixv, ether, 1, 1, 1); 180 181/* 182** TUNEABLE PARAMETERS: 183*/ 184 185/* 186** AIM: Adaptive Interrupt Moderation 187** which means that the interrupt rate 188** is varied over time based on the 189** traffic for that interrupt vector 190*/ 191static int ixv_enable_aim = FALSE; 192TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 193 194/* How many packets rxeof tries to clean at a time */ 195static int ixv_rx_process_limit = 128; 196TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 197 198/* Flow control setting, default to full */ 199static int ixv_flow_control = ixgbe_fc_full; 200TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control); 201 202/* 203 * Header split: this causes the hardware to DMA 204 * the header into a seperate mbuf from the payload, 205 * it can be a performance win in some workloads, but 206 * in others it actually hurts, its off by default. 207 */ 208static int ixv_header_split = FALSE; 209TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split); 210 211/* 212** Number of TX descriptors per ring, 213** setting higher than RX as this seems 214** the better performing choice. 215*/ 216static int ixv_txd = DEFAULT_TXD; 217TUNABLE_INT("hw.ixv.txd", &ixv_txd); 218 219/* Number of RX descriptors per ring */ 220static int ixv_rxd = DEFAULT_RXD; 221TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 222 223/* 224** Shadow VFTA table, this is needed because 225** the real filter table gets cleared during 226** a soft reset and we need to repopulate it. 227*/ 228static u32 ixv_shadow_vfta[VFTA_SIZE]; 229 230/********************************************************************* 231 * Device identification routine 232 * 233 * ixv_probe determines if the driver should be loaded on 234 * adapter based on PCI vendor/device id of the adapter. 235 * 236 * return BUS_PROBE_DEFAULT on success, positive on failure 237 *********************************************************************/ 238 239static int 240ixv_probe(device_t dev) 241{ 242 ixv_vendor_info_t *ent; 243 244 u16 pci_vendor_id = 0; 245 u16 pci_device_id = 0; 246 u16 pci_subvendor_id = 0; 247 u16 pci_subdevice_id = 0; 248 char adapter_name[256]; 249 250 251 pci_vendor_id = pci_get_vendor(dev); 252 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) 253 return (ENXIO); 254 255 pci_device_id = pci_get_device(dev); 256 pci_subvendor_id = pci_get_subvendor(dev); 257 pci_subdevice_id = pci_get_subdevice(dev); 258 259 ent = ixv_vendor_info_array; 260 while (ent->vendor_id != 0) { 261 if ((pci_vendor_id == ent->vendor_id) && 262 (pci_device_id == ent->device_id) && 263 264 ((pci_subvendor_id == ent->subvendor_id) || 265 (ent->subvendor_id == 0)) && 266 267 ((pci_subdevice_id == ent->subdevice_id) || 268 (ent->subdevice_id == 0))) { 269 sprintf(adapter_name, "%s, Version - %s", 270 ixv_strings[ent->index], 271 ixv_driver_version); 272 device_set_desc_copy(dev, adapter_name); 273 return (BUS_PROBE_DEFAULT); 274 } 275 ent++; 276 } 277 return (ENXIO); 278} 279 280/********************************************************************* 281 * Device initialization routine 282 * 283 * The attach entry point is called when the driver is being loaded. 284 * This routine identifies the type of hardware, allocates all resources 285 * and initializes the hardware. 286 * 287 * return 0 on success, positive on failure 288 *********************************************************************/ 289 290static int 291ixv_attach(device_t dev) 292{ 293 struct adapter *adapter; 294 struct ixgbe_hw *hw; 295 int error = 0; 296 297 INIT_DEBUGOUT("ixv_attach: begin"); 298 299 /* Allocate, clear, and link in our adapter structure */ 300 adapter = device_get_softc(dev); 301 adapter->dev = adapter->osdep.dev = dev; 302 hw = &adapter->hw; 303 304 /* Core Lock Init*/ 305 IXV_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 306 307 /* SYSCTL APIs */ 308 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 309 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 310 OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW, 311 adapter, 0, ixv_sysctl_stats, "I", "Statistics"); 312 313 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 314 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 315 OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW, 316 adapter, 0, ixv_sysctl_debug, "I", "Debug Info"); 317 318 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 319 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 320 OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW, 321 adapter, 0, ixv_set_flowcntl, "I", "Flow Control"); 322 323 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 324 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 325 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 326 &ixv_enable_aim, 1, "Interrupt Moderation"); 327 328 /* Set up the timer callout */ 329 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 330 331 /* Determine hardware revision */ 332 ixv_identify_hardware(adapter); 333 334 /* Do base PCI setup - map BAR0 */ 335 if (ixv_allocate_pci_resources(adapter)) { 336 device_printf(dev, "Allocation of PCI resources failed\n"); 337 error = ENXIO; 338 goto err_out; 339 } 340 341 /* Do descriptor calc and sanity checks */ 342 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 343 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 344 device_printf(dev, "TXD config issue, using default!\n"); 345 adapter->num_tx_desc = DEFAULT_TXD; 346 } else 347 adapter->num_tx_desc = ixv_txd; 348 349 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 350 ixv_rxd < MIN_TXD || ixv_rxd > MAX_TXD) { 351 device_printf(dev, "RXD config issue, using default!\n"); 352 adapter->num_rx_desc = DEFAULT_RXD; 353 } else 354 adapter->num_rx_desc = ixv_rxd; 355 356 /* Allocate our TX/RX Queues */ 357 if (ixv_allocate_queues(adapter)) { 358 error = ENOMEM; 359 goto err_out; 360 } 361 362 /* 363 ** Initialize the shared code: its 364 ** at this point the mac type is set. 365 */ 366 error = ixgbe_init_shared_code(hw); 367 if (error) { 368 device_printf(dev,"Shared Code Initialization Failure\n"); 369 error = EIO; 370 goto err_late; 371 } 372 373 /* Setup the mailbox */ 374 ixgbe_init_mbx_params_vf(hw); 375 376 ixgbe_reset_hw(hw); 377 378 /* Get Hardware Flow Control setting */ 379 hw->fc.requested_mode = ixgbe_fc_full; 380 hw->fc.pause_time = IXV_FC_PAUSE; 381 hw->fc.low_water[0] = IXV_FC_LO; 382 hw->fc.high_water[0] = IXV_FC_HI; 383 hw->fc.send_xon = TRUE; 384 385 error = ixgbe_init_hw(hw); 386 if (error) { 387 device_printf(dev,"Hardware Initialization Failure\n"); 388 error = EIO; 389 goto err_late; 390 } 391 392 error = ixv_allocate_msix(adapter); 393 if (error) 394 goto err_late; 395 396 /* Setup OS specific network interface */ 397 ixv_setup_interface(dev, adapter); 398 399 /* Sysctl for limiting the amount of work done in the taskqueue */ 400 ixv_add_rx_process_limit(adapter, "rx_processing_limit", 401 "max number of rx packets to process", &adapter->rx_process_limit, 402 ixv_rx_process_limit); 403 404 /* Do the stats setup */ 405 ixv_save_stats(adapter); 406 ixv_init_stats(adapter); 407 408 /* Register for VLAN events */ 409 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 410 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 411 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 412 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 413 414 INIT_DEBUGOUT("ixv_attach: end"); 415 return (0); 416 417err_late: 418 ixv_free_transmit_structures(adapter); 419 ixv_free_receive_structures(adapter); 420err_out: 421 ixv_free_pci_resources(adapter); 422 return (error); 423 424} 425 426/********************************************************************* 427 * Device removal routine 428 * 429 * The detach entry point is called when the driver is being removed. 430 * This routine stops the adapter and deallocates all the resources 431 * that were allocated for driver operation. 432 * 433 * return 0 on success, positive on failure 434 *********************************************************************/ 435 436static int 437ixv_detach(device_t dev) 438{ 439 struct adapter *adapter = device_get_softc(dev); 440 struct ix_queue *que = adapter->queues; 441 442 INIT_DEBUGOUT("ixv_detach: begin"); 443 444 /* Make sure VLANS are not using driver */ 445 if (adapter->ifp->if_vlantrunk != NULL) { 446 device_printf(dev,"Vlan in use, detach first\n"); 447 return (EBUSY); 448 } 449 450 IXV_CORE_LOCK(adapter); 451 ixv_stop(adapter); 452 IXV_CORE_UNLOCK(adapter); 453 454 for (int i = 0; i < adapter->num_queues; i++, que++) { 455 if (que->tq) { 456 taskqueue_drain(que->tq, &que->que_task); 457 taskqueue_free(que->tq); 458 } 459 } 460 461 /* Drain the Link queue */ 462 if (adapter->tq) { 463 taskqueue_drain(adapter->tq, &adapter->mbx_task); 464 taskqueue_free(adapter->tq); 465 } 466 467 /* Unregister VLAN events */ 468 if (adapter->vlan_attach != NULL) 469 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 470 if (adapter->vlan_detach != NULL) 471 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 472 473 ether_ifdetach(adapter->ifp); 474 callout_drain(&adapter->timer); 475 ixv_free_pci_resources(adapter); 476 bus_generic_detach(dev); 477 if_free(adapter->ifp); 478 479 ixv_free_transmit_structures(adapter); 480 ixv_free_receive_structures(adapter); 481 482 IXV_CORE_LOCK_DESTROY(adapter); 483 return (0); 484} 485 486/********************************************************************* 487 * 488 * Shutdown entry point 489 * 490 **********************************************************************/ 491static int 492ixv_shutdown(device_t dev) 493{ 494 struct adapter *adapter = device_get_softc(dev); 495 IXV_CORE_LOCK(adapter); 496 ixv_stop(adapter); 497 IXV_CORE_UNLOCK(adapter); 498 return (0); 499} 500 501#if __FreeBSD_version < 800000 502/********************************************************************* 503 * Transmit entry point 504 * 505 * ixv_start is called by the stack to initiate a transmit. 506 * The driver will remain in this routine as long as there are 507 * packets to transmit and transmit resources are available. 508 * In case resources are not available stack is notified and 509 * the packet is requeued. 510 **********************************************************************/ 511static void 512ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp) 513{ 514 struct mbuf *m_head; 515 struct adapter *adapter = txr->adapter; 516 517 IXV_TX_LOCK_ASSERT(txr); 518 519 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 520 IFF_DRV_RUNNING) 521 return; 522 if (!adapter->link_active) 523 return; 524 525 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 526 527 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 528 if (m_head == NULL) 529 break; 530 531 if (ixv_xmit(txr, &m_head)) { 532 if (m_head == NULL) 533 break; 534 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 535 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 536 break; 537 } 538 /* Send a copy of the frame to the BPF listener */ 539 ETHER_BPF_MTAP(ifp, m_head); 540 541 /* Set watchdog on */ 542 txr->watchdog_check = TRUE; 543 txr->watchdog_time = ticks; 544 545 } 546 return; 547} 548 549/* 550 * Legacy TX start - called by the stack, this 551 * always uses the first tx ring, and should 552 * not be used with multiqueue tx enabled. 553 */ 554static void 555ixv_start(struct ifnet *ifp) 556{ 557 struct adapter *adapter = ifp->if_softc; 558 struct tx_ring *txr = adapter->tx_rings; 559 560 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 561 IXV_TX_LOCK(txr); 562 ixv_start_locked(txr, ifp); 563 IXV_TX_UNLOCK(txr); 564 } 565 return; 566} 567 568#else 569 570/* 571** Multiqueue Transmit driver 572** 573*/ 574static int 575ixv_mq_start(struct ifnet *ifp, struct mbuf *m) 576{ 577 struct adapter *adapter = ifp->if_softc; 578 struct ix_queue *que; 579 struct tx_ring *txr; 580 int i = 0, err = 0; 581 582 /* Which queue to use */ 583 if ((m->m_flags & M_FLOWID) != 0) 584 i = m->m_pkthdr.flowid % adapter->num_queues; 585 586 txr = &adapter->tx_rings[i]; 587 que = &adapter->queues[i]; 588 589 if (IXV_TX_TRYLOCK(txr)) { 590 err = ixv_mq_start_locked(ifp, txr, m); 591 IXV_TX_UNLOCK(txr); 592 } else { 593 err = drbr_enqueue(ifp, txr->br, m); 594 taskqueue_enqueue(que->tq, &que->que_task); 595 } 596 597 return (err); 598} 599 600static int 601ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 602{ 603 struct adapter *adapter = txr->adapter; 604 struct mbuf *next; 605 int enqueued, err = 0; 606 607 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 608 IFF_DRV_RUNNING || adapter->link_active == 0) { 609 if (m != NULL) 610 err = drbr_enqueue(ifp, txr->br, m); 611 return (err); 612 } 613 614 /* Do a clean if descriptors are low */ 615 if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD) 616 ixv_txeof(txr); 617 618 enqueued = 0; 619 if (m != NULL) { 620 err = drbr_enqueue(ifp, txr->br, m); 621 if (err) { 622 return (err); 623 } 624 } 625 /* Process the queue */ 626 while ((next = drbr_peek(ifp, txr->br)) != NULL) { 627 if ((err = ixv_xmit(txr, &next)) != 0) { 628 if (next == NULL) { 629 drbr_advance(ifp, txr->br); 630 } else { 631 drbr_putback(ifp, txr->br, next); 632 } 633 break; 634 } 635 drbr_advance(ifp, txr->br); 636 enqueued++; 637 ifp->if_obytes += next->m_pkthdr.len; 638 if (next->m_flags & M_MCAST) 639 ifp->if_omcasts++; 640 /* Send a copy of the frame to the BPF listener */ 641 ETHER_BPF_MTAP(ifp, next); 642 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 643 break; 644 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) { 645 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 646 break; 647 } 648 } 649 650 if (enqueued > 0) { 651 /* Set watchdog on */ 652 txr->watchdog_check = TRUE; 653 txr->watchdog_time = ticks; 654 } 655 656 return (err); 657} 658 659/* 660** Flush all ring buffers 661*/ 662static void 663ixv_qflush(struct ifnet *ifp) 664{ 665 struct adapter *adapter = ifp->if_softc; 666 struct tx_ring *txr = adapter->tx_rings; 667 struct mbuf *m; 668 669 for (int i = 0; i < adapter->num_queues; i++, txr++) { 670 IXV_TX_LOCK(txr); 671 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 672 m_freem(m); 673 IXV_TX_UNLOCK(txr); 674 } 675 if_qflush(ifp); 676} 677 678#endif 679 680/********************************************************************* 681 * Ioctl entry point 682 * 683 * ixv_ioctl is called when the user wants to configure the 684 * interface. 685 * 686 * return 0 on success, positive on failure 687 **********************************************************************/ 688 689static int 690ixv_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 691{ 692 struct adapter *adapter = ifp->if_softc; 693 struct ifreq *ifr = (struct ifreq *) data; 694#if defined(INET) || defined(INET6) 695 struct ifaddr *ifa = (struct ifaddr *) data; 696 bool avoid_reset = FALSE; 697#endif 698 int error = 0; 699 700 switch (command) { 701 702 case SIOCSIFADDR: 703#ifdef INET 704 if (ifa->ifa_addr->sa_family == AF_INET) 705 avoid_reset = TRUE; 706#endif 707#ifdef INET6 708 if (ifa->ifa_addr->sa_family == AF_INET6) 709 avoid_reset = TRUE; 710#endif 711#if defined(INET) || defined(INET6) 712 /* 713 ** Calling init results in link renegotiation, 714 ** so we avoid doing it when possible. 715 */ 716 if (avoid_reset) { 717 ifp->if_flags |= IFF_UP; 718 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 719 ixv_init(adapter); 720 if (!(ifp->if_flags & IFF_NOARP)) 721 arp_ifinit(ifp, ifa); 722 } else 723 error = ether_ioctl(ifp, command, data); 724 break; 725#endif 726 case SIOCSIFMTU: 727 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 728 if (ifr->ifr_mtu > IXV_MAX_FRAME_SIZE - ETHER_HDR_LEN) { 729 error = EINVAL; 730 } else { 731 IXV_CORE_LOCK(adapter); 732 ifp->if_mtu = ifr->ifr_mtu; 733 adapter->max_frame_size = 734 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 735 ixv_init_locked(adapter); 736 IXV_CORE_UNLOCK(adapter); 737 } 738 break; 739 case SIOCSIFFLAGS: 740 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 741 IXV_CORE_LOCK(adapter); 742 if (ifp->if_flags & IFF_UP) { 743 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 744 ixv_init_locked(adapter); 745 } else 746 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 747 ixv_stop(adapter); 748 adapter->if_flags = ifp->if_flags; 749 IXV_CORE_UNLOCK(adapter); 750 break; 751 case SIOCADDMULTI: 752 case SIOCDELMULTI: 753 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 754 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 755 IXV_CORE_LOCK(adapter); 756 ixv_disable_intr(adapter); 757 ixv_set_multi(adapter); 758 ixv_enable_intr(adapter); 759 IXV_CORE_UNLOCK(adapter); 760 } 761 break; 762 case SIOCSIFMEDIA: 763 case SIOCGIFMEDIA: 764 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 765 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 766 break; 767 case SIOCSIFCAP: 768 { 769 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 770 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 771 if (mask & IFCAP_HWCSUM) 772 ifp->if_capenable ^= IFCAP_HWCSUM; 773 if (mask & IFCAP_TSO4) 774 ifp->if_capenable ^= IFCAP_TSO4; 775 if (mask & IFCAP_LRO) 776 ifp->if_capenable ^= IFCAP_LRO; 777 if (mask & IFCAP_VLAN_HWTAGGING) 778 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 779 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 780 IXV_CORE_LOCK(adapter); 781 ixv_init_locked(adapter); 782 IXV_CORE_UNLOCK(adapter); 783 } 784 VLAN_CAPABILITIES(ifp); 785 break; 786 } 787 788 default: 789 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 790 error = ether_ioctl(ifp, command, data); 791 break; 792 } 793 794 return (error); 795} 796 797/********************************************************************* 798 * Init entry point 799 * 800 * This routine is used in two ways. It is used by the stack as 801 * init entry point in network interface structure. It is also used 802 * by the driver as a hw/sw initialization routine to get to a 803 * consistent state. 804 * 805 * return 0 on success, positive on failure 806 **********************************************************************/ 807#define IXGBE_MHADD_MFS_SHIFT 16 808 809static void 810ixv_init_locked(struct adapter *adapter) 811{ 812 struct ifnet *ifp = adapter->ifp; 813 device_t dev = adapter->dev; 814 struct ixgbe_hw *hw = &adapter->hw; 815 u32 mhadd, gpie; 816 817 INIT_DEBUGOUT("ixv_init: begin"); 818 mtx_assert(&adapter->core_mtx, MA_OWNED); 819 hw->adapter_stopped = FALSE; 820 ixgbe_stop_adapter(hw); 821 callout_stop(&adapter->timer); 822 823 /* reprogram the RAR[0] in case user changed it. */ 824 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 825 826 /* Get the latest mac address, User can use a LAA */ 827 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr, 828 IXGBE_ETH_LENGTH_OF_ADDRESS); 829 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 830 hw->addr_ctrl.rar_used_count = 1; 831 832 /* Prepare transmit descriptors and buffers */ 833 if (ixv_setup_transmit_structures(adapter)) { 834 device_printf(dev,"Could not setup transmit structures\n"); 835 ixv_stop(adapter); 836 return; 837 } 838 839 ixgbe_reset_hw(hw); 840 ixv_initialize_transmit_units(adapter); 841 842 /* Setup Multicast table */ 843 ixv_set_multi(adapter); 844 845 /* 846 ** Determine the correct mbuf pool 847 ** for doing jumbo/headersplit 848 */ 849 if (ifp->if_mtu > ETHERMTU) 850 adapter->rx_mbuf_sz = MJUMPAGESIZE; 851 else 852 adapter->rx_mbuf_sz = MCLBYTES; 853 854 /* Prepare receive descriptors and buffers */ 855 if (ixv_setup_receive_structures(adapter)) { 856 device_printf(dev,"Could not setup receive structures\n"); 857 ixv_stop(adapter); 858 return; 859 } 860 861 /* Configure RX settings */ 862 ixv_initialize_receive_units(adapter); 863 864 /* Enable Enhanced MSIX mode */ 865 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 866 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME; 867 gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; 868 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 869 870 /* Set the various hardware offload abilities */ 871 ifp->if_hwassist = 0; 872 if (ifp->if_capenable & IFCAP_TSO4) 873 ifp->if_hwassist |= CSUM_TSO; 874 if (ifp->if_capenable & IFCAP_TXCSUM) { 875 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 876#if __FreeBSD_version >= 800000 877 ifp->if_hwassist |= CSUM_SCTP; 878#endif 879 } 880 881 /* Set MTU size */ 882 if (ifp->if_mtu > ETHERMTU) { 883 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 884 mhadd &= ~IXGBE_MHADD_MFS_MASK; 885 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 886 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 887 } 888 889 /* Set up VLAN offload and filter */ 890 ixv_setup_vlan_support(adapter); 891 892 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 893 894 /* Set up MSI/X routing */ 895 ixv_configure_ivars(adapter); 896 897 /* Set up auto-mask */ 898 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 899 900 /* Set moderation on the Link interrupt */ 901 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR); 902 903 /* Stats init */ 904 ixv_init_stats(adapter); 905 906 /* Config/Enable Link */ 907 ixv_config_link(adapter); 908 909 /* And now turn on interrupts */ 910 ixv_enable_intr(adapter); 911 912 /* Now inform the stack we're ready */ 913 ifp->if_drv_flags |= IFF_DRV_RUNNING; 914 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 915 916 return; 917} 918 919static void 920ixv_init(void *arg) 921{ 922 struct adapter *adapter = arg; 923 924 IXV_CORE_LOCK(adapter); 925 ixv_init_locked(adapter); 926 IXV_CORE_UNLOCK(adapter); 927 return; 928} 929 930 931/* 932** 933** MSIX Interrupt Handlers and Tasklets 934** 935*/ 936 937static inline void 938ixv_enable_queue(struct adapter *adapter, u32 vector) 939{ 940 struct ixgbe_hw *hw = &adapter->hw; 941 u32 queue = 1 << vector; 942 u32 mask; 943 944 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 945 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 946} 947 948static inline void 949ixv_disable_queue(struct adapter *adapter, u32 vector) 950{ 951 struct ixgbe_hw *hw = &adapter->hw; 952 u64 queue = (u64)(1 << vector); 953 u32 mask; 954 955 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 956 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 957} 958 959static inline void 960ixv_rearm_queues(struct adapter *adapter, u64 queues) 961{ 962 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 963 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 964} 965 966 967static void 968ixv_handle_que(void *context, int pending) 969{ 970 struct ix_queue *que = context; 971 struct adapter *adapter = que->adapter; 972 struct tx_ring *txr = que->txr; 973 struct ifnet *ifp = adapter->ifp; 974 bool more; 975 976 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 977 more = ixv_rxeof(que, adapter->rx_process_limit); 978 IXV_TX_LOCK(txr); 979 ixv_txeof(txr); 980#if __FreeBSD_version >= 800000 981 if (!drbr_empty(ifp, txr->br)) 982 ixv_mq_start_locked(ifp, txr, NULL); 983#else 984 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 985 ixv_start_locked(txr, ifp); 986#endif 987 IXV_TX_UNLOCK(txr); 988 if (more) { 989 taskqueue_enqueue(que->tq, &que->que_task); 990 return; 991 } 992 } 993 994 /* Reenable this interrupt */ 995 ixv_enable_queue(adapter, que->msix); 996 return; 997} 998 999/********************************************************************* 1000 * 1001 * MSI Queue Interrupt Service routine 1002 * 1003 **********************************************************************/ 1004void 1005ixv_msix_que(void *arg) 1006{ 1007 struct ix_queue *que = arg; 1008 struct adapter *adapter = que->adapter; 1009 struct tx_ring *txr = que->txr; 1010 struct rx_ring *rxr = que->rxr; 1011 bool more_tx, more_rx; 1012 u32 newitr = 0; 1013 1014 ixv_disable_queue(adapter, que->msix); 1015 ++que->irqs; 1016 1017 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1018 1019 IXV_TX_LOCK(txr); 1020 more_tx = ixv_txeof(txr); 1021 /* 1022 ** Make certain that if the stack 1023 ** has anything queued the task gets 1024 ** scheduled to handle it. 1025 */ 1026#if __FreeBSD_version < 800000 1027 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd)) 1028#else 1029 if (!drbr_empty(adapter->ifp, txr->br)) 1030#endif 1031 more_tx = 1; 1032 IXV_TX_UNLOCK(txr); 1033 1034 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1035 1036 /* Do AIM now? */ 1037 1038 if (ixv_enable_aim == FALSE) 1039 goto no_calc; 1040 /* 1041 ** Do Adaptive Interrupt Moderation: 1042 ** - Write out last calculated setting 1043 ** - Calculate based on average size over 1044 ** the last interval. 1045 */ 1046 if (que->eitr_setting) 1047 IXGBE_WRITE_REG(&adapter->hw, 1048 IXGBE_VTEITR(que->msix), 1049 que->eitr_setting); 1050 1051 que->eitr_setting = 0; 1052 1053 /* Idle, do nothing */ 1054 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1055 goto no_calc; 1056 1057 if ((txr->bytes) && (txr->packets)) 1058 newitr = txr->bytes/txr->packets; 1059 if ((rxr->bytes) && (rxr->packets)) 1060 newitr = max(newitr, 1061 (rxr->bytes / rxr->packets)); 1062 newitr += 24; /* account for hardware frame, crc */ 1063 1064 /* set an upper boundary */ 1065 newitr = min(newitr, 3000); 1066 1067 /* Be nice to the mid range */ 1068 if ((newitr > 300) && (newitr < 1200)) 1069 newitr = (newitr / 3); 1070 else 1071 newitr = (newitr / 2); 1072 1073 newitr |= newitr << 16; 1074 1075 /* save for next interrupt */ 1076 que->eitr_setting = newitr; 1077 1078 /* Reset state */ 1079 txr->bytes = 0; 1080 txr->packets = 0; 1081 rxr->bytes = 0; 1082 rxr->packets = 0; 1083 1084no_calc: 1085 if (more_tx || more_rx) 1086 taskqueue_enqueue(que->tq, &que->que_task); 1087 else /* Reenable this interrupt */ 1088 ixv_enable_queue(adapter, que->msix); 1089 return; 1090} 1091 1092static void 1093ixv_msix_mbx(void *arg) 1094{ 1095 struct adapter *adapter = arg; 1096 struct ixgbe_hw *hw = &adapter->hw; 1097 u32 reg; 1098 1099 ++adapter->mbx_irq; 1100 1101 /* First get the cause */ 1102 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 1103 /* Clear interrupt with write */ 1104 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 1105 1106 /* Link status change */ 1107 if (reg & IXGBE_EICR_LSC) 1108 taskqueue_enqueue(adapter->tq, &adapter->mbx_task); 1109 1110 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 1111 return; 1112} 1113 1114/********************************************************************* 1115 * 1116 * Media Ioctl callback 1117 * 1118 * This routine is called whenever the user queries the status of 1119 * the interface using ifconfig. 1120 * 1121 **********************************************************************/ 1122static void 1123ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1124{ 1125 struct adapter *adapter = ifp->if_softc; 1126 1127 INIT_DEBUGOUT("ixv_media_status: begin"); 1128 IXV_CORE_LOCK(adapter); 1129 ixv_update_link_status(adapter); 1130 1131 ifmr->ifm_status = IFM_AVALID; 1132 ifmr->ifm_active = IFM_ETHER; 1133 1134 if (!adapter->link_active) { 1135 IXV_CORE_UNLOCK(adapter); 1136 return; 1137 } 1138 1139 ifmr->ifm_status |= IFM_ACTIVE; 1140 1141 switch (adapter->link_speed) { 1142 case IXGBE_LINK_SPEED_1GB_FULL: 1143 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1144 break; 1145 case IXGBE_LINK_SPEED_10GB_FULL: 1146 ifmr->ifm_active |= IFM_FDX; 1147 break; 1148 } 1149 1150 IXV_CORE_UNLOCK(adapter); 1151 1152 return; 1153} 1154 1155/********************************************************************* 1156 * 1157 * Media Ioctl callback 1158 * 1159 * This routine is called when the user changes speed/duplex using 1160 * media/mediopt option with ifconfig. 1161 * 1162 **********************************************************************/ 1163static int 1164ixv_media_change(struct ifnet * ifp) 1165{ 1166 struct adapter *adapter = ifp->if_softc; 1167 struct ifmedia *ifm = &adapter->media; 1168 1169 INIT_DEBUGOUT("ixv_media_change: begin"); 1170 1171 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1172 return (EINVAL); 1173 1174 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1175 case IFM_AUTO: 1176 break; 1177 default: 1178 device_printf(adapter->dev, "Only auto media type\n"); 1179 return (EINVAL); 1180 } 1181 1182 return (0); 1183} 1184 1185/********************************************************************* 1186 * 1187 * This routine maps the mbufs to tx descriptors, allowing the 1188 * TX engine to transmit the packets. 1189 * - return 0 on success, positive on failure 1190 * 1191 **********************************************************************/ 1192 1193static int 1194ixv_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1195{ 1196 struct adapter *adapter = txr->adapter; 1197 u32 olinfo_status = 0, cmd_type_len; 1198 u32 paylen = 0; 1199 int i, j, error, nsegs; 1200 int first, last = 0; 1201 struct mbuf *m_head; 1202 bus_dma_segment_t segs[32]; 1203 bus_dmamap_t map; 1204 struct ixv_tx_buf *txbuf, *txbuf_mapped; 1205 union ixgbe_adv_tx_desc *txd = NULL; 1206 1207 m_head = *m_headp; 1208 1209 /* Basic descriptor defines */ 1210 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1211 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1212 1213 if (m_head->m_flags & M_VLANTAG) 1214 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1215 1216 /* 1217 * Important to capture the first descriptor 1218 * used because it will contain the index of 1219 * the one we tell the hardware to report back 1220 */ 1221 first = txr->next_avail_desc; 1222 txbuf = &txr->tx_buffers[first]; 1223 txbuf_mapped = txbuf; 1224 map = txbuf->map; 1225 1226 /* 1227 * Map the packet for DMA. 1228 */ 1229 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1230 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1231 1232 if (error == EFBIG) { 1233 struct mbuf *m; 1234 1235 m = m_defrag(*m_headp, M_NOWAIT); 1236 if (m == NULL) { 1237 adapter->mbuf_defrag_failed++; 1238 m_freem(*m_headp); 1239 *m_headp = NULL; 1240 return (ENOBUFS); 1241 } 1242 *m_headp = m; 1243 1244 /* Try it again */ 1245 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1246 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1247 1248 if (error == ENOMEM) { 1249 adapter->no_tx_dma_setup++; 1250 return (error); 1251 } else if (error != 0) { 1252 adapter->no_tx_dma_setup++; 1253 m_freem(*m_headp); 1254 *m_headp = NULL; 1255 return (error); 1256 } 1257 } else if (error == ENOMEM) { 1258 adapter->no_tx_dma_setup++; 1259 return (error); 1260 } else if (error != 0) { 1261 adapter->no_tx_dma_setup++; 1262 m_freem(*m_headp); 1263 *m_headp = NULL; 1264 return (error); 1265 } 1266 1267 /* Make certain there are enough descriptors */ 1268 if (nsegs > txr->tx_avail - 2) { 1269 txr->no_desc_avail++; 1270 error = ENOBUFS; 1271 goto xmit_fail; 1272 } 1273 m_head = *m_headp; 1274 1275 /* 1276 ** Set up the appropriate offload context 1277 ** this becomes the first descriptor of 1278 ** a packet. 1279 */ 1280 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1281 if (ixv_tso_setup(txr, m_head, &paylen)) { 1282 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1283 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1284 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1285 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1286 ++adapter->tso_tx; 1287 } else 1288 return (ENXIO); 1289 } else if (ixv_tx_ctx_setup(txr, m_head)) 1290 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1291 1292 /* Record payload length */ 1293 if (paylen == 0) 1294 olinfo_status |= m_head->m_pkthdr.len << 1295 IXGBE_ADVTXD_PAYLEN_SHIFT; 1296 1297 i = txr->next_avail_desc; 1298 for (j = 0; j < nsegs; j++) { 1299 bus_size_t seglen; 1300 bus_addr_t segaddr; 1301 1302 txbuf = &txr->tx_buffers[i]; 1303 txd = &txr->tx_base[i]; 1304 seglen = segs[j].ds_len; 1305 segaddr = htole64(segs[j].ds_addr); 1306 1307 txd->read.buffer_addr = segaddr; 1308 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1309 cmd_type_len |seglen); 1310 txd->read.olinfo_status = htole32(olinfo_status); 1311 last = i; /* descriptor that will get completion IRQ */ 1312 1313 if (++i == adapter->num_tx_desc) 1314 i = 0; 1315 1316 txbuf->m_head = NULL; 1317 txbuf->eop_index = -1; 1318 } 1319 1320 txd->read.cmd_type_len |= 1321 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1322 txr->tx_avail -= nsegs; 1323 txr->next_avail_desc = i; 1324 1325 txbuf->m_head = m_head; 1326 txr->tx_buffers[first].map = txbuf->map; 1327 txbuf->map = map; 1328 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1329 1330 /* Set the index of the descriptor that will be marked done */ 1331 txbuf = &txr->tx_buffers[first]; 1332 txbuf->eop_index = last; 1333 1334 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1335 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1336 /* 1337 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1338 * hardware that this frame is available to transmit. 1339 */ 1340 ++txr->total_packets; 1341 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i); 1342 1343 return (0); 1344 1345xmit_fail: 1346 bus_dmamap_unload(txr->txtag, txbuf->map); 1347 return (error); 1348 1349} 1350 1351 1352/********************************************************************* 1353 * Multicast Update 1354 * 1355 * This routine is called whenever multicast address list is updated. 1356 * 1357 **********************************************************************/ 1358#define IXGBE_RAR_ENTRIES 16 1359 1360static void 1361ixv_set_multi(struct adapter *adapter) 1362{ 1363 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1364 u8 *update_ptr; 1365 struct ifmultiaddr *ifma; 1366 int mcnt = 0; 1367 struct ifnet *ifp = adapter->ifp; 1368 1369 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1370 1371#if __FreeBSD_version < 800000 1372 IF_ADDR_LOCK(ifp); 1373#else 1374 if_maddr_rlock(ifp); 1375#endif 1376 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1377 if (ifma->ifma_addr->sa_family != AF_LINK) 1378 continue; 1379 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), 1380 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1381 IXGBE_ETH_LENGTH_OF_ADDRESS); 1382 mcnt++; 1383 } 1384#if __FreeBSD_version < 800000 1385 IF_ADDR_UNLOCK(ifp); 1386#else 1387 if_maddr_runlock(ifp); 1388#endif 1389 1390 update_ptr = mta; 1391 1392 ixgbe_update_mc_addr_list(&adapter->hw, 1393 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1394 1395 return; 1396} 1397 1398/* 1399 * This is an iterator function now needed by the multicast 1400 * shared code. It simply feeds the shared code routine the 1401 * addresses in the array of ixv_set_multi() one by one. 1402 */ 1403static u8 * 1404ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1405{ 1406 u8 *addr = *update_ptr; 1407 u8 *newptr; 1408 *vmdq = 0; 1409 1410 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1411 *update_ptr = newptr; 1412 return addr; 1413} 1414 1415/********************************************************************* 1416 * Timer routine 1417 * 1418 * This routine checks for link status,updates statistics, 1419 * and runs the watchdog check. 1420 * 1421 **********************************************************************/ 1422 1423static void 1424ixv_local_timer(void *arg) 1425{ 1426 struct adapter *adapter = arg; 1427 device_t dev = adapter->dev; 1428 struct tx_ring *txr = adapter->tx_rings; 1429 int i; 1430 1431 mtx_assert(&adapter->core_mtx, MA_OWNED); 1432 1433 ixv_update_link_status(adapter); 1434 1435 /* Stats Update */ 1436 ixv_update_stats(adapter); 1437 1438 /* 1439 * If the interface has been paused 1440 * then don't do the watchdog check 1441 */ 1442 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 1443 goto out; 1444 /* 1445 ** Check for time since any descriptor was cleaned 1446 */ 1447 for (i = 0; i < adapter->num_queues; i++, txr++) { 1448 IXV_TX_LOCK(txr); 1449 if (txr->watchdog_check == FALSE) { 1450 IXV_TX_UNLOCK(txr); 1451 continue; 1452 } 1453 if ((ticks - txr->watchdog_time) > IXV_WATCHDOG) 1454 goto hung; 1455 IXV_TX_UNLOCK(txr); 1456 } 1457out: 1458 ixv_rearm_queues(adapter, adapter->que_mask); 1459 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1460 return; 1461 1462hung: 1463 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1464 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1465 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)), 1466 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i))); 1467 device_printf(dev,"TX(%d) desc avail = %d," 1468 "Next TX to Clean = %d\n", 1469 txr->me, txr->tx_avail, txr->next_to_clean); 1470 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1471 adapter->watchdog_events++; 1472 IXV_TX_UNLOCK(txr); 1473 ixv_init_locked(adapter); 1474} 1475 1476/* 1477** Note: this routine updates the OS on the link state 1478** the real check of the hardware only happens with 1479** a link interrupt. 1480*/ 1481static void 1482ixv_update_link_status(struct adapter *adapter) 1483{ 1484 struct ifnet *ifp = adapter->ifp; 1485 struct tx_ring *txr = adapter->tx_rings; 1486 device_t dev = adapter->dev; 1487 1488 1489 if (adapter->link_up){ 1490 if (adapter->link_active == FALSE) { 1491 if (bootverbose) 1492 device_printf(dev,"Link is up %d Gbps %s \n", 1493 ((adapter->link_speed == 128)? 10:1), 1494 "Full Duplex"); 1495 adapter->link_active = TRUE; 1496 if_link_state_change(ifp, LINK_STATE_UP); 1497 } 1498 } else { /* Link down */ 1499 if (adapter->link_active == TRUE) { 1500 if (bootverbose) 1501 device_printf(dev,"Link is Down\n"); 1502 if_link_state_change(ifp, LINK_STATE_DOWN); 1503 adapter->link_active = FALSE; 1504 for (int i = 0; i < adapter->num_queues; 1505 i++, txr++) 1506 txr->watchdog_check = FALSE; 1507 } 1508 } 1509 1510 return; 1511} 1512 1513 1514/********************************************************************* 1515 * 1516 * This routine disables all traffic on the adapter by issuing a 1517 * global reset on the MAC and deallocates TX/RX buffers. 1518 * 1519 **********************************************************************/ 1520 1521static void 1522ixv_stop(void *arg) 1523{ 1524 struct ifnet *ifp; 1525 struct adapter *adapter = arg; 1526 struct ixgbe_hw *hw = &adapter->hw; 1527 ifp = adapter->ifp; 1528 1529 mtx_assert(&adapter->core_mtx, MA_OWNED); 1530 1531 INIT_DEBUGOUT("ixv_stop: begin\n"); 1532 ixv_disable_intr(adapter); 1533 1534 /* Tell the stack that the interface is no longer active */ 1535 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1536 1537 ixgbe_reset_hw(hw); 1538 adapter->hw.adapter_stopped = FALSE; 1539 ixgbe_stop_adapter(hw); 1540 callout_stop(&adapter->timer); 1541 1542 /* reprogram the RAR[0] in case user changed it. */ 1543 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1544 1545 return; 1546} 1547 1548 1549/********************************************************************* 1550 * 1551 * Determine hardware revision. 1552 * 1553 **********************************************************************/ 1554static void 1555ixv_identify_hardware(struct adapter *adapter) 1556{ 1557 device_t dev = adapter->dev; 1558 u16 pci_cmd_word; 1559 1560 /* 1561 ** Make sure BUSMASTER is set, on a VM under 1562 ** KVM it may not be and will break things. 1563 */ 1564 pci_enable_busmaster(dev); 1565 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 1566 1567 /* Save off the information about this board */ 1568 adapter->hw.vendor_id = pci_get_vendor(dev); 1569 adapter->hw.device_id = pci_get_device(dev); 1570 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1571 adapter->hw.subsystem_vendor_id = 1572 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1573 adapter->hw.subsystem_device_id = 1574 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1575 1576 return; 1577} 1578 1579/********************************************************************* 1580 * 1581 * Setup MSIX Interrupt resources and handlers 1582 * 1583 **********************************************************************/ 1584static int 1585ixv_allocate_msix(struct adapter *adapter) 1586{ 1587 device_t dev = adapter->dev; 1588 struct ix_queue *que = adapter->queues; 1589 int error, rid, vector = 0; 1590 1591 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 1592 rid = vector + 1; 1593 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 1594 RF_SHAREABLE | RF_ACTIVE); 1595 if (que->res == NULL) { 1596 device_printf(dev,"Unable to allocate" 1597 " bus resource: que interrupt [%d]\n", vector); 1598 return (ENXIO); 1599 } 1600 /* Set the handler function */ 1601 error = bus_setup_intr(dev, que->res, 1602 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1603 ixv_msix_que, que, &que->tag); 1604 if (error) { 1605 que->res = NULL; 1606 device_printf(dev, "Failed to register QUE handler"); 1607 return (error); 1608 } 1609#if __FreeBSD_version >= 800504 1610 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 1611#endif 1612 que->msix = vector; 1613 adapter->que_mask |= (u64)(1 << que->msix); 1614 /* 1615 ** Bind the msix vector, and thus the 1616 ** ring to the corresponding cpu. 1617 */ 1618 if (adapter->num_queues > 1) 1619 bus_bind_intr(dev, que->res, i); 1620 1621 TASK_INIT(&que->que_task, 0, ixv_handle_que, que); 1622 que->tq = taskqueue_create_fast("ixv_que", M_NOWAIT, 1623 taskqueue_thread_enqueue, &que->tq); 1624 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 1625 device_get_nameunit(adapter->dev)); 1626 } 1627 1628 /* and Mailbox */ 1629 rid = vector + 1; 1630 adapter->res = bus_alloc_resource_any(dev, 1631 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 1632 if (!adapter->res) { 1633 device_printf(dev,"Unable to allocate" 1634 " bus resource: MBX interrupt [%d]\n", rid); 1635 return (ENXIO); 1636 } 1637 /* Set the mbx handler function */ 1638 error = bus_setup_intr(dev, adapter->res, 1639 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1640 ixv_msix_mbx, adapter, &adapter->tag); 1641 if (error) { 1642 adapter->res = NULL; 1643 device_printf(dev, "Failed to register LINK handler"); 1644 return (error); 1645 } 1646#if __FreeBSD_version >= 800504 1647 bus_describe_intr(dev, adapter->res, adapter->tag, "mbx"); 1648#endif 1649 adapter->mbxvec = vector; 1650 /* Tasklets for Mailbox */ 1651 TASK_INIT(&adapter->mbx_task, 0, ixv_handle_mbx, adapter); 1652 adapter->tq = taskqueue_create_fast("ixv_mbx", M_NOWAIT, 1653 taskqueue_thread_enqueue, &adapter->tq); 1654 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s mbxq", 1655 device_get_nameunit(adapter->dev)); 1656 /* 1657 ** Due to a broken design QEMU will fail to properly 1658 ** enable the guest for MSIX unless the vectors in 1659 ** the table are all set up, so we must rewrite the 1660 ** ENABLE in the MSIX control register again at this 1661 ** point to cause it to successfully initialize us. 1662 */ 1663 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 1664 int msix_ctrl; 1665 pci_find_cap(dev, PCIY_MSIX, &rid); 1666 rid += PCIR_MSIX_CTRL; 1667 msix_ctrl = pci_read_config(dev, rid, 2); 1668 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 1669 pci_write_config(dev, rid, msix_ctrl, 2); 1670 } 1671 1672 return (0); 1673} 1674 1675/* 1676 * Setup MSIX resources, note that the VF 1677 * device MUST use MSIX, there is no fallback. 1678 */ 1679static int 1680ixv_setup_msix(struct adapter *adapter) 1681{ 1682 device_t dev = adapter->dev; 1683 int rid, want; 1684 1685 1686 /* First try MSI/X */ 1687 rid = PCIR_BAR(3); 1688 adapter->msix_mem = bus_alloc_resource_any(dev, 1689 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1690 if (adapter->msix_mem == NULL) { 1691 device_printf(adapter->dev, 1692 "Unable to map MSIX table \n"); 1693 goto out; 1694 } 1695 1696 /* 1697 ** Want two vectors: one for a queue, 1698 ** plus an additional for mailbox. 1699 */ 1700 want = 2; 1701 if ((pci_alloc_msix(dev, &want) == 0) && (want == 2)) { 1702 device_printf(adapter->dev, 1703 "Using MSIX interrupts with %d vectors\n", want); 1704 return (want); 1705 } 1706 /* Release in case alloc was insufficient */ 1707 pci_release_msi(dev); 1708out: 1709 if (adapter->msix_mem != NULL) { 1710 bus_release_resource(dev, SYS_RES_MEMORY, 1711 rid, adapter->msix_mem); 1712 adapter->msix_mem = NULL; 1713 } 1714 device_printf(adapter->dev,"MSIX config error\n"); 1715 return (ENXIO); 1716} 1717 1718 1719static int 1720ixv_allocate_pci_resources(struct adapter *adapter) 1721{ 1722 int rid; 1723 device_t dev = adapter->dev; 1724 1725 rid = PCIR_BAR(0); 1726 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1727 &rid, RF_ACTIVE); 1728 1729 if (!(adapter->pci_mem)) { 1730 device_printf(dev,"Unable to allocate bus resource: memory\n"); 1731 return (ENXIO); 1732 } 1733 1734 adapter->osdep.mem_bus_space_tag = 1735 rman_get_bustag(adapter->pci_mem); 1736 adapter->osdep.mem_bus_space_handle = 1737 rman_get_bushandle(adapter->pci_mem); 1738 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle; 1739 1740 adapter->num_queues = 1; 1741 adapter->hw.back = &adapter->osdep; 1742 1743 /* 1744 ** Now setup MSI/X, should 1745 ** return us the number of 1746 ** configured vectors. 1747 */ 1748 adapter->msix = ixv_setup_msix(adapter); 1749 if (adapter->msix == ENXIO) 1750 return (ENXIO); 1751 else 1752 return (0); 1753} 1754 1755static void 1756ixv_free_pci_resources(struct adapter * adapter) 1757{ 1758 struct ix_queue *que = adapter->queues; 1759 device_t dev = adapter->dev; 1760 int rid, memrid; 1761 1762 memrid = PCIR_BAR(MSIX_BAR); 1763 1764 /* 1765 ** There is a slight possibility of a failure mode 1766 ** in attach that will result in entering this function 1767 ** before interrupt resources have been initialized, and 1768 ** in that case we do not want to execute the loops below 1769 ** We can detect this reliably by the state of the adapter 1770 ** res pointer. 1771 */ 1772 if (adapter->res == NULL) 1773 goto mem; 1774 1775 /* 1776 ** Release all msix queue resources: 1777 */ 1778 for (int i = 0; i < adapter->num_queues; i++, que++) { 1779 rid = que->msix + 1; 1780 if (que->tag != NULL) { 1781 bus_teardown_intr(dev, que->res, que->tag); 1782 que->tag = NULL; 1783 } 1784 if (que->res != NULL) 1785 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 1786 } 1787 1788 1789 /* Clean the Legacy or Link interrupt last */ 1790 if (adapter->mbxvec) /* we are doing MSIX */ 1791 rid = adapter->mbxvec + 1; 1792 else 1793 (adapter->msix != 0) ? (rid = 1):(rid = 0); 1794 1795 if (adapter->tag != NULL) { 1796 bus_teardown_intr(dev, adapter->res, adapter->tag); 1797 adapter->tag = NULL; 1798 } 1799 if (adapter->res != NULL) 1800 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 1801 1802mem: 1803 if (adapter->msix) 1804 pci_release_msi(dev); 1805 1806 if (adapter->msix_mem != NULL) 1807 bus_release_resource(dev, SYS_RES_MEMORY, 1808 memrid, adapter->msix_mem); 1809 1810 if (adapter->pci_mem != NULL) 1811 bus_release_resource(dev, SYS_RES_MEMORY, 1812 PCIR_BAR(0), adapter->pci_mem); 1813 1814 return; 1815} 1816 1817/********************************************************************* 1818 * 1819 * Setup networking device structure and register an interface. 1820 * 1821 **********************************************************************/ 1822static void 1823ixv_setup_interface(device_t dev, struct adapter *adapter) 1824{ 1825 struct ifnet *ifp; 1826 1827 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1828 1829 ifp = adapter->ifp = if_alloc(IFT_ETHER); 1830 if (ifp == NULL) 1831 panic("%s: can not if_alloc()\n", device_get_nameunit(dev)); 1832 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1833 ifp->if_baudrate = 1000000000; 1834 ifp->if_init = ixv_init; 1835 ifp->if_softc = adapter; 1836 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1837 ifp->if_ioctl = ixv_ioctl; 1838#if __FreeBSD_version >= 800000 1839 ifp->if_transmit = ixv_mq_start; 1840 ifp->if_qflush = ixv_qflush; 1841#else 1842 ifp->if_start = ixv_start; 1843#endif 1844 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 1845 1846 ether_ifattach(ifp, adapter->hw.mac.addr); 1847 1848 adapter->max_frame_size = 1849 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1850 1851 /* 1852 * Tell the upper layer(s) we support long frames. 1853 */ 1854 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1855 1856 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM; 1857 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 1858 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING 1859 | IFCAP_VLAN_HWTSO 1860 | IFCAP_VLAN_MTU; 1861 ifp->if_capenable = ifp->if_capabilities; 1862 1863 /* Don't enable LRO by default */ 1864 ifp->if_capabilities |= IFCAP_LRO; 1865 1866 /* 1867 * Specify the media types supported by this adapter and register 1868 * callbacks to update media and link information 1869 */ 1870 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1871 ixv_media_status); 1872 ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL); 1873 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1874 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1875 1876 return; 1877} 1878 1879static void 1880ixv_config_link(struct adapter *adapter) 1881{ 1882 struct ixgbe_hw *hw = &adapter->hw; 1883 u32 autoneg, err = 0; 1884 1885 if (hw->mac.ops.check_link) 1886 err = hw->mac.ops.check_link(hw, &autoneg, 1887 &adapter->link_up, FALSE); 1888 if (err) 1889 goto out; 1890 1891 if (hw->mac.ops.setup_link) 1892 err = hw->mac.ops.setup_link(hw, 1893 autoneg, adapter->link_up); 1894out: 1895 return; 1896} 1897 1898/******************************************************************** 1899 * Manage DMA'able memory. 1900 *******************************************************************/ 1901static void 1902ixv_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) 1903{ 1904 if (error) 1905 return; 1906 *(bus_addr_t *) arg = segs->ds_addr; 1907 return; 1908} 1909 1910static int 1911ixv_dma_malloc(struct adapter *adapter, bus_size_t size, 1912 struct ixv_dma_alloc *dma, int mapflags) 1913{ 1914 device_t dev = adapter->dev; 1915 int r; 1916 1917 r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 1918 DBA_ALIGN, 0, /* alignment, bounds */ 1919 BUS_SPACE_MAXADDR, /* lowaddr */ 1920 BUS_SPACE_MAXADDR, /* highaddr */ 1921 NULL, NULL, /* filter, filterarg */ 1922 size, /* maxsize */ 1923 1, /* nsegments */ 1924 size, /* maxsegsize */ 1925 BUS_DMA_ALLOCNOW, /* flags */ 1926 NULL, /* lockfunc */ 1927 NULL, /* lockfuncarg */ 1928 &dma->dma_tag); 1929 if (r != 0) { 1930 device_printf(dev,"ixv_dma_malloc: bus_dma_tag_create failed; " 1931 "error %u\n", r); 1932 goto fail_0; 1933 } 1934 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 1935 BUS_DMA_NOWAIT, &dma->dma_map); 1936 if (r != 0) { 1937 device_printf(dev,"ixv_dma_malloc: bus_dmamem_alloc failed; " 1938 "error %u\n", r); 1939 goto fail_1; 1940 } 1941 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1942 size, 1943 ixv_dmamap_cb, 1944 &dma->dma_paddr, 1945 mapflags | BUS_DMA_NOWAIT); 1946 if (r != 0) { 1947 device_printf(dev,"ixv_dma_malloc: bus_dmamap_load failed; " 1948 "error %u\n", r); 1949 goto fail_2; 1950 } 1951 dma->dma_size = size; 1952 return (0); 1953fail_2: 1954 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1955fail_1: 1956 bus_dma_tag_destroy(dma->dma_tag); 1957fail_0: 1958 dma->dma_map = NULL; 1959 dma->dma_tag = NULL; 1960 return (r); 1961} 1962 1963static void 1964ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma) 1965{ 1966 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 1967 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1968 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1969 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1970 bus_dma_tag_destroy(dma->dma_tag); 1971} 1972 1973 1974/********************************************************************* 1975 * 1976 * Allocate memory for the transmit and receive rings, and then 1977 * the descriptors associated with each, called only once at attach. 1978 * 1979 **********************************************************************/ 1980static int 1981ixv_allocate_queues(struct adapter *adapter) 1982{ 1983 device_t dev = adapter->dev; 1984 struct ix_queue *que; 1985 struct tx_ring *txr; 1986 struct rx_ring *rxr; 1987 int rsize, tsize, error = 0; 1988 int txconf = 0, rxconf = 0; 1989 1990 /* First allocate the top level queue structs */ 1991 if (!(adapter->queues = 1992 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 1993 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 1994 device_printf(dev, "Unable to allocate queue memory\n"); 1995 error = ENOMEM; 1996 goto fail; 1997 } 1998 1999 /* First allocate the TX ring struct memory */ 2000 if (!(adapter->tx_rings = 2001 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2002 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2003 device_printf(dev, "Unable to allocate TX ring memory\n"); 2004 error = ENOMEM; 2005 goto tx_fail; 2006 } 2007 2008 /* Next allocate the RX */ 2009 if (!(adapter->rx_rings = 2010 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2011 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2012 device_printf(dev, "Unable to allocate RX ring memory\n"); 2013 error = ENOMEM; 2014 goto rx_fail; 2015 } 2016 2017 /* For the ring itself */ 2018 tsize = roundup2(adapter->num_tx_desc * 2019 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2020 2021 /* 2022 * Now set up the TX queues, txconf is needed to handle the 2023 * possibility that things fail midcourse and we need to 2024 * undo memory gracefully 2025 */ 2026 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2027 /* Set up some basics */ 2028 txr = &adapter->tx_rings[i]; 2029 txr->adapter = adapter; 2030 txr->me = i; 2031 2032 /* Initialize the TX side lock */ 2033 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2034 device_get_nameunit(dev), txr->me); 2035 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2036 2037 if (ixv_dma_malloc(adapter, tsize, 2038 &txr->txdma, BUS_DMA_NOWAIT)) { 2039 device_printf(dev, 2040 "Unable to allocate TX Descriptor memory\n"); 2041 error = ENOMEM; 2042 goto err_tx_desc; 2043 } 2044 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2045 bzero((void *)txr->tx_base, tsize); 2046 2047 /* Now allocate transmit buffers for the ring */ 2048 if (ixv_allocate_transmit_buffers(txr)) { 2049 device_printf(dev, 2050 "Critical Failure setting up transmit buffers\n"); 2051 error = ENOMEM; 2052 goto err_tx_desc; 2053 } 2054#if __FreeBSD_version >= 800000 2055 /* Allocate a buf ring */ 2056 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF, 2057 M_WAITOK, &txr->tx_mtx); 2058 if (txr->br == NULL) { 2059 device_printf(dev, 2060 "Critical Failure setting up buf ring\n"); 2061 error = ENOMEM; 2062 goto err_tx_desc; 2063 } 2064#endif 2065 } 2066 2067 /* 2068 * Next the RX queues... 2069 */ 2070 rsize = roundup2(adapter->num_rx_desc * 2071 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2072 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2073 rxr = &adapter->rx_rings[i]; 2074 /* Set up some basics */ 2075 rxr->adapter = adapter; 2076 rxr->me = i; 2077 2078 /* Initialize the RX side lock */ 2079 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2080 device_get_nameunit(dev), rxr->me); 2081 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2082 2083 if (ixv_dma_malloc(adapter, rsize, 2084 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2085 device_printf(dev, 2086 "Unable to allocate RxDescriptor memory\n"); 2087 error = ENOMEM; 2088 goto err_rx_desc; 2089 } 2090 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2091 bzero((void *)rxr->rx_base, rsize); 2092 2093 /* Allocate receive buffers for the ring*/ 2094 if (ixv_allocate_receive_buffers(rxr)) { 2095 device_printf(dev, 2096 "Critical Failure setting up receive buffers\n"); 2097 error = ENOMEM; 2098 goto err_rx_desc; 2099 } 2100 } 2101 2102 /* 2103 ** Finally set up the queue holding structs 2104 */ 2105 for (int i = 0; i < adapter->num_queues; i++) { 2106 que = &adapter->queues[i]; 2107 que->adapter = adapter; 2108 que->txr = &adapter->tx_rings[i]; 2109 que->rxr = &adapter->rx_rings[i]; 2110 } 2111 2112 return (0); 2113 2114err_rx_desc: 2115 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2116 ixv_dma_free(adapter, &rxr->rxdma); 2117err_tx_desc: 2118 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2119 ixv_dma_free(adapter, &txr->txdma); 2120 free(adapter->rx_rings, M_DEVBUF); 2121rx_fail: 2122 free(adapter->tx_rings, M_DEVBUF); 2123tx_fail: 2124 free(adapter->queues, M_DEVBUF); 2125fail: 2126 return (error); 2127} 2128 2129 2130/********************************************************************* 2131 * 2132 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2133 * the information needed to transmit a packet on the wire. This is 2134 * called only once at attach, setup is done every reset. 2135 * 2136 **********************************************************************/ 2137static int 2138ixv_allocate_transmit_buffers(struct tx_ring *txr) 2139{ 2140 struct adapter *adapter = txr->adapter; 2141 device_t dev = adapter->dev; 2142 struct ixv_tx_buf *txbuf; 2143 int error, i; 2144 2145 /* 2146 * Setup DMA descriptor areas. 2147 */ 2148 if ((error = bus_dma_tag_create( 2149 bus_get_dma_tag(adapter->dev), /* parent */ 2150 1, 0, /* alignment, bounds */ 2151 BUS_SPACE_MAXADDR, /* lowaddr */ 2152 BUS_SPACE_MAXADDR, /* highaddr */ 2153 NULL, NULL, /* filter, filterarg */ 2154 IXV_TSO_SIZE, /* maxsize */ 2155 32, /* nsegments */ 2156 PAGE_SIZE, /* maxsegsize */ 2157 0, /* flags */ 2158 NULL, /* lockfunc */ 2159 NULL, /* lockfuncarg */ 2160 &txr->txtag))) { 2161 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2162 goto fail; 2163 } 2164 2165 if (!(txr->tx_buffers = 2166 (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) * 2167 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2168 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2169 error = ENOMEM; 2170 goto fail; 2171 } 2172 2173 /* Create the descriptor buffer dma maps */ 2174 txbuf = txr->tx_buffers; 2175 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2176 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2177 if (error != 0) { 2178 device_printf(dev, "Unable to create TX DMA map\n"); 2179 goto fail; 2180 } 2181 } 2182 2183 return 0; 2184fail: 2185 /* We free all, it handles case where we are in the middle */ 2186 ixv_free_transmit_structures(adapter); 2187 return (error); 2188} 2189 2190/********************************************************************* 2191 * 2192 * Initialize a transmit ring. 2193 * 2194 **********************************************************************/ 2195static void 2196ixv_setup_transmit_ring(struct tx_ring *txr) 2197{ 2198 struct adapter *adapter = txr->adapter; 2199 struct ixv_tx_buf *txbuf; 2200 int i; 2201 2202 /* Clear the old ring contents */ 2203 IXV_TX_LOCK(txr); 2204 bzero((void *)txr->tx_base, 2205 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2206 /* Reset indices */ 2207 txr->next_avail_desc = 0; 2208 txr->next_to_clean = 0; 2209 2210 /* Free any existing tx buffers. */ 2211 txbuf = txr->tx_buffers; 2212 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2213 if (txbuf->m_head != NULL) { 2214 bus_dmamap_sync(txr->txtag, txbuf->map, 2215 BUS_DMASYNC_POSTWRITE); 2216 bus_dmamap_unload(txr->txtag, txbuf->map); 2217 m_freem(txbuf->m_head); 2218 txbuf->m_head = NULL; 2219 } 2220 /* Clear the EOP index */ 2221 txbuf->eop_index = -1; 2222 } 2223 2224 /* Set number of descriptors available */ 2225 txr->tx_avail = adapter->num_tx_desc; 2226 2227 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2228 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2229 IXV_TX_UNLOCK(txr); 2230} 2231 2232/********************************************************************* 2233 * 2234 * Initialize all transmit rings. 2235 * 2236 **********************************************************************/ 2237static int 2238ixv_setup_transmit_structures(struct adapter *adapter) 2239{ 2240 struct tx_ring *txr = adapter->tx_rings; 2241 2242 for (int i = 0; i < adapter->num_queues; i++, txr++) 2243 ixv_setup_transmit_ring(txr); 2244 2245 return (0); 2246} 2247 2248/********************************************************************* 2249 * 2250 * Enable transmit unit. 2251 * 2252 **********************************************************************/ 2253static void 2254ixv_initialize_transmit_units(struct adapter *adapter) 2255{ 2256 struct tx_ring *txr = adapter->tx_rings; 2257 struct ixgbe_hw *hw = &adapter->hw; 2258 2259 2260 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2261 u64 tdba = txr->txdma.dma_paddr; 2262 u32 txctrl, txdctl; 2263 2264 /* Set WTHRESH to 8, burst writeback */ 2265 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2266 txdctl |= (8 << 16); 2267 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2268 /* Now enable */ 2269 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2270 txdctl |= IXGBE_TXDCTL_ENABLE; 2271 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2272 2273 /* Set the HW Tx Head and Tail indices */ 2274 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0); 2275 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0); 2276 2277 /* Setup Transmit Descriptor Cmd Settings */ 2278 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2279 txr->watchdog_check = FALSE; 2280 2281 /* Set Ring parameters */ 2282 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 2283 (tdba & 0x00000000ffffffffULL)); 2284 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32)); 2285 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 2286 adapter->num_tx_desc * 2287 sizeof(struct ixgbe_legacy_tx_desc)); 2288 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i)); 2289 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2290 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl); 2291 break; 2292 } 2293 2294 return; 2295} 2296 2297/********************************************************************* 2298 * 2299 * Free all transmit rings. 2300 * 2301 **********************************************************************/ 2302static void 2303ixv_free_transmit_structures(struct adapter *adapter) 2304{ 2305 struct tx_ring *txr = adapter->tx_rings; 2306 2307 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2308 IXV_TX_LOCK(txr); 2309 ixv_free_transmit_buffers(txr); 2310 ixv_dma_free(adapter, &txr->txdma); 2311 IXV_TX_UNLOCK(txr); 2312 IXV_TX_LOCK_DESTROY(txr); 2313 } 2314 free(adapter->tx_rings, M_DEVBUF); 2315} 2316 2317/********************************************************************* 2318 * 2319 * Free transmit ring related data structures. 2320 * 2321 **********************************************************************/ 2322static void 2323ixv_free_transmit_buffers(struct tx_ring *txr) 2324{ 2325 struct adapter *adapter = txr->adapter; 2326 struct ixv_tx_buf *tx_buffer; 2327 int i; 2328 2329 INIT_DEBUGOUT("free_transmit_ring: begin"); 2330 2331 if (txr->tx_buffers == NULL) 2332 return; 2333 2334 tx_buffer = txr->tx_buffers; 2335 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2336 if (tx_buffer->m_head != NULL) { 2337 bus_dmamap_sync(txr->txtag, tx_buffer->map, 2338 BUS_DMASYNC_POSTWRITE); 2339 bus_dmamap_unload(txr->txtag, 2340 tx_buffer->map); 2341 m_freem(tx_buffer->m_head); 2342 tx_buffer->m_head = NULL; 2343 if (tx_buffer->map != NULL) { 2344 bus_dmamap_destroy(txr->txtag, 2345 tx_buffer->map); 2346 tx_buffer->map = NULL; 2347 } 2348 } else if (tx_buffer->map != NULL) { 2349 bus_dmamap_unload(txr->txtag, 2350 tx_buffer->map); 2351 bus_dmamap_destroy(txr->txtag, 2352 tx_buffer->map); 2353 tx_buffer->map = NULL; 2354 } 2355 } 2356#if __FreeBSD_version >= 800000 2357 if (txr->br != NULL) 2358 buf_ring_free(txr->br, M_DEVBUF); 2359#endif 2360 if (txr->tx_buffers != NULL) { 2361 free(txr->tx_buffers, M_DEVBUF); 2362 txr->tx_buffers = NULL; 2363 } 2364 if (txr->txtag != NULL) { 2365 bus_dma_tag_destroy(txr->txtag); 2366 txr->txtag = NULL; 2367 } 2368 return; 2369} 2370 2371/********************************************************************* 2372 * 2373 * Advanced Context Descriptor setup for VLAN or CSUM 2374 * 2375 **********************************************************************/ 2376 2377static bool 2378ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2379{ 2380 struct adapter *adapter = txr->adapter; 2381 struct ixgbe_adv_tx_context_desc *TXD; 2382 struct ixv_tx_buf *tx_buffer; 2383 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2384 struct ether_vlan_header *eh; 2385 struct ip *ip; 2386 struct ip6_hdr *ip6; 2387 int ehdrlen, ip_hlen = 0; 2388 u16 etype; 2389 u8 ipproto = 0; 2390 bool offload = TRUE; 2391 int ctxd = txr->next_avail_desc; 2392 u16 vtag = 0; 2393 2394 2395 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2396 offload = FALSE; 2397 2398 2399 tx_buffer = &txr->tx_buffers[ctxd]; 2400 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2401 2402 /* 2403 ** In advanced descriptors the vlan tag must 2404 ** be placed into the descriptor itself. 2405 */ 2406 if (mp->m_flags & M_VLANTAG) { 2407 vtag = htole16(mp->m_pkthdr.ether_vtag); 2408 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2409 } else if (offload == FALSE) 2410 return FALSE; 2411 2412 /* 2413 * Determine where frame payload starts. 2414 * Jump over vlan headers if already present, 2415 * helpful for QinQ too. 2416 */ 2417 eh = mtod(mp, struct ether_vlan_header *); 2418 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2419 etype = ntohs(eh->evl_proto); 2420 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2421 } else { 2422 etype = ntohs(eh->evl_encap_proto); 2423 ehdrlen = ETHER_HDR_LEN; 2424 } 2425 2426 /* Set the ether header length */ 2427 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2428 2429 switch (etype) { 2430 case ETHERTYPE_IP: 2431 ip = (struct ip *)(mp->m_data + ehdrlen); 2432 ip_hlen = ip->ip_hl << 2; 2433 if (mp->m_len < ehdrlen + ip_hlen) 2434 return (FALSE); 2435 ipproto = ip->ip_p; 2436 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2437 break; 2438 case ETHERTYPE_IPV6: 2439 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2440 ip_hlen = sizeof(struct ip6_hdr); 2441 if (mp->m_len < ehdrlen + ip_hlen) 2442 return (FALSE); 2443 ipproto = ip6->ip6_nxt; 2444 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2445 break; 2446 default: 2447 offload = FALSE; 2448 break; 2449 } 2450 2451 vlan_macip_lens |= ip_hlen; 2452 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2453 2454 switch (ipproto) { 2455 case IPPROTO_TCP: 2456 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2457 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2458 break; 2459 2460 case IPPROTO_UDP: 2461 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2462 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2463 break; 2464 2465#if __FreeBSD_version >= 800000 2466 case IPPROTO_SCTP: 2467 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 2468 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 2469 break; 2470#endif 2471 default: 2472 offload = FALSE; 2473 break; 2474 } 2475 2476 /* Now copy bits into descriptor */ 2477 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2478 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2479 TXD->seqnum_seed = htole32(0); 2480 TXD->mss_l4len_idx = htole32(0); 2481 2482 tx_buffer->m_head = NULL; 2483 tx_buffer->eop_index = -1; 2484 2485 /* We've consumed the first desc, adjust counters */ 2486 if (++ctxd == adapter->num_tx_desc) 2487 ctxd = 0; 2488 txr->next_avail_desc = ctxd; 2489 --txr->tx_avail; 2490 2491 return (offload); 2492} 2493 2494/********************************************************************** 2495 * 2496 * Setup work for hardware segmentation offload (TSO) on 2497 * adapters using advanced tx descriptors 2498 * 2499 **********************************************************************/ 2500static bool 2501ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2502{ 2503 struct adapter *adapter = txr->adapter; 2504 struct ixgbe_adv_tx_context_desc *TXD; 2505 struct ixv_tx_buf *tx_buffer; 2506 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2507 u32 mss_l4len_idx = 0; 2508 u16 vtag = 0; 2509 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2510 struct ether_vlan_header *eh; 2511 struct ip *ip; 2512 struct tcphdr *th; 2513 2514 2515 /* 2516 * Determine where frame payload starts. 2517 * Jump over vlan headers if already present 2518 */ 2519 eh = mtod(mp, struct ether_vlan_header *); 2520 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2521 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2522 else 2523 ehdrlen = ETHER_HDR_LEN; 2524 2525 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2526 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2527 return FALSE; 2528 2529 ctxd = txr->next_avail_desc; 2530 tx_buffer = &txr->tx_buffers[ctxd]; 2531 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2532 2533 ip = (struct ip *)(mp->m_data + ehdrlen); 2534 if (ip->ip_p != IPPROTO_TCP) 2535 return FALSE; /* 0 */ 2536 ip->ip_sum = 0; 2537 ip_hlen = ip->ip_hl << 2; 2538 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2539 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2540 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2541 tcp_hlen = th->th_off << 2; 2542 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2543 2544 /* This is used in the transmit desc in encap */ 2545 *paylen = mp->m_pkthdr.len - hdrlen; 2546 2547 /* VLAN MACLEN IPLEN */ 2548 if (mp->m_flags & M_VLANTAG) { 2549 vtag = htole16(mp->m_pkthdr.ether_vtag); 2550 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2551 } 2552 2553 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2554 vlan_macip_lens |= ip_hlen; 2555 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2556 2557 /* ADV DTYPE TUCMD */ 2558 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2559 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2560 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2561 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2562 2563 2564 /* MSS L4LEN IDX */ 2565 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2566 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2567 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2568 2569 TXD->seqnum_seed = htole32(0); 2570 tx_buffer->m_head = NULL; 2571 tx_buffer->eop_index = -1; 2572 2573 if (++ctxd == adapter->num_tx_desc) 2574 ctxd = 0; 2575 2576 txr->tx_avail--; 2577 txr->next_avail_desc = ctxd; 2578 return TRUE; 2579} 2580 2581 2582/********************************************************************** 2583 * 2584 * Examine each tx_buffer in the used queue. If the hardware is done 2585 * processing the packet then free associated resources. The 2586 * tx_buffer is put back on the free queue. 2587 * 2588 **********************************************************************/ 2589static bool 2590ixv_txeof(struct tx_ring *txr) 2591{ 2592 struct adapter *adapter = txr->adapter; 2593 struct ifnet *ifp = adapter->ifp; 2594 u32 first, last, done; 2595 struct ixv_tx_buf *tx_buffer; 2596 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2597 2598 mtx_assert(&txr->tx_mtx, MA_OWNED); 2599 2600 if (txr->tx_avail == adapter->num_tx_desc) 2601 return FALSE; 2602 2603 first = txr->next_to_clean; 2604 tx_buffer = &txr->tx_buffers[first]; 2605 /* For cleanup we just use legacy struct */ 2606 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2607 last = tx_buffer->eop_index; 2608 if (last == -1) 2609 return FALSE; 2610 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2611 2612 /* 2613 ** Get the index of the first descriptor 2614 ** BEYOND the EOP and call that 'done'. 2615 ** I do this so the comparison in the 2616 ** inner while loop below can be simple 2617 */ 2618 if (++last == adapter->num_tx_desc) last = 0; 2619 done = last; 2620 2621 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2622 BUS_DMASYNC_POSTREAD); 2623 /* 2624 ** Only the EOP descriptor of a packet now has the DD 2625 ** bit set, this is what we look for... 2626 */ 2627 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2628 /* We clean the range of the packet */ 2629 while (first != done) { 2630 tx_desc->upper.data = 0; 2631 tx_desc->lower.data = 0; 2632 tx_desc->buffer_addr = 0; 2633 ++txr->tx_avail; 2634 2635 if (tx_buffer->m_head) { 2636 bus_dmamap_sync(txr->txtag, 2637 tx_buffer->map, 2638 BUS_DMASYNC_POSTWRITE); 2639 bus_dmamap_unload(txr->txtag, 2640 tx_buffer->map); 2641 m_freem(tx_buffer->m_head); 2642 tx_buffer->m_head = NULL; 2643 tx_buffer->map = NULL; 2644 } 2645 tx_buffer->eop_index = -1; 2646 txr->watchdog_time = ticks; 2647 2648 if (++first == adapter->num_tx_desc) 2649 first = 0; 2650 2651 tx_buffer = &txr->tx_buffers[first]; 2652 tx_desc = 2653 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2654 } 2655 ++ifp->if_opackets; 2656 /* See if there is more work now */ 2657 last = tx_buffer->eop_index; 2658 if (last != -1) { 2659 eop_desc = 2660 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2661 /* Get next done point */ 2662 if (++last == adapter->num_tx_desc) last = 0; 2663 done = last; 2664 } else 2665 break; 2666 } 2667 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2668 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2669 2670 txr->next_to_clean = first; 2671 2672 /* 2673 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that 2674 * it is OK to send packets. If there are no pending descriptors, 2675 * clear the timeout. Otherwise, if some descriptors have been freed, 2676 * restart the timeout. 2677 */ 2678 if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) { 2679 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 2680 if (txr->tx_avail == adapter->num_tx_desc) { 2681 txr->watchdog_check = FALSE; 2682 return FALSE; 2683 } 2684 } 2685 2686 return TRUE; 2687} 2688 2689/********************************************************************* 2690 * 2691 * Refresh mbuf buffers for RX descriptor rings 2692 * - now keeps its own state so discards due to resource 2693 * exhaustion are unnecessary, if an mbuf cannot be obtained 2694 * it just returns, keeping its placeholder, thus it can simply 2695 * be recalled to try again. 2696 * 2697 **********************************************************************/ 2698static void 2699ixv_refresh_mbufs(struct rx_ring *rxr, int limit) 2700{ 2701 struct adapter *adapter = rxr->adapter; 2702 bus_dma_segment_t hseg[1]; 2703 bus_dma_segment_t pseg[1]; 2704 struct ixv_rx_buf *rxbuf; 2705 struct mbuf *mh, *mp; 2706 int i, j, nsegs, error; 2707 bool refreshed = FALSE; 2708 2709 i = j = rxr->next_to_refresh; 2710 /* Get the control variable, one beyond refresh point */ 2711 if (++j == adapter->num_rx_desc) 2712 j = 0; 2713 while (j != limit) { 2714 rxbuf = &rxr->rx_buffers[i]; 2715 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 2716 mh = m_gethdr(M_NOWAIT, MT_DATA); 2717 if (mh == NULL) 2718 goto update; 2719 mh->m_pkthdr.len = mh->m_len = MHLEN; 2720 mh->m_len = MHLEN; 2721 mh->m_flags |= M_PKTHDR; 2722 m_adj(mh, ETHER_ALIGN); 2723 /* Get the memory mapping */ 2724 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2725 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); 2726 if (error != 0) { 2727 printf("GET BUF: dmamap load" 2728 " failure - %d\n", error); 2729 m_free(mh); 2730 goto update; 2731 } 2732 rxbuf->m_head = mh; 2733 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2734 BUS_DMASYNC_PREREAD); 2735 rxr->rx_base[i].read.hdr_addr = 2736 htole64(hseg[0].ds_addr); 2737 } 2738 2739 if (rxbuf->m_pack == NULL) { 2740 mp = m_getjcl(M_NOWAIT, MT_DATA, 2741 M_PKTHDR, adapter->rx_mbuf_sz); 2742 if (mp == NULL) 2743 goto update; 2744 } else 2745 mp = rxbuf->m_pack; 2746 2747 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2748 /* Get the memory mapping */ 2749 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2750 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); 2751 if (error != 0) { 2752 printf("GET BUF: dmamap load" 2753 " failure - %d\n", error); 2754 m_free(mp); 2755 rxbuf->m_pack = NULL; 2756 goto update; 2757 } 2758 rxbuf->m_pack = mp; 2759 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2760 BUS_DMASYNC_PREREAD); 2761 rxr->rx_base[i].read.pkt_addr = 2762 htole64(pseg[0].ds_addr); 2763 2764 refreshed = TRUE; 2765 rxr->next_to_refresh = i = j; 2766 /* Calculate next index */ 2767 if (++j == adapter->num_rx_desc) 2768 j = 0; 2769 } 2770update: 2771 if (refreshed) /* update tail index */ 2772 IXGBE_WRITE_REG(&adapter->hw, 2773 IXGBE_VFRDT(rxr->me), rxr->next_to_refresh); 2774 return; 2775} 2776 2777/********************************************************************* 2778 * 2779 * Allocate memory for rx_buffer structures. Since we use one 2780 * rx_buffer per received packet, the maximum number of rx_buffer's 2781 * that we'll need is equal to the number of receive descriptors 2782 * that we've allocated. 2783 * 2784 **********************************************************************/ 2785static int 2786ixv_allocate_receive_buffers(struct rx_ring *rxr) 2787{ 2788 struct adapter *adapter = rxr->adapter; 2789 device_t dev = adapter->dev; 2790 struct ixv_rx_buf *rxbuf; 2791 int i, bsize, error; 2792 2793 bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc; 2794 if (!(rxr->rx_buffers = 2795 (struct ixv_rx_buf *) malloc(bsize, 2796 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2797 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 2798 error = ENOMEM; 2799 goto fail; 2800 } 2801 2802 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2803 1, 0, /* alignment, bounds */ 2804 BUS_SPACE_MAXADDR, /* lowaddr */ 2805 BUS_SPACE_MAXADDR, /* highaddr */ 2806 NULL, NULL, /* filter, filterarg */ 2807 MSIZE, /* maxsize */ 2808 1, /* nsegments */ 2809 MSIZE, /* maxsegsize */ 2810 0, /* flags */ 2811 NULL, /* lockfunc */ 2812 NULL, /* lockfuncarg */ 2813 &rxr->htag))) { 2814 device_printf(dev, "Unable to create RX DMA tag\n"); 2815 goto fail; 2816 } 2817 2818 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2819 1, 0, /* alignment, bounds */ 2820 BUS_SPACE_MAXADDR, /* lowaddr */ 2821 BUS_SPACE_MAXADDR, /* highaddr */ 2822 NULL, NULL, /* filter, filterarg */ 2823 MJUMPAGESIZE, /* maxsize */ 2824 1, /* nsegments */ 2825 MJUMPAGESIZE, /* maxsegsize */ 2826 0, /* flags */ 2827 NULL, /* lockfunc */ 2828 NULL, /* lockfuncarg */ 2829 &rxr->ptag))) { 2830 device_printf(dev, "Unable to create RX DMA tag\n"); 2831 goto fail; 2832 } 2833 2834 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 2835 rxbuf = &rxr->rx_buffers[i]; 2836 error = bus_dmamap_create(rxr->htag, 2837 BUS_DMA_NOWAIT, &rxbuf->hmap); 2838 if (error) { 2839 device_printf(dev, "Unable to create RX head map\n"); 2840 goto fail; 2841 } 2842 error = bus_dmamap_create(rxr->ptag, 2843 BUS_DMA_NOWAIT, &rxbuf->pmap); 2844 if (error) { 2845 device_printf(dev, "Unable to create RX pkt map\n"); 2846 goto fail; 2847 } 2848 } 2849 2850 return (0); 2851 2852fail: 2853 /* Frees all, but can handle partial completion */ 2854 ixv_free_receive_structures(adapter); 2855 return (error); 2856} 2857 2858static void 2859ixv_free_receive_ring(struct rx_ring *rxr) 2860{ 2861 struct adapter *adapter; 2862 struct ixv_rx_buf *rxbuf; 2863 int i; 2864 2865 adapter = rxr->adapter; 2866 for (i = 0; i < adapter->num_rx_desc; i++) { 2867 rxbuf = &rxr->rx_buffers[i]; 2868 if (rxbuf->m_head != NULL) { 2869 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2870 BUS_DMASYNC_POSTREAD); 2871 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 2872 rxbuf->m_head->m_flags |= M_PKTHDR; 2873 m_freem(rxbuf->m_head); 2874 } 2875 if (rxbuf->m_pack != NULL) { 2876 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2877 BUS_DMASYNC_POSTREAD); 2878 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 2879 rxbuf->m_pack->m_flags |= M_PKTHDR; 2880 m_freem(rxbuf->m_pack); 2881 } 2882 rxbuf->m_head = NULL; 2883 rxbuf->m_pack = NULL; 2884 } 2885} 2886 2887 2888/********************************************************************* 2889 * 2890 * Initialize a receive ring and its buffers. 2891 * 2892 **********************************************************************/ 2893static int 2894ixv_setup_receive_ring(struct rx_ring *rxr) 2895{ 2896 struct adapter *adapter; 2897 struct ifnet *ifp; 2898 device_t dev; 2899 struct ixv_rx_buf *rxbuf; 2900 bus_dma_segment_t pseg[1], hseg[1]; 2901 struct lro_ctrl *lro = &rxr->lro; 2902 int rsize, nsegs, error = 0; 2903 2904 adapter = rxr->adapter; 2905 ifp = adapter->ifp; 2906 dev = adapter->dev; 2907 2908 /* Clear the ring contents */ 2909 IXV_RX_LOCK(rxr); 2910 rsize = roundup2(adapter->num_rx_desc * 2911 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2912 bzero((void *)rxr->rx_base, rsize); 2913 2914 /* Free current RX buffer structs and their mbufs */ 2915 ixv_free_receive_ring(rxr); 2916 2917 /* Configure header split? */ 2918 if (ixv_header_split) 2919 rxr->hdr_split = TRUE; 2920 2921 /* Now replenish the mbufs */ 2922 for (int j = 0; j != adapter->num_rx_desc; ++j) { 2923 struct mbuf *mh, *mp; 2924 2925 rxbuf = &rxr->rx_buffers[j]; 2926 /* 2927 ** Dont allocate mbufs if not 2928 ** doing header split, its wasteful 2929 */ 2930 if (rxr->hdr_split == FALSE) 2931 goto skip_head; 2932 2933 /* First the header */ 2934 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA); 2935 if (rxbuf->m_head == NULL) { 2936 error = ENOBUFS; 2937 goto fail; 2938 } 2939 m_adj(rxbuf->m_head, ETHER_ALIGN); 2940 mh = rxbuf->m_head; 2941 mh->m_len = mh->m_pkthdr.len = MHLEN; 2942 mh->m_flags |= M_PKTHDR; 2943 /* Get the memory mapping */ 2944 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2945 rxbuf->hmap, rxbuf->m_head, hseg, 2946 &nsegs, BUS_DMA_NOWAIT); 2947 if (error != 0) /* Nothing elegant to do here */ 2948 goto fail; 2949 bus_dmamap_sync(rxr->htag, 2950 rxbuf->hmap, BUS_DMASYNC_PREREAD); 2951 /* Update descriptor */ 2952 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 2953 2954skip_head: 2955 /* Now the payload cluster */ 2956 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, 2957 M_PKTHDR, adapter->rx_mbuf_sz); 2958 if (rxbuf->m_pack == NULL) { 2959 error = ENOBUFS; 2960 goto fail; 2961 } 2962 mp = rxbuf->m_pack; 2963 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2964 /* Get the memory mapping */ 2965 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2966 rxbuf->pmap, mp, pseg, 2967 &nsegs, BUS_DMA_NOWAIT); 2968 if (error != 0) 2969 goto fail; 2970 bus_dmamap_sync(rxr->ptag, 2971 rxbuf->pmap, BUS_DMASYNC_PREREAD); 2972 /* Update descriptor */ 2973 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 2974 } 2975 2976 2977 /* Setup our descriptor indices */ 2978 rxr->next_to_check = 0; 2979 rxr->next_to_refresh = 0; 2980 rxr->lro_enabled = FALSE; 2981 rxr->rx_split_packets = 0; 2982 rxr->rx_bytes = 0; 2983 rxr->discard = FALSE; 2984 2985 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2986 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2987 2988 /* 2989 ** Now set up the LRO interface: 2990 */ 2991 if (ifp->if_capenable & IFCAP_LRO) { 2992 int err = tcp_lro_init(lro); 2993 if (err) { 2994 device_printf(dev, "LRO Initialization failed!\n"); 2995 goto fail; 2996 } 2997 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 2998 rxr->lro_enabled = TRUE; 2999 lro->ifp = adapter->ifp; 3000 } 3001 3002 IXV_RX_UNLOCK(rxr); 3003 return (0); 3004 3005fail: 3006 ixv_free_receive_ring(rxr); 3007 IXV_RX_UNLOCK(rxr); 3008 return (error); 3009} 3010 3011/********************************************************************* 3012 * 3013 * Initialize all receive rings. 3014 * 3015 **********************************************************************/ 3016static int 3017ixv_setup_receive_structures(struct adapter *adapter) 3018{ 3019 struct rx_ring *rxr = adapter->rx_rings; 3020 int j; 3021 3022 for (j = 0; j < adapter->num_queues; j++, rxr++) 3023 if (ixv_setup_receive_ring(rxr)) 3024 goto fail; 3025 3026 return (0); 3027fail: 3028 /* 3029 * Free RX buffers allocated so far, we will only handle 3030 * the rings that completed, the failing case will have 3031 * cleaned up for itself. 'j' failed, so its the terminus. 3032 */ 3033 for (int i = 0; i < j; ++i) { 3034 rxr = &adapter->rx_rings[i]; 3035 ixv_free_receive_ring(rxr); 3036 } 3037 3038 return (ENOBUFS); 3039} 3040 3041/********************************************************************* 3042 * 3043 * Setup receive registers and features. 3044 * 3045 **********************************************************************/ 3046#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3047 3048static void 3049ixv_initialize_receive_units(struct adapter *adapter) 3050{ 3051 struct rx_ring *rxr = adapter->rx_rings; 3052 struct ixgbe_hw *hw = &adapter->hw; 3053 struct ifnet *ifp = adapter->ifp; 3054 u32 bufsz, fctrl, rxcsum, hlreg; 3055 3056 3057 /* Enable broadcasts */ 3058 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3059 fctrl |= IXGBE_FCTRL_BAM; 3060 fctrl |= IXGBE_FCTRL_DPF; 3061 fctrl |= IXGBE_FCTRL_PMCF; 3062 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3063 3064 /* Set for Jumbo Frames? */ 3065 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3066 if (ifp->if_mtu > ETHERMTU) { 3067 hlreg |= IXGBE_HLREG0_JUMBOEN; 3068 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3069 } else { 3070 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 3071 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3072 } 3073 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 3074 3075 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3076 u64 rdba = rxr->rxdma.dma_paddr; 3077 u32 reg, rxdctl; 3078 3079 /* Do the queue enabling first */ 3080 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3081 rxdctl |= IXGBE_RXDCTL_ENABLE; 3082 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 3083 for (int k = 0; k < 10; k++) { 3084 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 3085 IXGBE_RXDCTL_ENABLE) 3086 break; 3087 else 3088 msec_delay(1); 3089 } 3090 wmb(); 3091 3092 /* Setup the Base and Length of the Rx Descriptor Ring */ 3093 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 3094 (rdba & 0x00000000ffffffffULL)); 3095 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 3096 (rdba >> 32)); 3097 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 3098 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3099 3100 /* Set up the SRRCTL register */ 3101 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i)); 3102 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3103 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 3104 reg |= bufsz; 3105 if (rxr->hdr_split) { 3106 /* Use a standard mbuf for the header */ 3107 reg |= ((IXV_RX_HDR << 3108 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3109 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3110 reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3111 } else 3112 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3113 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg); 3114 3115 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 3116 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 3117 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 3118 adapter->num_rx_desc - 1); 3119 } 3120 3121 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3122 3123 if (ifp->if_capenable & IFCAP_RXCSUM) 3124 rxcsum |= IXGBE_RXCSUM_PCSD; 3125 3126 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3127 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3128 3129 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3130 3131 return; 3132} 3133 3134/********************************************************************* 3135 * 3136 * Free all receive rings. 3137 * 3138 **********************************************************************/ 3139static void 3140ixv_free_receive_structures(struct adapter *adapter) 3141{ 3142 struct rx_ring *rxr = adapter->rx_rings; 3143 3144 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3145 struct lro_ctrl *lro = &rxr->lro; 3146 ixv_free_receive_buffers(rxr); 3147 /* Free LRO memory */ 3148 tcp_lro_free(lro); 3149 /* Free the ring memory as well */ 3150 ixv_dma_free(adapter, &rxr->rxdma); 3151 } 3152 3153 free(adapter->rx_rings, M_DEVBUF); 3154} 3155 3156 3157/********************************************************************* 3158 * 3159 * Free receive ring data structures 3160 * 3161 **********************************************************************/ 3162static void 3163ixv_free_receive_buffers(struct rx_ring *rxr) 3164{ 3165 struct adapter *adapter = rxr->adapter; 3166 struct ixv_rx_buf *rxbuf; 3167 3168 INIT_DEBUGOUT("free_receive_structures: begin"); 3169 3170 /* Cleanup any existing buffers */ 3171 if (rxr->rx_buffers != NULL) { 3172 for (int i = 0; i < adapter->num_rx_desc; i++) { 3173 rxbuf = &rxr->rx_buffers[i]; 3174 if (rxbuf->m_head != NULL) { 3175 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3176 BUS_DMASYNC_POSTREAD); 3177 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 3178 rxbuf->m_head->m_flags |= M_PKTHDR; 3179 m_freem(rxbuf->m_head); 3180 } 3181 if (rxbuf->m_pack != NULL) { 3182 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3183 BUS_DMASYNC_POSTREAD); 3184 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 3185 rxbuf->m_pack->m_flags |= M_PKTHDR; 3186 m_freem(rxbuf->m_pack); 3187 } 3188 rxbuf->m_head = NULL; 3189 rxbuf->m_pack = NULL; 3190 if (rxbuf->hmap != NULL) { 3191 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 3192 rxbuf->hmap = NULL; 3193 } 3194 if (rxbuf->pmap != NULL) { 3195 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 3196 rxbuf->pmap = NULL; 3197 } 3198 } 3199 if (rxr->rx_buffers != NULL) { 3200 free(rxr->rx_buffers, M_DEVBUF); 3201 rxr->rx_buffers = NULL; 3202 } 3203 } 3204 3205 if (rxr->htag != NULL) { 3206 bus_dma_tag_destroy(rxr->htag); 3207 rxr->htag = NULL; 3208 } 3209 if (rxr->ptag != NULL) { 3210 bus_dma_tag_destroy(rxr->ptag); 3211 rxr->ptag = NULL; 3212 } 3213 3214 return; 3215} 3216 3217static __inline void 3218ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 3219{ 3220 3221 /* 3222 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 3223 * should be computed by hardware. Also it should not have VLAN tag in 3224 * ethernet header. 3225 */ 3226 if (rxr->lro_enabled && 3227 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 3228 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3229 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 3230 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 3231 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 3232 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 3233 /* 3234 * Send to the stack if: 3235 ** - LRO not enabled, or 3236 ** - no LRO resources, or 3237 ** - lro enqueue fails 3238 */ 3239 if (rxr->lro.lro_cnt != 0) 3240 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 3241 return; 3242 } 3243 IXV_RX_UNLOCK(rxr); 3244 (*ifp->if_input)(ifp, m); 3245 IXV_RX_LOCK(rxr); 3246} 3247 3248static __inline void 3249ixv_rx_discard(struct rx_ring *rxr, int i) 3250{ 3251 struct ixv_rx_buf *rbuf; 3252 3253 rbuf = &rxr->rx_buffers[i]; 3254 3255 if (rbuf->fmp != NULL) {/* Partial chain ? */ 3256 rbuf->fmp->m_flags |= M_PKTHDR; 3257 m_freem(rbuf->fmp); 3258 rbuf->fmp = NULL; 3259 } 3260 3261 /* 3262 ** With advanced descriptors the writeback 3263 ** clobbers the buffer addrs, so its easier 3264 ** to just free the existing mbufs and take 3265 ** the normal refresh path to get new buffers 3266 ** and mapping. 3267 */ 3268 if (rbuf->m_head) { 3269 m_free(rbuf->m_head); 3270 rbuf->m_head = NULL; 3271 } 3272 3273 if (rbuf->m_pack) { 3274 m_free(rbuf->m_pack); 3275 rbuf->m_pack = NULL; 3276 } 3277 3278 return; 3279} 3280 3281 3282/********************************************************************* 3283 * 3284 * This routine executes in interrupt context. It replenishes 3285 * the mbufs in the descriptor and sends data which has been 3286 * dma'ed into host memory to upper layer. 3287 * 3288 * We loop at most count times if count is > 0, or until done if 3289 * count < 0. 3290 * 3291 * Return TRUE for more work, FALSE for all clean. 3292 *********************************************************************/ 3293static bool 3294ixv_rxeof(struct ix_queue *que, int count) 3295{ 3296 struct adapter *adapter = que->adapter; 3297 struct rx_ring *rxr = que->rxr; 3298 struct ifnet *ifp = adapter->ifp; 3299 struct lro_ctrl *lro = &rxr->lro; 3300 struct lro_entry *queued; 3301 int i, nextp, processed = 0; 3302 u32 staterr = 0; 3303 union ixgbe_adv_rx_desc *cur; 3304 struct ixv_rx_buf *rbuf, *nbuf; 3305 3306 IXV_RX_LOCK(rxr); 3307 3308 for (i = rxr->next_to_check; count != 0;) { 3309 struct mbuf *sendmp, *mh, *mp; 3310 u32 rsc, ptype; 3311 u16 hlen, plen, hdr, vtag; 3312 bool eop; 3313 3314 /* Sync the ring. */ 3315 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3316 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3317 3318 cur = &rxr->rx_base[i]; 3319 staterr = le32toh(cur->wb.upper.status_error); 3320 3321 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 3322 break; 3323 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 3324 break; 3325 3326 count--; 3327 sendmp = NULL; 3328 nbuf = NULL; 3329 rsc = 0; 3330 cur->wb.upper.status_error = 0; 3331 rbuf = &rxr->rx_buffers[i]; 3332 mh = rbuf->m_head; 3333 mp = rbuf->m_pack; 3334 3335 plen = le16toh(cur->wb.upper.length); 3336 ptype = le32toh(cur->wb.lower.lo_dword.data) & 3337 IXGBE_RXDADV_PKTTYPE_MASK; 3338 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 3339 vtag = le16toh(cur->wb.upper.vlan); 3340 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3341 3342 /* Make sure all parts of a bad packet are discarded */ 3343 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 3344 (rxr->discard)) { 3345 ifp->if_ierrors++; 3346 rxr->rx_discarded++; 3347 if (!eop) 3348 rxr->discard = TRUE; 3349 else 3350 rxr->discard = FALSE; 3351 ixv_rx_discard(rxr, i); 3352 goto next_desc; 3353 } 3354 3355 if (!eop) { 3356 nextp = i + 1; 3357 if (nextp == adapter->num_rx_desc) 3358 nextp = 0; 3359 nbuf = &rxr->rx_buffers[nextp]; 3360 prefetch(nbuf); 3361 } 3362 /* 3363 ** The header mbuf is ONLY used when header 3364 ** split is enabled, otherwise we get normal 3365 ** behavior, ie, both header and payload 3366 ** are DMA'd into the payload buffer. 3367 ** 3368 ** Rather than using the fmp/lmp global pointers 3369 ** we now keep the head of a packet chain in the 3370 ** buffer struct and pass this along from one 3371 ** descriptor to the next, until we get EOP. 3372 */ 3373 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 3374 /* This must be an initial descriptor */ 3375 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3376 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3377 if (hlen > IXV_RX_HDR) 3378 hlen = IXV_RX_HDR; 3379 mh->m_len = hlen; 3380 mh->m_flags |= M_PKTHDR; 3381 mh->m_next = NULL; 3382 mh->m_pkthdr.len = mh->m_len; 3383 /* Null buf pointer so it is refreshed */ 3384 rbuf->m_head = NULL; 3385 /* 3386 ** Check the payload length, this 3387 ** could be zero if its a small 3388 ** packet. 3389 */ 3390 if (plen > 0) { 3391 mp->m_len = plen; 3392 mp->m_next = NULL; 3393 mp->m_flags &= ~M_PKTHDR; 3394 mh->m_next = mp; 3395 mh->m_pkthdr.len += mp->m_len; 3396 /* Null buf pointer so it is refreshed */ 3397 rbuf->m_pack = NULL; 3398 rxr->rx_split_packets++; 3399 } 3400 /* 3401 ** Now create the forward 3402 ** chain so when complete 3403 ** we wont have to. 3404 */ 3405 if (eop == 0) { 3406 /* stash the chain head */ 3407 nbuf->fmp = mh; 3408 /* Make forward chain */ 3409 if (plen) 3410 mp->m_next = nbuf->m_pack; 3411 else 3412 mh->m_next = nbuf->m_pack; 3413 } else { 3414 /* Singlet, prepare to send */ 3415 sendmp = mh; 3416 if ((adapter->num_vlans) && 3417 (staterr & IXGBE_RXD_STAT_VP)) { 3418 sendmp->m_pkthdr.ether_vtag = vtag; 3419 sendmp->m_flags |= M_VLANTAG; 3420 } 3421 } 3422 } else { 3423 /* 3424 ** Either no header split, or a 3425 ** secondary piece of a fragmented 3426 ** split packet. 3427 */ 3428 mp->m_len = plen; 3429 /* 3430 ** See if there is a stored head 3431 ** that determines what we are 3432 */ 3433 sendmp = rbuf->fmp; 3434 rbuf->m_pack = rbuf->fmp = NULL; 3435 3436 if (sendmp != NULL) /* secondary frag */ 3437 sendmp->m_pkthdr.len += mp->m_len; 3438 else { 3439 /* first desc of a non-ps chain */ 3440 sendmp = mp; 3441 sendmp->m_flags |= M_PKTHDR; 3442 sendmp->m_pkthdr.len = mp->m_len; 3443 if (staterr & IXGBE_RXD_STAT_VP) { 3444 sendmp->m_pkthdr.ether_vtag = vtag; 3445 sendmp->m_flags |= M_VLANTAG; 3446 } 3447 } 3448 /* Pass the head pointer on */ 3449 if (eop == 0) { 3450 nbuf->fmp = sendmp; 3451 sendmp = NULL; 3452 mp->m_next = nbuf->m_pack; 3453 } 3454 } 3455 ++processed; 3456 /* Sending this frame? */ 3457 if (eop) { 3458 sendmp->m_pkthdr.rcvif = ifp; 3459 ifp->if_ipackets++; 3460 rxr->rx_packets++; 3461 /* capture data for AIM */ 3462 rxr->bytes += sendmp->m_pkthdr.len; 3463 rxr->rx_bytes += sendmp->m_pkthdr.len; 3464 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 3465 ixv_rx_checksum(staterr, sendmp, ptype); 3466#if __FreeBSD_version >= 800000 3467 sendmp->m_pkthdr.flowid = que->msix; 3468 sendmp->m_flags |= M_FLOWID; 3469#endif 3470 } 3471next_desc: 3472 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3473 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3474 3475 /* Advance our pointers to the next descriptor. */ 3476 if (++i == adapter->num_rx_desc) 3477 i = 0; 3478 3479 /* Now send to the stack or do LRO */ 3480 if (sendmp != NULL) 3481 ixv_rx_input(rxr, ifp, sendmp, ptype); 3482 3483 /* Every 8 descriptors we go to refresh mbufs */ 3484 if (processed == 8) { 3485 ixv_refresh_mbufs(rxr, i); 3486 processed = 0; 3487 } 3488 } 3489 3490 /* Refresh any remaining buf structs */ 3491 if (ixv_rx_unrefreshed(rxr)) 3492 ixv_refresh_mbufs(rxr, i); 3493 3494 rxr->next_to_check = i; 3495 3496 /* 3497 * Flush any outstanding LRO work 3498 */ 3499 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 3500 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3501 tcp_lro_flush(lro, queued); 3502 } 3503 3504 IXV_RX_UNLOCK(rxr); 3505 3506 /* 3507 ** We still have cleaning to do? 3508 ** Schedule another interrupt if so. 3509 */ 3510 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 3511 ixv_rearm_queues(adapter, (u64)(1 << que->msix)); 3512 return (TRUE); 3513 } 3514 3515 return (FALSE); 3516} 3517 3518 3519/********************************************************************* 3520 * 3521 * Verify that the hardware indicated that the checksum is valid. 3522 * Inform the stack about the status of checksum so that stack 3523 * doesn't spend time verifying the checksum. 3524 * 3525 *********************************************************************/ 3526static void 3527ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype) 3528{ 3529 u16 status = (u16) staterr; 3530 u8 errors = (u8) (staterr >> 24); 3531 bool sctp = FALSE; 3532 3533 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3534 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 3535 sctp = TRUE; 3536 3537 if (status & IXGBE_RXD_STAT_IPCS) { 3538 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3539 /* IP Checksum Good */ 3540 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 3541 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 3542 3543 } else 3544 mp->m_pkthdr.csum_flags = 0; 3545 } 3546 if (status & IXGBE_RXD_STAT_L4CS) { 3547 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3548#if __FreeBSD_version >= 800000 3549 if (sctp) 3550 type = CSUM_SCTP_VALID; 3551#endif 3552 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3553 mp->m_pkthdr.csum_flags |= type; 3554 if (!sctp) 3555 mp->m_pkthdr.csum_data = htons(0xffff); 3556 } 3557 } 3558 return; 3559} 3560 3561static void 3562ixv_setup_vlan_support(struct adapter *adapter) 3563{ 3564 struct ixgbe_hw *hw = &adapter->hw; 3565 u32 ctrl, vid, vfta, retry; 3566 3567 3568 /* 3569 ** We get here thru init_locked, meaning 3570 ** a soft reset, this has already cleared 3571 ** the VFTA and other state, so if there 3572 ** have been no vlan's registered do nothing. 3573 */ 3574 if (adapter->num_vlans == 0) 3575 return; 3576 3577 /* Enable the queues */ 3578 for (int i = 0; i < adapter->num_queues; i++) { 3579 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3580 ctrl |= IXGBE_RXDCTL_VME; 3581 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 3582 } 3583 3584 /* 3585 ** A soft reset zero's out the VFTA, so 3586 ** we need to repopulate it now. 3587 */ 3588 for (int i = 0; i < VFTA_SIZE; i++) { 3589 if (ixv_shadow_vfta[i] == 0) 3590 continue; 3591 vfta = ixv_shadow_vfta[i]; 3592 /* 3593 ** Reconstruct the vlan id's 3594 ** based on the bits set in each 3595 ** of the array ints. 3596 */ 3597 for ( int j = 0; j < 32; j++) { 3598 retry = 0; 3599 if ((vfta & (1 << j)) == 0) 3600 continue; 3601 vid = (i * 32) + j; 3602 /* Call the shared code mailbox routine */ 3603 while (ixgbe_set_vfta(hw, vid, 0, TRUE)) { 3604 if (++retry > 5) 3605 break; 3606 } 3607 } 3608 } 3609} 3610 3611/* 3612** This routine is run via an vlan config EVENT, 3613** it enables us to use the HW Filter table since 3614** we can get the vlan id. This just creates the 3615** entry in the soft version of the VFTA, init will 3616** repopulate the real table. 3617*/ 3618static void 3619ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3620{ 3621 struct adapter *adapter = ifp->if_softc; 3622 u16 index, bit; 3623 3624 if (ifp->if_softc != arg) /* Not our event */ 3625 return; 3626 3627 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3628 return; 3629 3630 IXV_CORE_LOCK(adapter); 3631 index = (vtag >> 5) & 0x7F; 3632 bit = vtag & 0x1F; 3633 ixv_shadow_vfta[index] |= (1 << bit); 3634 ++adapter->num_vlans; 3635 /* Re-init to load the changes */ 3636 ixv_init_locked(adapter); 3637 IXV_CORE_UNLOCK(adapter); 3638} 3639 3640/* 3641** This routine is run via an vlan 3642** unconfig EVENT, remove our entry 3643** in the soft vfta. 3644*/ 3645static void 3646ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3647{ 3648 struct adapter *adapter = ifp->if_softc; 3649 u16 index, bit; 3650 3651 if (ifp->if_softc != arg) 3652 return; 3653 3654 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3655 return; 3656 3657 IXV_CORE_LOCK(adapter); 3658 index = (vtag >> 5) & 0x7F; 3659 bit = vtag & 0x1F; 3660 ixv_shadow_vfta[index] &= ~(1 << bit); 3661 --adapter->num_vlans; 3662 /* Re-init to load the changes */ 3663 ixv_init_locked(adapter); 3664 IXV_CORE_UNLOCK(adapter); 3665} 3666 3667static void 3668ixv_enable_intr(struct adapter *adapter) 3669{ 3670 struct ixgbe_hw *hw = &adapter->hw; 3671 struct ix_queue *que = adapter->queues; 3672 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3673 3674 3675 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 3676 3677 mask = IXGBE_EIMS_ENABLE_MASK; 3678 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 3679 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 3680 3681 for (int i = 0; i < adapter->num_queues; i++, que++) 3682 ixv_enable_queue(adapter, que->msix); 3683 3684 IXGBE_WRITE_FLUSH(hw); 3685 3686 return; 3687} 3688 3689static void 3690ixv_disable_intr(struct adapter *adapter) 3691{ 3692 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 3693 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0); 3694 IXGBE_WRITE_FLUSH(&adapter->hw); 3695 return; 3696} 3697 3698/* 3699** Setup the correct IVAR register for a particular MSIX interrupt 3700** - entry is the register array entry 3701** - vector is the MSIX vector for this queue 3702** - type is RX/TX/MISC 3703*/ 3704static void 3705ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 3706{ 3707 struct ixgbe_hw *hw = &adapter->hw; 3708 u32 ivar, index; 3709 3710 vector |= IXGBE_IVAR_ALLOC_VAL; 3711 3712 if (type == -1) { /* MISC IVAR */ 3713 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 3714 ivar &= ~0xFF; 3715 ivar |= vector; 3716 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 3717 } else { /* RX/TX IVARS */ 3718 index = (16 * (entry & 1)) + (8 * type); 3719 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 3720 ivar &= ~(0xFF << index); 3721 ivar |= (vector << index); 3722 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 3723 } 3724} 3725 3726static void 3727ixv_configure_ivars(struct adapter *adapter) 3728{ 3729 struct ix_queue *que = adapter->queues; 3730 3731 for (int i = 0; i < adapter->num_queues; i++, que++) { 3732 /* First the RX queue entry */ 3733 ixv_set_ivar(adapter, i, que->msix, 0); 3734 /* ... and the TX */ 3735 ixv_set_ivar(adapter, i, que->msix, 1); 3736 /* Set an initial value in EITR */ 3737 IXGBE_WRITE_REG(&adapter->hw, 3738 IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT); 3739 } 3740 3741 /* For the Link interrupt */ 3742 ixv_set_ivar(adapter, 1, adapter->mbxvec, -1); 3743} 3744 3745 3746/* 3747** Tasklet handler for MSIX MBX interrupts 3748** - do outside interrupt since it might sleep 3749*/ 3750static void 3751ixv_handle_mbx(void *context, int pending) 3752{ 3753 struct adapter *adapter = context; 3754 3755 ixgbe_check_link(&adapter->hw, 3756 &adapter->link_speed, &adapter->link_up, 0); 3757 ixv_update_link_status(adapter); 3758} 3759 3760/* 3761** The VF stats registers never have a truely virgin 3762** starting point, so this routine tries to make an 3763** artificial one, marking ground zero on attach as 3764** it were. 3765*/ 3766static void 3767ixv_save_stats(struct adapter *adapter) 3768{ 3769 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 3770 adapter->stats.saved_reset_vfgprc += 3771 adapter->stats.vfgprc - adapter->stats.base_vfgprc; 3772 adapter->stats.saved_reset_vfgptc += 3773 adapter->stats.vfgptc - adapter->stats.base_vfgptc; 3774 adapter->stats.saved_reset_vfgorc += 3775 adapter->stats.vfgorc - adapter->stats.base_vfgorc; 3776 adapter->stats.saved_reset_vfgotc += 3777 adapter->stats.vfgotc - adapter->stats.base_vfgotc; 3778 adapter->stats.saved_reset_vfmprc += 3779 adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3780 } 3781} 3782 3783static void 3784ixv_init_stats(struct adapter *adapter) 3785{ 3786 struct ixgbe_hw *hw = &adapter->hw; 3787 3788 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 3789 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 3790 adapter->stats.last_vfgorc |= 3791 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 3792 3793 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 3794 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 3795 adapter->stats.last_vfgotc |= 3796 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 3797 3798 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 3799 3800 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 3801 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 3802 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 3803 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 3804 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 3805} 3806 3807#define UPDATE_STAT_32(reg, last, count) \ 3808{ \ 3809 u32 current = IXGBE_READ_REG(hw, reg); \ 3810 if (current < last) \ 3811 count += 0x100000000LL; \ 3812 last = current; \ 3813 count &= 0xFFFFFFFF00000000LL; \ 3814 count |= current; \ 3815} 3816 3817#define UPDATE_STAT_36(lsb, msb, last, count) \ 3818{ \ 3819 u64 cur_lsb = IXGBE_READ_REG(hw, lsb); \ 3820 u64 cur_msb = IXGBE_READ_REG(hw, msb); \ 3821 u64 current = ((cur_msb << 32) | cur_lsb); \ 3822 if (current < last) \ 3823 count += 0x1000000000LL; \ 3824 last = current; \ 3825 count &= 0xFFFFFFF000000000LL; \ 3826 count |= current; \ 3827} 3828 3829/* 3830** ixv_update_stats - Update the board statistics counters. 3831*/ 3832void 3833ixv_update_stats(struct adapter *adapter) 3834{ 3835 struct ixgbe_hw *hw = &adapter->hw; 3836 3837 UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3838 adapter->stats.vfgprc); 3839 UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3840 adapter->stats.vfgptc); 3841 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3842 adapter->stats.last_vfgorc, adapter->stats.vfgorc); 3843 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3844 adapter->stats.last_vfgotc, adapter->stats.vfgotc); 3845 UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3846 adapter->stats.vfmprc); 3847} 3848 3849/********************************************************************** 3850 * 3851 * This routine is called only when ixgbe_display_debug_stats is enabled. 3852 * This routine provides a way to take a look at important statistics 3853 * maintained by the driver and hardware. 3854 * 3855 **********************************************************************/ 3856static void 3857ixv_print_hw_stats(struct adapter * adapter) 3858{ 3859 device_t dev = adapter->dev; 3860 3861 device_printf(dev,"Std Mbuf Failed = %lu\n", 3862 adapter->mbuf_defrag_failed); 3863 device_printf(dev,"Driver dropped packets = %lu\n", 3864 adapter->dropped_pkts); 3865 device_printf(dev, "watchdog timeouts = %ld\n", 3866 adapter->watchdog_events); 3867 3868 device_printf(dev,"Good Packets Rcvd = %llu\n", 3869 (long long)adapter->stats.vfgprc); 3870 device_printf(dev,"Good Packets Xmtd = %llu\n", 3871 (long long)adapter->stats.vfgptc); 3872 device_printf(dev,"TSO Transmissions = %lu\n", 3873 adapter->tso_tx); 3874 3875} 3876 3877/********************************************************************** 3878 * 3879 * This routine is called only when em_display_debug_stats is enabled. 3880 * This routine provides a way to take a look at important statistics 3881 * maintained by the driver and hardware. 3882 * 3883 **********************************************************************/ 3884static void 3885ixv_print_debug_info(struct adapter *adapter) 3886{ 3887 device_t dev = adapter->dev; 3888 struct ixgbe_hw *hw = &adapter->hw; 3889 struct ix_queue *que = adapter->queues; 3890 struct rx_ring *rxr; 3891 struct tx_ring *txr; 3892 struct lro_ctrl *lro; 3893 3894 device_printf(dev,"Error Byte Count = %u \n", 3895 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 3896 3897 for (int i = 0; i < adapter->num_queues; i++, que++) { 3898 txr = que->txr; 3899 rxr = que->rxr; 3900 lro = &rxr->lro; 3901 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n", 3902 que->msix, (long)que->irqs); 3903 device_printf(dev,"RX(%d) Packets Received: %lld\n", 3904 rxr->me, (long long)rxr->rx_packets); 3905 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 3906 rxr->me, (long long)rxr->rx_split_packets); 3907 device_printf(dev,"RX(%d) Bytes Received: %lu\n", 3908 rxr->me, (long)rxr->rx_bytes); 3909 device_printf(dev,"RX(%d) LRO Queued= %d\n", 3910 rxr->me, lro->lro_queued); 3911 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 3912 rxr->me, lro->lro_flushed); 3913 device_printf(dev,"TX(%d) Packets Sent: %lu\n", 3914 txr->me, (long)txr->total_packets); 3915 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 3916 txr->me, (long)txr->no_desc_avail); 3917 } 3918 3919 device_printf(dev,"MBX IRQ Handled: %lu\n", 3920 (long)adapter->mbx_irq); 3921 return; 3922} 3923 3924static int 3925ixv_sysctl_stats(SYSCTL_HANDLER_ARGS) 3926{ 3927 int error; 3928 int result; 3929 struct adapter *adapter; 3930 3931 result = -1; 3932 error = sysctl_handle_int(oidp, &result, 0, req); 3933 3934 if (error || !req->newptr) 3935 return (error); 3936 3937 if (result == 1) { 3938 adapter = (struct adapter *) arg1; 3939 ixv_print_hw_stats(adapter); 3940 } 3941 return error; 3942} 3943 3944static int 3945ixv_sysctl_debug(SYSCTL_HANDLER_ARGS) 3946{ 3947 int error, result; 3948 struct adapter *adapter; 3949 3950 result = -1; 3951 error = sysctl_handle_int(oidp, &result, 0, req); 3952 3953 if (error || !req->newptr) 3954 return (error); 3955 3956 if (result == 1) { 3957 adapter = (struct adapter *) arg1; 3958 ixv_print_debug_info(adapter); 3959 } 3960 return error; 3961} 3962 3963/* 3964** Set flow control using sysctl: 3965** Flow control values: 3966** 0 - off 3967** 1 - rx pause 3968** 2 - tx pause 3969** 3 - full 3970*/ 3971static int 3972ixv_set_flowcntl(SYSCTL_HANDLER_ARGS) 3973{ 3974 int error; 3975 struct adapter *adapter; 3976 3977 error = sysctl_handle_int(oidp, &ixv_flow_control, 0, req); 3978 3979 if (error) 3980 return (error); 3981 3982 adapter = (struct adapter *) arg1; 3983 switch (ixv_flow_control) { 3984 case ixgbe_fc_rx_pause: 3985 case ixgbe_fc_tx_pause: 3986 case ixgbe_fc_full: 3987 adapter->hw.fc.requested_mode = ixv_flow_control; 3988 break; 3989 case ixgbe_fc_none: 3990 default: 3991 adapter->hw.fc.requested_mode = ixgbe_fc_none; 3992 } 3993 3994 ixgbe_fc_enable(&adapter->hw); 3995 return error; 3996} 3997 3998static void 3999ixv_add_rx_process_limit(struct adapter *adapter, const char *name, 4000 const char *description, int *limit, int value) 4001{ 4002 *limit = value; 4003 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 4004 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 4005 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 4006} 4007 4008