ixv.c revision 238149
1/****************************************************************************** 2 3 Copyright (c) 2001-2012, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ 33/*$FreeBSD: head/sys/dev/ixgbe/ixv.c 238149 2012-07-05 20:51:44Z jfv $*/ 34 35#ifdef HAVE_KERNEL_OPTION_HEADERS 36#include "opt_inet.h" 37#include "opt_inet6.h" 38#endif 39 40#include "ixv.h" 41 42/********************************************************************* 43 * Driver version 44 *********************************************************************/ 45char ixv_driver_version[] = "1.1.4"; 46 47/********************************************************************* 48 * PCI Device ID Table 49 * 50 * Used by probe to select devices to load on 51 * Last field stores an index into ixv_strings 52 * Last entry must be all 0s 53 * 54 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 55 *********************************************************************/ 56 57static ixv_vendor_info_t ixv_vendor_info_array[] = 58{ 59 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 60 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 61 /* required last entry */ 62 {0, 0, 0, 0, 0} 63}; 64 65/********************************************************************* 66 * Table of branding strings 67 *********************************************************************/ 68 69static char *ixv_strings[] = { 70 "Intel(R) PRO/10GbE Virtual Function Network Driver" 71}; 72 73/********************************************************************* 74 * Function prototypes 75 *********************************************************************/ 76static int ixv_probe(device_t); 77static int ixv_attach(device_t); 78static int ixv_detach(device_t); 79static int ixv_shutdown(device_t); 80#if __FreeBSD_version < 800000 81static void ixv_start(struct ifnet *); 82static void ixv_start_locked(struct tx_ring *, struct ifnet *); 83#else 84static int ixv_mq_start(struct ifnet *, struct mbuf *); 85static int ixv_mq_start_locked(struct ifnet *, 86 struct tx_ring *, struct mbuf *); 87static void ixv_qflush(struct ifnet *); 88#endif 89static int ixv_ioctl(struct ifnet *, u_long, caddr_t); 90static void ixv_init(void *); 91static void ixv_init_locked(struct adapter *); 92static void ixv_stop(void *); 93static void ixv_media_status(struct ifnet *, struct ifmediareq *); 94static int ixv_media_change(struct ifnet *); 95static void ixv_identify_hardware(struct adapter *); 96static int ixv_allocate_pci_resources(struct adapter *); 97static int ixv_allocate_msix(struct adapter *); 98static int ixv_allocate_queues(struct adapter *); 99static int ixv_setup_msix(struct adapter *); 100static void ixv_free_pci_resources(struct adapter *); 101static void ixv_local_timer(void *); 102static void ixv_setup_interface(device_t, struct adapter *); 103static void ixv_config_link(struct adapter *); 104 105static int ixv_allocate_transmit_buffers(struct tx_ring *); 106static int ixv_setup_transmit_structures(struct adapter *); 107static void ixv_setup_transmit_ring(struct tx_ring *); 108static void ixv_initialize_transmit_units(struct adapter *); 109static void ixv_free_transmit_structures(struct adapter *); 110static void ixv_free_transmit_buffers(struct tx_ring *); 111 112static int ixv_allocate_receive_buffers(struct rx_ring *); 113static int ixv_setup_receive_structures(struct adapter *); 114static int ixv_setup_receive_ring(struct rx_ring *); 115static void ixv_initialize_receive_units(struct adapter *); 116static void ixv_free_receive_structures(struct adapter *); 117static void ixv_free_receive_buffers(struct rx_ring *); 118 119static void ixv_enable_intr(struct adapter *); 120static void ixv_disable_intr(struct adapter *); 121static bool ixv_txeof(struct tx_ring *); 122static bool ixv_rxeof(struct ix_queue *, int); 123static void ixv_rx_checksum(u32, struct mbuf *, u32); 124static void ixv_set_multi(struct adapter *); 125static void ixv_update_link_status(struct adapter *); 126static void ixv_refresh_mbufs(struct rx_ring *, int); 127static int ixv_xmit(struct tx_ring *, struct mbuf **); 128static int ixv_sysctl_stats(SYSCTL_HANDLER_ARGS); 129static int ixv_sysctl_debug(SYSCTL_HANDLER_ARGS); 130static int ixv_set_flowcntl(SYSCTL_HANDLER_ARGS); 131static int ixv_dma_malloc(struct adapter *, bus_size_t, 132 struct ixv_dma_alloc *, int); 133static void ixv_dma_free(struct adapter *, struct ixv_dma_alloc *); 134static void ixv_add_rx_process_limit(struct adapter *, const char *, 135 const char *, int *, int); 136static bool ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *); 137static bool ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 138static void ixv_set_ivar(struct adapter *, u8, u8, s8); 139static void ixv_configure_ivars(struct adapter *); 140static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 141 142static void ixv_setup_vlan_support(struct adapter *); 143static void ixv_register_vlan(void *, struct ifnet *, u16); 144static void ixv_unregister_vlan(void *, struct ifnet *, u16); 145 146static void ixv_save_stats(struct adapter *); 147static void ixv_init_stats(struct adapter *); 148static void ixv_update_stats(struct adapter *); 149 150static __inline void ixv_rx_discard(struct rx_ring *, int); 151static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *, 152 struct mbuf *, u32); 153 154/* The MSI/X Interrupt handlers */ 155static void ixv_msix_que(void *); 156static void ixv_msix_mbx(void *); 157 158/* Deferred interrupt tasklets */ 159static void ixv_handle_que(void *, int); 160static void ixv_handle_mbx(void *, int); 161 162/********************************************************************* 163 * FreeBSD Device Interface Entry Points 164 *********************************************************************/ 165 166static device_method_t ixv_methods[] = { 167 /* Device interface */ 168 DEVMETHOD(device_probe, ixv_probe), 169 DEVMETHOD(device_attach, ixv_attach), 170 DEVMETHOD(device_detach, ixv_detach), 171 DEVMETHOD(device_shutdown, ixv_shutdown), 172 {0, 0} 173}; 174 175static driver_t ixv_driver = { 176 "ix", ixv_methods, sizeof(struct adapter), 177}; 178 179extern devclass_t ixgbe_devclass; 180DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0); 181MODULE_DEPEND(ixv, pci, 1, 1, 1); 182MODULE_DEPEND(ixv, ether, 1, 1, 1); 183 184/* 185** TUNEABLE PARAMETERS: 186*/ 187 188/* 189** AIM: Adaptive Interrupt Moderation 190** which means that the interrupt rate 191** is varied over time based on the 192** traffic for that interrupt vector 193*/ 194static int ixv_enable_aim = FALSE; 195TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 196 197/* How many packets rxeof tries to clean at a time */ 198static int ixv_rx_process_limit = 128; 199TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 200 201/* Flow control setting, default to full */ 202static int ixv_flow_control = ixgbe_fc_full; 203TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control); 204 205/* 206 * Header split: this causes the hardware to DMA 207 * the header into a seperate mbuf from the payload, 208 * it can be a performance win in some workloads, but 209 * in others it actually hurts, its off by default. 210 */ 211static int ixv_header_split = FALSE; 212TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split); 213 214/* 215** Number of TX descriptors per ring, 216** setting higher than RX as this seems 217** the better performing choice. 218*/ 219static int ixv_txd = DEFAULT_TXD; 220TUNABLE_INT("hw.ixv.txd", &ixv_txd); 221 222/* Number of RX descriptors per ring */ 223static int ixv_rxd = DEFAULT_RXD; 224TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 225 226/* 227** Shadow VFTA table, this is needed because 228** the real filter table gets cleared during 229** a soft reset and we need to repopulate it. 230*/ 231static u32 ixv_shadow_vfta[VFTA_SIZE]; 232 233/********************************************************************* 234 * Device identification routine 235 * 236 * ixv_probe determines if the driver should be loaded on 237 * adapter based on PCI vendor/device id of the adapter. 238 * 239 * return BUS_PROBE_DEFAULT on success, positive on failure 240 *********************************************************************/ 241 242static int 243ixv_probe(device_t dev) 244{ 245 ixv_vendor_info_t *ent; 246 247 u16 pci_vendor_id = 0; 248 u16 pci_device_id = 0; 249 u16 pci_subvendor_id = 0; 250 u16 pci_subdevice_id = 0; 251 char adapter_name[256]; 252 253 254 pci_vendor_id = pci_get_vendor(dev); 255 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) 256 return (ENXIO); 257 258 pci_device_id = pci_get_device(dev); 259 pci_subvendor_id = pci_get_subvendor(dev); 260 pci_subdevice_id = pci_get_subdevice(dev); 261 262 ent = ixv_vendor_info_array; 263 while (ent->vendor_id != 0) { 264 if ((pci_vendor_id == ent->vendor_id) && 265 (pci_device_id == ent->device_id) && 266 267 ((pci_subvendor_id == ent->subvendor_id) || 268 (ent->subvendor_id == 0)) && 269 270 ((pci_subdevice_id == ent->subdevice_id) || 271 (ent->subdevice_id == 0))) { 272 sprintf(adapter_name, "%s, Version - %s", 273 ixv_strings[ent->index], 274 ixv_driver_version); 275 device_set_desc_copy(dev, adapter_name); 276 return (BUS_PROBE_DEFAULT); 277 } 278 ent++; 279 } 280 return (ENXIO); 281} 282 283/********************************************************************* 284 * Device initialization routine 285 * 286 * The attach entry point is called when the driver is being loaded. 287 * This routine identifies the type of hardware, allocates all resources 288 * and initializes the hardware. 289 * 290 * return 0 on success, positive on failure 291 *********************************************************************/ 292 293static int 294ixv_attach(device_t dev) 295{ 296 struct adapter *adapter; 297 struct ixgbe_hw *hw; 298 int error = 0; 299 300 INIT_DEBUGOUT("ixv_attach: begin"); 301 302 if (resource_disabled("ixgbe", device_get_unit(dev))) { 303 device_printf(dev, "Disabled by device hint\n"); 304 return (ENXIO); 305 } 306 307 /* Allocate, clear, and link in our adapter structure */ 308 adapter = device_get_softc(dev); 309 adapter->dev = adapter->osdep.dev = dev; 310 hw = &adapter->hw; 311 312 /* Core Lock Init*/ 313 IXV_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 314 315 /* SYSCTL APIs */ 316 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 317 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 318 OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW, 319 adapter, 0, ixv_sysctl_stats, "I", "Statistics"); 320 321 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 322 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 323 OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW, 324 adapter, 0, ixv_sysctl_debug, "I", "Debug Info"); 325 326 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 327 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 328 OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW, 329 adapter, 0, ixv_set_flowcntl, "I", "Flow Control"); 330 331 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 332 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 333 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 334 &ixv_enable_aim, 1, "Interrupt Moderation"); 335 336 /* Set up the timer callout */ 337 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 338 339 /* Determine hardware revision */ 340 ixv_identify_hardware(adapter); 341 342 /* Do base PCI setup - map BAR0 */ 343 if (ixv_allocate_pci_resources(adapter)) { 344 device_printf(dev, "Allocation of PCI resources failed\n"); 345 error = ENXIO; 346 goto err_out; 347 } 348 349 /* Do descriptor calc and sanity checks */ 350 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 351 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 352 device_printf(dev, "TXD config issue, using default!\n"); 353 adapter->num_tx_desc = DEFAULT_TXD; 354 } else 355 adapter->num_tx_desc = ixv_txd; 356 357 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 358 ixv_rxd < MIN_TXD || ixv_rxd > MAX_TXD) { 359 device_printf(dev, "RXD config issue, using default!\n"); 360 adapter->num_rx_desc = DEFAULT_RXD; 361 } else 362 adapter->num_rx_desc = ixv_rxd; 363 364 /* Allocate our TX/RX Queues */ 365 if (ixv_allocate_queues(adapter)) { 366 error = ENOMEM; 367 goto err_out; 368 } 369 370 /* 371 ** Initialize the shared code: its 372 ** at this point the mac type is set. 373 */ 374 error = ixgbe_init_shared_code(hw); 375 if (error) { 376 device_printf(dev,"Shared Code Initialization Failure\n"); 377 error = EIO; 378 goto err_late; 379 } 380 381 /* Setup the mailbox */ 382 ixgbe_init_mbx_params_vf(hw); 383 384 ixgbe_reset_hw(hw); 385 386 /* Get Hardware Flow Control setting */ 387 hw->fc.requested_mode = ixgbe_fc_full; 388 hw->fc.pause_time = IXV_FC_PAUSE; 389 hw->fc.low_water[0] = IXV_FC_LO; 390 hw->fc.high_water[0] = IXV_FC_HI; 391 hw->fc.send_xon = TRUE; 392 393 error = ixgbe_init_hw(hw); 394 if (error) { 395 device_printf(dev,"Hardware Initialization Failure\n"); 396 error = EIO; 397 goto err_late; 398 } 399 400 error = ixv_allocate_msix(adapter); 401 if (error) 402 goto err_late; 403 404 /* Setup OS specific network interface */ 405 ixv_setup_interface(dev, adapter); 406 407 /* Sysctl for limiting the amount of work done in the taskqueue */ 408 ixv_add_rx_process_limit(adapter, "rx_processing_limit", 409 "max number of rx packets to process", &adapter->rx_process_limit, 410 ixv_rx_process_limit); 411 412 /* Do the stats setup */ 413 ixv_save_stats(adapter); 414 ixv_init_stats(adapter); 415 416 /* Register for VLAN events */ 417 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 418 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 419 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 420 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 421 422 INIT_DEBUGOUT("ixv_attach: end"); 423 return (0); 424 425err_late: 426 ixv_free_transmit_structures(adapter); 427 ixv_free_receive_structures(adapter); 428err_out: 429 ixv_free_pci_resources(adapter); 430 return (error); 431 432} 433 434/********************************************************************* 435 * Device removal routine 436 * 437 * The detach entry point is called when the driver is being removed. 438 * This routine stops the adapter and deallocates all the resources 439 * that were allocated for driver operation. 440 * 441 * return 0 on success, positive on failure 442 *********************************************************************/ 443 444static int 445ixv_detach(device_t dev) 446{ 447 struct adapter *adapter = device_get_softc(dev); 448 struct ix_queue *que = adapter->queues; 449 450 INIT_DEBUGOUT("ixv_detach: begin"); 451 452 /* Make sure VLANS are not using driver */ 453 if (adapter->ifp->if_vlantrunk != NULL) { 454 device_printf(dev,"Vlan in use, detach first\n"); 455 return (EBUSY); 456 } 457 458 IXV_CORE_LOCK(adapter); 459 ixv_stop(adapter); 460 IXV_CORE_UNLOCK(adapter); 461 462 for (int i = 0; i < adapter->num_queues; i++, que++) { 463 if (que->tq) { 464 taskqueue_drain(que->tq, &que->que_task); 465 taskqueue_free(que->tq); 466 } 467 } 468 469 /* Drain the Link queue */ 470 if (adapter->tq) { 471 taskqueue_drain(adapter->tq, &adapter->mbx_task); 472 taskqueue_free(adapter->tq); 473 } 474 475 /* Unregister VLAN events */ 476 if (adapter->vlan_attach != NULL) 477 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 478 if (adapter->vlan_detach != NULL) 479 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 480 481 ether_ifdetach(adapter->ifp); 482 callout_drain(&adapter->timer); 483 ixv_free_pci_resources(adapter); 484 bus_generic_detach(dev); 485 if_free(adapter->ifp); 486 487 ixv_free_transmit_structures(adapter); 488 ixv_free_receive_structures(adapter); 489 490 IXV_CORE_LOCK_DESTROY(adapter); 491 return (0); 492} 493 494/********************************************************************* 495 * 496 * Shutdown entry point 497 * 498 **********************************************************************/ 499static int 500ixv_shutdown(device_t dev) 501{ 502 struct adapter *adapter = device_get_softc(dev); 503 IXV_CORE_LOCK(adapter); 504 ixv_stop(adapter); 505 IXV_CORE_UNLOCK(adapter); 506 return (0); 507} 508 509#if __FreeBSD_version < 800000 510/********************************************************************* 511 * Transmit entry point 512 * 513 * ixv_start is called by the stack to initiate a transmit. 514 * The driver will remain in this routine as long as there are 515 * packets to transmit and transmit resources are available. 516 * In case resources are not available stack is notified and 517 * the packet is requeued. 518 **********************************************************************/ 519static void 520ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp) 521{ 522 struct mbuf *m_head; 523 struct adapter *adapter = txr->adapter; 524 525 IXV_TX_LOCK_ASSERT(txr); 526 527 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 528 IFF_DRV_RUNNING) 529 return; 530 if (!adapter->link_active) 531 return; 532 533 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 534 535 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 536 if (m_head == NULL) 537 break; 538 539 if (ixv_xmit(txr, &m_head)) { 540 if (m_head == NULL) 541 break; 542 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 543 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 544 break; 545 } 546 /* Send a copy of the frame to the BPF listener */ 547 ETHER_BPF_MTAP(ifp, m_head); 548 549 /* Set watchdog on */ 550 txr->watchdog_check = TRUE; 551 txr->watchdog_time = ticks; 552 553 } 554 return; 555} 556 557/* 558 * Legacy TX start - called by the stack, this 559 * always uses the first tx ring, and should 560 * not be used with multiqueue tx enabled. 561 */ 562static void 563ixv_start(struct ifnet *ifp) 564{ 565 struct adapter *adapter = ifp->if_softc; 566 struct tx_ring *txr = adapter->tx_rings; 567 568 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 569 IXV_TX_LOCK(txr); 570 ixv_start_locked(txr, ifp); 571 IXV_TX_UNLOCK(txr); 572 } 573 return; 574} 575 576#else 577 578/* 579** Multiqueue Transmit driver 580** 581*/ 582static int 583ixv_mq_start(struct ifnet *ifp, struct mbuf *m) 584{ 585 struct adapter *adapter = ifp->if_softc; 586 struct ix_queue *que; 587 struct tx_ring *txr; 588 int i = 0, err = 0; 589 590 /* Which queue to use */ 591 if ((m->m_flags & M_FLOWID) != 0) 592 i = m->m_pkthdr.flowid % adapter->num_queues; 593 594 txr = &adapter->tx_rings[i]; 595 que = &adapter->queues[i]; 596 597 if (IXV_TX_TRYLOCK(txr)) { 598 err = ixv_mq_start_locked(ifp, txr, m); 599 IXV_TX_UNLOCK(txr); 600 } else { 601 err = drbr_enqueue(ifp, txr->br, m); 602 taskqueue_enqueue(que->tq, &que->que_task); 603 } 604 605 return (err); 606} 607 608static int 609ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 610{ 611 struct adapter *adapter = txr->adapter; 612 struct mbuf *next; 613 int enqueued, err = 0; 614 615 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 616 IFF_DRV_RUNNING || adapter->link_active == 0) { 617 if (m != NULL) 618 err = drbr_enqueue(ifp, txr->br, m); 619 return (err); 620 } 621 622 /* Do a clean if descriptors are low */ 623 if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD) 624 ixv_txeof(txr); 625 626 enqueued = 0; 627 if (m == NULL) { 628 next = drbr_dequeue(ifp, txr->br); 629 } else if (drbr_needs_enqueue(ifp, txr->br)) { 630 if ((err = drbr_enqueue(ifp, txr->br, m)) != 0) 631 return (err); 632 next = drbr_dequeue(ifp, txr->br); 633 } else 634 next = m; 635 636 /* Process the queue */ 637 while (next != NULL) { 638 if ((err = ixv_xmit(txr, &next)) != 0) { 639 if (next != NULL) 640 err = drbr_enqueue(ifp, txr->br, next); 641 break; 642 } 643 enqueued++; 644 drbr_stats_update(ifp, next->m_pkthdr.len, next->m_flags); 645 /* Send a copy of the frame to the BPF listener */ 646 ETHER_BPF_MTAP(ifp, next); 647 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 648 break; 649 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) { 650 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 651 break; 652 } 653 next = drbr_dequeue(ifp, txr->br); 654 } 655 656 if (enqueued > 0) { 657 /* Set watchdog on */ 658 txr->watchdog_check = TRUE; 659 txr->watchdog_time = ticks; 660 } 661 662 return (err); 663} 664 665/* 666** Flush all ring buffers 667*/ 668static void 669ixv_qflush(struct ifnet *ifp) 670{ 671 struct adapter *adapter = ifp->if_softc; 672 struct tx_ring *txr = adapter->tx_rings; 673 struct mbuf *m; 674 675 for (int i = 0; i < adapter->num_queues; i++, txr++) { 676 IXV_TX_LOCK(txr); 677 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 678 m_freem(m); 679 IXV_TX_UNLOCK(txr); 680 } 681 if_qflush(ifp); 682} 683 684#endif 685 686/********************************************************************* 687 * Ioctl entry point 688 * 689 * ixv_ioctl is called when the user wants to configure the 690 * interface. 691 * 692 * return 0 on success, positive on failure 693 **********************************************************************/ 694 695static int 696ixv_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 697{ 698 struct adapter *adapter = ifp->if_softc; 699 struct ifreq *ifr = (struct ifreq *) data; 700#if defined(INET) || defined(INET6) 701 struct ifaddr *ifa = (struct ifaddr *) data; 702 bool avoid_reset = FALSE; 703#endif 704 int error = 0; 705 706 switch (command) { 707 708 case SIOCSIFADDR: 709#ifdef INET 710 if (ifa->ifa_addr->sa_family == AF_INET) 711 avoid_reset = TRUE; 712#endif 713#ifdef INET6 714 if (ifa->ifa_addr->sa_family == AF_INET6) 715 avoid_reset = TRUE; 716#endif 717#if defined(INET) || defined(INET6) 718 /* 719 ** Calling init results in link renegotiation, 720 ** so we avoid doing it when possible. 721 */ 722 if (avoid_reset) { 723 ifp->if_flags |= IFF_UP; 724 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 725 ixv_init(adapter); 726 if (!(ifp->if_flags & IFF_NOARP)) 727 arp_ifinit(ifp, ifa); 728 } else 729 error = ether_ioctl(ifp, command, data); 730 break; 731#endif 732 case SIOCSIFMTU: 733 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 734 if (ifr->ifr_mtu > IXV_MAX_FRAME_SIZE - ETHER_HDR_LEN) { 735 error = EINVAL; 736 } else { 737 IXV_CORE_LOCK(adapter); 738 ifp->if_mtu = ifr->ifr_mtu; 739 adapter->max_frame_size = 740 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 741 ixv_init_locked(adapter); 742 IXV_CORE_UNLOCK(adapter); 743 } 744 break; 745 case SIOCSIFFLAGS: 746 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 747 IXV_CORE_LOCK(adapter); 748 if (ifp->if_flags & IFF_UP) { 749 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 750 ixv_init_locked(adapter); 751 } else 752 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 753 ixv_stop(adapter); 754 adapter->if_flags = ifp->if_flags; 755 IXV_CORE_UNLOCK(adapter); 756 break; 757 case SIOCADDMULTI: 758 case SIOCDELMULTI: 759 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 760 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 761 IXV_CORE_LOCK(adapter); 762 ixv_disable_intr(adapter); 763 ixv_set_multi(adapter); 764 ixv_enable_intr(adapter); 765 IXV_CORE_UNLOCK(adapter); 766 } 767 break; 768 case SIOCSIFMEDIA: 769 case SIOCGIFMEDIA: 770 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 771 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 772 break; 773 case SIOCSIFCAP: 774 { 775 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 776 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 777 if (mask & IFCAP_HWCSUM) 778 ifp->if_capenable ^= IFCAP_HWCSUM; 779 if (mask & IFCAP_TSO4) 780 ifp->if_capenable ^= IFCAP_TSO4; 781 if (mask & IFCAP_LRO) 782 ifp->if_capenable ^= IFCAP_LRO; 783 if (mask & IFCAP_VLAN_HWTAGGING) 784 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 785 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 786 IXV_CORE_LOCK(adapter); 787 ixv_init_locked(adapter); 788 IXV_CORE_UNLOCK(adapter); 789 } 790 VLAN_CAPABILITIES(ifp); 791 break; 792 } 793 794 default: 795 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 796 error = ether_ioctl(ifp, command, data); 797 break; 798 } 799 800 return (error); 801} 802 803/********************************************************************* 804 * Init entry point 805 * 806 * This routine is used in two ways. It is used by the stack as 807 * init entry point in network interface structure. It is also used 808 * by the driver as a hw/sw initialization routine to get to a 809 * consistent state. 810 * 811 * return 0 on success, positive on failure 812 **********************************************************************/ 813#define IXGBE_MHADD_MFS_SHIFT 16 814 815static void 816ixv_init_locked(struct adapter *adapter) 817{ 818 struct ifnet *ifp = adapter->ifp; 819 device_t dev = adapter->dev; 820 struct ixgbe_hw *hw = &adapter->hw; 821 u32 mhadd, gpie; 822 823 INIT_DEBUGOUT("ixv_init: begin"); 824 mtx_assert(&adapter->core_mtx, MA_OWNED); 825 hw->adapter_stopped = FALSE; 826 ixgbe_stop_adapter(hw); 827 callout_stop(&adapter->timer); 828 829 /* reprogram the RAR[0] in case user changed it. */ 830 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 831 832 /* Get the latest mac address, User can use a LAA */ 833 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr, 834 IXGBE_ETH_LENGTH_OF_ADDRESS); 835 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 836 hw->addr_ctrl.rar_used_count = 1; 837 838 /* Prepare transmit descriptors and buffers */ 839 if (ixv_setup_transmit_structures(adapter)) { 840 device_printf(dev,"Could not setup transmit structures\n"); 841 ixv_stop(adapter); 842 return; 843 } 844 845 ixgbe_reset_hw(hw); 846 ixv_initialize_transmit_units(adapter); 847 848 /* Setup Multicast table */ 849 ixv_set_multi(adapter); 850 851 /* 852 ** Determine the correct mbuf pool 853 ** for doing jumbo/headersplit 854 */ 855 if (ifp->if_mtu > ETHERMTU) 856 adapter->rx_mbuf_sz = MJUMPAGESIZE; 857 else 858 adapter->rx_mbuf_sz = MCLBYTES; 859 860 /* Prepare receive descriptors and buffers */ 861 if (ixv_setup_receive_structures(adapter)) { 862 device_printf(dev,"Could not setup receive structures\n"); 863 ixv_stop(adapter); 864 return; 865 } 866 867 /* Configure RX settings */ 868 ixv_initialize_receive_units(adapter); 869 870 /* Enable Enhanced MSIX mode */ 871 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 872 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME; 873 gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; 874 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 875 876 /* Set the various hardware offload abilities */ 877 ifp->if_hwassist = 0; 878 if (ifp->if_capenable & IFCAP_TSO4) 879 ifp->if_hwassist |= CSUM_TSO; 880 if (ifp->if_capenable & IFCAP_TXCSUM) { 881 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 882#if __FreeBSD_version >= 800000 883 ifp->if_hwassist |= CSUM_SCTP; 884#endif 885 } 886 887 /* Set MTU size */ 888 if (ifp->if_mtu > ETHERMTU) { 889 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 890 mhadd &= ~IXGBE_MHADD_MFS_MASK; 891 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 892 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 893 } 894 895 /* Set up VLAN offload and filter */ 896 ixv_setup_vlan_support(adapter); 897 898 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 899 900 /* Set up MSI/X routing */ 901 ixv_configure_ivars(adapter); 902 903 /* Set up auto-mask */ 904 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 905 906 /* Set moderation on the Link interrupt */ 907 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR); 908 909 /* Stats init */ 910 ixv_init_stats(adapter); 911 912 /* Config/Enable Link */ 913 ixv_config_link(adapter); 914 915 /* And now turn on interrupts */ 916 ixv_enable_intr(adapter); 917 918 /* Now inform the stack we're ready */ 919 ifp->if_drv_flags |= IFF_DRV_RUNNING; 920 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 921 922 return; 923} 924 925static void 926ixv_init(void *arg) 927{ 928 struct adapter *adapter = arg; 929 930 IXV_CORE_LOCK(adapter); 931 ixv_init_locked(adapter); 932 IXV_CORE_UNLOCK(adapter); 933 return; 934} 935 936 937/* 938** 939** MSIX Interrupt Handlers and Tasklets 940** 941*/ 942 943static inline void 944ixv_enable_queue(struct adapter *adapter, u32 vector) 945{ 946 struct ixgbe_hw *hw = &adapter->hw; 947 u32 queue = 1 << vector; 948 u32 mask; 949 950 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 951 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 952} 953 954static inline void 955ixv_disable_queue(struct adapter *adapter, u32 vector) 956{ 957 struct ixgbe_hw *hw = &adapter->hw; 958 u64 queue = (u64)(1 << vector); 959 u32 mask; 960 961 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 962 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 963} 964 965static inline void 966ixv_rearm_queues(struct adapter *adapter, u64 queues) 967{ 968 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 969 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 970} 971 972 973static void 974ixv_handle_que(void *context, int pending) 975{ 976 struct ix_queue *que = context; 977 struct adapter *adapter = que->adapter; 978 struct tx_ring *txr = que->txr; 979 struct ifnet *ifp = adapter->ifp; 980 bool more; 981 982 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 983 more = ixv_rxeof(que, adapter->rx_process_limit); 984 IXV_TX_LOCK(txr); 985 ixv_txeof(txr); 986#if __FreeBSD_version >= 800000 987 if (!drbr_empty(ifp, txr->br)) 988 ixv_mq_start_locked(ifp, txr, NULL); 989#else 990 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 991 ixv_start_locked(txr, ifp); 992#endif 993 IXV_TX_UNLOCK(txr); 994 if (more) { 995 taskqueue_enqueue(que->tq, &que->que_task); 996 return; 997 } 998 } 999 1000 /* Reenable this interrupt */ 1001 ixv_enable_queue(adapter, que->msix); 1002 return; 1003} 1004 1005/********************************************************************* 1006 * 1007 * MSI Queue Interrupt Service routine 1008 * 1009 **********************************************************************/ 1010void 1011ixv_msix_que(void *arg) 1012{ 1013 struct ix_queue *que = arg; 1014 struct adapter *adapter = que->adapter; 1015 struct tx_ring *txr = que->txr; 1016 struct rx_ring *rxr = que->rxr; 1017 bool more_tx, more_rx; 1018 u32 newitr = 0; 1019 1020 ixv_disable_queue(adapter, que->msix); 1021 ++que->irqs; 1022 1023 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1024 1025 IXV_TX_LOCK(txr); 1026 more_tx = ixv_txeof(txr); 1027 /* 1028 ** Make certain that if the stack 1029 ** has anything queued the task gets 1030 ** scheduled to handle it. 1031 */ 1032#if __FreeBSD_version < 800000 1033 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd)) 1034#else 1035 if (!drbr_empty(adapter->ifp, txr->br)) 1036#endif 1037 more_tx = 1; 1038 IXV_TX_UNLOCK(txr); 1039 1040 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1041 1042 /* Do AIM now? */ 1043 1044 if (ixv_enable_aim == FALSE) 1045 goto no_calc; 1046 /* 1047 ** Do Adaptive Interrupt Moderation: 1048 ** - Write out last calculated setting 1049 ** - Calculate based on average size over 1050 ** the last interval. 1051 */ 1052 if (que->eitr_setting) 1053 IXGBE_WRITE_REG(&adapter->hw, 1054 IXGBE_VTEITR(que->msix), 1055 que->eitr_setting); 1056 1057 que->eitr_setting = 0; 1058 1059 /* Idle, do nothing */ 1060 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1061 goto no_calc; 1062 1063 if ((txr->bytes) && (txr->packets)) 1064 newitr = txr->bytes/txr->packets; 1065 if ((rxr->bytes) && (rxr->packets)) 1066 newitr = max(newitr, 1067 (rxr->bytes / rxr->packets)); 1068 newitr += 24; /* account for hardware frame, crc */ 1069 1070 /* set an upper boundary */ 1071 newitr = min(newitr, 3000); 1072 1073 /* Be nice to the mid range */ 1074 if ((newitr > 300) && (newitr < 1200)) 1075 newitr = (newitr / 3); 1076 else 1077 newitr = (newitr / 2); 1078 1079 newitr |= newitr << 16; 1080 1081 /* save for next interrupt */ 1082 que->eitr_setting = newitr; 1083 1084 /* Reset state */ 1085 txr->bytes = 0; 1086 txr->packets = 0; 1087 rxr->bytes = 0; 1088 rxr->packets = 0; 1089 1090no_calc: 1091 if (more_tx || more_rx) 1092 taskqueue_enqueue(que->tq, &que->que_task); 1093 else /* Reenable this interrupt */ 1094 ixv_enable_queue(adapter, que->msix); 1095 return; 1096} 1097 1098static void 1099ixv_msix_mbx(void *arg) 1100{ 1101 struct adapter *adapter = arg; 1102 struct ixgbe_hw *hw = &adapter->hw; 1103 u32 reg; 1104 1105 ++adapter->mbx_irq; 1106 1107 /* First get the cause */ 1108 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 1109 /* Clear interrupt with write */ 1110 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 1111 1112 /* Link status change */ 1113 if (reg & IXGBE_EICR_LSC) 1114 taskqueue_enqueue(adapter->tq, &adapter->mbx_task); 1115 1116 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 1117 return; 1118} 1119 1120/********************************************************************* 1121 * 1122 * Media Ioctl callback 1123 * 1124 * This routine is called whenever the user queries the status of 1125 * the interface using ifconfig. 1126 * 1127 **********************************************************************/ 1128static void 1129ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1130{ 1131 struct adapter *adapter = ifp->if_softc; 1132 1133 INIT_DEBUGOUT("ixv_media_status: begin"); 1134 IXV_CORE_LOCK(adapter); 1135 ixv_update_link_status(adapter); 1136 1137 ifmr->ifm_status = IFM_AVALID; 1138 ifmr->ifm_active = IFM_ETHER; 1139 1140 if (!adapter->link_active) { 1141 IXV_CORE_UNLOCK(adapter); 1142 return; 1143 } 1144 1145 ifmr->ifm_status |= IFM_ACTIVE; 1146 1147 switch (adapter->link_speed) { 1148 case IXGBE_LINK_SPEED_1GB_FULL: 1149 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1150 break; 1151 case IXGBE_LINK_SPEED_10GB_FULL: 1152 ifmr->ifm_active |= IFM_FDX; 1153 break; 1154 } 1155 1156 IXV_CORE_UNLOCK(adapter); 1157 1158 return; 1159} 1160 1161/********************************************************************* 1162 * 1163 * Media Ioctl callback 1164 * 1165 * This routine is called when the user changes speed/duplex using 1166 * media/mediopt option with ifconfig. 1167 * 1168 **********************************************************************/ 1169static int 1170ixv_media_change(struct ifnet * ifp) 1171{ 1172 struct adapter *adapter = ifp->if_softc; 1173 struct ifmedia *ifm = &adapter->media; 1174 1175 INIT_DEBUGOUT("ixv_media_change: begin"); 1176 1177 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1178 return (EINVAL); 1179 1180 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1181 case IFM_AUTO: 1182 break; 1183 default: 1184 device_printf(adapter->dev, "Only auto media type\n"); 1185 return (EINVAL); 1186 } 1187 1188 return (0); 1189} 1190 1191/********************************************************************* 1192 * 1193 * This routine maps the mbufs to tx descriptors, allowing the 1194 * TX engine to transmit the packets. 1195 * - return 0 on success, positive on failure 1196 * 1197 **********************************************************************/ 1198 1199static int 1200ixv_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1201{ 1202 struct adapter *adapter = txr->adapter; 1203 u32 olinfo_status = 0, cmd_type_len; 1204 u32 paylen = 0; 1205 int i, j, error, nsegs; 1206 int first, last = 0; 1207 struct mbuf *m_head; 1208 bus_dma_segment_t segs[32]; 1209 bus_dmamap_t map; 1210 struct ixv_tx_buf *txbuf, *txbuf_mapped; 1211 union ixgbe_adv_tx_desc *txd = NULL; 1212 1213 m_head = *m_headp; 1214 1215 /* Basic descriptor defines */ 1216 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1217 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1218 1219 if (m_head->m_flags & M_VLANTAG) 1220 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1221 1222 /* 1223 * Important to capture the first descriptor 1224 * used because it will contain the index of 1225 * the one we tell the hardware to report back 1226 */ 1227 first = txr->next_avail_desc; 1228 txbuf = &txr->tx_buffers[first]; 1229 txbuf_mapped = txbuf; 1230 map = txbuf->map; 1231 1232 /* 1233 * Map the packet for DMA. 1234 */ 1235 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1236 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1237 1238 if (error == EFBIG) { 1239 struct mbuf *m; 1240 1241 m = m_defrag(*m_headp, M_DONTWAIT); 1242 if (m == NULL) { 1243 adapter->mbuf_defrag_failed++; 1244 m_freem(*m_headp); 1245 *m_headp = NULL; 1246 return (ENOBUFS); 1247 } 1248 *m_headp = m; 1249 1250 /* Try it again */ 1251 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1252 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1253 1254 if (error == ENOMEM) { 1255 adapter->no_tx_dma_setup++; 1256 return (error); 1257 } else if (error != 0) { 1258 adapter->no_tx_dma_setup++; 1259 m_freem(*m_headp); 1260 *m_headp = NULL; 1261 return (error); 1262 } 1263 } else if (error == ENOMEM) { 1264 adapter->no_tx_dma_setup++; 1265 return (error); 1266 } else if (error != 0) { 1267 adapter->no_tx_dma_setup++; 1268 m_freem(*m_headp); 1269 *m_headp = NULL; 1270 return (error); 1271 } 1272 1273 /* Make certain there are enough descriptors */ 1274 if (nsegs > txr->tx_avail - 2) { 1275 txr->no_desc_avail++; 1276 error = ENOBUFS; 1277 goto xmit_fail; 1278 } 1279 m_head = *m_headp; 1280 1281 /* 1282 ** Set up the appropriate offload context 1283 ** this becomes the first descriptor of 1284 ** a packet. 1285 */ 1286 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1287 if (ixv_tso_setup(txr, m_head, &paylen)) { 1288 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1289 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1290 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1291 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1292 ++adapter->tso_tx; 1293 } else 1294 return (ENXIO); 1295 } else if (ixv_tx_ctx_setup(txr, m_head)) 1296 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1297 1298 /* Record payload length */ 1299 if (paylen == 0) 1300 olinfo_status |= m_head->m_pkthdr.len << 1301 IXGBE_ADVTXD_PAYLEN_SHIFT; 1302 1303 i = txr->next_avail_desc; 1304 for (j = 0; j < nsegs; j++) { 1305 bus_size_t seglen; 1306 bus_addr_t segaddr; 1307 1308 txbuf = &txr->tx_buffers[i]; 1309 txd = &txr->tx_base[i]; 1310 seglen = segs[j].ds_len; 1311 segaddr = htole64(segs[j].ds_addr); 1312 1313 txd->read.buffer_addr = segaddr; 1314 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1315 cmd_type_len |seglen); 1316 txd->read.olinfo_status = htole32(olinfo_status); 1317 last = i; /* descriptor that will get completion IRQ */ 1318 1319 if (++i == adapter->num_tx_desc) 1320 i = 0; 1321 1322 txbuf->m_head = NULL; 1323 txbuf->eop_index = -1; 1324 } 1325 1326 txd->read.cmd_type_len |= 1327 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1328 txr->tx_avail -= nsegs; 1329 txr->next_avail_desc = i; 1330 1331 txbuf->m_head = m_head; 1332 txr->tx_buffers[first].map = txbuf->map; 1333 txbuf->map = map; 1334 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1335 1336 /* Set the index of the descriptor that will be marked done */ 1337 txbuf = &txr->tx_buffers[first]; 1338 txbuf->eop_index = last; 1339 1340 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1341 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1342 /* 1343 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1344 * hardware that this frame is available to transmit. 1345 */ 1346 ++txr->total_packets; 1347 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i); 1348 1349 return (0); 1350 1351xmit_fail: 1352 bus_dmamap_unload(txr->txtag, txbuf->map); 1353 return (error); 1354 1355} 1356 1357 1358/********************************************************************* 1359 * Multicast Update 1360 * 1361 * This routine is called whenever multicast address list is updated. 1362 * 1363 **********************************************************************/ 1364#define IXGBE_RAR_ENTRIES 16 1365 1366static void 1367ixv_set_multi(struct adapter *adapter) 1368{ 1369 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1370 u8 *update_ptr; 1371 struct ifmultiaddr *ifma; 1372 int mcnt = 0; 1373 struct ifnet *ifp = adapter->ifp; 1374 1375 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1376 1377#if __FreeBSD_version < 800000 1378 IF_ADDR_LOCK(ifp); 1379#else 1380 if_maddr_rlock(ifp); 1381#endif 1382 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1383 if (ifma->ifma_addr->sa_family != AF_LINK) 1384 continue; 1385 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), 1386 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1387 IXGBE_ETH_LENGTH_OF_ADDRESS); 1388 mcnt++; 1389 } 1390#if __FreeBSD_version < 800000 1391 IF_ADDR_UNLOCK(ifp); 1392#else 1393 if_maddr_runlock(ifp); 1394#endif 1395 1396 update_ptr = mta; 1397 1398 ixgbe_update_mc_addr_list(&adapter->hw, 1399 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1400 1401 return; 1402} 1403 1404/* 1405 * This is an iterator function now needed by the multicast 1406 * shared code. It simply feeds the shared code routine the 1407 * addresses in the array of ixv_set_multi() one by one. 1408 */ 1409static u8 * 1410ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1411{ 1412 u8 *addr = *update_ptr; 1413 u8 *newptr; 1414 *vmdq = 0; 1415 1416 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1417 *update_ptr = newptr; 1418 return addr; 1419} 1420 1421/********************************************************************* 1422 * Timer routine 1423 * 1424 * This routine checks for link status,updates statistics, 1425 * and runs the watchdog check. 1426 * 1427 **********************************************************************/ 1428 1429static void 1430ixv_local_timer(void *arg) 1431{ 1432 struct adapter *adapter = arg; 1433 device_t dev = adapter->dev; 1434 struct tx_ring *txr = adapter->tx_rings; 1435 int i; 1436 1437 mtx_assert(&adapter->core_mtx, MA_OWNED); 1438 1439 ixv_update_link_status(adapter); 1440 1441 /* Stats Update */ 1442 ixv_update_stats(adapter); 1443 1444 /* 1445 * If the interface has been paused 1446 * then don't do the watchdog check 1447 */ 1448 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 1449 goto out; 1450 /* 1451 ** Check for time since any descriptor was cleaned 1452 */ 1453 for (i = 0; i < adapter->num_queues; i++, txr++) { 1454 IXV_TX_LOCK(txr); 1455 if (txr->watchdog_check == FALSE) { 1456 IXV_TX_UNLOCK(txr); 1457 continue; 1458 } 1459 if ((ticks - txr->watchdog_time) > IXV_WATCHDOG) 1460 goto hung; 1461 IXV_TX_UNLOCK(txr); 1462 } 1463out: 1464 ixv_rearm_queues(adapter, adapter->que_mask); 1465 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1466 return; 1467 1468hung: 1469 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1470 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1471 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)), 1472 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i))); 1473 device_printf(dev,"TX(%d) desc avail = %d," 1474 "Next TX to Clean = %d\n", 1475 txr->me, txr->tx_avail, txr->next_to_clean); 1476 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1477 adapter->watchdog_events++; 1478 IXV_TX_UNLOCK(txr); 1479 ixv_init_locked(adapter); 1480} 1481 1482/* 1483** Note: this routine updates the OS on the link state 1484** the real check of the hardware only happens with 1485** a link interrupt. 1486*/ 1487static void 1488ixv_update_link_status(struct adapter *adapter) 1489{ 1490 struct ifnet *ifp = adapter->ifp; 1491 struct tx_ring *txr = adapter->tx_rings; 1492 device_t dev = adapter->dev; 1493 1494 1495 if (adapter->link_up){ 1496 if (adapter->link_active == FALSE) { 1497 if (bootverbose) 1498 device_printf(dev,"Link is up %d Gbps %s \n", 1499 ((adapter->link_speed == 128)? 10:1), 1500 "Full Duplex"); 1501 adapter->link_active = TRUE; 1502 if_link_state_change(ifp, LINK_STATE_UP); 1503 } 1504 } else { /* Link down */ 1505 if (adapter->link_active == TRUE) { 1506 if (bootverbose) 1507 device_printf(dev,"Link is Down\n"); 1508 if_link_state_change(ifp, LINK_STATE_DOWN); 1509 adapter->link_active = FALSE; 1510 for (int i = 0; i < adapter->num_queues; 1511 i++, txr++) 1512 txr->watchdog_check = FALSE; 1513 } 1514 } 1515 1516 return; 1517} 1518 1519 1520/********************************************************************* 1521 * 1522 * This routine disables all traffic on the adapter by issuing a 1523 * global reset on the MAC and deallocates TX/RX buffers. 1524 * 1525 **********************************************************************/ 1526 1527static void 1528ixv_stop(void *arg) 1529{ 1530 struct ifnet *ifp; 1531 struct adapter *adapter = arg; 1532 struct ixgbe_hw *hw = &adapter->hw; 1533 ifp = adapter->ifp; 1534 1535 mtx_assert(&adapter->core_mtx, MA_OWNED); 1536 1537 INIT_DEBUGOUT("ixv_stop: begin\n"); 1538 ixv_disable_intr(adapter); 1539 1540 /* Tell the stack that the interface is no longer active */ 1541 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1542 1543 ixgbe_reset_hw(hw); 1544 adapter->hw.adapter_stopped = FALSE; 1545 ixgbe_stop_adapter(hw); 1546 callout_stop(&adapter->timer); 1547 1548 /* reprogram the RAR[0] in case user changed it. */ 1549 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1550 1551 return; 1552} 1553 1554 1555/********************************************************************* 1556 * 1557 * Determine hardware revision. 1558 * 1559 **********************************************************************/ 1560static void 1561ixv_identify_hardware(struct adapter *adapter) 1562{ 1563 device_t dev = adapter->dev; 1564 u16 pci_cmd_word; 1565 1566 /* 1567 ** Make sure BUSMASTER is set, on a VM under 1568 ** KVM it may not be and will break things. 1569 */ 1570 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 1571 if (!((pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 1572 (pci_cmd_word & PCIM_CMD_MEMEN))) { 1573 INIT_DEBUGOUT("Memory Access and/or Bus Master " 1574 "bits were not set!\n"); 1575 pci_cmd_word |= (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 1576 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); 1577 } 1578 1579 /* Save off the information about this board */ 1580 adapter->hw.vendor_id = pci_get_vendor(dev); 1581 adapter->hw.device_id = pci_get_device(dev); 1582 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1583 adapter->hw.subsystem_vendor_id = 1584 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1585 adapter->hw.subsystem_device_id = 1586 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1587 1588 return; 1589} 1590 1591/********************************************************************* 1592 * 1593 * Setup MSIX Interrupt resources and handlers 1594 * 1595 **********************************************************************/ 1596static int 1597ixv_allocate_msix(struct adapter *adapter) 1598{ 1599 device_t dev = adapter->dev; 1600 struct ix_queue *que = adapter->queues; 1601 int error, rid, vector = 0; 1602 1603 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 1604 rid = vector + 1; 1605 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 1606 RF_SHAREABLE | RF_ACTIVE); 1607 if (que->res == NULL) { 1608 device_printf(dev,"Unable to allocate" 1609 " bus resource: que interrupt [%d]\n", vector); 1610 return (ENXIO); 1611 } 1612 /* Set the handler function */ 1613 error = bus_setup_intr(dev, que->res, 1614 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1615 ixv_msix_que, que, &que->tag); 1616 if (error) { 1617 que->res = NULL; 1618 device_printf(dev, "Failed to register QUE handler"); 1619 return (error); 1620 } 1621#if __FreeBSD_version >= 800504 1622 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 1623#endif 1624 que->msix = vector; 1625 adapter->que_mask |= (u64)(1 << que->msix); 1626 /* 1627 ** Bind the msix vector, and thus the 1628 ** ring to the corresponding cpu. 1629 */ 1630 if (adapter->num_queues > 1) 1631 bus_bind_intr(dev, que->res, i); 1632 1633 TASK_INIT(&que->que_task, 0, ixv_handle_que, que); 1634 que->tq = taskqueue_create_fast("ixv_que", M_NOWAIT, 1635 taskqueue_thread_enqueue, &que->tq); 1636 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 1637 device_get_nameunit(adapter->dev)); 1638 } 1639 1640 /* and Mailbox */ 1641 rid = vector + 1; 1642 adapter->res = bus_alloc_resource_any(dev, 1643 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 1644 if (!adapter->res) { 1645 device_printf(dev,"Unable to allocate" 1646 " bus resource: MBX interrupt [%d]\n", rid); 1647 return (ENXIO); 1648 } 1649 /* Set the mbx handler function */ 1650 error = bus_setup_intr(dev, adapter->res, 1651 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1652 ixv_msix_mbx, adapter, &adapter->tag); 1653 if (error) { 1654 adapter->res = NULL; 1655 device_printf(dev, "Failed to register LINK handler"); 1656 return (error); 1657 } 1658#if __FreeBSD_version >= 800504 1659 bus_describe_intr(dev, adapter->res, adapter->tag, "mbx"); 1660#endif 1661 adapter->mbxvec = vector; 1662 /* Tasklets for Mailbox */ 1663 TASK_INIT(&adapter->mbx_task, 0, ixv_handle_mbx, adapter); 1664 adapter->tq = taskqueue_create_fast("ixv_mbx", M_NOWAIT, 1665 taskqueue_thread_enqueue, &adapter->tq); 1666 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s mbxq", 1667 device_get_nameunit(adapter->dev)); 1668 /* 1669 ** Due to a broken design QEMU will fail to properly 1670 ** enable the guest for MSIX unless the vectors in 1671 ** the table are all set up, so we must rewrite the 1672 ** ENABLE in the MSIX control register again at this 1673 ** point to cause it to successfully initialize us. 1674 */ 1675 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 1676 int msix_ctrl; 1677 pci_find_cap(dev, PCIY_MSIX, &rid); 1678 rid += PCIR_MSIX_CTRL; 1679 msix_ctrl = pci_read_config(dev, rid, 2); 1680 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 1681 pci_write_config(dev, rid, msix_ctrl, 2); 1682 } 1683 1684 return (0); 1685} 1686 1687/* 1688 * Setup MSIX resources, note that the VF 1689 * device MUST use MSIX, there is no fallback. 1690 */ 1691static int 1692ixv_setup_msix(struct adapter *adapter) 1693{ 1694 device_t dev = adapter->dev; 1695 int rid, vectors, want = 2; 1696 1697 1698 /* First try MSI/X */ 1699 rid = PCIR_BAR(3); 1700 adapter->msix_mem = bus_alloc_resource_any(dev, 1701 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1702 if (!adapter->msix_mem) { 1703 device_printf(adapter->dev, 1704 "Unable to map MSIX table \n"); 1705 goto out; 1706 } 1707 1708 vectors = pci_msix_count(dev); 1709 if (vectors < 2) { 1710 bus_release_resource(dev, SYS_RES_MEMORY, 1711 rid, adapter->msix_mem); 1712 adapter->msix_mem = NULL; 1713 goto out; 1714 } 1715 1716 /* 1717 ** Want two vectors: one for a queue, 1718 ** plus an additional for mailbox. 1719 */ 1720 if (pci_alloc_msix(dev, &want) == 0) { 1721 device_printf(adapter->dev, 1722 "Using MSIX interrupts with %d vectors\n", want); 1723 return (want); 1724 } 1725out: 1726 device_printf(adapter->dev,"MSIX config error\n"); 1727 return (ENXIO); 1728} 1729 1730 1731static int 1732ixv_allocate_pci_resources(struct adapter *adapter) 1733{ 1734 int rid; 1735 device_t dev = adapter->dev; 1736 1737 rid = PCIR_BAR(0); 1738 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1739 &rid, RF_ACTIVE); 1740 1741 if (!(adapter->pci_mem)) { 1742 device_printf(dev,"Unable to allocate bus resource: memory\n"); 1743 return (ENXIO); 1744 } 1745 1746 adapter->osdep.mem_bus_space_tag = 1747 rman_get_bustag(adapter->pci_mem); 1748 adapter->osdep.mem_bus_space_handle = 1749 rman_get_bushandle(adapter->pci_mem); 1750 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle; 1751 1752 adapter->num_queues = 1; 1753 adapter->hw.back = &adapter->osdep; 1754 1755 /* 1756 ** Now setup MSI/X, should 1757 ** return us the number of 1758 ** configured vectors. 1759 */ 1760 adapter->msix = ixv_setup_msix(adapter); 1761 if (adapter->msix == ENXIO) 1762 return (ENXIO); 1763 else 1764 return (0); 1765} 1766 1767static void 1768ixv_free_pci_resources(struct adapter * adapter) 1769{ 1770 struct ix_queue *que = adapter->queues; 1771 device_t dev = adapter->dev; 1772 int rid, memrid; 1773 1774 memrid = PCIR_BAR(MSIX_BAR); 1775 1776 /* 1777 ** There is a slight possibility of a failure mode 1778 ** in attach that will result in entering this function 1779 ** before interrupt resources have been initialized, and 1780 ** in that case we do not want to execute the loops below 1781 ** We can detect this reliably by the state of the adapter 1782 ** res pointer. 1783 */ 1784 if (adapter->res == NULL) 1785 goto mem; 1786 1787 /* 1788 ** Release all msix queue resources: 1789 */ 1790 for (int i = 0; i < adapter->num_queues; i++, que++) { 1791 rid = que->msix + 1; 1792 if (que->tag != NULL) { 1793 bus_teardown_intr(dev, que->res, que->tag); 1794 que->tag = NULL; 1795 } 1796 if (que->res != NULL) 1797 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 1798 } 1799 1800 1801 /* Clean the Legacy or Link interrupt last */ 1802 if (adapter->mbxvec) /* we are doing MSIX */ 1803 rid = adapter->mbxvec + 1; 1804 else 1805 (adapter->msix != 0) ? (rid = 1):(rid = 0); 1806 1807 if (adapter->tag != NULL) { 1808 bus_teardown_intr(dev, adapter->res, adapter->tag); 1809 adapter->tag = NULL; 1810 } 1811 if (adapter->res != NULL) 1812 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 1813 1814mem: 1815 if (adapter->msix) 1816 pci_release_msi(dev); 1817 1818 if (adapter->msix_mem != NULL) 1819 bus_release_resource(dev, SYS_RES_MEMORY, 1820 memrid, adapter->msix_mem); 1821 1822 if (adapter->pci_mem != NULL) 1823 bus_release_resource(dev, SYS_RES_MEMORY, 1824 PCIR_BAR(0), adapter->pci_mem); 1825 1826 return; 1827} 1828 1829/********************************************************************* 1830 * 1831 * Setup networking device structure and register an interface. 1832 * 1833 **********************************************************************/ 1834static void 1835ixv_setup_interface(device_t dev, struct adapter *adapter) 1836{ 1837 struct ifnet *ifp; 1838 1839 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1840 1841 ifp = adapter->ifp = if_alloc(IFT_ETHER); 1842 if (ifp == NULL) 1843 panic("%s: can not if_alloc()\n", device_get_nameunit(dev)); 1844 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1845 ifp->if_baudrate = 1000000000; 1846 ifp->if_init = ixv_init; 1847 ifp->if_softc = adapter; 1848 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1849 ifp->if_ioctl = ixv_ioctl; 1850#if __FreeBSD_version >= 800000 1851 ifp->if_transmit = ixv_mq_start; 1852 ifp->if_qflush = ixv_qflush; 1853#else 1854 ifp->if_start = ixv_start; 1855#endif 1856 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 1857 1858 ether_ifattach(ifp, adapter->hw.mac.addr); 1859 1860 adapter->max_frame_size = 1861 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1862 1863 /* 1864 * Tell the upper layer(s) we support long frames. 1865 */ 1866 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1867 1868 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM; 1869 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 1870 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING 1871 | IFCAP_VLAN_HWTSO 1872 | IFCAP_VLAN_MTU; 1873 ifp->if_capenable = ifp->if_capabilities; 1874 1875 /* Don't enable LRO by default */ 1876 ifp->if_capabilities |= IFCAP_LRO; 1877 1878 /* 1879 * Specify the media types supported by this adapter and register 1880 * callbacks to update media and link information 1881 */ 1882 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1883 ixv_media_status); 1884 ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL); 1885 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1886 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1887 1888 return; 1889} 1890 1891static void 1892ixv_config_link(struct adapter *adapter) 1893{ 1894 struct ixgbe_hw *hw = &adapter->hw; 1895 u32 autoneg, err = 0; 1896 bool negotiate = TRUE; 1897 1898 if (hw->mac.ops.check_link) 1899 err = hw->mac.ops.check_link(hw, &autoneg, 1900 &adapter->link_up, FALSE); 1901 if (err) 1902 goto out; 1903 1904 if (hw->mac.ops.setup_link) 1905 err = hw->mac.ops.setup_link(hw, autoneg, 1906 negotiate, adapter->link_up); 1907out: 1908 return; 1909} 1910 1911/******************************************************************** 1912 * Manage DMA'able memory. 1913 *******************************************************************/ 1914static void 1915ixv_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) 1916{ 1917 if (error) 1918 return; 1919 *(bus_addr_t *) arg = segs->ds_addr; 1920 return; 1921} 1922 1923static int 1924ixv_dma_malloc(struct adapter *adapter, bus_size_t size, 1925 struct ixv_dma_alloc *dma, int mapflags) 1926{ 1927 device_t dev = adapter->dev; 1928 int r; 1929 1930 r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 1931 DBA_ALIGN, 0, /* alignment, bounds */ 1932 BUS_SPACE_MAXADDR, /* lowaddr */ 1933 BUS_SPACE_MAXADDR, /* highaddr */ 1934 NULL, NULL, /* filter, filterarg */ 1935 size, /* maxsize */ 1936 1, /* nsegments */ 1937 size, /* maxsegsize */ 1938 BUS_DMA_ALLOCNOW, /* flags */ 1939 NULL, /* lockfunc */ 1940 NULL, /* lockfuncarg */ 1941 &dma->dma_tag); 1942 if (r != 0) { 1943 device_printf(dev,"ixv_dma_malloc: bus_dma_tag_create failed; " 1944 "error %u\n", r); 1945 goto fail_0; 1946 } 1947 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 1948 BUS_DMA_NOWAIT, &dma->dma_map); 1949 if (r != 0) { 1950 device_printf(dev,"ixv_dma_malloc: bus_dmamem_alloc failed; " 1951 "error %u\n", r); 1952 goto fail_1; 1953 } 1954 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1955 size, 1956 ixv_dmamap_cb, 1957 &dma->dma_paddr, 1958 mapflags | BUS_DMA_NOWAIT); 1959 if (r != 0) { 1960 device_printf(dev,"ixv_dma_malloc: bus_dmamap_load failed; " 1961 "error %u\n", r); 1962 goto fail_2; 1963 } 1964 dma->dma_size = size; 1965 return (0); 1966fail_2: 1967 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1968fail_1: 1969 bus_dma_tag_destroy(dma->dma_tag); 1970fail_0: 1971 dma->dma_map = NULL; 1972 dma->dma_tag = NULL; 1973 return (r); 1974} 1975 1976static void 1977ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma) 1978{ 1979 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 1980 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1981 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1982 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1983 bus_dma_tag_destroy(dma->dma_tag); 1984} 1985 1986 1987/********************************************************************* 1988 * 1989 * Allocate memory for the transmit and receive rings, and then 1990 * the descriptors associated with each, called only once at attach. 1991 * 1992 **********************************************************************/ 1993static int 1994ixv_allocate_queues(struct adapter *adapter) 1995{ 1996 device_t dev = adapter->dev; 1997 struct ix_queue *que; 1998 struct tx_ring *txr; 1999 struct rx_ring *rxr; 2000 int rsize, tsize, error = 0; 2001 int txconf = 0, rxconf = 0; 2002 2003 /* First allocate the top level queue structs */ 2004 if (!(adapter->queues = 2005 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 2006 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2007 device_printf(dev, "Unable to allocate queue memory\n"); 2008 error = ENOMEM; 2009 goto fail; 2010 } 2011 2012 /* First allocate the TX ring struct memory */ 2013 if (!(adapter->tx_rings = 2014 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2015 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2016 device_printf(dev, "Unable to allocate TX ring memory\n"); 2017 error = ENOMEM; 2018 goto tx_fail; 2019 } 2020 2021 /* Next allocate the RX */ 2022 if (!(adapter->rx_rings = 2023 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2024 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2025 device_printf(dev, "Unable to allocate RX ring memory\n"); 2026 error = ENOMEM; 2027 goto rx_fail; 2028 } 2029 2030 /* For the ring itself */ 2031 tsize = roundup2(adapter->num_tx_desc * 2032 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2033 2034 /* 2035 * Now set up the TX queues, txconf is needed to handle the 2036 * possibility that things fail midcourse and we need to 2037 * undo memory gracefully 2038 */ 2039 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2040 /* Set up some basics */ 2041 txr = &adapter->tx_rings[i]; 2042 txr->adapter = adapter; 2043 txr->me = i; 2044 2045 /* Initialize the TX side lock */ 2046 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2047 device_get_nameunit(dev), txr->me); 2048 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2049 2050 if (ixv_dma_malloc(adapter, tsize, 2051 &txr->txdma, BUS_DMA_NOWAIT)) { 2052 device_printf(dev, 2053 "Unable to allocate TX Descriptor memory\n"); 2054 error = ENOMEM; 2055 goto err_tx_desc; 2056 } 2057 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2058 bzero((void *)txr->tx_base, tsize); 2059 2060 /* Now allocate transmit buffers for the ring */ 2061 if (ixv_allocate_transmit_buffers(txr)) { 2062 device_printf(dev, 2063 "Critical Failure setting up transmit buffers\n"); 2064 error = ENOMEM; 2065 goto err_tx_desc; 2066 } 2067#if __FreeBSD_version >= 800000 2068 /* Allocate a buf ring */ 2069 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF, 2070 M_WAITOK, &txr->tx_mtx); 2071 if (txr->br == NULL) { 2072 device_printf(dev, 2073 "Critical Failure setting up buf ring\n"); 2074 error = ENOMEM; 2075 goto err_tx_desc; 2076 } 2077#endif 2078 } 2079 2080 /* 2081 * Next the RX queues... 2082 */ 2083 rsize = roundup2(adapter->num_rx_desc * 2084 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2085 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2086 rxr = &adapter->rx_rings[i]; 2087 /* Set up some basics */ 2088 rxr->adapter = adapter; 2089 rxr->me = i; 2090 2091 /* Initialize the RX side lock */ 2092 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2093 device_get_nameunit(dev), rxr->me); 2094 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2095 2096 if (ixv_dma_malloc(adapter, rsize, 2097 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2098 device_printf(dev, 2099 "Unable to allocate RxDescriptor memory\n"); 2100 error = ENOMEM; 2101 goto err_rx_desc; 2102 } 2103 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2104 bzero((void *)rxr->rx_base, rsize); 2105 2106 /* Allocate receive buffers for the ring*/ 2107 if (ixv_allocate_receive_buffers(rxr)) { 2108 device_printf(dev, 2109 "Critical Failure setting up receive buffers\n"); 2110 error = ENOMEM; 2111 goto err_rx_desc; 2112 } 2113 } 2114 2115 /* 2116 ** Finally set up the queue holding structs 2117 */ 2118 for (int i = 0; i < adapter->num_queues; i++) { 2119 que = &adapter->queues[i]; 2120 que->adapter = adapter; 2121 que->txr = &adapter->tx_rings[i]; 2122 que->rxr = &adapter->rx_rings[i]; 2123 } 2124 2125 return (0); 2126 2127err_rx_desc: 2128 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2129 ixv_dma_free(adapter, &rxr->rxdma); 2130err_tx_desc: 2131 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2132 ixv_dma_free(adapter, &txr->txdma); 2133 free(adapter->rx_rings, M_DEVBUF); 2134rx_fail: 2135 free(adapter->tx_rings, M_DEVBUF); 2136tx_fail: 2137 free(adapter->queues, M_DEVBUF); 2138fail: 2139 return (error); 2140} 2141 2142 2143/********************************************************************* 2144 * 2145 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2146 * the information needed to transmit a packet on the wire. This is 2147 * called only once at attach, setup is done every reset. 2148 * 2149 **********************************************************************/ 2150static int 2151ixv_allocate_transmit_buffers(struct tx_ring *txr) 2152{ 2153 struct adapter *adapter = txr->adapter; 2154 device_t dev = adapter->dev; 2155 struct ixv_tx_buf *txbuf; 2156 int error, i; 2157 2158 /* 2159 * Setup DMA descriptor areas. 2160 */ 2161 if ((error = bus_dma_tag_create( 2162 bus_get_dma_tag(adapter->dev), /* parent */ 2163 1, 0, /* alignment, bounds */ 2164 BUS_SPACE_MAXADDR, /* lowaddr */ 2165 BUS_SPACE_MAXADDR, /* highaddr */ 2166 NULL, NULL, /* filter, filterarg */ 2167 IXV_TSO_SIZE, /* maxsize */ 2168 32, /* nsegments */ 2169 PAGE_SIZE, /* maxsegsize */ 2170 0, /* flags */ 2171 NULL, /* lockfunc */ 2172 NULL, /* lockfuncarg */ 2173 &txr->txtag))) { 2174 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2175 goto fail; 2176 } 2177 2178 if (!(txr->tx_buffers = 2179 (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) * 2180 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2181 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2182 error = ENOMEM; 2183 goto fail; 2184 } 2185 2186 /* Create the descriptor buffer dma maps */ 2187 txbuf = txr->tx_buffers; 2188 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2189 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2190 if (error != 0) { 2191 device_printf(dev, "Unable to create TX DMA map\n"); 2192 goto fail; 2193 } 2194 } 2195 2196 return 0; 2197fail: 2198 /* We free all, it handles case where we are in the middle */ 2199 ixv_free_transmit_structures(adapter); 2200 return (error); 2201} 2202 2203/********************************************************************* 2204 * 2205 * Initialize a transmit ring. 2206 * 2207 **********************************************************************/ 2208static void 2209ixv_setup_transmit_ring(struct tx_ring *txr) 2210{ 2211 struct adapter *adapter = txr->adapter; 2212 struct ixv_tx_buf *txbuf; 2213 int i; 2214 2215 /* Clear the old ring contents */ 2216 IXV_TX_LOCK(txr); 2217 bzero((void *)txr->tx_base, 2218 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2219 /* Reset indices */ 2220 txr->next_avail_desc = 0; 2221 txr->next_to_clean = 0; 2222 2223 /* Free any existing tx buffers. */ 2224 txbuf = txr->tx_buffers; 2225 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2226 if (txbuf->m_head != NULL) { 2227 bus_dmamap_sync(txr->txtag, txbuf->map, 2228 BUS_DMASYNC_POSTWRITE); 2229 bus_dmamap_unload(txr->txtag, txbuf->map); 2230 m_freem(txbuf->m_head); 2231 txbuf->m_head = NULL; 2232 } 2233 /* Clear the EOP index */ 2234 txbuf->eop_index = -1; 2235 } 2236 2237 /* Set number of descriptors available */ 2238 txr->tx_avail = adapter->num_tx_desc; 2239 2240 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2241 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2242 IXV_TX_UNLOCK(txr); 2243} 2244 2245/********************************************************************* 2246 * 2247 * Initialize all transmit rings. 2248 * 2249 **********************************************************************/ 2250static int 2251ixv_setup_transmit_structures(struct adapter *adapter) 2252{ 2253 struct tx_ring *txr = adapter->tx_rings; 2254 2255 for (int i = 0; i < adapter->num_queues; i++, txr++) 2256 ixv_setup_transmit_ring(txr); 2257 2258 return (0); 2259} 2260 2261/********************************************************************* 2262 * 2263 * Enable transmit unit. 2264 * 2265 **********************************************************************/ 2266static void 2267ixv_initialize_transmit_units(struct adapter *adapter) 2268{ 2269 struct tx_ring *txr = adapter->tx_rings; 2270 struct ixgbe_hw *hw = &adapter->hw; 2271 2272 2273 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2274 u64 tdba = txr->txdma.dma_paddr; 2275 u32 txctrl, txdctl; 2276 2277 /* Set WTHRESH to 8, burst writeback */ 2278 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2279 txdctl |= (8 << 16); 2280 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2281 /* Now enable */ 2282 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2283 txdctl |= IXGBE_TXDCTL_ENABLE; 2284 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2285 2286 /* Set the HW Tx Head and Tail indices */ 2287 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0); 2288 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0); 2289 2290 /* Setup Transmit Descriptor Cmd Settings */ 2291 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2292 txr->watchdog_check = FALSE; 2293 2294 /* Set Ring parameters */ 2295 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 2296 (tdba & 0x00000000ffffffffULL)); 2297 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32)); 2298 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 2299 adapter->num_tx_desc * 2300 sizeof(struct ixgbe_legacy_tx_desc)); 2301 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i)); 2302 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2303 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl); 2304 break; 2305 } 2306 2307 return; 2308} 2309 2310/********************************************************************* 2311 * 2312 * Free all transmit rings. 2313 * 2314 **********************************************************************/ 2315static void 2316ixv_free_transmit_structures(struct adapter *adapter) 2317{ 2318 struct tx_ring *txr = adapter->tx_rings; 2319 2320 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2321 IXV_TX_LOCK(txr); 2322 ixv_free_transmit_buffers(txr); 2323 ixv_dma_free(adapter, &txr->txdma); 2324 IXV_TX_UNLOCK(txr); 2325 IXV_TX_LOCK_DESTROY(txr); 2326 } 2327 free(adapter->tx_rings, M_DEVBUF); 2328} 2329 2330/********************************************************************* 2331 * 2332 * Free transmit ring related data structures. 2333 * 2334 **********************************************************************/ 2335static void 2336ixv_free_transmit_buffers(struct tx_ring *txr) 2337{ 2338 struct adapter *adapter = txr->adapter; 2339 struct ixv_tx_buf *tx_buffer; 2340 int i; 2341 2342 INIT_DEBUGOUT("free_transmit_ring: begin"); 2343 2344 if (txr->tx_buffers == NULL) 2345 return; 2346 2347 tx_buffer = txr->tx_buffers; 2348 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2349 if (tx_buffer->m_head != NULL) { 2350 bus_dmamap_sync(txr->txtag, tx_buffer->map, 2351 BUS_DMASYNC_POSTWRITE); 2352 bus_dmamap_unload(txr->txtag, 2353 tx_buffer->map); 2354 m_freem(tx_buffer->m_head); 2355 tx_buffer->m_head = NULL; 2356 if (tx_buffer->map != NULL) { 2357 bus_dmamap_destroy(txr->txtag, 2358 tx_buffer->map); 2359 tx_buffer->map = NULL; 2360 } 2361 } else if (tx_buffer->map != NULL) { 2362 bus_dmamap_unload(txr->txtag, 2363 tx_buffer->map); 2364 bus_dmamap_destroy(txr->txtag, 2365 tx_buffer->map); 2366 tx_buffer->map = NULL; 2367 } 2368 } 2369#if __FreeBSD_version >= 800000 2370 if (txr->br != NULL) 2371 buf_ring_free(txr->br, M_DEVBUF); 2372#endif 2373 if (txr->tx_buffers != NULL) { 2374 free(txr->tx_buffers, M_DEVBUF); 2375 txr->tx_buffers = NULL; 2376 } 2377 if (txr->txtag != NULL) { 2378 bus_dma_tag_destroy(txr->txtag); 2379 txr->txtag = NULL; 2380 } 2381 return; 2382} 2383 2384/********************************************************************* 2385 * 2386 * Advanced Context Descriptor setup for VLAN or CSUM 2387 * 2388 **********************************************************************/ 2389 2390static bool 2391ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2392{ 2393 struct adapter *adapter = txr->adapter; 2394 struct ixgbe_adv_tx_context_desc *TXD; 2395 struct ixv_tx_buf *tx_buffer; 2396 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2397 struct ether_vlan_header *eh; 2398 struct ip *ip; 2399 struct ip6_hdr *ip6; 2400 int ehdrlen, ip_hlen = 0; 2401 u16 etype; 2402 u8 ipproto = 0; 2403 bool offload = TRUE; 2404 int ctxd = txr->next_avail_desc; 2405 u16 vtag = 0; 2406 2407 2408 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2409 offload = FALSE; 2410 2411 2412 tx_buffer = &txr->tx_buffers[ctxd]; 2413 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2414 2415 /* 2416 ** In advanced descriptors the vlan tag must 2417 ** be placed into the descriptor itself. 2418 */ 2419 if (mp->m_flags & M_VLANTAG) { 2420 vtag = htole16(mp->m_pkthdr.ether_vtag); 2421 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2422 } else if (offload == FALSE) 2423 return FALSE; 2424 2425 /* 2426 * Determine where frame payload starts. 2427 * Jump over vlan headers if already present, 2428 * helpful for QinQ too. 2429 */ 2430 eh = mtod(mp, struct ether_vlan_header *); 2431 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2432 etype = ntohs(eh->evl_proto); 2433 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2434 } else { 2435 etype = ntohs(eh->evl_encap_proto); 2436 ehdrlen = ETHER_HDR_LEN; 2437 } 2438 2439 /* Set the ether header length */ 2440 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2441 2442 switch (etype) { 2443 case ETHERTYPE_IP: 2444 ip = (struct ip *)(mp->m_data + ehdrlen); 2445 ip_hlen = ip->ip_hl << 2; 2446 if (mp->m_len < ehdrlen + ip_hlen) 2447 return (FALSE); 2448 ipproto = ip->ip_p; 2449 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2450 break; 2451 case ETHERTYPE_IPV6: 2452 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2453 ip_hlen = sizeof(struct ip6_hdr); 2454 if (mp->m_len < ehdrlen + ip_hlen) 2455 return (FALSE); 2456 ipproto = ip6->ip6_nxt; 2457 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2458 break; 2459 default: 2460 offload = FALSE; 2461 break; 2462 } 2463 2464 vlan_macip_lens |= ip_hlen; 2465 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2466 2467 switch (ipproto) { 2468 case IPPROTO_TCP: 2469 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2470 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2471 break; 2472 2473 case IPPROTO_UDP: 2474 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2475 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2476 break; 2477 2478#if __FreeBSD_version >= 800000 2479 case IPPROTO_SCTP: 2480 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 2481 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 2482 break; 2483#endif 2484 default: 2485 offload = FALSE; 2486 break; 2487 } 2488 2489 /* Now copy bits into descriptor */ 2490 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2491 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2492 TXD->seqnum_seed = htole32(0); 2493 TXD->mss_l4len_idx = htole32(0); 2494 2495 tx_buffer->m_head = NULL; 2496 tx_buffer->eop_index = -1; 2497 2498 /* We've consumed the first desc, adjust counters */ 2499 if (++ctxd == adapter->num_tx_desc) 2500 ctxd = 0; 2501 txr->next_avail_desc = ctxd; 2502 --txr->tx_avail; 2503 2504 return (offload); 2505} 2506 2507/********************************************************************** 2508 * 2509 * Setup work for hardware segmentation offload (TSO) on 2510 * adapters using advanced tx descriptors 2511 * 2512 **********************************************************************/ 2513static bool 2514ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2515{ 2516 struct adapter *adapter = txr->adapter; 2517 struct ixgbe_adv_tx_context_desc *TXD; 2518 struct ixv_tx_buf *tx_buffer; 2519 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2520 u32 mss_l4len_idx = 0; 2521 u16 vtag = 0; 2522 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2523 struct ether_vlan_header *eh; 2524 struct ip *ip; 2525 struct tcphdr *th; 2526 2527 2528 /* 2529 * Determine where frame payload starts. 2530 * Jump over vlan headers if already present 2531 */ 2532 eh = mtod(mp, struct ether_vlan_header *); 2533 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2534 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2535 else 2536 ehdrlen = ETHER_HDR_LEN; 2537 2538 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2539 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2540 return FALSE; 2541 2542 ctxd = txr->next_avail_desc; 2543 tx_buffer = &txr->tx_buffers[ctxd]; 2544 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2545 2546 ip = (struct ip *)(mp->m_data + ehdrlen); 2547 if (ip->ip_p != IPPROTO_TCP) 2548 return FALSE; /* 0 */ 2549 ip->ip_sum = 0; 2550 ip_hlen = ip->ip_hl << 2; 2551 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2552 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2553 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2554 tcp_hlen = th->th_off << 2; 2555 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2556 2557 /* This is used in the transmit desc in encap */ 2558 *paylen = mp->m_pkthdr.len - hdrlen; 2559 2560 /* VLAN MACLEN IPLEN */ 2561 if (mp->m_flags & M_VLANTAG) { 2562 vtag = htole16(mp->m_pkthdr.ether_vtag); 2563 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2564 } 2565 2566 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2567 vlan_macip_lens |= ip_hlen; 2568 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2569 2570 /* ADV DTYPE TUCMD */ 2571 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2572 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2573 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2574 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2575 2576 2577 /* MSS L4LEN IDX */ 2578 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2579 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2580 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2581 2582 TXD->seqnum_seed = htole32(0); 2583 tx_buffer->m_head = NULL; 2584 tx_buffer->eop_index = -1; 2585 2586 if (++ctxd == adapter->num_tx_desc) 2587 ctxd = 0; 2588 2589 txr->tx_avail--; 2590 txr->next_avail_desc = ctxd; 2591 return TRUE; 2592} 2593 2594 2595/********************************************************************** 2596 * 2597 * Examine each tx_buffer in the used queue. If the hardware is done 2598 * processing the packet then free associated resources. The 2599 * tx_buffer is put back on the free queue. 2600 * 2601 **********************************************************************/ 2602static bool 2603ixv_txeof(struct tx_ring *txr) 2604{ 2605 struct adapter *adapter = txr->adapter; 2606 struct ifnet *ifp = adapter->ifp; 2607 u32 first, last, done; 2608 struct ixv_tx_buf *tx_buffer; 2609 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2610 2611 mtx_assert(&txr->tx_mtx, MA_OWNED); 2612 2613 if (txr->tx_avail == adapter->num_tx_desc) 2614 return FALSE; 2615 2616 first = txr->next_to_clean; 2617 tx_buffer = &txr->tx_buffers[first]; 2618 /* For cleanup we just use legacy struct */ 2619 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2620 last = tx_buffer->eop_index; 2621 if (last == -1) 2622 return FALSE; 2623 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2624 2625 /* 2626 ** Get the index of the first descriptor 2627 ** BEYOND the EOP and call that 'done'. 2628 ** I do this so the comparison in the 2629 ** inner while loop below can be simple 2630 */ 2631 if (++last == adapter->num_tx_desc) last = 0; 2632 done = last; 2633 2634 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2635 BUS_DMASYNC_POSTREAD); 2636 /* 2637 ** Only the EOP descriptor of a packet now has the DD 2638 ** bit set, this is what we look for... 2639 */ 2640 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2641 /* We clean the range of the packet */ 2642 while (first != done) { 2643 tx_desc->upper.data = 0; 2644 tx_desc->lower.data = 0; 2645 tx_desc->buffer_addr = 0; 2646 ++txr->tx_avail; 2647 2648 if (tx_buffer->m_head) { 2649 bus_dmamap_sync(txr->txtag, 2650 tx_buffer->map, 2651 BUS_DMASYNC_POSTWRITE); 2652 bus_dmamap_unload(txr->txtag, 2653 tx_buffer->map); 2654 m_freem(tx_buffer->m_head); 2655 tx_buffer->m_head = NULL; 2656 tx_buffer->map = NULL; 2657 } 2658 tx_buffer->eop_index = -1; 2659 txr->watchdog_time = ticks; 2660 2661 if (++first == adapter->num_tx_desc) 2662 first = 0; 2663 2664 tx_buffer = &txr->tx_buffers[first]; 2665 tx_desc = 2666 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2667 } 2668 ++ifp->if_opackets; 2669 /* See if there is more work now */ 2670 last = tx_buffer->eop_index; 2671 if (last != -1) { 2672 eop_desc = 2673 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2674 /* Get next done point */ 2675 if (++last == adapter->num_tx_desc) last = 0; 2676 done = last; 2677 } else 2678 break; 2679 } 2680 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2681 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2682 2683 txr->next_to_clean = first; 2684 2685 /* 2686 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that 2687 * it is OK to send packets. If there are no pending descriptors, 2688 * clear the timeout. Otherwise, if some descriptors have been freed, 2689 * restart the timeout. 2690 */ 2691 if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) { 2692 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 2693 if (txr->tx_avail == adapter->num_tx_desc) { 2694 txr->watchdog_check = FALSE; 2695 return FALSE; 2696 } 2697 } 2698 2699 return TRUE; 2700} 2701 2702/********************************************************************* 2703 * 2704 * Refresh mbuf buffers for RX descriptor rings 2705 * - now keeps its own state so discards due to resource 2706 * exhaustion are unnecessary, if an mbuf cannot be obtained 2707 * it just returns, keeping its placeholder, thus it can simply 2708 * be recalled to try again. 2709 * 2710 **********************************************************************/ 2711static void 2712ixv_refresh_mbufs(struct rx_ring *rxr, int limit) 2713{ 2714 struct adapter *adapter = rxr->adapter; 2715 bus_dma_segment_t hseg[1]; 2716 bus_dma_segment_t pseg[1]; 2717 struct ixv_rx_buf *rxbuf; 2718 struct mbuf *mh, *mp; 2719 int i, j, nsegs, error; 2720 bool refreshed = FALSE; 2721 2722 i = j = rxr->next_to_refresh; 2723 /* Get the control variable, one beyond refresh point */ 2724 if (++j == adapter->num_rx_desc) 2725 j = 0; 2726 while (j != limit) { 2727 rxbuf = &rxr->rx_buffers[i]; 2728 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 2729 mh = m_gethdr(M_DONTWAIT, MT_DATA); 2730 if (mh == NULL) 2731 goto update; 2732 mh->m_pkthdr.len = mh->m_len = MHLEN; 2733 mh->m_len = MHLEN; 2734 mh->m_flags |= M_PKTHDR; 2735 m_adj(mh, ETHER_ALIGN); 2736 /* Get the memory mapping */ 2737 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2738 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); 2739 if (error != 0) { 2740 printf("GET BUF: dmamap load" 2741 " failure - %d\n", error); 2742 m_free(mh); 2743 goto update; 2744 } 2745 rxbuf->m_head = mh; 2746 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2747 BUS_DMASYNC_PREREAD); 2748 rxr->rx_base[i].read.hdr_addr = 2749 htole64(hseg[0].ds_addr); 2750 } 2751 2752 if (rxbuf->m_pack == NULL) { 2753 mp = m_getjcl(M_DONTWAIT, MT_DATA, 2754 M_PKTHDR, adapter->rx_mbuf_sz); 2755 if (mp == NULL) 2756 goto update; 2757 } else 2758 mp = rxbuf->m_pack; 2759 2760 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2761 /* Get the memory mapping */ 2762 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2763 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); 2764 if (error != 0) { 2765 printf("GET BUF: dmamap load" 2766 " failure - %d\n", error); 2767 m_free(mp); 2768 rxbuf->m_pack = NULL; 2769 goto update; 2770 } 2771 rxbuf->m_pack = mp; 2772 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2773 BUS_DMASYNC_PREREAD); 2774 rxr->rx_base[i].read.pkt_addr = 2775 htole64(pseg[0].ds_addr); 2776 2777 refreshed = TRUE; 2778 rxr->next_to_refresh = i = j; 2779 /* Calculate next index */ 2780 if (++j == adapter->num_rx_desc) 2781 j = 0; 2782 } 2783update: 2784 if (refreshed) /* update tail index */ 2785 IXGBE_WRITE_REG(&adapter->hw, 2786 IXGBE_VFRDT(rxr->me), rxr->next_to_refresh); 2787 return; 2788} 2789 2790/********************************************************************* 2791 * 2792 * Allocate memory for rx_buffer structures. Since we use one 2793 * rx_buffer per received packet, the maximum number of rx_buffer's 2794 * that we'll need is equal to the number of receive descriptors 2795 * that we've allocated. 2796 * 2797 **********************************************************************/ 2798static int 2799ixv_allocate_receive_buffers(struct rx_ring *rxr) 2800{ 2801 struct adapter *adapter = rxr->adapter; 2802 device_t dev = adapter->dev; 2803 struct ixv_rx_buf *rxbuf; 2804 int i, bsize, error; 2805 2806 bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc; 2807 if (!(rxr->rx_buffers = 2808 (struct ixv_rx_buf *) malloc(bsize, 2809 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2810 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 2811 error = ENOMEM; 2812 goto fail; 2813 } 2814 2815 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2816 1, 0, /* alignment, bounds */ 2817 BUS_SPACE_MAXADDR, /* lowaddr */ 2818 BUS_SPACE_MAXADDR, /* highaddr */ 2819 NULL, NULL, /* filter, filterarg */ 2820 MSIZE, /* maxsize */ 2821 1, /* nsegments */ 2822 MSIZE, /* maxsegsize */ 2823 0, /* flags */ 2824 NULL, /* lockfunc */ 2825 NULL, /* lockfuncarg */ 2826 &rxr->htag))) { 2827 device_printf(dev, "Unable to create RX DMA tag\n"); 2828 goto fail; 2829 } 2830 2831 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2832 1, 0, /* alignment, bounds */ 2833 BUS_SPACE_MAXADDR, /* lowaddr */ 2834 BUS_SPACE_MAXADDR, /* highaddr */ 2835 NULL, NULL, /* filter, filterarg */ 2836 MJUMPAGESIZE, /* maxsize */ 2837 1, /* nsegments */ 2838 MJUMPAGESIZE, /* maxsegsize */ 2839 0, /* flags */ 2840 NULL, /* lockfunc */ 2841 NULL, /* lockfuncarg */ 2842 &rxr->ptag))) { 2843 device_printf(dev, "Unable to create RX DMA tag\n"); 2844 goto fail; 2845 } 2846 2847 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 2848 rxbuf = &rxr->rx_buffers[i]; 2849 error = bus_dmamap_create(rxr->htag, 2850 BUS_DMA_NOWAIT, &rxbuf->hmap); 2851 if (error) { 2852 device_printf(dev, "Unable to create RX head map\n"); 2853 goto fail; 2854 } 2855 error = bus_dmamap_create(rxr->ptag, 2856 BUS_DMA_NOWAIT, &rxbuf->pmap); 2857 if (error) { 2858 device_printf(dev, "Unable to create RX pkt map\n"); 2859 goto fail; 2860 } 2861 } 2862 2863 return (0); 2864 2865fail: 2866 /* Frees all, but can handle partial completion */ 2867 ixv_free_receive_structures(adapter); 2868 return (error); 2869} 2870 2871static void 2872ixv_free_receive_ring(struct rx_ring *rxr) 2873{ 2874 struct adapter *adapter; 2875 struct ixv_rx_buf *rxbuf; 2876 int i; 2877 2878 adapter = rxr->adapter; 2879 for (i = 0; i < adapter->num_rx_desc; i++) { 2880 rxbuf = &rxr->rx_buffers[i]; 2881 if (rxbuf->m_head != NULL) { 2882 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2883 BUS_DMASYNC_POSTREAD); 2884 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 2885 rxbuf->m_head->m_flags |= M_PKTHDR; 2886 m_freem(rxbuf->m_head); 2887 } 2888 if (rxbuf->m_pack != NULL) { 2889 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2890 BUS_DMASYNC_POSTREAD); 2891 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 2892 rxbuf->m_pack->m_flags |= M_PKTHDR; 2893 m_freem(rxbuf->m_pack); 2894 } 2895 rxbuf->m_head = NULL; 2896 rxbuf->m_pack = NULL; 2897 } 2898} 2899 2900 2901/********************************************************************* 2902 * 2903 * Initialize a receive ring and its buffers. 2904 * 2905 **********************************************************************/ 2906static int 2907ixv_setup_receive_ring(struct rx_ring *rxr) 2908{ 2909 struct adapter *adapter; 2910 struct ifnet *ifp; 2911 device_t dev; 2912 struct ixv_rx_buf *rxbuf; 2913 bus_dma_segment_t pseg[1], hseg[1]; 2914 struct lro_ctrl *lro = &rxr->lro; 2915 int rsize, nsegs, error = 0; 2916 2917 adapter = rxr->adapter; 2918 ifp = adapter->ifp; 2919 dev = adapter->dev; 2920 2921 /* Clear the ring contents */ 2922 IXV_RX_LOCK(rxr); 2923 rsize = roundup2(adapter->num_rx_desc * 2924 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2925 bzero((void *)rxr->rx_base, rsize); 2926 2927 /* Free current RX buffer structs and their mbufs */ 2928 ixv_free_receive_ring(rxr); 2929 2930 /* Configure header split? */ 2931 if (ixv_header_split) 2932 rxr->hdr_split = TRUE; 2933 2934 /* Now replenish the mbufs */ 2935 for (int j = 0; j != adapter->num_rx_desc; ++j) { 2936 struct mbuf *mh, *mp; 2937 2938 rxbuf = &rxr->rx_buffers[j]; 2939 /* 2940 ** Dont allocate mbufs if not 2941 ** doing header split, its wasteful 2942 */ 2943 if (rxr->hdr_split == FALSE) 2944 goto skip_head; 2945 2946 /* First the header */ 2947 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA); 2948 if (rxbuf->m_head == NULL) { 2949 error = ENOBUFS; 2950 goto fail; 2951 } 2952 m_adj(rxbuf->m_head, ETHER_ALIGN); 2953 mh = rxbuf->m_head; 2954 mh->m_len = mh->m_pkthdr.len = MHLEN; 2955 mh->m_flags |= M_PKTHDR; 2956 /* Get the memory mapping */ 2957 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2958 rxbuf->hmap, rxbuf->m_head, hseg, 2959 &nsegs, BUS_DMA_NOWAIT); 2960 if (error != 0) /* Nothing elegant to do here */ 2961 goto fail; 2962 bus_dmamap_sync(rxr->htag, 2963 rxbuf->hmap, BUS_DMASYNC_PREREAD); 2964 /* Update descriptor */ 2965 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 2966 2967skip_head: 2968 /* Now the payload cluster */ 2969 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, 2970 M_PKTHDR, adapter->rx_mbuf_sz); 2971 if (rxbuf->m_pack == NULL) { 2972 error = ENOBUFS; 2973 goto fail; 2974 } 2975 mp = rxbuf->m_pack; 2976 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2977 /* Get the memory mapping */ 2978 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2979 rxbuf->pmap, mp, pseg, 2980 &nsegs, BUS_DMA_NOWAIT); 2981 if (error != 0) 2982 goto fail; 2983 bus_dmamap_sync(rxr->ptag, 2984 rxbuf->pmap, BUS_DMASYNC_PREREAD); 2985 /* Update descriptor */ 2986 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 2987 } 2988 2989 2990 /* Setup our descriptor indices */ 2991 rxr->next_to_check = 0; 2992 rxr->next_to_refresh = 0; 2993 rxr->lro_enabled = FALSE; 2994 rxr->rx_split_packets = 0; 2995 rxr->rx_bytes = 0; 2996 rxr->discard = FALSE; 2997 2998 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2999 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3000 3001 /* 3002 ** Now set up the LRO interface: 3003 */ 3004 if (ifp->if_capenable & IFCAP_LRO) { 3005 int err = tcp_lro_init(lro); 3006 if (err) { 3007 device_printf(dev, "LRO Initialization failed!\n"); 3008 goto fail; 3009 } 3010 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 3011 rxr->lro_enabled = TRUE; 3012 lro->ifp = adapter->ifp; 3013 } 3014 3015 IXV_RX_UNLOCK(rxr); 3016 return (0); 3017 3018fail: 3019 ixv_free_receive_ring(rxr); 3020 IXV_RX_UNLOCK(rxr); 3021 return (error); 3022} 3023 3024/********************************************************************* 3025 * 3026 * Initialize all receive rings. 3027 * 3028 **********************************************************************/ 3029static int 3030ixv_setup_receive_structures(struct adapter *adapter) 3031{ 3032 struct rx_ring *rxr = adapter->rx_rings; 3033 int j; 3034 3035 for (j = 0; j < adapter->num_queues; j++, rxr++) 3036 if (ixv_setup_receive_ring(rxr)) 3037 goto fail; 3038 3039 return (0); 3040fail: 3041 /* 3042 * Free RX buffers allocated so far, we will only handle 3043 * the rings that completed, the failing case will have 3044 * cleaned up for itself. 'j' failed, so its the terminus. 3045 */ 3046 for (int i = 0; i < j; ++i) { 3047 rxr = &adapter->rx_rings[i]; 3048 ixv_free_receive_ring(rxr); 3049 } 3050 3051 return (ENOBUFS); 3052} 3053 3054/********************************************************************* 3055 * 3056 * Setup receive registers and features. 3057 * 3058 **********************************************************************/ 3059#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3060 3061static void 3062ixv_initialize_receive_units(struct adapter *adapter) 3063{ 3064 struct rx_ring *rxr = adapter->rx_rings; 3065 struct ixgbe_hw *hw = &adapter->hw; 3066 struct ifnet *ifp = adapter->ifp; 3067 u32 bufsz, fctrl, rxcsum, hlreg; 3068 3069 3070 /* Enable broadcasts */ 3071 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3072 fctrl |= IXGBE_FCTRL_BAM; 3073 fctrl |= IXGBE_FCTRL_DPF; 3074 fctrl |= IXGBE_FCTRL_PMCF; 3075 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3076 3077 /* Set for Jumbo Frames? */ 3078 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3079 if (ifp->if_mtu > ETHERMTU) { 3080 hlreg |= IXGBE_HLREG0_JUMBOEN; 3081 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3082 } else { 3083 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 3084 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3085 } 3086 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 3087 3088 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3089 u64 rdba = rxr->rxdma.dma_paddr; 3090 u32 reg, rxdctl; 3091 3092 /* Do the queue enabling first */ 3093 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3094 rxdctl |= IXGBE_RXDCTL_ENABLE; 3095 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 3096 for (int k = 0; k < 10; k++) { 3097 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 3098 IXGBE_RXDCTL_ENABLE) 3099 break; 3100 else 3101 msec_delay(1); 3102 } 3103 wmb(); 3104 3105 /* Setup the Base and Length of the Rx Descriptor Ring */ 3106 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 3107 (rdba & 0x00000000ffffffffULL)); 3108 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 3109 (rdba >> 32)); 3110 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 3111 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3112 3113 /* Set up the SRRCTL register */ 3114 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i)); 3115 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3116 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 3117 reg |= bufsz; 3118 if (rxr->hdr_split) { 3119 /* Use a standard mbuf for the header */ 3120 reg |= ((IXV_RX_HDR << 3121 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3122 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3123 reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3124 } else 3125 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3126 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg); 3127 3128 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 3129 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 3130 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 3131 adapter->num_rx_desc - 1); 3132 } 3133 3134 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3135 3136 if (ifp->if_capenable & IFCAP_RXCSUM) 3137 rxcsum |= IXGBE_RXCSUM_PCSD; 3138 3139 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3140 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3141 3142 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3143 3144 return; 3145} 3146 3147/********************************************************************* 3148 * 3149 * Free all receive rings. 3150 * 3151 **********************************************************************/ 3152static void 3153ixv_free_receive_structures(struct adapter *adapter) 3154{ 3155 struct rx_ring *rxr = adapter->rx_rings; 3156 3157 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3158 struct lro_ctrl *lro = &rxr->lro; 3159 ixv_free_receive_buffers(rxr); 3160 /* Free LRO memory */ 3161 tcp_lro_free(lro); 3162 /* Free the ring memory as well */ 3163 ixv_dma_free(adapter, &rxr->rxdma); 3164 } 3165 3166 free(adapter->rx_rings, M_DEVBUF); 3167} 3168 3169 3170/********************************************************************* 3171 * 3172 * Free receive ring data structures 3173 * 3174 **********************************************************************/ 3175static void 3176ixv_free_receive_buffers(struct rx_ring *rxr) 3177{ 3178 struct adapter *adapter = rxr->adapter; 3179 struct ixv_rx_buf *rxbuf; 3180 3181 INIT_DEBUGOUT("free_receive_structures: begin"); 3182 3183 /* Cleanup any existing buffers */ 3184 if (rxr->rx_buffers != NULL) { 3185 for (int i = 0; i < adapter->num_rx_desc; i++) { 3186 rxbuf = &rxr->rx_buffers[i]; 3187 if (rxbuf->m_head != NULL) { 3188 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3189 BUS_DMASYNC_POSTREAD); 3190 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 3191 rxbuf->m_head->m_flags |= M_PKTHDR; 3192 m_freem(rxbuf->m_head); 3193 } 3194 if (rxbuf->m_pack != NULL) { 3195 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3196 BUS_DMASYNC_POSTREAD); 3197 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 3198 rxbuf->m_pack->m_flags |= M_PKTHDR; 3199 m_freem(rxbuf->m_pack); 3200 } 3201 rxbuf->m_head = NULL; 3202 rxbuf->m_pack = NULL; 3203 if (rxbuf->hmap != NULL) { 3204 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 3205 rxbuf->hmap = NULL; 3206 } 3207 if (rxbuf->pmap != NULL) { 3208 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 3209 rxbuf->pmap = NULL; 3210 } 3211 } 3212 if (rxr->rx_buffers != NULL) { 3213 free(rxr->rx_buffers, M_DEVBUF); 3214 rxr->rx_buffers = NULL; 3215 } 3216 } 3217 3218 if (rxr->htag != NULL) { 3219 bus_dma_tag_destroy(rxr->htag); 3220 rxr->htag = NULL; 3221 } 3222 if (rxr->ptag != NULL) { 3223 bus_dma_tag_destroy(rxr->ptag); 3224 rxr->ptag = NULL; 3225 } 3226 3227 return; 3228} 3229 3230static __inline void 3231ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 3232{ 3233 3234 /* 3235 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 3236 * should be computed by hardware. Also it should not have VLAN tag in 3237 * ethernet header. 3238 */ 3239 if (rxr->lro_enabled && 3240 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 3241 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3242 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 3243 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 3244 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 3245 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 3246 /* 3247 * Send to the stack if: 3248 ** - LRO not enabled, or 3249 ** - no LRO resources, or 3250 ** - lro enqueue fails 3251 */ 3252 if (rxr->lro.lro_cnt != 0) 3253 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 3254 return; 3255 } 3256 IXV_RX_UNLOCK(rxr); 3257 (*ifp->if_input)(ifp, m); 3258 IXV_RX_LOCK(rxr); 3259} 3260 3261static __inline void 3262ixv_rx_discard(struct rx_ring *rxr, int i) 3263{ 3264 struct ixv_rx_buf *rbuf; 3265 3266 rbuf = &rxr->rx_buffers[i]; 3267 3268 if (rbuf->fmp != NULL) {/* Partial chain ? */ 3269 rbuf->fmp->m_flags |= M_PKTHDR; 3270 m_freem(rbuf->fmp); 3271 rbuf->fmp = NULL; 3272 } 3273 3274 /* 3275 ** With advanced descriptors the writeback 3276 ** clobbers the buffer addrs, so its easier 3277 ** to just free the existing mbufs and take 3278 ** the normal refresh path to get new buffers 3279 ** and mapping. 3280 */ 3281 if (rbuf->m_head) { 3282 m_free(rbuf->m_head); 3283 rbuf->m_head = NULL; 3284 } 3285 3286 if (rbuf->m_pack) { 3287 m_free(rbuf->m_pack); 3288 rbuf->m_pack = NULL; 3289 } 3290 3291 return; 3292} 3293 3294 3295/********************************************************************* 3296 * 3297 * This routine executes in interrupt context. It replenishes 3298 * the mbufs in the descriptor and sends data which has been 3299 * dma'ed into host memory to upper layer. 3300 * 3301 * We loop at most count times if count is > 0, or until done if 3302 * count < 0. 3303 * 3304 * Return TRUE for more work, FALSE for all clean. 3305 *********************************************************************/ 3306static bool 3307ixv_rxeof(struct ix_queue *que, int count) 3308{ 3309 struct adapter *adapter = que->adapter; 3310 struct rx_ring *rxr = que->rxr; 3311 struct ifnet *ifp = adapter->ifp; 3312 struct lro_ctrl *lro = &rxr->lro; 3313 struct lro_entry *queued; 3314 int i, nextp, processed = 0; 3315 u32 staterr = 0; 3316 union ixgbe_adv_rx_desc *cur; 3317 struct ixv_rx_buf *rbuf, *nbuf; 3318 3319 IXV_RX_LOCK(rxr); 3320 3321 for (i = rxr->next_to_check; count != 0;) { 3322 struct mbuf *sendmp, *mh, *mp; 3323 u32 rsc, ptype; 3324 u16 hlen, plen, hdr, vtag; 3325 bool eop; 3326 3327 /* Sync the ring. */ 3328 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3329 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3330 3331 cur = &rxr->rx_base[i]; 3332 staterr = le32toh(cur->wb.upper.status_error); 3333 3334 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 3335 break; 3336 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 3337 break; 3338 3339 count--; 3340 sendmp = NULL; 3341 nbuf = NULL; 3342 rsc = 0; 3343 cur->wb.upper.status_error = 0; 3344 rbuf = &rxr->rx_buffers[i]; 3345 mh = rbuf->m_head; 3346 mp = rbuf->m_pack; 3347 3348 plen = le16toh(cur->wb.upper.length); 3349 ptype = le32toh(cur->wb.lower.lo_dword.data) & 3350 IXGBE_RXDADV_PKTTYPE_MASK; 3351 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 3352 vtag = le16toh(cur->wb.upper.vlan); 3353 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3354 3355 /* Make sure all parts of a bad packet are discarded */ 3356 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 3357 (rxr->discard)) { 3358 ifp->if_ierrors++; 3359 rxr->rx_discarded++; 3360 if (!eop) 3361 rxr->discard = TRUE; 3362 else 3363 rxr->discard = FALSE; 3364 ixv_rx_discard(rxr, i); 3365 goto next_desc; 3366 } 3367 3368 if (!eop) { 3369 nextp = i + 1; 3370 if (nextp == adapter->num_rx_desc) 3371 nextp = 0; 3372 nbuf = &rxr->rx_buffers[nextp]; 3373 prefetch(nbuf); 3374 } 3375 /* 3376 ** The header mbuf is ONLY used when header 3377 ** split is enabled, otherwise we get normal 3378 ** behavior, ie, both header and payload 3379 ** are DMA'd into the payload buffer. 3380 ** 3381 ** Rather than using the fmp/lmp global pointers 3382 ** we now keep the head of a packet chain in the 3383 ** buffer struct and pass this along from one 3384 ** descriptor to the next, until we get EOP. 3385 */ 3386 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 3387 /* This must be an initial descriptor */ 3388 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3389 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3390 if (hlen > IXV_RX_HDR) 3391 hlen = IXV_RX_HDR; 3392 mh->m_len = hlen; 3393 mh->m_flags |= M_PKTHDR; 3394 mh->m_next = NULL; 3395 mh->m_pkthdr.len = mh->m_len; 3396 /* Null buf pointer so it is refreshed */ 3397 rbuf->m_head = NULL; 3398 /* 3399 ** Check the payload length, this 3400 ** could be zero if its a small 3401 ** packet. 3402 */ 3403 if (plen > 0) { 3404 mp->m_len = plen; 3405 mp->m_next = NULL; 3406 mp->m_flags &= ~M_PKTHDR; 3407 mh->m_next = mp; 3408 mh->m_pkthdr.len += mp->m_len; 3409 /* Null buf pointer so it is refreshed */ 3410 rbuf->m_pack = NULL; 3411 rxr->rx_split_packets++; 3412 } 3413 /* 3414 ** Now create the forward 3415 ** chain so when complete 3416 ** we wont have to. 3417 */ 3418 if (eop == 0) { 3419 /* stash the chain head */ 3420 nbuf->fmp = mh; 3421 /* Make forward chain */ 3422 if (plen) 3423 mp->m_next = nbuf->m_pack; 3424 else 3425 mh->m_next = nbuf->m_pack; 3426 } else { 3427 /* Singlet, prepare to send */ 3428 sendmp = mh; 3429 if ((adapter->num_vlans) && 3430 (staterr & IXGBE_RXD_STAT_VP)) { 3431 sendmp->m_pkthdr.ether_vtag = vtag; 3432 sendmp->m_flags |= M_VLANTAG; 3433 } 3434 } 3435 } else { 3436 /* 3437 ** Either no header split, or a 3438 ** secondary piece of a fragmented 3439 ** split packet. 3440 */ 3441 mp->m_len = plen; 3442 /* 3443 ** See if there is a stored head 3444 ** that determines what we are 3445 */ 3446 sendmp = rbuf->fmp; 3447 rbuf->m_pack = rbuf->fmp = NULL; 3448 3449 if (sendmp != NULL) /* secondary frag */ 3450 sendmp->m_pkthdr.len += mp->m_len; 3451 else { 3452 /* first desc of a non-ps chain */ 3453 sendmp = mp; 3454 sendmp->m_flags |= M_PKTHDR; 3455 sendmp->m_pkthdr.len = mp->m_len; 3456 if (staterr & IXGBE_RXD_STAT_VP) { 3457 sendmp->m_pkthdr.ether_vtag = vtag; 3458 sendmp->m_flags |= M_VLANTAG; 3459 } 3460 } 3461 /* Pass the head pointer on */ 3462 if (eop == 0) { 3463 nbuf->fmp = sendmp; 3464 sendmp = NULL; 3465 mp->m_next = nbuf->m_pack; 3466 } 3467 } 3468 ++processed; 3469 /* Sending this frame? */ 3470 if (eop) { 3471 sendmp->m_pkthdr.rcvif = ifp; 3472 ifp->if_ipackets++; 3473 rxr->rx_packets++; 3474 /* capture data for AIM */ 3475 rxr->bytes += sendmp->m_pkthdr.len; 3476 rxr->rx_bytes += sendmp->m_pkthdr.len; 3477 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 3478 ixv_rx_checksum(staterr, sendmp, ptype); 3479#if __FreeBSD_version >= 800000 3480 sendmp->m_pkthdr.flowid = que->msix; 3481 sendmp->m_flags |= M_FLOWID; 3482#endif 3483 } 3484next_desc: 3485 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3486 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3487 3488 /* Advance our pointers to the next descriptor. */ 3489 if (++i == adapter->num_rx_desc) 3490 i = 0; 3491 3492 /* Now send to the stack or do LRO */ 3493 if (sendmp != NULL) 3494 ixv_rx_input(rxr, ifp, sendmp, ptype); 3495 3496 /* Every 8 descriptors we go to refresh mbufs */ 3497 if (processed == 8) { 3498 ixv_refresh_mbufs(rxr, i); 3499 processed = 0; 3500 } 3501 } 3502 3503 /* Refresh any remaining buf structs */ 3504 if (ixv_rx_unrefreshed(rxr)) 3505 ixv_refresh_mbufs(rxr, i); 3506 3507 rxr->next_to_check = i; 3508 3509 /* 3510 * Flush any outstanding LRO work 3511 */ 3512 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 3513 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3514 tcp_lro_flush(lro, queued); 3515 } 3516 3517 IXV_RX_UNLOCK(rxr); 3518 3519 /* 3520 ** We still have cleaning to do? 3521 ** Schedule another interrupt if so. 3522 */ 3523 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 3524 ixv_rearm_queues(adapter, (u64)(1 << que->msix)); 3525 return (TRUE); 3526 } 3527 3528 return (FALSE); 3529} 3530 3531 3532/********************************************************************* 3533 * 3534 * Verify that the hardware indicated that the checksum is valid. 3535 * Inform the stack about the status of checksum so that stack 3536 * doesn't spend time verifying the checksum. 3537 * 3538 *********************************************************************/ 3539static void 3540ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype) 3541{ 3542 u16 status = (u16) staterr; 3543 u8 errors = (u8) (staterr >> 24); 3544 bool sctp = FALSE; 3545 3546 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3547 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 3548 sctp = TRUE; 3549 3550 if (status & IXGBE_RXD_STAT_IPCS) { 3551 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3552 /* IP Checksum Good */ 3553 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 3554 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 3555 3556 } else 3557 mp->m_pkthdr.csum_flags = 0; 3558 } 3559 if (status & IXGBE_RXD_STAT_L4CS) { 3560 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3561#if __FreeBSD_version >= 800000 3562 if (sctp) 3563 type = CSUM_SCTP_VALID; 3564#endif 3565 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3566 mp->m_pkthdr.csum_flags |= type; 3567 if (!sctp) 3568 mp->m_pkthdr.csum_data = htons(0xffff); 3569 } 3570 } 3571 return; 3572} 3573 3574static void 3575ixv_setup_vlan_support(struct adapter *adapter) 3576{ 3577 struct ixgbe_hw *hw = &adapter->hw; 3578 u32 ctrl, vid, vfta, retry; 3579 3580 3581 /* 3582 ** We get here thru init_locked, meaning 3583 ** a soft reset, this has already cleared 3584 ** the VFTA and other state, so if there 3585 ** have been no vlan's registered do nothing. 3586 */ 3587 if (adapter->num_vlans == 0) 3588 return; 3589 3590 /* Enable the queues */ 3591 for (int i = 0; i < adapter->num_queues; i++) { 3592 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3593 ctrl |= IXGBE_RXDCTL_VME; 3594 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 3595 } 3596 3597 /* 3598 ** A soft reset zero's out the VFTA, so 3599 ** we need to repopulate it now. 3600 */ 3601 for (int i = 0; i < VFTA_SIZE; i++) { 3602 if (ixv_shadow_vfta[i] == 0) 3603 continue; 3604 vfta = ixv_shadow_vfta[i]; 3605 /* 3606 ** Reconstruct the vlan id's 3607 ** based on the bits set in each 3608 ** of the array ints. 3609 */ 3610 for ( int j = 0; j < 32; j++) { 3611 retry = 0; 3612 if ((vfta & (1 << j)) == 0) 3613 continue; 3614 vid = (i * 32) + j; 3615 /* Call the shared code mailbox routine */ 3616 while (ixgbe_set_vfta(hw, vid, 0, TRUE)) { 3617 if (++retry > 5) 3618 break; 3619 } 3620 } 3621 } 3622} 3623 3624/* 3625** This routine is run via an vlan config EVENT, 3626** it enables us to use the HW Filter table since 3627** we can get the vlan id. This just creates the 3628** entry in the soft version of the VFTA, init will 3629** repopulate the real table. 3630*/ 3631static void 3632ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3633{ 3634 struct adapter *adapter = ifp->if_softc; 3635 u16 index, bit; 3636 3637 if (ifp->if_softc != arg) /* Not our event */ 3638 return; 3639 3640 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3641 return; 3642 3643 IXV_CORE_LOCK(adapter); 3644 index = (vtag >> 5) & 0x7F; 3645 bit = vtag & 0x1F; 3646 ixv_shadow_vfta[index] |= (1 << bit); 3647 ++adapter->num_vlans; 3648 /* Re-init to load the changes */ 3649 ixv_init_locked(adapter); 3650 IXV_CORE_UNLOCK(adapter); 3651} 3652 3653/* 3654** This routine is run via an vlan 3655** unconfig EVENT, remove our entry 3656** in the soft vfta. 3657*/ 3658static void 3659ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3660{ 3661 struct adapter *adapter = ifp->if_softc; 3662 u16 index, bit; 3663 3664 if (ifp->if_softc != arg) 3665 return; 3666 3667 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3668 return; 3669 3670 IXV_CORE_LOCK(adapter); 3671 index = (vtag >> 5) & 0x7F; 3672 bit = vtag & 0x1F; 3673 ixv_shadow_vfta[index] &= ~(1 << bit); 3674 --adapter->num_vlans; 3675 /* Re-init to load the changes */ 3676 ixv_init_locked(adapter); 3677 IXV_CORE_UNLOCK(adapter); 3678} 3679 3680static void 3681ixv_enable_intr(struct adapter *adapter) 3682{ 3683 struct ixgbe_hw *hw = &adapter->hw; 3684 struct ix_queue *que = adapter->queues; 3685 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3686 3687 3688 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 3689 3690 mask = IXGBE_EIMS_ENABLE_MASK; 3691 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 3692 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 3693 3694 for (int i = 0; i < adapter->num_queues; i++, que++) 3695 ixv_enable_queue(adapter, que->msix); 3696 3697 IXGBE_WRITE_FLUSH(hw); 3698 3699 return; 3700} 3701 3702static void 3703ixv_disable_intr(struct adapter *adapter) 3704{ 3705 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 3706 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0); 3707 IXGBE_WRITE_FLUSH(&adapter->hw); 3708 return; 3709} 3710 3711/* 3712** Setup the correct IVAR register for a particular MSIX interrupt 3713** - entry is the register array entry 3714** - vector is the MSIX vector for this queue 3715** - type is RX/TX/MISC 3716*/ 3717static void 3718ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 3719{ 3720 struct ixgbe_hw *hw = &adapter->hw; 3721 u32 ivar, index; 3722 3723 vector |= IXGBE_IVAR_ALLOC_VAL; 3724 3725 if (type == -1) { /* MISC IVAR */ 3726 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 3727 ivar &= ~0xFF; 3728 ivar |= vector; 3729 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 3730 } else { /* RX/TX IVARS */ 3731 index = (16 * (entry & 1)) + (8 * type); 3732 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 3733 ivar &= ~(0xFF << index); 3734 ivar |= (vector << index); 3735 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 3736 } 3737} 3738 3739static void 3740ixv_configure_ivars(struct adapter *adapter) 3741{ 3742 struct ix_queue *que = adapter->queues; 3743 3744 for (int i = 0; i < adapter->num_queues; i++, que++) { 3745 /* First the RX queue entry */ 3746 ixv_set_ivar(adapter, i, que->msix, 0); 3747 /* ... and the TX */ 3748 ixv_set_ivar(adapter, i, que->msix, 1); 3749 /* Set an initial value in EITR */ 3750 IXGBE_WRITE_REG(&adapter->hw, 3751 IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT); 3752 } 3753 3754 /* For the Link interrupt */ 3755 ixv_set_ivar(adapter, 1, adapter->mbxvec, -1); 3756} 3757 3758 3759/* 3760** Tasklet handler for MSIX MBX interrupts 3761** - do outside interrupt since it might sleep 3762*/ 3763static void 3764ixv_handle_mbx(void *context, int pending) 3765{ 3766 struct adapter *adapter = context; 3767 3768 ixgbe_check_link(&adapter->hw, 3769 &adapter->link_speed, &adapter->link_up, 0); 3770 ixv_update_link_status(adapter); 3771} 3772 3773/* 3774** The VF stats registers never have a truely virgin 3775** starting point, so this routine tries to make an 3776** artificial one, marking ground zero on attach as 3777** it were. 3778*/ 3779static void 3780ixv_save_stats(struct adapter *adapter) 3781{ 3782 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 3783 adapter->stats.saved_reset_vfgprc += 3784 adapter->stats.vfgprc - adapter->stats.base_vfgprc; 3785 adapter->stats.saved_reset_vfgptc += 3786 adapter->stats.vfgptc - adapter->stats.base_vfgptc; 3787 adapter->stats.saved_reset_vfgorc += 3788 adapter->stats.vfgorc - adapter->stats.base_vfgorc; 3789 adapter->stats.saved_reset_vfgotc += 3790 adapter->stats.vfgotc - adapter->stats.base_vfgotc; 3791 adapter->stats.saved_reset_vfmprc += 3792 adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3793 } 3794} 3795 3796static void 3797ixv_init_stats(struct adapter *adapter) 3798{ 3799 struct ixgbe_hw *hw = &adapter->hw; 3800 3801 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 3802 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 3803 adapter->stats.last_vfgorc |= 3804 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 3805 3806 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 3807 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 3808 adapter->stats.last_vfgotc |= 3809 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 3810 3811 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 3812 3813 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 3814 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 3815 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 3816 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 3817 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 3818} 3819 3820#define UPDATE_STAT_32(reg, last, count) \ 3821{ \ 3822 u32 current = IXGBE_READ_REG(hw, reg); \ 3823 if (current < last) \ 3824 count += 0x100000000LL; \ 3825 last = current; \ 3826 count &= 0xFFFFFFFF00000000LL; \ 3827 count |= current; \ 3828} 3829 3830#define UPDATE_STAT_36(lsb, msb, last, count) \ 3831{ \ 3832 u64 cur_lsb = IXGBE_READ_REG(hw, lsb); \ 3833 u64 cur_msb = IXGBE_READ_REG(hw, msb); \ 3834 u64 current = ((cur_msb << 32) | cur_lsb); \ 3835 if (current < last) \ 3836 count += 0x1000000000LL; \ 3837 last = current; \ 3838 count &= 0xFFFFFFF000000000LL; \ 3839 count |= current; \ 3840} 3841 3842/* 3843** ixv_update_stats - Update the board statistics counters. 3844*/ 3845void 3846ixv_update_stats(struct adapter *adapter) 3847{ 3848 struct ixgbe_hw *hw = &adapter->hw; 3849 3850 UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3851 adapter->stats.vfgprc); 3852 UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3853 adapter->stats.vfgptc); 3854 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3855 adapter->stats.last_vfgorc, adapter->stats.vfgorc); 3856 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3857 adapter->stats.last_vfgotc, adapter->stats.vfgotc); 3858 UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3859 adapter->stats.vfmprc); 3860} 3861 3862/********************************************************************** 3863 * 3864 * This routine is called only when ixgbe_display_debug_stats is enabled. 3865 * This routine provides a way to take a look at important statistics 3866 * maintained by the driver and hardware. 3867 * 3868 **********************************************************************/ 3869static void 3870ixv_print_hw_stats(struct adapter * adapter) 3871{ 3872 device_t dev = adapter->dev; 3873 3874 device_printf(dev,"Std Mbuf Failed = %lu\n", 3875 adapter->mbuf_defrag_failed); 3876 device_printf(dev,"Driver dropped packets = %lu\n", 3877 adapter->dropped_pkts); 3878 device_printf(dev, "watchdog timeouts = %ld\n", 3879 adapter->watchdog_events); 3880 3881 device_printf(dev,"Good Packets Rcvd = %llu\n", 3882 (long long)adapter->stats.vfgprc); 3883 device_printf(dev,"Good Packets Xmtd = %llu\n", 3884 (long long)adapter->stats.vfgptc); 3885 device_printf(dev,"TSO Transmissions = %lu\n", 3886 adapter->tso_tx); 3887 3888} 3889 3890/********************************************************************** 3891 * 3892 * This routine is called only when em_display_debug_stats is enabled. 3893 * This routine provides a way to take a look at important statistics 3894 * maintained by the driver and hardware. 3895 * 3896 **********************************************************************/ 3897static void 3898ixv_print_debug_info(struct adapter *adapter) 3899{ 3900 device_t dev = adapter->dev; 3901 struct ixgbe_hw *hw = &adapter->hw; 3902 struct ix_queue *que = adapter->queues; 3903 struct rx_ring *rxr; 3904 struct tx_ring *txr; 3905 struct lro_ctrl *lro; 3906 3907 device_printf(dev,"Error Byte Count = %u \n", 3908 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 3909 3910 for (int i = 0; i < adapter->num_queues; i++, que++) { 3911 txr = que->txr; 3912 rxr = que->rxr; 3913 lro = &rxr->lro; 3914 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n", 3915 que->msix, (long)que->irqs); 3916 device_printf(dev,"RX(%d) Packets Received: %lld\n", 3917 rxr->me, (long long)rxr->rx_packets); 3918 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 3919 rxr->me, (long long)rxr->rx_split_packets); 3920 device_printf(dev,"RX(%d) Bytes Received: %lu\n", 3921 rxr->me, (long)rxr->rx_bytes); 3922 device_printf(dev,"RX(%d) LRO Queued= %d\n", 3923 rxr->me, lro->lro_queued); 3924 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 3925 rxr->me, lro->lro_flushed); 3926 device_printf(dev,"TX(%d) Packets Sent: %lu\n", 3927 txr->me, (long)txr->total_packets); 3928 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 3929 txr->me, (long)txr->no_desc_avail); 3930 } 3931 3932 device_printf(dev,"MBX IRQ Handled: %lu\n", 3933 (long)adapter->mbx_irq); 3934 return; 3935} 3936 3937static int 3938ixv_sysctl_stats(SYSCTL_HANDLER_ARGS) 3939{ 3940 int error; 3941 int result; 3942 struct adapter *adapter; 3943 3944 result = -1; 3945 error = sysctl_handle_int(oidp, &result, 0, req); 3946 3947 if (error || !req->newptr) 3948 return (error); 3949 3950 if (result == 1) { 3951 adapter = (struct adapter *) arg1; 3952 ixv_print_hw_stats(adapter); 3953 } 3954 return error; 3955} 3956 3957static int 3958ixv_sysctl_debug(SYSCTL_HANDLER_ARGS) 3959{ 3960 int error, result; 3961 struct adapter *adapter; 3962 3963 result = -1; 3964 error = sysctl_handle_int(oidp, &result, 0, req); 3965 3966 if (error || !req->newptr) 3967 return (error); 3968 3969 if (result == 1) { 3970 adapter = (struct adapter *) arg1; 3971 ixv_print_debug_info(adapter); 3972 } 3973 return error; 3974} 3975 3976/* 3977** Set flow control using sysctl: 3978** Flow control values: 3979** 0 - off 3980** 1 - rx pause 3981** 2 - tx pause 3982** 3 - full 3983*/ 3984static int 3985ixv_set_flowcntl(SYSCTL_HANDLER_ARGS) 3986{ 3987 int error; 3988 struct adapter *adapter; 3989 3990 error = sysctl_handle_int(oidp, &ixv_flow_control, 0, req); 3991 3992 if (error) 3993 return (error); 3994 3995 adapter = (struct adapter *) arg1; 3996 switch (ixv_flow_control) { 3997 case ixgbe_fc_rx_pause: 3998 case ixgbe_fc_tx_pause: 3999 case ixgbe_fc_full: 4000 adapter->hw.fc.requested_mode = ixv_flow_control; 4001 break; 4002 case ixgbe_fc_none: 4003 default: 4004 adapter->hw.fc.requested_mode = ixgbe_fc_none; 4005 } 4006 4007 ixgbe_fc_enable(&adapter->hw); 4008 return error; 4009} 4010 4011static void 4012ixv_add_rx_process_limit(struct adapter *adapter, const char *name, 4013 const char *description, int *limit, int value) 4014{ 4015 *limit = value; 4016 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 4017 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 4018 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 4019} 4020 4021