ixv.c revision 253374
1/****************************************************************************** 2 3 Copyright (c) 2001-2013, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32******************************************************************************/ 33/*$FreeBSD: stable/9/sys/dev/ixgbe/ixv.c 253374 2013-07-15 23:27:48Z jfv $*/ 34 35#include "opt_inet.h" 36#include "opt_inet6.h" 37#include "ixv.h" 38 39/********************************************************************* 40 * Driver version 41 *********************************************************************/ 42char ixv_driver_version[] = "1.1.4"; 43 44/********************************************************************* 45 * PCI Device ID Table 46 * 47 * Used by probe to select devices to load on 48 * Last field stores an index into ixv_strings 49 * Last entry must be all 0s 50 * 51 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index } 52 *********************************************************************/ 53 54static ixv_vendor_info_t ixv_vendor_info_array[] = 55{ 56 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF, 0, 0, 0}, 57 {IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF, 0, 0, 0}, 58 /* required last entry */ 59 {0, 0, 0, 0, 0} 60}; 61 62/********************************************************************* 63 * Table of branding strings 64 *********************************************************************/ 65 66static char *ixv_strings[] = { 67 "Intel(R) PRO/10GbE Virtual Function Network Driver" 68}; 69 70/********************************************************************* 71 * Function prototypes 72 *********************************************************************/ 73static int ixv_probe(device_t); 74static int ixv_attach(device_t); 75static int ixv_detach(device_t); 76static int ixv_shutdown(device_t); 77#if __FreeBSD_version < 800000 78static void ixv_start(struct ifnet *); 79static void ixv_start_locked(struct tx_ring *, struct ifnet *); 80#else 81static int ixv_mq_start(struct ifnet *, struct mbuf *); 82static int ixv_mq_start_locked(struct ifnet *, 83 struct tx_ring *, struct mbuf *); 84static void ixv_qflush(struct ifnet *); 85#endif 86static int ixv_ioctl(struct ifnet *, u_long, caddr_t); 87static void ixv_init(void *); 88static void ixv_init_locked(struct adapter *); 89static void ixv_stop(void *); 90static void ixv_media_status(struct ifnet *, struct ifmediareq *); 91static int ixv_media_change(struct ifnet *); 92static void ixv_identify_hardware(struct adapter *); 93static int ixv_allocate_pci_resources(struct adapter *); 94static int ixv_allocate_msix(struct adapter *); 95static int ixv_allocate_queues(struct adapter *); 96static int ixv_setup_msix(struct adapter *); 97static void ixv_free_pci_resources(struct adapter *); 98static void ixv_local_timer(void *); 99static void ixv_setup_interface(device_t, struct adapter *); 100static void ixv_config_link(struct adapter *); 101 102static int ixv_allocate_transmit_buffers(struct tx_ring *); 103static int ixv_setup_transmit_structures(struct adapter *); 104static void ixv_setup_transmit_ring(struct tx_ring *); 105static void ixv_initialize_transmit_units(struct adapter *); 106static void ixv_free_transmit_structures(struct adapter *); 107static void ixv_free_transmit_buffers(struct tx_ring *); 108 109static int ixv_allocate_receive_buffers(struct rx_ring *); 110static int ixv_setup_receive_structures(struct adapter *); 111static int ixv_setup_receive_ring(struct rx_ring *); 112static void ixv_initialize_receive_units(struct adapter *); 113static void ixv_free_receive_structures(struct adapter *); 114static void ixv_free_receive_buffers(struct rx_ring *); 115 116static void ixv_enable_intr(struct adapter *); 117static void ixv_disable_intr(struct adapter *); 118static bool ixv_txeof(struct tx_ring *); 119static bool ixv_rxeof(struct ix_queue *, int); 120static void ixv_rx_checksum(u32, struct mbuf *, u32); 121static void ixv_set_multi(struct adapter *); 122static void ixv_update_link_status(struct adapter *); 123static void ixv_refresh_mbufs(struct rx_ring *, int); 124static int ixv_xmit(struct tx_ring *, struct mbuf **); 125static int ixv_sysctl_stats(SYSCTL_HANDLER_ARGS); 126static int ixv_sysctl_debug(SYSCTL_HANDLER_ARGS); 127static int ixv_set_flowcntl(SYSCTL_HANDLER_ARGS); 128static int ixv_dma_malloc(struct adapter *, bus_size_t, 129 struct ixv_dma_alloc *, int); 130static void ixv_dma_free(struct adapter *, struct ixv_dma_alloc *); 131static void ixv_add_rx_process_limit(struct adapter *, const char *, 132 const char *, int *, int); 133static bool ixv_tx_ctx_setup(struct tx_ring *, struct mbuf *); 134static bool ixv_tso_setup(struct tx_ring *, struct mbuf *, u32 *); 135static void ixv_set_ivar(struct adapter *, u8, u8, s8); 136static void ixv_configure_ivars(struct adapter *); 137static u8 * ixv_mc_array_itr(struct ixgbe_hw *, u8 **, u32 *); 138 139static void ixv_setup_vlan_support(struct adapter *); 140static void ixv_register_vlan(void *, struct ifnet *, u16); 141static void ixv_unregister_vlan(void *, struct ifnet *, u16); 142 143static void ixv_save_stats(struct adapter *); 144static void ixv_init_stats(struct adapter *); 145static void ixv_update_stats(struct adapter *); 146 147static __inline void ixv_rx_discard(struct rx_ring *, int); 148static __inline void ixv_rx_input(struct rx_ring *, struct ifnet *, 149 struct mbuf *, u32); 150 151/* The MSI/X Interrupt handlers */ 152static void ixv_msix_que(void *); 153static void ixv_msix_mbx(void *); 154 155/* Deferred interrupt tasklets */ 156static void ixv_handle_que(void *, int); 157static void ixv_handle_mbx(void *, int); 158 159/********************************************************************* 160 * FreeBSD Device Interface Entry Points 161 *********************************************************************/ 162 163static device_method_t ixv_methods[] = { 164 /* Device interface */ 165 DEVMETHOD(device_probe, ixv_probe), 166 DEVMETHOD(device_attach, ixv_attach), 167 DEVMETHOD(device_detach, ixv_detach), 168 DEVMETHOD(device_shutdown, ixv_shutdown), 169 DEVMETHOD_END 170}; 171 172static driver_t ixv_driver = { 173 "ix", ixv_methods, sizeof(struct adapter), 174}; 175 176extern devclass_t ixgbe_devclass; 177DRIVER_MODULE(ixv, pci, ixv_driver, ixgbe_devclass, 0, 0); 178MODULE_DEPEND(ixv, pci, 1, 1, 1); 179MODULE_DEPEND(ixv, ether, 1, 1, 1); 180 181/* 182** TUNEABLE PARAMETERS: 183*/ 184 185/* 186** AIM: Adaptive Interrupt Moderation 187** which means that the interrupt rate 188** is varied over time based on the 189** traffic for that interrupt vector 190*/ 191static int ixv_enable_aim = FALSE; 192TUNABLE_INT("hw.ixv.enable_aim", &ixv_enable_aim); 193 194/* How many packets rxeof tries to clean at a time */ 195static int ixv_rx_process_limit = 128; 196TUNABLE_INT("hw.ixv.rx_process_limit", &ixv_rx_process_limit); 197 198/* Flow control setting, default to full */ 199static int ixv_flow_control = ixgbe_fc_full; 200TUNABLE_INT("hw.ixv.flow_control", &ixv_flow_control); 201 202/* 203 * Header split: this causes the hardware to DMA 204 * the header into a seperate mbuf from the payload, 205 * it can be a performance win in some workloads, but 206 * in others it actually hurts, its off by default. 207 */ 208static int ixv_header_split = FALSE; 209TUNABLE_INT("hw.ixv.hdr_split", &ixv_header_split); 210 211/* 212** Number of TX descriptors per ring, 213** setting higher than RX as this seems 214** the better performing choice. 215*/ 216static int ixv_txd = DEFAULT_TXD; 217TUNABLE_INT("hw.ixv.txd", &ixv_txd); 218 219/* Number of RX descriptors per ring */ 220static int ixv_rxd = DEFAULT_RXD; 221TUNABLE_INT("hw.ixv.rxd", &ixv_rxd); 222 223/* 224** Shadow VFTA table, this is needed because 225** the real filter table gets cleared during 226** a soft reset and we need to repopulate it. 227*/ 228static u32 ixv_shadow_vfta[VFTA_SIZE]; 229 230/********************************************************************* 231 * Device identification routine 232 * 233 * ixv_probe determines if the driver should be loaded on 234 * adapter based on PCI vendor/device id of the adapter. 235 * 236 * return BUS_PROBE_DEFAULT on success, positive on failure 237 *********************************************************************/ 238 239static int 240ixv_probe(device_t dev) 241{ 242 ixv_vendor_info_t *ent; 243 244 u16 pci_vendor_id = 0; 245 u16 pci_device_id = 0; 246 u16 pci_subvendor_id = 0; 247 u16 pci_subdevice_id = 0; 248 char adapter_name[256]; 249 250 251 pci_vendor_id = pci_get_vendor(dev); 252 if (pci_vendor_id != IXGBE_INTEL_VENDOR_ID) 253 return (ENXIO); 254 255 pci_device_id = pci_get_device(dev); 256 pci_subvendor_id = pci_get_subvendor(dev); 257 pci_subdevice_id = pci_get_subdevice(dev); 258 259 ent = ixv_vendor_info_array; 260 while (ent->vendor_id != 0) { 261 if ((pci_vendor_id == ent->vendor_id) && 262 (pci_device_id == ent->device_id) && 263 264 ((pci_subvendor_id == ent->subvendor_id) || 265 (ent->subvendor_id == 0)) && 266 267 ((pci_subdevice_id == ent->subdevice_id) || 268 (ent->subdevice_id == 0))) { 269 sprintf(adapter_name, "%s, Version - %s", 270 ixv_strings[ent->index], 271 ixv_driver_version); 272 device_set_desc_copy(dev, adapter_name); 273 return (BUS_PROBE_DEFAULT); 274 } 275 ent++; 276 } 277 return (ENXIO); 278} 279 280/********************************************************************* 281 * Device initialization routine 282 * 283 * The attach entry point is called when the driver is being loaded. 284 * This routine identifies the type of hardware, allocates all resources 285 * and initializes the hardware. 286 * 287 * return 0 on success, positive on failure 288 *********************************************************************/ 289 290static int 291ixv_attach(device_t dev) 292{ 293 struct adapter *adapter; 294 struct ixgbe_hw *hw; 295 int error = 0; 296 297 INIT_DEBUGOUT("ixv_attach: begin"); 298 299 /* Allocate, clear, and link in our adapter structure */ 300 adapter = device_get_softc(dev); 301 adapter->dev = adapter->osdep.dev = dev; 302 hw = &adapter->hw; 303 304 /* Core Lock Init*/ 305 IXV_CORE_LOCK_INIT(adapter, device_get_nameunit(dev)); 306 307 /* SYSCTL APIs */ 308 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 309 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 310 OID_AUTO, "stats", CTLTYPE_INT | CTLFLAG_RW, 311 adapter, 0, ixv_sysctl_stats, "I", "Statistics"); 312 313 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 314 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 315 OID_AUTO, "debug", CTLTYPE_INT | CTLFLAG_RW, 316 adapter, 0, ixv_sysctl_debug, "I", "Debug Info"); 317 318 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 319 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 320 OID_AUTO, "flow_control", CTLTYPE_INT | CTLFLAG_RW, 321 adapter, 0, ixv_set_flowcntl, "I", "Flow Control"); 322 323 SYSCTL_ADD_INT(device_get_sysctl_ctx(dev), 324 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 325 OID_AUTO, "enable_aim", CTLTYPE_INT|CTLFLAG_RW, 326 &ixv_enable_aim, 1, "Interrupt Moderation"); 327 328 /* Set up the timer callout */ 329 callout_init_mtx(&adapter->timer, &adapter->core_mtx, 0); 330 331 /* Determine hardware revision */ 332 ixv_identify_hardware(adapter); 333 334 /* Do base PCI setup - map BAR0 */ 335 if (ixv_allocate_pci_resources(adapter)) { 336 device_printf(dev, "Allocation of PCI resources failed\n"); 337 error = ENXIO; 338 goto err_out; 339 } 340 341 /* Do descriptor calc and sanity checks */ 342 if (((ixv_txd * sizeof(union ixgbe_adv_tx_desc)) % DBA_ALIGN) != 0 || 343 ixv_txd < MIN_TXD || ixv_txd > MAX_TXD) { 344 device_printf(dev, "TXD config issue, using default!\n"); 345 adapter->num_tx_desc = DEFAULT_TXD; 346 } else 347 adapter->num_tx_desc = ixv_txd; 348 349 if (((ixv_rxd * sizeof(union ixgbe_adv_rx_desc)) % DBA_ALIGN) != 0 || 350 ixv_rxd < MIN_TXD || ixv_rxd > MAX_TXD) { 351 device_printf(dev, "RXD config issue, using default!\n"); 352 adapter->num_rx_desc = DEFAULT_RXD; 353 } else 354 adapter->num_rx_desc = ixv_rxd; 355 356 /* Allocate our TX/RX Queues */ 357 if (ixv_allocate_queues(adapter)) { 358 error = ENOMEM; 359 goto err_out; 360 } 361 362 /* 363 ** Initialize the shared code: its 364 ** at this point the mac type is set. 365 */ 366 error = ixgbe_init_shared_code(hw); 367 if (error) { 368 device_printf(dev,"Shared Code Initialization Failure\n"); 369 error = EIO; 370 goto err_late; 371 } 372 373 /* Setup the mailbox */ 374 ixgbe_init_mbx_params_vf(hw); 375 376 ixgbe_reset_hw(hw); 377 378 /* Get Hardware Flow Control setting */ 379 hw->fc.requested_mode = ixgbe_fc_full; 380 hw->fc.pause_time = IXV_FC_PAUSE; 381 hw->fc.low_water[0] = IXV_FC_LO; 382 hw->fc.high_water[0] = IXV_FC_HI; 383 hw->fc.send_xon = TRUE; 384 385 error = ixgbe_init_hw(hw); 386 if (error) { 387 device_printf(dev,"Hardware Initialization Failure\n"); 388 error = EIO; 389 goto err_late; 390 } 391 392 error = ixv_allocate_msix(adapter); 393 if (error) 394 goto err_late; 395 396 /* Setup OS specific network interface */ 397 ixv_setup_interface(dev, adapter); 398 399 /* Sysctl for limiting the amount of work done in the taskqueue */ 400 ixv_add_rx_process_limit(adapter, "rx_processing_limit", 401 "max number of rx packets to process", &adapter->rx_process_limit, 402 ixv_rx_process_limit); 403 404 /* Do the stats setup */ 405 ixv_save_stats(adapter); 406 ixv_init_stats(adapter); 407 408 /* Register for VLAN events */ 409 adapter->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 410 ixv_register_vlan, adapter, EVENTHANDLER_PRI_FIRST); 411 adapter->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 412 ixv_unregister_vlan, adapter, EVENTHANDLER_PRI_FIRST); 413 414 INIT_DEBUGOUT("ixv_attach: end"); 415 return (0); 416 417err_late: 418 ixv_free_transmit_structures(adapter); 419 ixv_free_receive_structures(adapter); 420err_out: 421 ixv_free_pci_resources(adapter); 422 return (error); 423 424} 425 426/********************************************************************* 427 * Device removal routine 428 * 429 * The detach entry point is called when the driver is being removed. 430 * This routine stops the adapter and deallocates all the resources 431 * that were allocated for driver operation. 432 * 433 * return 0 on success, positive on failure 434 *********************************************************************/ 435 436static int 437ixv_detach(device_t dev) 438{ 439 struct adapter *adapter = device_get_softc(dev); 440 struct ix_queue *que = adapter->queues; 441 442 INIT_DEBUGOUT("ixv_detach: begin"); 443 444 /* Make sure VLANS are not using driver */ 445 if (adapter->ifp->if_vlantrunk != NULL) { 446 device_printf(dev,"Vlan in use, detach first\n"); 447 return (EBUSY); 448 } 449 450 IXV_CORE_LOCK(adapter); 451 ixv_stop(adapter); 452 IXV_CORE_UNLOCK(adapter); 453 454 for (int i = 0; i < adapter->num_queues; i++, que++) { 455 if (que->tq) { 456 taskqueue_drain(que->tq, &que->que_task); 457 taskqueue_free(que->tq); 458 } 459 } 460 461 /* Drain the Link queue */ 462 if (adapter->tq) { 463 taskqueue_drain(adapter->tq, &adapter->mbx_task); 464 taskqueue_free(adapter->tq); 465 } 466 467 /* Unregister VLAN events */ 468 if (adapter->vlan_attach != NULL) 469 EVENTHANDLER_DEREGISTER(vlan_config, adapter->vlan_attach); 470 if (adapter->vlan_detach != NULL) 471 EVENTHANDLER_DEREGISTER(vlan_unconfig, adapter->vlan_detach); 472 473 ether_ifdetach(adapter->ifp); 474 callout_drain(&adapter->timer); 475 ixv_free_pci_resources(adapter); 476 bus_generic_detach(dev); 477 if_free(adapter->ifp); 478 479 ixv_free_transmit_structures(adapter); 480 ixv_free_receive_structures(adapter); 481 482 IXV_CORE_LOCK_DESTROY(adapter); 483 return (0); 484} 485 486/********************************************************************* 487 * 488 * Shutdown entry point 489 * 490 **********************************************************************/ 491static int 492ixv_shutdown(device_t dev) 493{ 494 struct adapter *adapter = device_get_softc(dev); 495 IXV_CORE_LOCK(adapter); 496 ixv_stop(adapter); 497 IXV_CORE_UNLOCK(adapter); 498 return (0); 499} 500 501#if __FreeBSD_version < 800000 502/********************************************************************* 503 * Transmit entry point 504 * 505 * ixv_start is called by the stack to initiate a transmit. 506 * The driver will remain in this routine as long as there are 507 * packets to transmit and transmit resources are available. 508 * In case resources are not available stack is notified and 509 * the packet is requeued. 510 **********************************************************************/ 511static void 512ixv_start_locked(struct tx_ring *txr, struct ifnet * ifp) 513{ 514 struct mbuf *m_head; 515 struct adapter *adapter = txr->adapter; 516 517 IXV_TX_LOCK_ASSERT(txr); 518 519 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != 520 IFF_DRV_RUNNING) 521 return; 522 if (!adapter->link_active) 523 return; 524 525 while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) { 526 527 IFQ_DRV_DEQUEUE(&ifp->if_snd, m_head); 528 if (m_head == NULL) 529 break; 530 531 if (ixv_xmit(txr, &m_head)) { 532 if (m_head == NULL) 533 break; 534 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 535 IFQ_DRV_PREPEND(&ifp->if_snd, m_head); 536 break; 537 } 538 /* Send a copy of the frame to the BPF listener */ 539 ETHER_BPF_MTAP(ifp, m_head); 540 541 /* Set watchdog on */ 542 txr->watchdog_check = TRUE; 543 txr->watchdog_time = ticks; 544 545 } 546 return; 547} 548 549/* 550 * Legacy TX start - called by the stack, this 551 * always uses the first tx ring, and should 552 * not be used with multiqueue tx enabled. 553 */ 554static void 555ixv_start(struct ifnet *ifp) 556{ 557 struct adapter *adapter = ifp->if_softc; 558 struct tx_ring *txr = adapter->tx_rings; 559 560 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 561 IXV_TX_LOCK(txr); 562 ixv_start_locked(txr, ifp); 563 IXV_TX_UNLOCK(txr); 564 } 565 return; 566} 567 568#else 569 570/* 571** Multiqueue Transmit driver 572** 573*/ 574static int 575ixv_mq_start(struct ifnet *ifp, struct mbuf *m) 576{ 577 struct adapter *adapter = ifp->if_softc; 578 struct ix_queue *que; 579 struct tx_ring *txr; 580 int i = 0, err = 0; 581 582 /* Which queue to use */ 583 if ((m->m_flags & M_FLOWID) != 0) 584 i = m->m_pkthdr.flowid % adapter->num_queues; 585 586 txr = &adapter->tx_rings[i]; 587 que = &adapter->queues[i]; 588 589 if (IXV_TX_TRYLOCK(txr)) { 590 err = ixv_mq_start_locked(ifp, txr, m); 591 IXV_TX_UNLOCK(txr); 592 } else { 593 err = drbr_enqueue(ifp, txr->br, m); 594 taskqueue_enqueue(que->tq, &que->que_task); 595 } 596 597 return (err); 598} 599 600static int 601ixv_mq_start_locked(struct ifnet *ifp, struct tx_ring *txr, struct mbuf *m) 602{ 603 struct adapter *adapter = txr->adapter; 604 struct mbuf *next; 605 int enqueued, err = 0; 606 607 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 608 IFF_DRV_RUNNING || adapter->link_active == 0) { 609 if (m != NULL) 610 err = drbr_enqueue(ifp, txr->br, m); 611 return (err); 612 } 613 614 /* Do a clean if descriptors are low */ 615 if (txr->tx_avail <= IXV_TX_CLEANUP_THRESHOLD) 616 ixv_txeof(txr); 617 618 enqueued = 0; 619 if (m != NULL) { 620 err = drbr_enqueue(ifp, txr->br, m); 621 if (err) { 622 return (err); 623 } 624 } 625 /* Process the queue */ 626 while ((next = drbr_peek(ifp, txr->br)) != NULL) { 627 if ((err = ixv_xmit(txr, &next)) != 0) { 628 if (next == NULL) { 629 drbr_advance(ifp, txr->br); 630 } else { 631 drbr_putback(ifp, txr->br, next); 632 } 633 break; 634 } 635 drbr_advance(ifp, txr->br); 636 enqueued++; 637 ifp->if_obytes += next->m_pkthdr.len; 638 if (next->m_flags & M_MCAST) 639 ifp->if_omcasts++; 640 /* Send a copy of the frame to the BPF listener */ 641 ETHER_BPF_MTAP(ifp, next); 642 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 643 break; 644 if (txr->tx_avail <= IXV_TX_OP_THRESHOLD) { 645 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 646 break; 647 } 648 } 649 650 if (enqueued > 0) { 651 /* Set watchdog on */ 652 txr->watchdog_check = TRUE; 653 txr->watchdog_time = ticks; 654 } 655 656 return (err); 657} 658 659/* 660** Flush all ring buffers 661*/ 662static void 663ixv_qflush(struct ifnet *ifp) 664{ 665 struct adapter *adapter = ifp->if_softc; 666 struct tx_ring *txr = adapter->tx_rings; 667 struct mbuf *m; 668 669 for (int i = 0; i < adapter->num_queues; i++, txr++) { 670 IXV_TX_LOCK(txr); 671 while ((m = buf_ring_dequeue_sc(txr->br)) != NULL) 672 m_freem(m); 673 IXV_TX_UNLOCK(txr); 674 } 675 if_qflush(ifp); 676} 677 678#endif 679 680/********************************************************************* 681 * Ioctl entry point 682 * 683 * ixv_ioctl is called when the user wants to configure the 684 * interface. 685 * 686 * return 0 on success, positive on failure 687 **********************************************************************/ 688 689static int 690ixv_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 691{ 692 struct adapter *adapter = ifp->if_softc; 693 struct ifreq *ifr = (struct ifreq *) data; 694#if defined(INET) || defined(INET6) 695 struct ifaddr *ifa = (struct ifaddr *) data; 696 bool avoid_reset = FALSE; 697#endif 698 int error = 0; 699 700 switch (command) { 701 702 case SIOCSIFADDR: 703#ifdef INET 704 if (ifa->ifa_addr->sa_family == AF_INET) 705 avoid_reset = TRUE; 706#endif 707#ifdef INET6 708 if (ifa->ifa_addr->sa_family == AF_INET6) 709 avoid_reset = TRUE; 710#endif 711#if defined(INET) || defined(INET6) 712 /* 713 ** Calling init results in link renegotiation, 714 ** so we avoid doing it when possible. 715 */ 716 if (avoid_reset) { 717 ifp->if_flags |= IFF_UP; 718 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) 719 ixv_init(adapter); 720 if (!(ifp->if_flags & IFF_NOARP)) 721 arp_ifinit(ifp, ifa); 722 } else 723 error = ether_ioctl(ifp, command, data); 724 break; 725#endif 726 case SIOCSIFMTU: 727 IOCTL_DEBUGOUT("ioctl: SIOCSIFMTU (Set Interface MTU)"); 728 if (ifr->ifr_mtu > IXV_MAX_FRAME_SIZE - ETHER_HDR_LEN) { 729 error = EINVAL; 730 } else { 731 IXV_CORE_LOCK(adapter); 732 ifp->if_mtu = ifr->ifr_mtu; 733 adapter->max_frame_size = 734 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 735 ixv_init_locked(adapter); 736 IXV_CORE_UNLOCK(adapter); 737 } 738 break; 739 case SIOCSIFFLAGS: 740 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 741 IXV_CORE_LOCK(adapter); 742 if (ifp->if_flags & IFF_UP) { 743 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 744 ixv_init_locked(adapter); 745 } else 746 if (ifp->if_drv_flags & IFF_DRV_RUNNING) 747 ixv_stop(adapter); 748 adapter->if_flags = ifp->if_flags; 749 IXV_CORE_UNLOCK(adapter); 750 break; 751 case SIOCADDMULTI: 752 case SIOCDELMULTI: 753 IOCTL_DEBUGOUT("ioctl: SIOC(ADD|DEL)MULTI"); 754 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 755 IXV_CORE_LOCK(adapter); 756 ixv_disable_intr(adapter); 757 ixv_set_multi(adapter); 758 ixv_enable_intr(adapter); 759 IXV_CORE_UNLOCK(adapter); 760 } 761 break; 762 case SIOCSIFMEDIA: 763 case SIOCGIFMEDIA: 764 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 765 error = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 766 break; 767 case SIOCSIFCAP: 768 { 769 int mask = ifr->ifr_reqcap ^ ifp->if_capenable; 770 IOCTL_DEBUGOUT("ioctl: SIOCSIFCAP (Set Capabilities)"); 771 if (mask & IFCAP_HWCSUM) 772 ifp->if_capenable ^= IFCAP_HWCSUM; 773 if (mask & IFCAP_TSO4) 774 ifp->if_capenable ^= IFCAP_TSO4; 775 if (mask & IFCAP_LRO) 776 ifp->if_capenable ^= IFCAP_LRO; 777 if (mask & IFCAP_VLAN_HWTAGGING) 778 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 779 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 780 IXV_CORE_LOCK(adapter); 781 ixv_init_locked(adapter); 782 IXV_CORE_UNLOCK(adapter); 783 } 784 VLAN_CAPABILITIES(ifp); 785 break; 786 } 787 788 default: 789 IOCTL_DEBUGOUT1("ioctl: UNKNOWN (0x%X)\n", (int)command); 790 error = ether_ioctl(ifp, command, data); 791 break; 792 } 793 794 return (error); 795} 796 797/********************************************************************* 798 * Init entry point 799 * 800 * This routine is used in two ways. It is used by the stack as 801 * init entry point in network interface structure. It is also used 802 * by the driver as a hw/sw initialization routine to get to a 803 * consistent state. 804 * 805 * return 0 on success, positive on failure 806 **********************************************************************/ 807#define IXGBE_MHADD_MFS_SHIFT 16 808 809static void 810ixv_init_locked(struct adapter *adapter) 811{ 812 struct ifnet *ifp = adapter->ifp; 813 device_t dev = adapter->dev; 814 struct ixgbe_hw *hw = &adapter->hw; 815 u32 mhadd, gpie; 816 817 INIT_DEBUGOUT("ixv_init: begin"); 818 mtx_assert(&adapter->core_mtx, MA_OWNED); 819 hw->adapter_stopped = FALSE; 820 ixgbe_stop_adapter(hw); 821 callout_stop(&adapter->timer); 822 823 /* reprogram the RAR[0] in case user changed it. */ 824 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 825 826 /* Get the latest mac address, User can use a LAA */ 827 bcopy(IF_LLADDR(adapter->ifp), hw->mac.addr, 828 IXGBE_ETH_LENGTH_OF_ADDRESS); 829 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, 1); 830 hw->addr_ctrl.rar_used_count = 1; 831 832 /* Prepare transmit descriptors and buffers */ 833 if (ixv_setup_transmit_structures(adapter)) { 834 device_printf(dev,"Could not setup transmit structures\n"); 835 ixv_stop(adapter); 836 return; 837 } 838 839 ixgbe_reset_hw(hw); 840 ixv_initialize_transmit_units(adapter); 841 842 /* Setup Multicast table */ 843 ixv_set_multi(adapter); 844 845 /* 846 ** Determine the correct mbuf pool 847 ** for doing jumbo/headersplit 848 */ 849 if (ifp->if_mtu > ETHERMTU) 850 adapter->rx_mbuf_sz = MJUMPAGESIZE; 851 else 852 adapter->rx_mbuf_sz = MCLBYTES; 853 854 /* Prepare receive descriptors and buffers */ 855 if (ixv_setup_receive_structures(adapter)) { 856 device_printf(dev,"Could not setup receive structures\n"); 857 ixv_stop(adapter); 858 return; 859 } 860 861 /* Configure RX settings */ 862 ixv_initialize_receive_units(adapter); 863 864 /* Enable Enhanced MSIX mode */ 865 gpie = IXGBE_READ_REG(&adapter->hw, IXGBE_GPIE); 866 gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_EIAME; 867 gpie |= IXGBE_GPIE_PBA_SUPPORT | IXGBE_GPIE_OCD; 868 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 869 870 /* Set the various hardware offload abilities */ 871 ifp->if_hwassist = 0; 872 if (ifp->if_capenable & IFCAP_TSO4) 873 ifp->if_hwassist |= CSUM_TSO; 874 if (ifp->if_capenable & IFCAP_TXCSUM) { 875 ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP); 876#if __FreeBSD_version >= 800000 877 ifp->if_hwassist |= CSUM_SCTP; 878#endif 879 } 880 881 /* Set MTU size */ 882 if (ifp->if_mtu > ETHERMTU) { 883 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); 884 mhadd &= ~IXGBE_MHADD_MFS_MASK; 885 mhadd |= adapter->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 886 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); 887 } 888 889 /* Set up VLAN offload and filter */ 890 ixv_setup_vlan_support(adapter); 891 892 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 893 894 /* Set up MSI/X routing */ 895 ixv_configure_ivars(adapter); 896 897 /* Set up auto-mask */ 898 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_EICS_RTX_QUEUE); 899 900 /* Set moderation on the Link interrupt */ 901 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(adapter->mbxvec), IXV_LINK_ITR); 902 903 /* Stats init */ 904 ixv_init_stats(adapter); 905 906 /* Config/Enable Link */ 907 ixv_config_link(adapter); 908 909 /* And now turn on interrupts */ 910 ixv_enable_intr(adapter); 911 912 /* Now inform the stack we're ready */ 913 ifp->if_drv_flags |= IFF_DRV_RUNNING; 914 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 915 916 return; 917} 918 919static void 920ixv_init(void *arg) 921{ 922 struct adapter *adapter = arg; 923 924 IXV_CORE_LOCK(adapter); 925 ixv_init_locked(adapter); 926 IXV_CORE_UNLOCK(adapter); 927 return; 928} 929 930 931/* 932** 933** MSIX Interrupt Handlers and Tasklets 934** 935*/ 936 937static inline void 938ixv_enable_queue(struct adapter *adapter, u32 vector) 939{ 940 struct ixgbe_hw *hw = &adapter->hw; 941 u32 queue = 1 << vector; 942 u32 mask; 943 944 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 945 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 946} 947 948static inline void 949ixv_disable_queue(struct adapter *adapter, u32 vector) 950{ 951 struct ixgbe_hw *hw = &adapter->hw; 952 u64 queue = (u64)(1 << vector); 953 u32 mask; 954 955 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 956 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, mask); 957} 958 959static inline void 960ixv_rearm_queues(struct adapter *adapter, u64 queues) 961{ 962 u32 mask = (IXGBE_EIMS_RTX_QUEUE & queues); 963 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEICS, mask); 964} 965 966 967static void 968ixv_handle_que(void *context, int pending) 969{ 970 struct ix_queue *que = context; 971 struct adapter *adapter = que->adapter; 972 struct tx_ring *txr = que->txr; 973 struct ifnet *ifp = adapter->ifp; 974 bool more; 975 976 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 977 more = ixv_rxeof(que, adapter->rx_process_limit); 978 IXV_TX_LOCK(txr); 979 ixv_txeof(txr); 980#if __FreeBSD_version >= 800000 981 if (!drbr_empty(ifp, txr->br)) 982 ixv_mq_start_locked(ifp, txr, NULL); 983#else 984 if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) 985 ixv_start_locked(txr, ifp); 986#endif 987 IXV_TX_UNLOCK(txr); 988 if (more) { 989 taskqueue_enqueue(que->tq, &que->que_task); 990 return; 991 } 992 } 993 994 /* Reenable this interrupt */ 995 ixv_enable_queue(adapter, que->msix); 996 return; 997} 998 999/********************************************************************* 1000 * 1001 * MSI Queue Interrupt Service routine 1002 * 1003 **********************************************************************/ 1004void 1005ixv_msix_que(void *arg) 1006{ 1007 struct ix_queue *que = arg; 1008 struct adapter *adapter = que->adapter; 1009 struct tx_ring *txr = que->txr; 1010 struct rx_ring *rxr = que->rxr; 1011 bool more_tx, more_rx; 1012 u32 newitr = 0; 1013 1014 ixv_disable_queue(adapter, que->msix); 1015 ++que->irqs; 1016 1017 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1018 1019 IXV_TX_LOCK(txr); 1020 more_tx = ixv_txeof(txr); 1021 /* 1022 ** Make certain that if the stack 1023 ** has anything queued the task gets 1024 ** scheduled to handle it. 1025 */ 1026#if __FreeBSD_version < 800000 1027 if (!IFQ_DRV_IS_EMPTY(&adapter->ifp->if_snd)) 1028#else 1029 if (!drbr_empty(adapter->ifp, txr->br)) 1030#endif 1031 more_tx = 1; 1032 IXV_TX_UNLOCK(txr); 1033 1034 more_rx = ixv_rxeof(que, adapter->rx_process_limit); 1035 1036 /* Do AIM now? */ 1037 1038 if (ixv_enable_aim == FALSE) 1039 goto no_calc; 1040 /* 1041 ** Do Adaptive Interrupt Moderation: 1042 ** - Write out last calculated setting 1043 ** - Calculate based on average size over 1044 ** the last interval. 1045 */ 1046 if (que->eitr_setting) 1047 IXGBE_WRITE_REG(&adapter->hw, 1048 IXGBE_VTEITR(que->msix), 1049 que->eitr_setting); 1050 1051 que->eitr_setting = 0; 1052 1053 /* Idle, do nothing */ 1054 if ((txr->bytes == 0) && (rxr->bytes == 0)) 1055 goto no_calc; 1056 1057 if ((txr->bytes) && (txr->packets)) 1058 newitr = txr->bytes/txr->packets; 1059 if ((rxr->bytes) && (rxr->packets)) 1060 newitr = max(newitr, 1061 (rxr->bytes / rxr->packets)); 1062 newitr += 24; /* account for hardware frame, crc */ 1063 1064 /* set an upper boundary */ 1065 newitr = min(newitr, 3000); 1066 1067 /* Be nice to the mid range */ 1068 if ((newitr > 300) && (newitr < 1200)) 1069 newitr = (newitr / 3); 1070 else 1071 newitr = (newitr / 2); 1072 1073 newitr |= newitr << 16; 1074 1075 /* save for next interrupt */ 1076 que->eitr_setting = newitr; 1077 1078 /* Reset state */ 1079 txr->bytes = 0; 1080 txr->packets = 0; 1081 rxr->bytes = 0; 1082 rxr->packets = 0; 1083 1084no_calc: 1085 if (more_tx || more_rx) 1086 taskqueue_enqueue(que->tq, &que->que_task); 1087 else /* Reenable this interrupt */ 1088 ixv_enable_queue(adapter, que->msix); 1089 return; 1090} 1091 1092static void 1093ixv_msix_mbx(void *arg) 1094{ 1095 struct adapter *adapter = arg; 1096 struct ixgbe_hw *hw = &adapter->hw; 1097 u32 reg; 1098 1099 ++adapter->mbx_irq; 1100 1101 /* First get the cause */ 1102 reg = IXGBE_READ_REG(hw, IXGBE_VTEICS); 1103 /* Clear interrupt with write */ 1104 IXGBE_WRITE_REG(hw, IXGBE_VTEICR, reg); 1105 1106 /* Link status change */ 1107 if (reg & IXGBE_EICR_LSC) 1108 taskqueue_enqueue(adapter->tq, &adapter->mbx_task); 1109 1110 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_EIMS_OTHER); 1111 return; 1112} 1113 1114/********************************************************************* 1115 * 1116 * Media Ioctl callback 1117 * 1118 * This routine is called whenever the user queries the status of 1119 * the interface using ifconfig. 1120 * 1121 **********************************************************************/ 1122static void 1123ixv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr) 1124{ 1125 struct adapter *adapter = ifp->if_softc; 1126 1127 INIT_DEBUGOUT("ixv_media_status: begin"); 1128 IXV_CORE_LOCK(adapter); 1129 ixv_update_link_status(adapter); 1130 1131 ifmr->ifm_status = IFM_AVALID; 1132 ifmr->ifm_active = IFM_ETHER; 1133 1134 if (!adapter->link_active) { 1135 IXV_CORE_UNLOCK(adapter); 1136 return; 1137 } 1138 1139 ifmr->ifm_status |= IFM_ACTIVE; 1140 1141 switch (adapter->link_speed) { 1142 case IXGBE_LINK_SPEED_1GB_FULL: 1143 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1144 break; 1145 case IXGBE_LINK_SPEED_10GB_FULL: 1146 ifmr->ifm_active |= IFM_FDX; 1147 break; 1148 } 1149 1150 IXV_CORE_UNLOCK(adapter); 1151 1152 return; 1153} 1154 1155/********************************************************************* 1156 * 1157 * Media Ioctl callback 1158 * 1159 * This routine is called when the user changes speed/duplex using 1160 * media/mediopt option with ifconfig. 1161 * 1162 **********************************************************************/ 1163static int 1164ixv_media_change(struct ifnet * ifp) 1165{ 1166 struct adapter *adapter = ifp->if_softc; 1167 struct ifmedia *ifm = &adapter->media; 1168 1169 INIT_DEBUGOUT("ixv_media_change: begin"); 1170 1171 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1172 return (EINVAL); 1173 1174 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1175 case IFM_AUTO: 1176 break; 1177 default: 1178 device_printf(adapter->dev, "Only auto media type\n"); 1179 return (EINVAL); 1180 } 1181 1182 return (0); 1183} 1184 1185/********************************************************************* 1186 * 1187 * This routine maps the mbufs to tx descriptors, allowing the 1188 * TX engine to transmit the packets. 1189 * - return 0 on success, positive on failure 1190 * 1191 **********************************************************************/ 1192 1193static int 1194ixv_xmit(struct tx_ring *txr, struct mbuf **m_headp) 1195{ 1196 struct adapter *adapter = txr->adapter; 1197 u32 olinfo_status = 0, cmd_type_len; 1198 u32 paylen = 0; 1199 int i, j, error, nsegs; 1200 int first, last = 0; 1201 struct mbuf *m_head; 1202 bus_dma_segment_t segs[32]; 1203 bus_dmamap_t map; 1204 struct ixv_tx_buf *txbuf, *txbuf_mapped; 1205 union ixgbe_adv_tx_desc *txd = NULL; 1206 1207 m_head = *m_headp; 1208 1209 /* Basic descriptor defines */ 1210 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1211 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1212 1213 if (m_head->m_flags & M_VLANTAG) 1214 cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 1215 1216 /* 1217 * Important to capture the first descriptor 1218 * used because it will contain the index of 1219 * the one we tell the hardware to report back 1220 */ 1221 first = txr->next_avail_desc; 1222 txbuf = &txr->tx_buffers[first]; 1223 txbuf_mapped = txbuf; 1224 map = txbuf->map; 1225 1226 /* 1227 * Map the packet for DMA. 1228 */ 1229 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1230 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1231 1232 if (error == EFBIG) { 1233 struct mbuf *m; 1234 1235 m = m_defrag(*m_headp, M_NOWAIT); 1236 if (m == NULL) { 1237 adapter->mbuf_defrag_failed++; 1238 m_freem(*m_headp); 1239 *m_headp = NULL; 1240 return (ENOBUFS); 1241 } 1242 *m_headp = m; 1243 1244 /* Try it again */ 1245 error = bus_dmamap_load_mbuf_sg(txr->txtag, map, 1246 *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); 1247 1248 if (error == ENOMEM) { 1249 adapter->no_tx_dma_setup++; 1250 return (error); 1251 } else if (error != 0) { 1252 adapter->no_tx_dma_setup++; 1253 m_freem(*m_headp); 1254 *m_headp = NULL; 1255 return (error); 1256 } 1257 } else if (error == ENOMEM) { 1258 adapter->no_tx_dma_setup++; 1259 return (error); 1260 } else if (error != 0) { 1261 adapter->no_tx_dma_setup++; 1262 m_freem(*m_headp); 1263 *m_headp = NULL; 1264 return (error); 1265 } 1266 1267 /* Make certain there are enough descriptors */ 1268 if (nsegs > txr->tx_avail - 2) { 1269 txr->no_desc_avail++; 1270 error = ENOBUFS; 1271 goto xmit_fail; 1272 } 1273 m_head = *m_headp; 1274 1275 /* 1276 ** Set up the appropriate offload context 1277 ** this becomes the first descriptor of 1278 ** a packet. 1279 */ 1280 if (m_head->m_pkthdr.csum_flags & CSUM_TSO) { 1281 if (ixv_tso_setup(txr, m_head, &paylen)) { 1282 cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 1283 olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 1284 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1285 olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 1286 ++adapter->tso_tx; 1287 } else 1288 return (ENXIO); 1289 } else if (ixv_tx_ctx_setup(txr, m_head)) 1290 olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 1291 1292 /* Record payload length */ 1293 if (paylen == 0) 1294 olinfo_status |= m_head->m_pkthdr.len << 1295 IXGBE_ADVTXD_PAYLEN_SHIFT; 1296 1297 i = txr->next_avail_desc; 1298 for (j = 0; j < nsegs; j++) { 1299 bus_size_t seglen; 1300 bus_addr_t segaddr; 1301 1302 txbuf = &txr->tx_buffers[i]; 1303 txd = &txr->tx_base[i]; 1304 seglen = segs[j].ds_len; 1305 segaddr = htole64(segs[j].ds_addr); 1306 1307 txd->read.buffer_addr = segaddr; 1308 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1309 cmd_type_len |seglen); 1310 txd->read.olinfo_status = htole32(olinfo_status); 1311 last = i; /* descriptor that will get completion IRQ */ 1312 1313 if (++i == adapter->num_tx_desc) 1314 i = 0; 1315 1316 txbuf->m_head = NULL; 1317 txbuf->eop_index = -1; 1318 } 1319 1320 txd->read.cmd_type_len |= 1321 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1322 txr->tx_avail -= nsegs; 1323 txr->next_avail_desc = i; 1324 1325 txbuf->m_head = m_head; 1326 txr->tx_buffers[first].map = txbuf->map; 1327 txbuf->map = map; 1328 bus_dmamap_sync(txr->txtag, map, BUS_DMASYNC_PREWRITE); 1329 1330 /* Set the index of the descriptor that will be marked done */ 1331 txbuf = &txr->tx_buffers[first]; 1332 txbuf->eop_index = last; 1333 1334 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 1335 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1336 /* 1337 * Advance the Transmit Descriptor Tail (Tdt), this tells the 1338 * hardware that this frame is available to transmit. 1339 */ 1340 ++txr->total_packets; 1341 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(txr->me), i); 1342 1343 return (0); 1344 1345xmit_fail: 1346 bus_dmamap_unload(txr->txtag, txbuf->map); 1347 return (error); 1348 1349} 1350 1351 1352/********************************************************************* 1353 * Multicast Update 1354 * 1355 * This routine is called whenever multicast address list is updated. 1356 * 1357 **********************************************************************/ 1358#define IXGBE_RAR_ENTRIES 16 1359 1360static void 1361ixv_set_multi(struct adapter *adapter) 1362{ 1363 u8 mta[MAX_NUM_MULTICAST_ADDRESSES * IXGBE_ETH_LENGTH_OF_ADDRESS]; 1364 u8 *update_ptr; 1365 struct ifmultiaddr *ifma; 1366 int mcnt = 0; 1367 struct ifnet *ifp = adapter->ifp; 1368 1369 IOCTL_DEBUGOUT("ixv_set_multi: begin"); 1370 1371#if __FreeBSD_version < 800000 1372 IF_ADDR_LOCK(ifp); 1373#else 1374 if_maddr_rlock(ifp); 1375#endif 1376 TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { 1377 if (ifma->ifma_addr->sa_family != AF_LINK) 1378 continue; 1379 bcopy(LLADDR((struct sockaddr_dl *) ifma->ifma_addr), 1380 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1381 IXGBE_ETH_LENGTH_OF_ADDRESS); 1382 mcnt++; 1383 } 1384#if __FreeBSD_version < 800000 1385 IF_ADDR_UNLOCK(ifp); 1386#else 1387 if_maddr_runlock(ifp); 1388#endif 1389 1390 update_ptr = mta; 1391 1392 ixgbe_update_mc_addr_list(&adapter->hw, 1393 update_ptr, mcnt, ixv_mc_array_itr, TRUE); 1394 1395 return; 1396} 1397 1398/* 1399 * This is an iterator function now needed by the multicast 1400 * shared code. It simply feeds the shared code routine the 1401 * addresses in the array of ixv_set_multi() one by one. 1402 */ 1403static u8 * 1404ixv_mc_array_itr(struct ixgbe_hw *hw, u8 **update_ptr, u32 *vmdq) 1405{ 1406 u8 *addr = *update_ptr; 1407 u8 *newptr; 1408 *vmdq = 0; 1409 1410 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1411 *update_ptr = newptr; 1412 return addr; 1413} 1414 1415/********************************************************************* 1416 * Timer routine 1417 * 1418 * This routine checks for link status,updates statistics, 1419 * and runs the watchdog check. 1420 * 1421 **********************************************************************/ 1422 1423static void 1424ixv_local_timer(void *arg) 1425{ 1426 struct adapter *adapter = arg; 1427 device_t dev = adapter->dev; 1428 struct tx_ring *txr = adapter->tx_rings; 1429 int i; 1430 1431 mtx_assert(&adapter->core_mtx, MA_OWNED); 1432 1433 ixv_update_link_status(adapter); 1434 1435 /* Stats Update */ 1436 ixv_update_stats(adapter); 1437 1438 /* 1439 * If the interface has been paused 1440 * then don't do the watchdog check 1441 */ 1442 if (IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF) 1443 goto out; 1444 /* 1445 ** Check for time since any descriptor was cleaned 1446 */ 1447 for (i = 0; i < adapter->num_queues; i++, txr++) { 1448 IXV_TX_LOCK(txr); 1449 if (txr->watchdog_check == FALSE) { 1450 IXV_TX_UNLOCK(txr); 1451 continue; 1452 } 1453 if ((ticks - txr->watchdog_time) > IXV_WATCHDOG) 1454 goto hung; 1455 IXV_TX_UNLOCK(txr); 1456 } 1457out: 1458 ixv_rearm_queues(adapter, adapter->que_mask); 1459 callout_reset(&adapter->timer, hz, ixv_local_timer, adapter); 1460 return; 1461 1462hung: 1463 device_printf(adapter->dev, "Watchdog timeout -- resetting\n"); 1464 device_printf(dev,"Queue(%d) tdh = %d, hw tdt = %d\n", txr->me, 1465 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDH(i)), 1466 IXGBE_READ_REG(&adapter->hw, IXGBE_VFTDT(i))); 1467 device_printf(dev,"TX(%d) desc avail = %d," 1468 "Next TX to Clean = %d\n", 1469 txr->me, txr->tx_avail, txr->next_to_clean); 1470 adapter->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 1471 adapter->watchdog_events++; 1472 IXV_TX_UNLOCK(txr); 1473 ixv_init_locked(adapter); 1474} 1475 1476/* 1477** Note: this routine updates the OS on the link state 1478** the real check of the hardware only happens with 1479** a link interrupt. 1480*/ 1481static void 1482ixv_update_link_status(struct adapter *adapter) 1483{ 1484 struct ifnet *ifp = adapter->ifp; 1485 struct tx_ring *txr = adapter->tx_rings; 1486 device_t dev = adapter->dev; 1487 1488 1489 if (adapter->link_up){ 1490 if (adapter->link_active == FALSE) { 1491 if (bootverbose) 1492 device_printf(dev,"Link is up %d Gbps %s \n", 1493 ((adapter->link_speed == 128)? 10:1), 1494 "Full Duplex"); 1495 adapter->link_active = TRUE; 1496 if_link_state_change(ifp, LINK_STATE_UP); 1497 } 1498 } else { /* Link down */ 1499 if (adapter->link_active == TRUE) { 1500 if (bootverbose) 1501 device_printf(dev,"Link is Down\n"); 1502 if_link_state_change(ifp, LINK_STATE_DOWN); 1503 adapter->link_active = FALSE; 1504 for (int i = 0; i < adapter->num_queues; 1505 i++, txr++) 1506 txr->watchdog_check = FALSE; 1507 } 1508 } 1509 1510 return; 1511} 1512 1513 1514/********************************************************************* 1515 * 1516 * This routine disables all traffic on the adapter by issuing a 1517 * global reset on the MAC and deallocates TX/RX buffers. 1518 * 1519 **********************************************************************/ 1520 1521static void 1522ixv_stop(void *arg) 1523{ 1524 struct ifnet *ifp; 1525 struct adapter *adapter = arg; 1526 struct ixgbe_hw *hw = &adapter->hw; 1527 ifp = adapter->ifp; 1528 1529 mtx_assert(&adapter->core_mtx, MA_OWNED); 1530 1531 INIT_DEBUGOUT("ixv_stop: begin\n"); 1532 ixv_disable_intr(adapter); 1533 1534 /* Tell the stack that the interface is no longer active */ 1535 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 1536 1537 ixgbe_reset_hw(hw); 1538 adapter->hw.adapter_stopped = FALSE; 1539 ixgbe_stop_adapter(hw); 1540 callout_stop(&adapter->timer); 1541 1542 /* reprogram the RAR[0] in case user changed it. */ 1543 ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV); 1544 1545 return; 1546} 1547 1548 1549/********************************************************************* 1550 * 1551 * Determine hardware revision. 1552 * 1553 **********************************************************************/ 1554static void 1555ixv_identify_hardware(struct adapter *adapter) 1556{ 1557 device_t dev = adapter->dev; 1558 u16 pci_cmd_word; 1559 1560 /* 1561 ** Make sure BUSMASTER is set, on a VM under 1562 ** KVM it may not be and will break things. 1563 */ 1564 pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2); 1565 if (!((pci_cmd_word & PCIM_CMD_BUSMASTEREN) && 1566 (pci_cmd_word & PCIM_CMD_MEMEN))) { 1567 INIT_DEBUGOUT("Memory Access and/or Bus Master " 1568 "bits were not set!\n"); 1569 pci_cmd_word |= (PCIM_CMD_BUSMASTEREN | PCIM_CMD_MEMEN); 1570 pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2); 1571 } 1572 1573 /* Save off the information about this board */ 1574 adapter->hw.vendor_id = pci_get_vendor(dev); 1575 adapter->hw.device_id = pci_get_device(dev); 1576 adapter->hw.revision_id = pci_read_config(dev, PCIR_REVID, 1); 1577 adapter->hw.subsystem_vendor_id = 1578 pci_read_config(dev, PCIR_SUBVEND_0, 2); 1579 adapter->hw.subsystem_device_id = 1580 pci_read_config(dev, PCIR_SUBDEV_0, 2); 1581 1582 return; 1583} 1584 1585/********************************************************************* 1586 * 1587 * Setup MSIX Interrupt resources and handlers 1588 * 1589 **********************************************************************/ 1590static int 1591ixv_allocate_msix(struct adapter *adapter) 1592{ 1593 device_t dev = adapter->dev; 1594 struct ix_queue *que = adapter->queues; 1595 int error, rid, vector = 0; 1596 1597 for (int i = 0; i < adapter->num_queues; i++, vector++, que++) { 1598 rid = vector + 1; 1599 que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 1600 RF_SHAREABLE | RF_ACTIVE); 1601 if (que->res == NULL) { 1602 device_printf(dev,"Unable to allocate" 1603 " bus resource: que interrupt [%d]\n", vector); 1604 return (ENXIO); 1605 } 1606 /* Set the handler function */ 1607 error = bus_setup_intr(dev, que->res, 1608 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1609 ixv_msix_que, que, &que->tag); 1610 if (error) { 1611 que->res = NULL; 1612 device_printf(dev, "Failed to register QUE handler"); 1613 return (error); 1614 } 1615#if __FreeBSD_version >= 800504 1616 bus_describe_intr(dev, que->res, que->tag, "que %d", i); 1617#endif 1618 que->msix = vector; 1619 adapter->que_mask |= (u64)(1 << que->msix); 1620 /* 1621 ** Bind the msix vector, and thus the 1622 ** ring to the corresponding cpu. 1623 */ 1624 if (adapter->num_queues > 1) 1625 bus_bind_intr(dev, que->res, i); 1626 1627 TASK_INIT(&que->que_task, 0, ixv_handle_que, que); 1628 que->tq = taskqueue_create_fast("ixv_que", M_NOWAIT, 1629 taskqueue_thread_enqueue, &que->tq); 1630 taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que", 1631 device_get_nameunit(adapter->dev)); 1632 } 1633 1634 /* and Mailbox */ 1635 rid = vector + 1; 1636 adapter->res = bus_alloc_resource_any(dev, 1637 SYS_RES_IRQ, &rid, RF_SHAREABLE | RF_ACTIVE); 1638 if (!adapter->res) { 1639 device_printf(dev,"Unable to allocate" 1640 " bus resource: MBX interrupt [%d]\n", rid); 1641 return (ENXIO); 1642 } 1643 /* Set the mbx handler function */ 1644 error = bus_setup_intr(dev, adapter->res, 1645 INTR_TYPE_NET | INTR_MPSAFE, NULL, 1646 ixv_msix_mbx, adapter, &adapter->tag); 1647 if (error) { 1648 adapter->res = NULL; 1649 device_printf(dev, "Failed to register LINK handler"); 1650 return (error); 1651 } 1652#if __FreeBSD_version >= 800504 1653 bus_describe_intr(dev, adapter->res, adapter->tag, "mbx"); 1654#endif 1655 adapter->mbxvec = vector; 1656 /* Tasklets for Mailbox */ 1657 TASK_INIT(&adapter->mbx_task, 0, ixv_handle_mbx, adapter); 1658 adapter->tq = taskqueue_create_fast("ixv_mbx", M_NOWAIT, 1659 taskqueue_thread_enqueue, &adapter->tq); 1660 taskqueue_start_threads(&adapter->tq, 1, PI_NET, "%s mbxq", 1661 device_get_nameunit(adapter->dev)); 1662 /* 1663 ** Due to a broken design QEMU will fail to properly 1664 ** enable the guest for MSIX unless the vectors in 1665 ** the table are all set up, so we must rewrite the 1666 ** ENABLE in the MSIX control register again at this 1667 ** point to cause it to successfully initialize us. 1668 */ 1669 if (adapter->hw.mac.type == ixgbe_mac_82599_vf) { 1670 int msix_ctrl; 1671 pci_find_cap(dev, PCIY_MSIX, &rid); 1672 rid += PCIR_MSIX_CTRL; 1673 msix_ctrl = pci_read_config(dev, rid, 2); 1674 msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE; 1675 pci_write_config(dev, rid, msix_ctrl, 2); 1676 } 1677 1678 return (0); 1679} 1680 1681/* 1682 * Setup MSIX resources, note that the VF 1683 * device MUST use MSIX, there is no fallback. 1684 */ 1685static int 1686ixv_setup_msix(struct adapter *adapter) 1687{ 1688 device_t dev = adapter->dev; 1689 int rid, vectors, want = 2; 1690 1691 1692 /* First try MSI/X */ 1693 rid = PCIR_BAR(3); 1694 adapter->msix_mem = bus_alloc_resource_any(dev, 1695 SYS_RES_MEMORY, &rid, RF_ACTIVE); 1696 if (!adapter->msix_mem) { 1697 device_printf(adapter->dev, 1698 "Unable to map MSIX table \n"); 1699 goto out; 1700 } 1701 1702 vectors = pci_msix_count(dev); 1703 if (vectors < 2) { 1704 bus_release_resource(dev, SYS_RES_MEMORY, 1705 rid, adapter->msix_mem); 1706 adapter->msix_mem = NULL; 1707 goto out; 1708 } 1709 1710 /* 1711 ** Want two vectors: one for a queue, 1712 ** plus an additional for mailbox. 1713 */ 1714 if (pci_alloc_msix(dev, &want) == 0) { 1715 device_printf(adapter->dev, 1716 "Using MSIX interrupts with %d vectors\n", want); 1717 return (want); 1718 } 1719out: 1720 device_printf(adapter->dev,"MSIX config error\n"); 1721 return (ENXIO); 1722} 1723 1724 1725static int 1726ixv_allocate_pci_resources(struct adapter *adapter) 1727{ 1728 int rid; 1729 device_t dev = adapter->dev; 1730 1731 rid = PCIR_BAR(0); 1732 adapter->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 1733 &rid, RF_ACTIVE); 1734 1735 if (!(adapter->pci_mem)) { 1736 device_printf(dev,"Unable to allocate bus resource: memory\n"); 1737 return (ENXIO); 1738 } 1739 1740 adapter->osdep.mem_bus_space_tag = 1741 rman_get_bustag(adapter->pci_mem); 1742 adapter->osdep.mem_bus_space_handle = 1743 rman_get_bushandle(adapter->pci_mem); 1744 adapter->hw.hw_addr = (u8 *) &adapter->osdep.mem_bus_space_handle; 1745 1746 adapter->num_queues = 1; 1747 adapter->hw.back = &adapter->osdep; 1748 1749 /* 1750 ** Now setup MSI/X, should 1751 ** return us the number of 1752 ** configured vectors. 1753 */ 1754 adapter->msix = ixv_setup_msix(adapter); 1755 if (adapter->msix == ENXIO) 1756 return (ENXIO); 1757 else 1758 return (0); 1759} 1760 1761static void 1762ixv_free_pci_resources(struct adapter * adapter) 1763{ 1764 struct ix_queue *que = adapter->queues; 1765 device_t dev = adapter->dev; 1766 int rid, memrid; 1767 1768 memrid = PCIR_BAR(MSIX_BAR); 1769 1770 /* 1771 ** There is a slight possibility of a failure mode 1772 ** in attach that will result in entering this function 1773 ** before interrupt resources have been initialized, and 1774 ** in that case we do not want to execute the loops below 1775 ** We can detect this reliably by the state of the adapter 1776 ** res pointer. 1777 */ 1778 if (adapter->res == NULL) 1779 goto mem; 1780 1781 /* 1782 ** Release all msix queue resources: 1783 */ 1784 for (int i = 0; i < adapter->num_queues; i++, que++) { 1785 rid = que->msix + 1; 1786 if (que->tag != NULL) { 1787 bus_teardown_intr(dev, que->res, que->tag); 1788 que->tag = NULL; 1789 } 1790 if (que->res != NULL) 1791 bus_release_resource(dev, SYS_RES_IRQ, rid, que->res); 1792 } 1793 1794 1795 /* Clean the Legacy or Link interrupt last */ 1796 if (adapter->mbxvec) /* we are doing MSIX */ 1797 rid = adapter->mbxvec + 1; 1798 else 1799 (adapter->msix != 0) ? (rid = 1):(rid = 0); 1800 1801 if (adapter->tag != NULL) { 1802 bus_teardown_intr(dev, adapter->res, adapter->tag); 1803 adapter->tag = NULL; 1804 } 1805 if (adapter->res != NULL) 1806 bus_release_resource(dev, SYS_RES_IRQ, rid, adapter->res); 1807 1808mem: 1809 if (adapter->msix) 1810 pci_release_msi(dev); 1811 1812 if (adapter->msix_mem != NULL) 1813 bus_release_resource(dev, SYS_RES_MEMORY, 1814 memrid, adapter->msix_mem); 1815 1816 if (adapter->pci_mem != NULL) 1817 bus_release_resource(dev, SYS_RES_MEMORY, 1818 PCIR_BAR(0), adapter->pci_mem); 1819 1820 return; 1821} 1822 1823/********************************************************************* 1824 * 1825 * Setup networking device structure and register an interface. 1826 * 1827 **********************************************************************/ 1828static void 1829ixv_setup_interface(device_t dev, struct adapter *adapter) 1830{ 1831 struct ifnet *ifp; 1832 1833 INIT_DEBUGOUT("ixv_setup_interface: begin"); 1834 1835 ifp = adapter->ifp = if_alloc(IFT_ETHER); 1836 if (ifp == NULL) 1837 panic("%s: can not if_alloc()\n", device_get_nameunit(dev)); 1838 if_initname(ifp, device_get_name(dev), device_get_unit(dev)); 1839 ifp->if_baudrate = 1000000000; 1840 ifp->if_init = ixv_init; 1841 ifp->if_softc = adapter; 1842 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1843 ifp->if_ioctl = ixv_ioctl; 1844#if __FreeBSD_version >= 800000 1845 ifp->if_transmit = ixv_mq_start; 1846 ifp->if_qflush = ixv_qflush; 1847#else 1848 ifp->if_start = ixv_start; 1849#endif 1850 ifp->if_snd.ifq_maxlen = adapter->num_tx_desc - 2; 1851 1852 ether_ifattach(ifp, adapter->hw.mac.addr); 1853 1854 adapter->max_frame_size = 1855 ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; 1856 1857 /* 1858 * Tell the upper layer(s) we support long frames. 1859 */ 1860 ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header); 1861 1862 ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_TSO4 | IFCAP_VLAN_HWCSUM; 1863 ifp->if_capabilities |= IFCAP_JUMBO_MTU; 1864 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING 1865 | IFCAP_VLAN_HWTSO 1866 | IFCAP_VLAN_MTU; 1867 ifp->if_capenable = ifp->if_capabilities; 1868 1869 /* Don't enable LRO by default */ 1870 ifp->if_capabilities |= IFCAP_LRO; 1871 1872 /* 1873 * Specify the media types supported by this adapter and register 1874 * callbacks to update media and link information 1875 */ 1876 ifmedia_init(&adapter->media, IFM_IMASK, ixv_media_change, 1877 ixv_media_status); 1878 ifmedia_add(&adapter->media, IFM_ETHER | IFM_FDX, 0, NULL); 1879 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 1880 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 1881 1882 return; 1883} 1884 1885static void 1886ixv_config_link(struct adapter *adapter) 1887{ 1888 struct ixgbe_hw *hw = &adapter->hw; 1889 u32 autoneg, err = 0; 1890 1891 if (hw->mac.ops.check_link) 1892 err = hw->mac.ops.check_link(hw, &autoneg, 1893 &adapter->link_up, FALSE); 1894 if (err) 1895 goto out; 1896 1897 if (hw->mac.ops.setup_link) 1898 err = hw->mac.ops.setup_link(hw, 1899 autoneg, adapter->link_up); 1900out: 1901 return; 1902} 1903 1904/******************************************************************** 1905 * Manage DMA'able memory. 1906 *******************************************************************/ 1907static void 1908ixv_dmamap_cb(void *arg, bus_dma_segment_t * segs, int nseg, int error) 1909{ 1910 if (error) 1911 return; 1912 *(bus_addr_t *) arg = segs->ds_addr; 1913 return; 1914} 1915 1916static int 1917ixv_dma_malloc(struct adapter *adapter, bus_size_t size, 1918 struct ixv_dma_alloc *dma, int mapflags) 1919{ 1920 device_t dev = adapter->dev; 1921 int r; 1922 1923 r = bus_dma_tag_create(bus_get_dma_tag(adapter->dev), /* parent */ 1924 DBA_ALIGN, 0, /* alignment, bounds */ 1925 BUS_SPACE_MAXADDR, /* lowaddr */ 1926 BUS_SPACE_MAXADDR, /* highaddr */ 1927 NULL, NULL, /* filter, filterarg */ 1928 size, /* maxsize */ 1929 1, /* nsegments */ 1930 size, /* maxsegsize */ 1931 BUS_DMA_ALLOCNOW, /* flags */ 1932 NULL, /* lockfunc */ 1933 NULL, /* lockfuncarg */ 1934 &dma->dma_tag); 1935 if (r != 0) { 1936 device_printf(dev,"ixv_dma_malloc: bus_dma_tag_create failed; " 1937 "error %u\n", r); 1938 goto fail_0; 1939 } 1940 r = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr, 1941 BUS_DMA_NOWAIT, &dma->dma_map); 1942 if (r != 0) { 1943 device_printf(dev,"ixv_dma_malloc: bus_dmamem_alloc failed; " 1944 "error %u\n", r); 1945 goto fail_1; 1946 } 1947 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 1948 size, 1949 ixv_dmamap_cb, 1950 &dma->dma_paddr, 1951 mapflags | BUS_DMA_NOWAIT); 1952 if (r != 0) { 1953 device_printf(dev,"ixv_dma_malloc: bus_dmamap_load failed; " 1954 "error %u\n", r); 1955 goto fail_2; 1956 } 1957 dma->dma_size = size; 1958 return (0); 1959fail_2: 1960 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1961fail_1: 1962 bus_dma_tag_destroy(dma->dma_tag); 1963fail_0: 1964 dma->dma_map = NULL; 1965 dma->dma_tag = NULL; 1966 return (r); 1967} 1968 1969static void 1970ixv_dma_free(struct adapter *adapter, struct ixv_dma_alloc *dma) 1971{ 1972 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 1973 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1974 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 1975 bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); 1976 bus_dma_tag_destroy(dma->dma_tag); 1977} 1978 1979 1980/********************************************************************* 1981 * 1982 * Allocate memory for the transmit and receive rings, and then 1983 * the descriptors associated with each, called only once at attach. 1984 * 1985 **********************************************************************/ 1986static int 1987ixv_allocate_queues(struct adapter *adapter) 1988{ 1989 device_t dev = adapter->dev; 1990 struct ix_queue *que; 1991 struct tx_ring *txr; 1992 struct rx_ring *rxr; 1993 int rsize, tsize, error = 0; 1994 int txconf = 0, rxconf = 0; 1995 1996 /* First allocate the top level queue structs */ 1997 if (!(adapter->queues = 1998 (struct ix_queue *) malloc(sizeof(struct ix_queue) * 1999 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2000 device_printf(dev, "Unable to allocate queue memory\n"); 2001 error = ENOMEM; 2002 goto fail; 2003 } 2004 2005 /* First allocate the TX ring struct memory */ 2006 if (!(adapter->tx_rings = 2007 (struct tx_ring *) malloc(sizeof(struct tx_ring) * 2008 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2009 device_printf(dev, "Unable to allocate TX ring memory\n"); 2010 error = ENOMEM; 2011 goto tx_fail; 2012 } 2013 2014 /* Next allocate the RX */ 2015 if (!(adapter->rx_rings = 2016 (struct rx_ring *) malloc(sizeof(struct rx_ring) * 2017 adapter->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2018 device_printf(dev, "Unable to allocate RX ring memory\n"); 2019 error = ENOMEM; 2020 goto rx_fail; 2021 } 2022 2023 /* For the ring itself */ 2024 tsize = roundup2(adapter->num_tx_desc * 2025 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2026 2027 /* 2028 * Now set up the TX queues, txconf is needed to handle the 2029 * possibility that things fail midcourse and we need to 2030 * undo memory gracefully 2031 */ 2032 for (int i = 0; i < adapter->num_queues; i++, txconf++) { 2033 /* Set up some basics */ 2034 txr = &adapter->tx_rings[i]; 2035 txr->adapter = adapter; 2036 txr->me = i; 2037 2038 /* Initialize the TX side lock */ 2039 snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)", 2040 device_get_nameunit(dev), txr->me); 2041 mtx_init(&txr->tx_mtx, txr->mtx_name, NULL, MTX_DEF); 2042 2043 if (ixv_dma_malloc(adapter, tsize, 2044 &txr->txdma, BUS_DMA_NOWAIT)) { 2045 device_printf(dev, 2046 "Unable to allocate TX Descriptor memory\n"); 2047 error = ENOMEM; 2048 goto err_tx_desc; 2049 } 2050 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2051 bzero((void *)txr->tx_base, tsize); 2052 2053 /* Now allocate transmit buffers for the ring */ 2054 if (ixv_allocate_transmit_buffers(txr)) { 2055 device_printf(dev, 2056 "Critical Failure setting up transmit buffers\n"); 2057 error = ENOMEM; 2058 goto err_tx_desc; 2059 } 2060#if __FreeBSD_version >= 800000 2061 /* Allocate a buf ring */ 2062 txr->br = buf_ring_alloc(IXV_BR_SIZE, M_DEVBUF, 2063 M_WAITOK, &txr->tx_mtx); 2064 if (txr->br == NULL) { 2065 device_printf(dev, 2066 "Critical Failure setting up buf ring\n"); 2067 error = ENOMEM; 2068 goto err_tx_desc; 2069 } 2070#endif 2071 } 2072 2073 /* 2074 * Next the RX queues... 2075 */ 2076 rsize = roundup2(adapter->num_rx_desc * 2077 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2078 for (int i = 0; i < adapter->num_queues; i++, rxconf++) { 2079 rxr = &adapter->rx_rings[i]; 2080 /* Set up some basics */ 2081 rxr->adapter = adapter; 2082 rxr->me = i; 2083 2084 /* Initialize the RX side lock */ 2085 snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)", 2086 device_get_nameunit(dev), rxr->me); 2087 mtx_init(&rxr->rx_mtx, rxr->mtx_name, NULL, MTX_DEF); 2088 2089 if (ixv_dma_malloc(adapter, rsize, 2090 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2091 device_printf(dev, 2092 "Unable to allocate RxDescriptor memory\n"); 2093 error = ENOMEM; 2094 goto err_rx_desc; 2095 } 2096 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2097 bzero((void *)rxr->rx_base, rsize); 2098 2099 /* Allocate receive buffers for the ring*/ 2100 if (ixv_allocate_receive_buffers(rxr)) { 2101 device_printf(dev, 2102 "Critical Failure setting up receive buffers\n"); 2103 error = ENOMEM; 2104 goto err_rx_desc; 2105 } 2106 } 2107 2108 /* 2109 ** Finally set up the queue holding structs 2110 */ 2111 for (int i = 0; i < adapter->num_queues; i++) { 2112 que = &adapter->queues[i]; 2113 que->adapter = adapter; 2114 que->txr = &adapter->tx_rings[i]; 2115 que->rxr = &adapter->rx_rings[i]; 2116 } 2117 2118 return (0); 2119 2120err_rx_desc: 2121 for (rxr = adapter->rx_rings; rxconf > 0; rxr++, rxconf--) 2122 ixv_dma_free(adapter, &rxr->rxdma); 2123err_tx_desc: 2124 for (txr = adapter->tx_rings; txconf > 0; txr++, txconf--) 2125 ixv_dma_free(adapter, &txr->txdma); 2126 free(adapter->rx_rings, M_DEVBUF); 2127rx_fail: 2128 free(adapter->tx_rings, M_DEVBUF); 2129tx_fail: 2130 free(adapter->queues, M_DEVBUF); 2131fail: 2132 return (error); 2133} 2134 2135 2136/********************************************************************* 2137 * 2138 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2139 * the information needed to transmit a packet on the wire. This is 2140 * called only once at attach, setup is done every reset. 2141 * 2142 **********************************************************************/ 2143static int 2144ixv_allocate_transmit_buffers(struct tx_ring *txr) 2145{ 2146 struct adapter *adapter = txr->adapter; 2147 device_t dev = adapter->dev; 2148 struct ixv_tx_buf *txbuf; 2149 int error, i; 2150 2151 /* 2152 * Setup DMA descriptor areas. 2153 */ 2154 if ((error = bus_dma_tag_create( 2155 bus_get_dma_tag(adapter->dev), /* parent */ 2156 1, 0, /* alignment, bounds */ 2157 BUS_SPACE_MAXADDR, /* lowaddr */ 2158 BUS_SPACE_MAXADDR, /* highaddr */ 2159 NULL, NULL, /* filter, filterarg */ 2160 IXV_TSO_SIZE, /* maxsize */ 2161 32, /* nsegments */ 2162 PAGE_SIZE, /* maxsegsize */ 2163 0, /* flags */ 2164 NULL, /* lockfunc */ 2165 NULL, /* lockfuncarg */ 2166 &txr->txtag))) { 2167 device_printf(dev,"Unable to allocate TX DMA tag\n"); 2168 goto fail; 2169 } 2170 2171 if (!(txr->tx_buffers = 2172 (struct ixv_tx_buf *) malloc(sizeof(struct ixv_tx_buf) * 2173 adapter->num_tx_desc, M_DEVBUF, M_NOWAIT | M_ZERO))) { 2174 device_printf(dev, "Unable to allocate tx_buffer memory\n"); 2175 error = ENOMEM; 2176 goto fail; 2177 } 2178 2179 /* Create the descriptor buffer dma maps */ 2180 txbuf = txr->tx_buffers; 2181 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2182 error = bus_dmamap_create(txr->txtag, 0, &txbuf->map); 2183 if (error != 0) { 2184 device_printf(dev, "Unable to create TX DMA map\n"); 2185 goto fail; 2186 } 2187 } 2188 2189 return 0; 2190fail: 2191 /* We free all, it handles case where we are in the middle */ 2192 ixv_free_transmit_structures(adapter); 2193 return (error); 2194} 2195 2196/********************************************************************* 2197 * 2198 * Initialize a transmit ring. 2199 * 2200 **********************************************************************/ 2201static void 2202ixv_setup_transmit_ring(struct tx_ring *txr) 2203{ 2204 struct adapter *adapter = txr->adapter; 2205 struct ixv_tx_buf *txbuf; 2206 int i; 2207 2208 /* Clear the old ring contents */ 2209 IXV_TX_LOCK(txr); 2210 bzero((void *)txr->tx_base, 2211 (sizeof(union ixgbe_adv_tx_desc)) * adapter->num_tx_desc); 2212 /* Reset indices */ 2213 txr->next_avail_desc = 0; 2214 txr->next_to_clean = 0; 2215 2216 /* Free any existing tx buffers. */ 2217 txbuf = txr->tx_buffers; 2218 for (i = 0; i < adapter->num_tx_desc; i++, txbuf++) { 2219 if (txbuf->m_head != NULL) { 2220 bus_dmamap_sync(txr->txtag, txbuf->map, 2221 BUS_DMASYNC_POSTWRITE); 2222 bus_dmamap_unload(txr->txtag, txbuf->map); 2223 m_freem(txbuf->m_head); 2224 txbuf->m_head = NULL; 2225 } 2226 /* Clear the EOP index */ 2227 txbuf->eop_index = -1; 2228 } 2229 2230 /* Set number of descriptors available */ 2231 txr->tx_avail = adapter->num_tx_desc; 2232 2233 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2234 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2235 IXV_TX_UNLOCK(txr); 2236} 2237 2238/********************************************************************* 2239 * 2240 * Initialize all transmit rings. 2241 * 2242 **********************************************************************/ 2243static int 2244ixv_setup_transmit_structures(struct adapter *adapter) 2245{ 2246 struct tx_ring *txr = adapter->tx_rings; 2247 2248 for (int i = 0; i < adapter->num_queues; i++, txr++) 2249 ixv_setup_transmit_ring(txr); 2250 2251 return (0); 2252} 2253 2254/********************************************************************* 2255 * 2256 * Enable transmit unit. 2257 * 2258 **********************************************************************/ 2259static void 2260ixv_initialize_transmit_units(struct adapter *adapter) 2261{ 2262 struct tx_ring *txr = adapter->tx_rings; 2263 struct ixgbe_hw *hw = &adapter->hw; 2264 2265 2266 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2267 u64 tdba = txr->txdma.dma_paddr; 2268 u32 txctrl, txdctl; 2269 2270 /* Set WTHRESH to 8, burst writeback */ 2271 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2272 txdctl |= (8 << 16); 2273 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2274 /* Now enable */ 2275 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(i)); 2276 txdctl |= IXGBE_TXDCTL_ENABLE; 2277 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(i), txdctl); 2278 2279 /* Set the HW Tx Head and Tail indices */ 2280 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDH(i), 0); 2281 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VFTDT(i), 0); 2282 2283 /* Setup Transmit Descriptor Cmd Settings */ 2284 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2285 txr->watchdog_check = FALSE; 2286 2287 /* Set Ring parameters */ 2288 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(i), 2289 (tdba & 0x00000000ffffffffULL)); 2290 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(i), (tdba >> 32)); 2291 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(i), 2292 adapter->num_tx_desc * 2293 sizeof(struct ixgbe_legacy_tx_desc)); 2294 txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(i)); 2295 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2296 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(i), txctrl); 2297 break; 2298 } 2299 2300 return; 2301} 2302 2303/********************************************************************* 2304 * 2305 * Free all transmit rings. 2306 * 2307 **********************************************************************/ 2308static void 2309ixv_free_transmit_structures(struct adapter *adapter) 2310{ 2311 struct tx_ring *txr = adapter->tx_rings; 2312 2313 for (int i = 0; i < adapter->num_queues; i++, txr++) { 2314 IXV_TX_LOCK(txr); 2315 ixv_free_transmit_buffers(txr); 2316 ixv_dma_free(adapter, &txr->txdma); 2317 IXV_TX_UNLOCK(txr); 2318 IXV_TX_LOCK_DESTROY(txr); 2319 } 2320 free(adapter->tx_rings, M_DEVBUF); 2321} 2322 2323/********************************************************************* 2324 * 2325 * Free transmit ring related data structures. 2326 * 2327 **********************************************************************/ 2328static void 2329ixv_free_transmit_buffers(struct tx_ring *txr) 2330{ 2331 struct adapter *adapter = txr->adapter; 2332 struct ixv_tx_buf *tx_buffer; 2333 int i; 2334 2335 INIT_DEBUGOUT("free_transmit_ring: begin"); 2336 2337 if (txr->tx_buffers == NULL) 2338 return; 2339 2340 tx_buffer = txr->tx_buffers; 2341 for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { 2342 if (tx_buffer->m_head != NULL) { 2343 bus_dmamap_sync(txr->txtag, tx_buffer->map, 2344 BUS_DMASYNC_POSTWRITE); 2345 bus_dmamap_unload(txr->txtag, 2346 tx_buffer->map); 2347 m_freem(tx_buffer->m_head); 2348 tx_buffer->m_head = NULL; 2349 if (tx_buffer->map != NULL) { 2350 bus_dmamap_destroy(txr->txtag, 2351 tx_buffer->map); 2352 tx_buffer->map = NULL; 2353 } 2354 } else if (tx_buffer->map != NULL) { 2355 bus_dmamap_unload(txr->txtag, 2356 tx_buffer->map); 2357 bus_dmamap_destroy(txr->txtag, 2358 tx_buffer->map); 2359 tx_buffer->map = NULL; 2360 } 2361 } 2362#if __FreeBSD_version >= 800000 2363 if (txr->br != NULL) 2364 buf_ring_free(txr->br, M_DEVBUF); 2365#endif 2366 if (txr->tx_buffers != NULL) { 2367 free(txr->tx_buffers, M_DEVBUF); 2368 txr->tx_buffers = NULL; 2369 } 2370 if (txr->txtag != NULL) { 2371 bus_dma_tag_destroy(txr->txtag); 2372 txr->txtag = NULL; 2373 } 2374 return; 2375} 2376 2377/********************************************************************* 2378 * 2379 * Advanced Context Descriptor setup for VLAN or CSUM 2380 * 2381 **********************************************************************/ 2382 2383static bool 2384ixv_tx_ctx_setup(struct tx_ring *txr, struct mbuf *mp) 2385{ 2386 struct adapter *adapter = txr->adapter; 2387 struct ixgbe_adv_tx_context_desc *TXD; 2388 struct ixv_tx_buf *tx_buffer; 2389 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2390 struct ether_vlan_header *eh; 2391 struct ip *ip; 2392 struct ip6_hdr *ip6; 2393 int ehdrlen, ip_hlen = 0; 2394 u16 etype; 2395 u8 ipproto = 0; 2396 bool offload = TRUE; 2397 int ctxd = txr->next_avail_desc; 2398 u16 vtag = 0; 2399 2400 2401 if ((mp->m_pkthdr.csum_flags & CSUM_OFFLOAD) == 0) 2402 offload = FALSE; 2403 2404 2405 tx_buffer = &txr->tx_buffers[ctxd]; 2406 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2407 2408 /* 2409 ** In advanced descriptors the vlan tag must 2410 ** be placed into the descriptor itself. 2411 */ 2412 if (mp->m_flags & M_VLANTAG) { 2413 vtag = htole16(mp->m_pkthdr.ether_vtag); 2414 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2415 } else if (offload == FALSE) 2416 return FALSE; 2417 2418 /* 2419 * Determine where frame payload starts. 2420 * Jump over vlan headers if already present, 2421 * helpful for QinQ too. 2422 */ 2423 eh = mtod(mp, struct ether_vlan_header *); 2424 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 2425 etype = ntohs(eh->evl_proto); 2426 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2427 } else { 2428 etype = ntohs(eh->evl_encap_proto); 2429 ehdrlen = ETHER_HDR_LEN; 2430 } 2431 2432 /* Set the ether header length */ 2433 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2434 2435 switch (etype) { 2436 case ETHERTYPE_IP: 2437 ip = (struct ip *)(mp->m_data + ehdrlen); 2438 ip_hlen = ip->ip_hl << 2; 2439 if (mp->m_len < ehdrlen + ip_hlen) 2440 return (FALSE); 2441 ipproto = ip->ip_p; 2442 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2443 break; 2444 case ETHERTYPE_IPV6: 2445 ip6 = (struct ip6_hdr *)(mp->m_data + ehdrlen); 2446 ip_hlen = sizeof(struct ip6_hdr); 2447 if (mp->m_len < ehdrlen + ip_hlen) 2448 return (FALSE); 2449 ipproto = ip6->ip6_nxt; 2450 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2451 break; 2452 default: 2453 offload = FALSE; 2454 break; 2455 } 2456 2457 vlan_macip_lens |= ip_hlen; 2458 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2459 2460 switch (ipproto) { 2461 case IPPROTO_TCP: 2462 if (mp->m_pkthdr.csum_flags & CSUM_TCP) 2463 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2464 break; 2465 2466 case IPPROTO_UDP: 2467 if (mp->m_pkthdr.csum_flags & CSUM_UDP) 2468 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2469 break; 2470 2471#if __FreeBSD_version >= 800000 2472 case IPPROTO_SCTP: 2473 if (mp->m_pkthdr.csum_flags & CSUM_SCTP) 2474 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP; 2475 break; 2476#endif 2477 default: 2478 offload = FALSE; 2479 break; 2480 } 2481 2482 /* Now copy bits into descriptor */ 2483 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2484 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2485 TXD->seqnum_seed = htole32(0); 2486 TXD->mss_l4len_idx = htole32(0); 2487 2488 tx_buffer->m_head = NULL; 2489 tx_buffer->eop_index = -1; 2490 2491 /* We've consumed the first desc, adjust counters */ 2492 if (++ctxd == adapter->num_tx_desc) 2493 ctxd = 0; 2494 txr->next_avail_desc = ctxd; 2495 --txr->tx_avail; 2496 2497 return (offload); 2498} 2499 2500/********************************************************************** 2501 * 2502 * Setup work for hardware segmentation offload (TSO) on 2503 * adapters using advanced tx descriptors 2504 * 2505 **********************************************************************/ 2506static bool 2507ixv_tso_setup(struct tx_ring *txr, struct mbuf *mp, u32 *paylen) 2508{ 2509 struct adapter *adapter = txr->adapter; 2510 struct ixgbe_adv_tx_context_desc *TXD; 2511 struct ixv_tx_buf *tx_buffer; 2512 u32 vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2513 u32 mss_l4len_idx = 0; 2514 u16 vtag = 0; 2515 int ctxd, ehdrlen, hdrlen, ip_hlen, tcp_hlen; 2516 struct ether_vlan_header *eh; 2517 struct ip *ip; 2518 struct tcphdr *th; 2519 2520 2521 /* 2522 * Determine where frame payload starts. 2523 * Jump over vlan headers if already present 2524 */ 2525 eh = mtod(mp, struct ether_vlan_header *); 2526 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) 2527 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 2528 else 2529 ehdrlen = ETHER_HDR_LEN; 2530 2531 /* Ensure we have at least the IP+TCP header in the first mbuf. */ 2532 if (mp->m_len < ehdrlen + sizeof(struct ip) + sizeof(struct tcphdr)) 2533 return FALSE; 2534 2535 ctxd = txr->next_avail_desc; 2536 tx_buffer = &txr->tx_buffers[ctxd]; 2537 TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[ctxd]; 2538 2539 ip = (struct ip *)(mp->m_data + ehdrlen); 2540 if (ip->ip_p != IPPROTO_TCP) 2541 return FALSE; /* 0 */ 2542 ip->ip_sum = 0; 2543 ip_hlen = ip->ip_hl << 2; 2544 th = (struct tcphdr *)((caddr_t)ip + ip_hlen); 2545 th->th_sum = in_pseudo(ip->ip_src.s_addr, 2546 ip->ip_dst.s_addr, htons(IPPROTO_TCP)); 2547 tcp_hlen = th->th_off << 2; 2548 hdrlen = ehdrlen + ip_hlen + tcp_hlen; 2549 2550 /* This is used in the transmit desc in encap */ 2551 *paylen = mp->m_pkthdr.len - hdrlen; 2552 2553 /* VLAN MACLEN IPLEN */ 2554 if (mp->m_flags & M_VLANTAG) { 2555 vtag = htole16(mp->m_pkthdr.ether_vtag); 2556 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2557 } 2558 2559 vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT; 2560 vlan_macip_lens |= ip_hlen; 2561 TXD->vlan_macip_lens |= htole32(vlan_macip_lens); 2562 2563 /* ADV DTYPE TUCMD */ 2564 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2565 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2566 type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2567 TXD->type_tucmd_mlhl |= htole32(type_tucmd_mlhl); 2568 2569 2570 /* MSS L4LEN IDX */ 2571 mss_l4len_idx |= (mp->m_pkthdr.tso_segsz << IXGBE_ADVTXD_MSS_SHIFT); 2572 mss_l4len_idx |= (tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT); 2573 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2574 2575 TXD->seqnum_seed = htole32(0); 2576 tx_buffer->m_head = NULL; 2577 tx_buffer->eop_index = -1; 2578 2579 if (++ctxd == adapter->num_tx_desc) 2580 ctxd = 0; 2581 2582 txr->tx_avail--; 2583 txr->next_avail_desc = ctxd; 2584 return TRUE; 2585} 2586 2587 2588/********************************************************************** 2589 * 2590 * Examine each tx_buffer in the used queue. If the hardware is done 2591 * processing the packet then free associated resources. The 2592 * tx_buffer is put back on the free queue. 2593 * 2594 **********************************************************************/ 2595static bool 2596ixv_txeof(struct tx_ring *txr) 2597{ 2598 struct adapter *adapter = txr->adapter; 2599 struct ifnet *ifp = adapter->ifp; 2600 u32 first, last, done; 2601 struct ixv_tx_buf *tx_buffer; 2602 struct ixgbe_legacy_tx_desc *tx_desc, *eop_desc; 2603 2604 mtx_assert(&txr->tx_mtx, MA_OWNED); 2605 2606 if (txr->tx_avail == adapter->num_tx_desc) 2607 return FALSE; 2608 2609 first = txr->next_to_clean; 2610 tx_buffer = &txr->tx_buffers[first]; 2611 /* For cleanup we just use legacy struct */ 2612 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2613 last = tx_buffer->eop_index; 2614 if (last == -1) 2615 return FALSE; 2616 eop_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2617 2618 /* 2619 ** Get the index of the first descriptor 2620 ** BEYOND the EOP and call that 'done'. 2621 ** I do this so the comparison in the 2622 ** inner while loop below can be simple 2623 */ 2624 if (++last == adapter->num_tx_desc) last = 0; 2625 done = last; 2626 2627 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2628 BUS_DMASYNC_POSTREAD); 2629 /* 2630 ** Only the EOP descriptor of a packet now has the DD 2631 ** bit set, this is what we look for... 2632 */ 2633 while (eop_desc->upper.fields.status & IXGBE_TXD_STAT_DD) { 2634 /* We clean the range of the packet */ 2635 while (first != done) { 2636 tx_desc->upper.data = 0; 2637 tx_desc->lower.data = 0; 2638 tx_desc->buffer_addr = 0; 2639 ++txr->tx_avail; 2640 2641 if (tx_buffer->m_head) { 2642 bus_dmamap_sync(txr->txtag, 2643 tx_buffer->map, 2644 BUS_DMASYNC_POSTWRITE); 2645 bus_dmamap_unload(txr->txtag, 2646 tx_buffer->map); 2647 m_freem(tx_buffer->m_head); 2648 tx_buffer->m_head = NULL; 2649 tx_buffer->map = NULL; 2650 } 2651 tx_buffer->eop_index = -1; 2652 txr->watchdog_time = ticks; 2653 2654 if (++first == adapter->num_tx_desc) 2655 first = 0; 2656 2657 tx_buffer = &txr->tx_buffers[first]; 2658 tx_desc = 2659 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[first]; 2660 } 2661 ++ifp->if_opackets; 2662 /* See if there is more work now */ 2663 last = tx_buffer->eop_index; 2664 if (last != -1) { 2665 eop_desc = 2666 (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2667 /* Get next done point */ 2668 if (++last == adapter->num_tx_desc) last = 0; 2669 done = last; 2670 } else 2671 break; 2672 } 2673 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2674 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2675 2676 txr->next_to_clean = first; 2677 2678 /* 2679 * If we have enough room, clear IFF_DRV_OACTIVE to tell the stack that 2680 * it is OK to send packets. If there are no pending descriptors, 2681 * clear the timeout. Otherwise, if some descriptors have been freed, 2682 * restart the timeout. 2683 */ 2684 if (txr->tx_avail > IXV_TX_CLEANUP_THRESHOLD) { 2685 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 2686 if (txr->tx_avail == adapter->num_tx_desc) { 2687 txr->watchdog_check = FALSE; 2688 return FALSE; 2689 } 2690 } 2691 2692 return TRUE; 2693} 2694 2695/********************************************************************* 2696 * 2697 * Refresh mbuf buffers for RX descriptor rings 2698 * - now keeps its own state so discards due to resource 2699 * exhaustion are unnecessary, if an mbuf cannot be obtained 2700 * it just returns, keeping its placeholder, thus it can simply 2701 * be recalled to try again. 2702 * 2703 **********************************************************************/ 2704static void 2705ixv_refresh_mbufs(struct rx_ring *rxr, int limit) 2706{ 2707 struct adapter *adapter = rxr->adapter; 2708 bus_dma_segment_t hseg[1]; 2709 bus_dma_segment_t pseg[1]; 2710 struct ixv_rx_buf *rxbuf; 2711 struct mbuf *mh, *mp; 2712 int i, j, nsegs, error; 2713 bool refreshed = FALSE; 2714 2715 i = j = rxr->next_to_refresh; 2716 /* Get the control variable, one beyond refresh point */ 2717 if (++j == adapter->num_rx_desc) 2718 j = 0; 2719 while (j != limit) { 2720 rxbuf = &rxr->rx_buffers[i]; 2721 if ((rxbuf->m_head == NULL) && (rxr->hdr_split)) { 2722 mh = m_gethdr(M_NOWAIT, MT_DATA); 2723 if (mh == NULL) 2724 goto update; 2725 mh->m_pkthdr.len = mh->m_len = MHLEN; 2726 mh->m_len = MHLEN; 2727 mh->m_flags |= M_PKTHDR; 2728 m_adj(mh, ETHER_ALIGN); 2729 /* Get the memory mapping */ 2730 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2731 rxbuf->hmap, mh, hseg, &nsegs, BUS_DMA_NOWAIT); 2732 if (error != 0) { 2733 printf("GET BUF: dmamap load" 2734 " failure - %d\n", error); 2735 m_free(mh); 2736 goto update; 2737 } 2738 rxbuf->m_head = mh; 2739 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2740 BUS_DMASYNC_PREREAD); 2741 rxr->rx_base[i].read.hdr_addr = 2742 htole64(hseg[0].ds_addr); 2743 } 2744 2745 if (rxbuf->m_pack == NULL) { 2746 mp = m_getjcl(M_NOWAIT, MT_DATA, 2747 M_PKTHDR, adapter->rx_mbuf_sz); 2748 if (mp == NULL) 2749 goto update; 2750 } else 2751 mp = rxbuf->m_pack; 2752 2753 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2754 /* Get the memory mapping */ 2755 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2756 rxbuf->pmap, mp, pseg, &nsegs, BUS_DMA_NOWAIT); 2757 if (error != 0) { 2758 printf("GET BUF: dmamap load" 2759 " failure - %d\n", error); 2760 m_free(mp); 2761 rxbuf->m_pack = NULL; 2762 goto update; 2763 } 2764 rxbuf->m_pack = mp; 2765 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2766 BUS_DMASYNC_PREREAD); 2767 rxr->rx_base[i].read.pkt_addr = 2768 htole64(pseg[0].ds_addr); 2769 2770 refreshed = TRUE; 2771 rxr->next_to_refresh = i = j; 2772 /* Calculate next index */ 2773 if (++j == adapter->num_rx_desc) 2774 j = 0; 2775 } 2776update: 2777 if (refreshed) /* update tail index */ 2778 IXGBE_WRITE_REG(&adapter->hw, 2779 IXGBE_VFRDT(rxr->me), rxr->next_to_refresh); 2780 return; 2781} 2782 2783/********************************************************************* 2784 * 2785 * Allocate memory for rx_buffer structures. Since we use one 2786 * rx_buffer per received packet, the maximum number of rx_buffer's 2787 * that we'll need is equal to the number of receive descriptors 2788 * that we've allocated. 2789 * 2790 **********************************************************************/ 2791static int 2792ixv_allocate_receive_buffers(struct rx_ring *rxr) 2793{ 2794 struct adapter *adapter = rxr->adapter; 2795 device_t dev = adapter->dev; 2796 struct ixv_rx_buf *rxbuf; 2797 int i, bsize, error; 2798 2799 bsize = sizeof(struct ixv_rx_buf) * adapter->num_rx_desc; 2800 if (!(rxr->rx_buffers = 2801 (struct ixv_rx_buf *) malloc(bsize, 2802 M_DEVBUF, M_NOWAIT | M_ZERO))) { 2803 device_printf(dev, "Unable to allocate rx_buffer memory\n"); 2804 error = ENOMEM; 2805 goto fail; 2806 } 2807 2808 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2809 1, 0, /* alignment, bounds */ 2810 BUS_SPACE_MAXADDR, /* lowaddr */ 2811 BUS_SPACE_MAXADDR, /* highaddr */ 2812 NULL, NULL, /* filter, filterarg */ 2813 MSIZE, /* maxsize */ 2814 1, /* nsegments */ 2815 MSIZE, /* maxsegsize */ 2816 0, /* flags */ 2817 NULL, /* lockfunc */ 2818 NULL, /* lockfuncarg */ 2819 &rxr->htag))) { 2820 device_printf(dev, "Unable to create RX DMA tag\n"); 2821 goto fail; 2822 } 2823 2824 if ((error = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */ 2825 1, 0, /* alignment, bounds */ 2826 BUS_SPACE_MAXADDR, /* lowaddr */ 2827 BUS_SPACE_MAXADDR, /* highaddr */ 2828 NULL, NULL, /* filter, filterarg */ 2829 MJUMPAGESIZE, /* maxsize */ 2830 1, /* nsegments */ 2831 MJUMPAGESIZE, /* maxsegsize */ 2832 0, /* flags */ 2833 NULL, /* lockfunc */ 2834 NULL, /* lockfuncarg */ 2835 &rxr->ptag))) { 2836 device_printf(dev, "Unable to create RX DMA tag\n"); 2837 goto fail; 2838 } 2839 2840 for (i = 0; i < adapter->num_rx_desc; i++, rxbuf++) { 2841 rxbuf = &rxr->rx_buffers[i]; 2842 error = bus_dmamap_create(rxr->htag, 2843 BUS_DMA_NOWAIT, &rxbuf->hmap); 2844 if (error) { 2845 device_printf(dev, "Unable to create RX head map\n"); 2846 goto fail; 2847 } 2848 error = bus_dmamap_create(rxr->ptag, 2849 BUS_DMA_NOWAIT, &rxbuf->pmap); 2850 if (error) { 2851 device_printf(dev, "Unable to create RX pkt map\n"); 2852 goto fail; 2853 } 2854 } 2855 2856 return (0); 2857 2858fail: 2859 /* Frees all, but can handle partial completion */ 2860 ixv_free_receive_structures(adapter); 2861 return (error); 2862} 2863 2864static void 2865ixv_free_receive_ring(struct rx_ring *rxr) 2866{ 2867 struct adapter *adapter; 2868 struct ixv_rx_buf *rxbuf; 2869 int i; 2870 2871 adapter = rxr->adapter; 2872 for (i = 0; i < adapter->num_rx_desc; i++) { 2873 rxbuf = &rxr->rx_buffers[i]; 2874 if (rxbuf->m_head != NULL) { 2875 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 2876 BUS_DMASYNC_POSTREAD); 2877 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 2878 rxbuf->m_head->m_flags |= M_PKTHDR; 2879 m_freem(rxbuf->m_head); 2880 } 2881 if (rxbuf->m_pack != NULL) { 2882 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 2883 BUS_DMASYNC_POSTREAD); 2884 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 2885 rxbuf->m_pack->m_flags |= M_PKTHDR; 2886 m_freem(rxbuf->m_pack); 2887 } 2888 rxbuf->m_head = NULL; 2889 rxbuf->m_pack = NULL; 2890 } 2891} 2892 2893 2894/********************************************************************* 2895 * 2896 * Initialize a receive ring and its buffers. 2897 * 2898 **********************************************************************/ 2899static int 2900ixv_setup_receive_ring(struct rx_ring *rxr) 2901{ 2902 struct adapter *adapter; 2903 struct ifnet *ifp; 2904 device_t dev; 2905 struct ixv_rx_buf *rxbuf; 2906 bus_dma_segment_t pseg[1], hseg[1]; 2907 struct lro_ctrl *lro = &rxr->lro; 2908 int rsize, nsegs, error = 0; 2909 2910 adapter = rxr->adapter; 2911 ifp = adapter->ifp; 2912 dev = adapter->dev; 2913 2914 /* Clear the ring contents */ 2915 IXV_RX_LOCK(rxr); 2916 rsize = roundup2(adapter->num_rx_desc * 2917 sizeof(union ixgbe_adv_rx_desc), DBA_ALIGN); 2918 bzero((void *)rxr->rx_base, rsize); 2919 2920 /* Free current RX buffer structs and their mbufs */ 2921 ixv_free_receive_ring(rxr); 2922 2923 /* Configure header split? */ 2924 if (ixv_header_split) 2925 rxr->hdr_split = TRUE; 2926 2927 /* Now replenish the mbufs */ 2928 for (int j = 0; j != adapter->num_rx_desc; ++j) { 2929 struct mbuf *mh, *mp; 2930 2931 rxbuf = &rxr->rx_buffers[j]; 2932 /* 2933 ** Dont allocate mbufs if not 2934 ** doing header split, its wasteful 2935 */ 2936 if (rxr->hdr_split == FALSE) 2937 goto skip_head; 2938 2939 /* First the header */ 2940 rxbuf->m_head = m_gethdr(M_NOWAIT, MT_DATA); 2941 if (rxbuf->m_head == NULL) { 2942 error = ENOBUFS; 2943 goto fail; 2944 } 2945 m_adj(rxbuf->m_head, ETHER_ALIGN); 2946 mh = rxbuf->m_head; 2947 mh->m_len = mh->m_pkthdr.len = MHLEN; 2948 mh->m_flags |= M_PKTHDR; 2949 /* Get the memory mapping */ 2950 error = bus_dmamap_load_mbuf_sg(rxr->htag, 2951 rxbuf->hmap, rxbuf->m_head, hseg, 2952 &nsegs, BUS_DMA_NOWAIT); 2953 if (error != 0) /* Nothing elegant to do here */ 2954 goto fail; 2955 bus_dmamap_sync(rxr->htag, 2956 rxbuf->hmap, BUS_DMASYNC_PREREAD); 2957 /* Update descriptor */ 2958 rxr->rx_base[j].read.hdr_addr = htole64(hseg[0].ds_addr); 2959 2960skip_head: 2961 /* Now the payload cluster */ 2962 rxbuf->m_pack = m_getjcl(M_NOWAIT, MT_DATA, 2963 M_PKTHDR, adapter->rx_mbuf_sz); 2964 if (rxbuf->m_pack == NULL) { 2965 error = ENOBUFS; 2966 goto fail; 2967 } 2968 mp = rxbuf->m_pack; 2969 mp->m_pkthdr.len = mp->m_len = adapter->rx_mbuf_sz; 2970 /* Get the memory mapping */ 2971 error = bus_dmamap_load_mbuf_sg(rxr->ptag, 2972 rxbuf->pmap, mp, pseg, 2973 &nsegs, BUS_DMA_NOWAIT); 2974 if (error != 0) 2975 goto fail; 2976 bus_dmamap_sync(rxr->ptag, 2977 rxbuf->pmap, BUS_DMASYNC_PREREAD); 2978 /* Update descriptor */ 2979 rxr->rx_base[j].read.pkt_addr = htole64(pseg[0].ds_addr); 2980 } 2981 2982 2983 /* Setup our descriptor indices */ 2984 rxr->next_to_check = 0; 2985 rxr->next_to_refresh = 0; 2986 rxr->lro_enabled = FALSE; 2987 rxr->rx_split_packets = 0; 2988 rxr->rx_bytes = 0; 2989 rxr->discard = FALSE; 2990 2991 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2992 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2993 2994 /* 2995 ** Now set up the LRO interface: 2996 */ 2997 if (ifp->if_capenable & IFCAP_LRO) { 2998 int err = tcp_lro_init(lro); 2999 if (err) { 3000 device_printf(dev, "LRO Initialization failed!\n"); 3001 goto fail; 3002 } 3003 INIT_DEBUGOUT("RX Soft LRO Initialized\n"); 3004 rxr->lro_enabled = TRUE; 3005 lro->ifp = adapter->ifp; 3006 } 3007 3008 IXV_RX_UNLOCK(rxr); 3009 return (0); 3010 3011fail: 3012 ixv_free_receive_ring(rxr); 3013 IXV_RX_UNLOCK(rxr); 3014 return (error); 3015} 3016 3017/********************************************************************* 3018 * 3019 * Initialize all receive rings. 3020 * 3021 **********************************************************************/ 3022static int 3023ixv_setup_receive_structures(struct adapter *adapter) 3024{ 3025 struct rx_ring *rxr = adapter->rx_rings; 3026 int j; 3027 3028 for (j = 0; j < adapter->num_queues; j++, rxr++) 3029 if (ixv_setup_receive_ring(rxr)) 3030 goto fail; 3031 3032 return (0); 3033fail: 3034 /* 3035 * Free RX buffers allocated so far, we will only handle 3036 * the rings that completed, the failing case will have 3037 * cleaned up for itself. 'j' failed, so its the terminus. 3038 */ 3039 for (int i = 0; i < j; ++i) { 3040 rxr = &adapter->rx_rings[i]; 3041 ixv_free_receive_ring(rxr); 3042 } 3043 3044 return (ENOBUFS); 3045} 3046 3047/********************************************************************* 3048 * 3049 * Setup receive registers and features. 3050 * 3051 **********************************************************************/ 3052#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 3053 3054static void 3055ixv_initialize_receive_units(struct adapter *adapter) 3056{ 3057 struct rx_ring *rxr = adapter->rx_rings; 3058 struct ixgbe_hw *hw = &adapter->hw; 3059 struct ifnet *ifp = adapter->ifp; 3060 u32 bufsz, fctrl, rxcsum, hlreg; 3061 3062 3063 /* Enable broadcasts */ 3064 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 3065 fctrl |= IXGBE_FCTRL_BAM; 3066 fctrl |= IXGBE_FCTRL_DPF; 3067 fctrl |= IXGBE_FCTRL_PMCF; 3068 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 3069 3070 /* Set for Jumbo Frames? */ 3071 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 3072 if (ifp->if_mtu > ETHERMTU) { 3073 hlreg |= IXGBE_HLREG0_JUMBOEN; 3074 bufsz = 4096 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3075 } else { 3076 hlreg &= ~IXGBE_HLREG0_JUMBOEN; 3077 bufsz = 2048 >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 3078 } 3079 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 3080 3081 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3082 u64 rdba = rxr->rxdma.dma_paddr; 3083 u32 reg, rxdctl; 3084 3085 /* Do the queue enabling first */ 3086 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3087 rxdctl |= IXGBE_RXDCTL_ENABLE; 3088 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), rxdctl); 3089 for (int k = 0; k < 10; k++) { 3090 if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)) & 3091 IXGBE_RXDCTL_ENABLE) 3092 break; 3093 else 3094 msec_delay(1); 3095 } 3096 wmb(); 3097 3098 /* Setup the Base and Length of the Rx Descriptor Ring */ 3099 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(i), 3100 (rdba & 0x00000000ffffffffULL)); 3101 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(i), 3102 (rdba >> 32)); 3103 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(i), 3104 adapter->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 3105 3106 /* Set up the SRRCTL register */ 3107 reg = IXGBE_READ_REG(hw, IXGBE_VFSRRCTL(i)); 3108 reg &= ~IXGBE_SRRCTL_BSIZEHDR_MASK; 3109 reg &= ~IXGBE_SRRCTL_BSIZEPKT_MASK; 3110 reg |= bufsz; 3111 if (rxr->hdr_split) { 3112 /* Use a standard mbuf for the header */ 3113 reg |= ((IXV_RX_HDR << 3114 IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) 3115 & IXGBE_SRRCTL_BSIZEHDR_MASK); 3116 reg |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS; 3117 } else 3118 reg |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 3119 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(i), reg); 3120 3121 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 3122 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(rxr->me), 0); 3123 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rxr->me), 3124 adapter->num_rx_desc - 1); 3125 } 3126 3127 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3128 3129 if (ifp->if_capenable & IFCAP_RXCSUM) 3130 rxcsum |= IXGBE_RXCSUM_PCSD; 3131 3132 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 3133 rxcsum |= IXGBE_RXCSUM_IPPCSE; 3134 3135 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3136 3137 return; 3138} 3139 3140/********************************************************************* 3141 * 3142 * Free all receive rings. 3143 * 3144 **********************************************************************/ 3145static void 3146ixv_free_receive_structures(struct adapter *adapter) 3147{ 3148 struct rx_ring *rxr = adapter->rx_rings; 3149 3150 for (int i = 0; i < adapter->num_queues; i++, rxr++) { 3151 struct lro_ctrl *lro = &rxr->lro; 3152 ixv_free_receive_buffers(rxr); 3153 /* Free LRO memory */ 3154 tcp_lro_free(lro); 3155 /* Free the ring memory as well */ 3156 ixv_dma_free(adapter, &rxr->rxdma); 3157 } 3158 3159 free(adapter->rx_rings, M_DEVBUF); 3160} 3161 3162 3163/********************************************************************* 3164 * 3165 * Free receive ring data structures 3166 * 3167 **********************************************************************/ 3168static void 3169ixv_free_receive_buffers(struct rx_ring *rxr) 3170{ 3171 struct adapter *adapter = rxr->adapter; 3172 struct ixv_rx_buf *rxbuf; 3173 3174 INIT_DEBUGOUT("free_receive_structures: begin"); 3175 3176 /* Cleanup any existing buffers */ 3177 if (rxr->rx_buffers != NULL) { 3178 for (int i = 0; i < adapter->num_rx_desc; i++) { 3179 rxbuf = &rxr->rx_buffers[i]; 3180 if (rxbuf->m_head != NULL) { 3181 bus_dmamap_sync(rxr->htag, rxbuf->hmap, 3182 BUS_DMASYNC_POSTREAD); 3183 bus_dmamap_unload(rxr->htag, rxbuf->hmap); 3184 rxbuf->m_head->m_flags |= M_PKTHDR; 3185 m_freem(rxbuf->m_head); 3186 } 3187 if (rxbuf->m_pack != NULL) { 3188 bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 3189 BUS_DMASYNC_POSTREAD); 3190 bus_dmamap_unload(rxr->ptag, rxbuf->pmap); 3191 rxbuf->m_pack->m_flags |= M_PKTHDR; 3192 m_freem(rxbuf->m_pack); 3193 } 3194 rxbuf->m_head = NULL; 3195 rxbuf->m_pack = NULL; 3196 if (rxbuf->hmap != NULL) { 3197 bus_dmamap_destroy(rxr->htag, rxbuf->hmap); 3198 rxbuf->hmap = NULL; 3199 } 3200 if (rxbuf->pmap != NULL) { 3201 bus_dmamap_destroy(rxr->ptag, rxbuf->pmap); 3202 rxbuf->pmap = NULL; 3203 } 3204 } 3205 if (rxr->rx_buffers != NULL) { 3206 free(rxr->rx_buffers, M_DEVBUF); 3207 rxr->rx_buffers = NULL; 3208 } 3209 } 3210 3211 if (rxr->htag != NULL) { 3212 bus_dma_tag_destroy(rxr->htag); 3213 rxr->htag = NULL; 3214 } 3215 if (rxr->ptag != NULL) { 3216 bus_dma_tag_destroy(rxr->ptag); 3217 rxr->ptag = NULL; 3218 } 3219 3220 return; 3221} 3222 3223static __inline void 3224ixv_rx_input(struct rx_ring *rxr, struct ifnet *ifp, struct mbuf *m, u32 ptype) 3225{ 3226 3227 /* 3228 * ATM LRO is only for IPv4/TCP packets and TCP checksum of the packet 3229 * should be computed by hardware. Also it should not have VLAN tag in 3230 * ethernet header. 3231 */ 3232 if (rxr->lro_enabled && 3233 (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 3234 (ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3235 (ptype & (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP)) == 3236 (IXGBE_RXDADV_PKTTYPE_IPV4 | IXGBE_RXDADV_PKTTYPE_TCP) && 3237 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 3238 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 3239 /* 3240 * Send to the stack if: 3241 ** - LRO not enabled, or 3242 ** - no LRO resources, or 3243 ** - lro enqueue fails 3244 */ 3245 if (rxr->lro.lro_cnt != 0) 3246 if (tcp_lro_rx(&rxr->lro, m, 0) == 0) 3247 return; 3248 } 3249 IXV_RX_UNLOCK(rxr); 3250 (*ifp->if_input)(ifp, m); 3251 IXV_RX_LOCK(rxr); 3252} 3253 3254static __inline void 3255ixv_rx_discard(struct rx_ring *rxr, int i) 3256{ 3257 struct ixv_rx_buf *rbuf; 3258 3259 rbuf = &rxr->rx_buffers[i]; 3260 3261 if (rbuf->fmp != NULL) {/* Partial chain ? */ 3262 rbuf->fmp->m_flags |= M_PKTHDR; 3263 m_freem(rbuf->fmp); 3264 rbuf->fmp = NULL; 3265 } 3266 3267 /* 3268 ** With advanced descriptors the writeback 3269 ** clobbers the buffer addrs, so its easier 3270 ** to just free the existing mbufs and take 3271 ** the normal refresh path to get new buffers 3272 ** and mapping. 3273 */ 3274 if (rbuf->m_head) { 3275 m_free(rbuf->m_head); 3276 rbuf->m_head = NULL; 3277 } 3278 3279 if (rbuf->m_pack) { 3280 m_free(rbuf->m_pack); 3281 rbuf->m_pack = NULL; 3282 } 3283 3284 return; 3285} 3286 3287 3288/********************************************************************* 3289 * 3290 * This routine executes in interrupt context. It replenishes 3291 * the mbufs in the descriptor and sends data which has been 3292 * dma'ed into host memory to upper layer. 3293 * 3294 * We loop at most count times if count is > 0, or until done if 3295 * count < 0. 3296 * 3297 * Return TRUE for more work, FALSE for all clean. 3298 *********************************************************************/ 3299static bool 3300ixv_rxeof(struct ix_queue *que, int count) 3301{ 3302 struct adapter *adapter = que->adapter; 3303 struct rx_ring *rxr = que->rxr; 3304 struct ifnet *ifp = adapter->ifp; 3305 struct lro_ctrl *lro = &rxr->lro; 3306 struct lro_entry *queued; 3307 int i, nextp, processed = 0; 3308 u32 staterr = 0; 3309 union ixgbe_adv_rx_desc *cur; 3310 struct ixv_rx_buf *rbuf, *nbuf; 3311 3312 IXV_RX_LOCK(rxr); 3313 3314 for (i = rxr->next_to_check; count != 0;) { 3315 struct mbuf *sendmp, *mh, *mp; 3316 u32 rsc, ptype; 3317 u16 hlen, plen, hdr, vtag; 3318 bool eop; 3319 3320 /* Sync the ring. */ 3321 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3322 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 3323 3324 cur = &rxr->rx_base[i]; 3325 staterr = le32toh(cur->wb.upper.status_error); 3326 3327 if ((staterr & IXGBE_RXD_STAT_DD) == 0) 3328 break; 3329 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 3330 break; 3331 3332 count--; 3333 sendmp = NULL; 3334 nbuf = NULL; 3335 rsc = 0; 3336 cur->wb.upper.status_error = 0; 3337 rbuf = &rxr->rx_buffers[i]; 3338 mh = rbuf->m_head; 3339 mp = rbuf->m_pack; 3340 3341 plen = le16toh(cur->wb.upper.length); 3342 ptype = le32toh(cur->wb.lower.lo_dword.data) & 3343 IXGBE_RXDADV_PKTTYPE_MASK; 3344 hdr = le16toh(cur->wb.lower.lo_dword.hs_rss.hdr_info); 3345 vtag = le16toh(cur->wb.upper.vlan); 3346 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3347 3348 /* Make sure all parts of a bad packet are discarded */ 3349 if (((staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) || 3350 (rxr->discard)) { 3351 ifp->if_ierrors++; 3352 rxr->rx_discarded++; 3353 if (!eop) 3354 rxr->discard = TRUE; 3355 else 3356 rxr->discard = FALSE; 3357 ixv_rx_discard(rxr, i); 3358 goto next_desc; 3359 } 3360 3361 if (!eop) { 3362 nextp = i + 1; 3363 if (nextp == adapter->num_rx_desc) 3364 nextp = 0; 3365 nbuf = &rxr->rx_buffers[nextp]; 3366 prefetch(nbuf); 3367 } 3368 /* 3369 ** The header mbuf is ONLY used when header 3370 ** split is enabled, otherwise we get normal 3371 ** behavior, ie, both header and payload 3372 ** are DMA'd into the payload buffer. 3373 ** 3374 ** Rather than using the fmp/lmp global pointers 3375 ** we now keep the head of a packet chain in the 3376 ** buffer struct and pass this along from one 3377 ** descriptor to the next, until we get EOP. 3378 */ 3379 if (rxr->hdr_split && (rbuf->fmp == NULL)) { 3380 /* This must be an initial descriptor */ 3381 hlen = (hdr & IXGBE_RXDADV_HDRBUFLEN_MASK) >> 3382 IXGBE_RXDADV_HDRBUFLEN_SHIFT; 3383 if (hlen > IXV_RX_HDR) 3384 hlen = IXV_RX_HDR; 3385 mh->m_len = hlen; 3386 mh->m_flags |= M_PKTHDR; 3387 mh->m_next = NULL; 3388 mh->m_pkthdr.len = mh->m_len; 3389 /* Null buf pointer so it is refreshed */ 3390 rbuf->m_head = NULL; 3391 /* 3392 ** Check the payload length, this 3393 ** could be zero if its a small 3394 ** packet. 3395 */ 3396 if (plen > 0) { 3397 mp->m_len = plen; 3398 mp->m_next = NULL; 3399 mp->m_flags &= ~M_PKTHDR; 3400 mh->m_next = mp; 3401 mh->m_pkthdr.len += mp->m_len; 3402 /* Null buf pointer so it is refreshed */ 3403 rbuf->m_pack = NULL; 3404 rxr->rx_split_packets++; 3405 } 3406 /* 3407 ** Now create the forward 3408 ** chain so when complete 3409 ** we wont have to. 3410 */ 3411 if (eop == 0) { 3412 /* stash the chain head */ 3413 nbuf->fmp = mh; 3414 /* Make forward chain */ 3415 if (plen) 3416 mp->m_next = nbuf->m_pack; 3417 else 3418 mh->m_next = nbuf->m_pack; 3419 } else { 3420 /* Singlet, prepare to send */ 3421 sendmp = mh; 3422 if ((adapter->num_vlans) && 3423 (staterr & IXGBE_RXD_STAT_VP)) { 3424 sendmp->m_pkthdr.ether_vtag = vtag; 3425 sendmp->m_flags |= M_VLANTAG; 3426 } 3427 } 3428 } else { 3429 /* 3430 ** Either no header split, or a 3431 ** secondary piece of a fragmented 3432 ** split packet. 3433 */ 3434 mp->m_len = plen; 3435 /* 3436 ** See if there is a stored head 3437 ** that determines what we are 3438 */ 3439 sendmp = rbuf->fmp; 3440 rbuf->m_pack = rbuf->fmp = NULL; 3441 3442 if (sendmp != NULL) /* secondary frag */ 3443 sendmp->m_pkthdr.len += mp->m_len; 3444 else { 3445 /* first desc of a non-ps chain */ 3446 sendmp = mp; 3447 sendmp->m_flags |= M_PKTHDR; 3448 sendmp->m_pkthdr.len = mp->m_len; 3449 if (staterr & IXGBE_RXD_STAT_VP) { 3450 sendmp->m_pkthdr.ether_vtag = vtag; 3451 sendmp->m_flags |= M_VLANTAG; 3452 } 3453 } 3454 /* Pass the head pointer on */ 3455 if (eop == 0) { 3456 nbuf->fmp = sendmp; 3457 sendmp = NULL; 3458 mp->m_next = nbuf->m_pack; 3459 } 3460 } 3461 ++processed; 3462 /* Sending this frame? */ 3463 if (eop) { 3464 sendmp->m_pkthdr.rcvif = ifp; 3465 ifp->if_ipackets++; 3466 rxr->rx_packets++; 3467 /* capture data for AIM */ 3468 rxr->bytes += sendmp->m_pkthdr.len; 3469 rxr->rx_bytes += sendmp->m_pkthdr.len; 3470 if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 3471 ixv_rx_checksum(staterr, sendmp, ptype); 3472#if __FreeBSD_version >= 800000 3473 sendmp->m_pkthdr.flowid = que->msix; 3474 sendmp->m_flags |= M_FLOWID; 3475#endif 3476 } 3477next_desc: 3478 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3479 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 3480 3481 /* Advance our pointers to the next descriptor. */ 3482 if (++i == adapter->num_rx_desc) 3483 i = 0; 3484 3485 /* Now send to the stack or do LRO */ 3486 if (sendmp != NULL) 3487 ixv_rx_input(rxr, ifp, sendmp, ptype); 3488 3489 /* Every 8 descriptors we go to refresh mbufs */ 3490 if (processed == 8) { 3491 ixv_refresh_mbufs(rxr, i); 3492 processed = 0; 3493 } 3494 } 3495 3496 /* Refresh any remaining buf structs */ 3497 if (ixv_rx_unrefreshed(rxr)) 3498 ixv_refresh_mbufs(rxr, i); 3499 3500 rxr->next_to_check = i; 3501 3502 /* 3503 * Flush any outstanding LRO work 3504 */ 3505 while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 3506 SLIST_REMOVE_HEAD(&lro->lro_active, next); 3507 tcp_lro_flush(lro, queued); 3508 } 3509 3510 IXV_RX_UNLOCK(rxr); 3511 3512 /* 3513 ** We still have cleaning to do? 3514 ** Schedule another interrupt if so. 3515 */ 3516 if ((staterr & IXGBE_RXD_STAT_DD) != 0) { 3517 ixv_rearm_queues(adapter, (u64)(1 << que->msix)); 3518 return (TRUE); 3519 } 3520 3521 return (FALSE); 3522} 3523 3524 3525/********************************************************************* 3526 * 3527 * Verify that the hardware indicated that the checksum is valid. 3528 * Inform the stack about the status of checksum so that stack 3529 * doesn't spend time verifying the checksum. 3530 * 3531 *********************************************************************/ 3532static void 3533ixv_rx_checksum(u32 staterr, struct mbuf * mp, u32 ptype) 3534{ 3535 u16 status = (u16) staterr; 3536 u8 errors = (u8) (staterr >> 24); 3537 bool sctp = FALSE; 3538 3539 if ((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 && 3540 (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0) 3541 sctp = TRUE; 3542 3543 if (status & IXGBE_RXD_STAT_IPCS) { 3544 if (!(errors & IXGBE_RXD_ERR_IPE)) { 3545 /* IP Checksum Good */ 3546 mp->m_pkthdr.csum_flags = CSUM_IP_CHECKED; 3547 mp->m_pkthdr.csum_flags |= CSUM_IP_VALID; 3548 3549 } else 3550 mp->m_pkthdr.csum_flags = 0; 3551 } 3552 if (status & IXGBE_RXD_STAT_L4CS) { 3553 u16 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 3554#if __FreeBSD_version >= 800000 3555 if (sctp) 3556 type = CSUM_SCTP_VALID; 3557#endif 3558 if (!(errors & IXGBE_RXD_ERR_TCPE)) { 3559 mp->m_pkthdr.csum_flags |= type; 3560 if (!sctp) 3561 mp->m_pkthdr.csum_data = htons(0xffff); 3562 } 3563 } 3564 return; 3565} 3566 3567static void 3568ixv_setup_vlan_support(struct adapter *adapter) 3569{ 3570 struct ixgbe_hw *hw = &adapter->hw; 3571 u32 ctrl, vid, vfta, retry; 3572 3573 3574 /* 3575 ** We get here thru init_locked, meaning 3576 ** a soft reset, this has already cleared 3577 ** the VFTA and other state, so if there 3578 ** have been no vlan's registered do nothing. 3579 */ 3580 if (adapter->num_vlans == 0) 3581 return; 3582 3583 /* Enable the queues */ 3584 for (int i = 0; i < adapter->num_queues; i++) { 3585 ctrl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(i)); 3586 ctrl |= IXGBE_RXDCTL_VME; 3587 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(i), ctrl); 3588 } 3589 3590 /* 3591 ** A soft reset zero's out the VFTA, so 3592 ** we need to repopulate it now. 3593 */ 3594 for (int i = 0; i < VFTA_SIZE; i++) { 3595 if (ixv_shadow_vfta[i] == 0) 3596 continue; 3597 vfta = ixv_shadow_vfta[i]; 3598 /* 3599 ** Reconstruct the vlan id's 3600 ** based on the bits set in each 3601 ** of the array ints. 3602 */ 3603 for ( int j = 0; j < 32; j++) { 3604 retry = 0; 3605 if ((vfta & (1 << j)) == 0) 3606 continue; 3607 vid = (i * 32) + j; 3608 /* Call the shared code mailbox routine */ 3609 while (ixgbe_set_vfta(hw, vid, 0, TRUE)) { 3610 if (++retry > 5) 3611 break; 3612 } 3613 } 3614 } 3615} 3616 3617/* 3618** This routine is run via an vlan config EVENT, 3619** it enables us to use the HW Filter table since 3620** we can get the vlan id. This just creates the 3621** entry in the soft version of the VFTA, init will 3622** repopulate the real table. 3623*/ 3624static void 3625ixv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3626{ 3627 struct adapter *adapter = ifp->if_softc; 3628 u16 index, bit; 3629 3630 if (ifp->if_softc != arg) /* Not our event */ 3631 return; 3632 3633 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3634 return; 3635 3636 IXV_CORE_LOCK(adapter); 3637 index = (vtag >> 5) & 0x7F; 3638 bit = vtag & 0x1F; 3639 ixv_shadow_vfta[index] |= (1 << bit); 3640 ++adapter->num_vlans; 3641 /* Re-init to load the changes */ 3642 ixv_init_locked(adapter); 3643 IXV_CORE_UNLOCK(adapter); 3644} 3645 3646/* 3647** This routine is run via an vlan 3648** unconfig EVENT, remove our entry 3649** in the soft vfta. 3650*/ 3651static void 3652ixv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag) 3653{ 3654 struct adapter *adapter = ifp->if_softc; 3655 u16 index, bit; 3656 3657 if (ifp->if_softc != arg) 3658 return; 3659 3660 if ((vtag == 0) || (vtag > 4095)) /* Invalid */ 3661 return; 3662 3663 IXV_CORE_LOCK(adapter); 3664 index = (vtag >> 5) & 0x7F; 3665 bit = vtag & 0x1F; 3666 ixv_shadow_vfta[index] &= ~(1 << bit); 3667 --adapter->num_vlans; 3668 /* Re-init to load the changes */ 3669 ixv_init_locked(adapter); 3670 IXV_CORE_UNLOCK(adapter); 3671} 3672 3673static void 3674ixv_enable_intr(struct adapter *adapter) 3675{ 3676 struct ixgbe_hw *hw = &adapter->hw; 3677 struct ix_queue *que = adapter->queues; 3678 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3679 3680 3681 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask); 3682 3683 mask = IXGBE_EIMS_ENABLE_MASK; 3684 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 3685 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, mask); 3686 3687 for (int i = 0; i < adapter->num_queues; i++, que++) 3688 ixv_enable_queue(adapter, que->msix); 3689 3690 IXGBE_WRITE_FLUSH(hw); 3691 3692 return; 3693} 3694 3695static void 3696ixv_disable_intr(struct adapter *adapter) 3697{ 3698 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIAC, 0); 3699 IXGBE_WRITE_REG(&adapter->hw, IXGBE_VTEIMC, ~0); 3700 IXGBE_WRITE_FLUSH(&adapter->hw); 3701 return; 3702} 3703 3704/* 3705** Setup the correct IVAR register for a particular MSIX interrupt 3706** - entry is the register array entry 3707** - vector is the MSIX vector for this queue 3708** - type is RX/TX/MISC 3709*/ 3710static void 3711ixv_set_ivar(struct adapter *adapter, u8 entry, u8 vector, s8 type) 3712{ 3713 struct ixgbe_hw *hw = &adapter->hw; 3714 u32 ivar, index; 3715 3716 vector |= IXGBE_IVAR_ALLOC_VAL; 3717 3718 if (type == -1) { /* MISC IVAR */ 3719 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 3720 ivar &= ~0xFF; 3721 ivar |= vector; 3722 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 3723 } else { /* RX/TX IVARS */ 3724 index = (16 * (entry & 1)) + (8 * type); 3725 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(entry >> 1)); 3726 ivar &= ~(0xFF << index); 3727 ivar |= (vector << index); 3728 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(entry >> 1), ivar); 3729 } 3730} 3731 3732static void 3733ixv_configure_ivars(struct adapter *adapter) 3734{ 3735 struct ix_queue *que = adapter->queues; 3736 3737 for (int i = 0; i < adapter->num_queues; i++, que++) { 3738 /* First the RX queue entry */ 3739 ixv_set_ivar(adapter, i, que->msix, 0); 3740 /* ... and the TX */ 3741 ixv_set_ivar(adapter, i, que->msix, 1); 3742 /* Set an initial value in EITR */ 3743 IXGBE_WRITE_REG(&adapter->hw, 3744 IXGBE_VTEITR(que->msix), IXV_EITR_DEFAULT); 3745 } 3746 3747 /* For the Link interrupt */ 3748 ixv_set_ivar(adapter, 1, adapter->mbxvec, -1); 3749} 3750 3751 3752/* 3753** Tasklet handler for MSIX MBX interrupts 3754** - do outside interrupt since it might sleep 3755*/ 3756static void 3757ixv_handle_mbx(void *context, int pending) 3758{ 3759 struct adapter *adapter = context; 3760 3761 ixgbe_check_link(&adapter->hw, 3762 &adapter->link_speed, &adapter->link_up, 0); 3763 ixv_update_link_status(adapter); 3764} 3765 3766/* 3767** The VF stats registers never have a truely virgin 3768** starting point, so this routine tries to make an 3769** artificial one, marking ground zero on attach as 3770** it were. 3771*/ 3772static void 3773ixv_save_stats(struct adapter *adapter) 3774{ 3775 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 3776 adapter->stats.saved_reset_vfgprc += 3777 adapter->stats.vfgprc - adapter->stats.base_vfgprc; 3778 adapter->stats.saved_reset_vfgptc += 3779 adapter->stats.vfgptc - adapter->stats.base_vfgptc; 3780 adapter->stats.saved_reset_vfgorc += 3781 adapter->stats.vfgorc - adapter->stats.base_vfgorc; 3782 adapter->stats.saved_reset_vfgotc += 3783 adapter->stats.vfgotc - adapter->stats.base_vfgotc; 3784 adapter->stats.saved_reset_vfmprc += 3785 adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3786 } 3787} 3788 3789static void 3790ixv_init_stats(struct adapter *adapter) 3791{ 3792 struct ixgbe_hw *hw = &adapter->hw; 3793 3794 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 3795 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 3796 adapter->stats.last_vfgorc |= 3797 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 3798 3799 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 3800 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 3801 adapter->stats.last_vfgotc |= 3802 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 3803 3804 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 3805 3806 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 3807 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 3808 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 3809 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 3810 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 3811} 3812 3813#define UPDATE_STAT_32(reg, last, count) \ 3814{ \ 3815 u32 current = IXGBE_READ_REG(hw, reg); \ 3816 if (current < last) \ 3817 count += 0x100000000LL; \ 3818 last = current; \ 3819 count &= 0xFFFFFFFF00000000LL; \ 3820 count |= current; \ 3821} 3822 3823#define UPDATE_STAT_36(lsb, msb, last, count) \ 3824{ \ 3825 u64 cur_lsb = IXGBE_READ_REG(hw, lsb); \ 3826 u64 cur_msb = IXGBE_READ_REG(hw, msb); \ 3827 u64 current = ((cur_msb << 32) | cur_lsb); \ 3828 if (current < last) \ 3829 count += 0x1000000000LL; \ 3830 last = current; \ 3831 count &= 0xFFFFFFF000000000LL; \ 3832 count |= current; \ 3833} 3834 3835/* 3836** ixv_update_stats - Update the board statistics counters. 3837*/ 3838void 3839ixv_update_stats(struct adapter *adapter) 3840{ 3841 struct ixgbe_hw *hw = &adapter->hw; 3842 3843 UPDATE_STAT_32(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 3844 adapter->stats.vfgprc); 3845 UPDATE_STAT_32(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 3846 adapter->stats.vfgptc); 3847 UPDATE_STAT_36(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 3848 adapter->stats.last_vfgorc, adapter->stats.vfgorc); 3849 UPDATE_STAT_36(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 3850 adapter->stats.last_vfgotc, adapter->stats.vfgotc); 3851 UPDATE_STAT_32(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 3852 adapter->stats.vfmprc); 3853} 3854 3855/********************************************************************** 3856 * 3857 * This routine is called only when ixgbe_display_debug_stats is enabled. 3858 * This routine provides a way to take a look at important statistics 3859 * maintained by the driver and hardware. 3860 * 3861 **********************************************************************/ 3862static void 3863ixv_print_hw_stats(struct adapter * adapter) 3864{ 3865 device_t dev = adapter->dev; 3866 3867 device_printf(dev,"Std Mbuf Failed = %lu\n", 3868 adapter->mbuf_defrag_failed); 3869 device_printf(dev,"Driver dropped packets = %lu\n", 3870 adapter->dropped_pkts); 3871 device_printf(dev, "watchdog timeouts = %ld\n", 3872 adapter->watchdog_events); 3873 3874 device_printf(dev,"Good Packets Rcvd = %llu\n", 3875 (long long)adapter->stats.vfgprc); 3876 device_printf(dev,"Good Packets Xmtd = %llu\n", 3877 (long long)adapter->stats.vfgptc); 3878 device_printf(dev,"TSO Transmissions = %lu\n", 3879 adapter->tso_tx); 3880 3881} 3882 3883/********************************************************************** 3884 * 3885 * This routine is called only when em_display_debug_stats is enabled. 3886 * This routine provides a way to take a look at important statistics 3887 * maintained by the driver and hardware. 3888 * 3889 **********************************************************************/ 3890static void 3891ixv_print_debug_info(struct adapter *adapter) 3892{ 3893 device_t dev = adapter->dev; 3894 struct ixgbe_hw *hw = &adapter->hw; 3895 struct ix_queue *que = adapter->queues; 3896 struct rx_ring *rxr; 3897 struct tx_ring *txr; 3898 struct lro_ctrl *lro; 3899 3900 device_printf(dev,"Error Byte Count = %u \n", 3901 IXGBE_READ_REG(hw, IXGBE_ERRBC)); 3902 3903 for (int i = 0; i < adapter->num_queues; i++, que++) { 3904 txr = que->txr; 3905 rxr = que->rxr; 3906 lro = &rxr->lro; 3907 device_printf(dev,"QUE(%d) IRQs Handled: %lu\n", 3908 que->msix, (long)que->irqs); 3909 device_printf(dev,"RX(%d) Packets Received: %lld\n", 3910 rxr->me, (long long)rxr->rx_packets); 3911 device_printf(dev,"RX(%d) Split RX Packets: %lld\n", 3912 rxr->me, (long long)rxr->rx_split_packets); 3913 device_printf(dev,"RX(%d) Bytes Received: %lu\n", 3914 rxr->me, (long)rxr->rx_bytes); 3915 device_printf(dev,"RX(%d) LRO Queued= %d\n", 3916 rxr->me, lro->lro_queued); 3917 device_printf(dev,"RX(%d) LRO Flushed= %d\n", 3918 rxr->me, lro->lro_flushed); 3919 device_printf(dev,"TX(%d) Packets Sent: %lu\n", 3920 txr->me, (long)txr->total_packets); 3921 device_printf(dev,"TX(%d) NO Desc Avail: %lu\n", 3922 txr->me, (long)txr->no_desc_avail); 3923 } 3924 3925 device_printf(dev,"MBX IRQ Handled: %lu\n", 3926 (long)adapter->mbx_irq); 3927 return; 3928} 3929 3930static int 3931ixv_sysctl_stats(SYSCTL_HANDLER_ARGS) 3932{ 3933 int error; 3934 int result; 3935 struct adapter *adapter; 3936 3937 result = -1; 3938 error = sysctl_handle_int(oidp, &result, 0, req); 3939 3940 if (error || !req->newptr) 3941 return (error); 3942 3943 if (result == 1) { 3944 adapter = (struct adapter *) arg1; 3945 ixv_print_hw_stats(adapter); 3946 } 3947 return error; 3948} 3949 3950static int 3951ixv_sysctl_debug(SYSCTL_HANDLER_ARGS) 3952{ 3953 int error, result; 3954 struct adapter *adapter; 3955 3956 result = -1; 3957 error = sysctl_handle_int(oidp, &result, 0, req); 3958 3959 if (error || !req->newptr) 3960 return (error); 3961 3962 if (result == 1) { 3963 adapter = (struct adapter *) arg1; 3964 ixv_print_debug_info(adapter); 3965 } 3966 return error; 3967} 3968 3969/* 3970** Set flow control using sysctl: 3971** Flow control values: 3972** 0 - off 3973** 1 - rx pause 3974** 2 - tx pause 3975** 3 - full 3976*/ 3977static int 3978ixv_set_flowcntl(SYSCTL_HANDLER_ARGS) 3979{ 3980 int error; 3981 struct adapter *adapter; 3982 3983 error = sysctl_handle_int(oidp, &ixv_flow_control, 0, req); 3984 3985 if (error) 3986 return (error); 3987 3988 adapter = (struct adapter *) arg1; 3989 switch (ixv_flow_control) { 3990 case ixgbe_fc_rx_pause: 3991 case ixgbe_fc_tx_pause: 3992 case ixgbe_fc_full: 3993 adapter->hw.fc.requested_mode = ixv_flow_control; 3994 break; 3995 case ixgbe_fc_none: 3996 default: 3997 adapter->hw.fc.requested_mode = ixgbe_fc_none; 3998 } 3999 4000 ixgbe_fc_enable(&adapter->hw); 4001 return error; 4002} 4003 4004static void 4005ixv_add_rx_process_limit(struct adapter *adapter, const char *name, 4006 const char *description, int *limit, int value) 4007{ 4008 *limit = value; 4009 SYSCTL_ADD_INT(device_get_sysctl_ctx(adapter->dev), 4010 SYSCTL_CHILDREN(device_get_sysctl_tree(adapter->dev)), 4011 OID_AUTO, name, CTLTYPE_INT|CTLFLAG_RW, limit, value, description); 4012} 4013 4014