1/* $OpenBSD: if_ix.c,v 1.215 2024/05/21 11:19:39 bluhm Exp $ */ 2 3/****************************************************************************** 4 5 Copyright (c) 2001-2013, Intel Corporation 6 All rights reserved. 7 8 Redistribution and use in source and binary forms, with or without 9 modification, are permitted provided that the following conditions are met: 10 11 1. Redistributions of source code must retain the above copyright notice, 12 this list of conditions and the following disclaimer. 13 14 2. Redistributions in binary form must reproduce the above copyright 15 notice, this list of conditions and the following disclaimer in the 16 documentation and/or other materials provided with the distribution. 17 18 3. Neither the name of the Intel Corporation nor the names of its 19 contributors may be used to endorse or promote products derived from 20 this software without specific prior written permission. 21 22 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 26 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 27 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 28 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 30 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 31 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 32 POSSIBILITY OF SUCH DAMAGE. 33 34******************************************************************************/ 35/* FreeBSD: src/sys/dev/ixgbe/ixgbe.c 251964 Jun 18 21:28:19 2013 UTC */ 36 37#include <dev/pci/if_ix.h> 38#include <dev/pci/ixgbe_type.h> 39 40/* 41 * Our TCP/IP Stack is unable to handle packets greater than MAXMCLBYTES. 42 * This interface is unable to handle packets greater than IXGBE_TSO_SIZE. 43 */ 44CTASSERT(MAXMCLBYTES <= IXGBE_TSO_SIZE); 45 46/********************************************************************* 47 * Driver version 48 *********************************************************************/ 49/* char ixgbe_driver_version[] = "2.5.13"; */ 50 51/********************************************************************* 52 * PCI Device ID Table 53 * 54 * Used by probe to select devices to load on 55 *********************************************************************/ 56 57const struct pci_matchid ixgbe_devices[] = { 58 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598 }, 59 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_BX }, 60 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF_DUAL }, 61 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AF }, 62 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT }, 63 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT2 }, 64 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598AT_DUAL }, 65 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4 }, 66 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_CX4_DUAL }, 67 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_XF_LR }, 68 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598EB_SFP }, 69 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_SR_DUAL_EM }, 70 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82598_DA_DUAL }, 71 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4 }, 72 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_KX4_MEZZ }, 73 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_XAUI }, 74 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_COMBO_BP }, 75 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_BPLANE_FCOE }, 76 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_CX4 }, 77 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_T3_LOM }, 78 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP }, 79 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_EM }, 80 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF_QP }, 81 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_SF2 }, 82 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_SFP_FCOE }, 83 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599EN_SFP }, 84 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_82599_QSFP_SF_QP }, 85 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T }, 86 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X540T1 }, 87 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T }, 88 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550T1 }, 89 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KX4 }, 90 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_KR }, 91 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_SFP }, 92 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_10G_T }, 93 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_X_1G_T }, 94 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR }, 95 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_KR_L }, 96 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP_N }, 97 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SFP }, 98 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII }, 99 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_SGMII_L }, 100 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_10G_T }, 101 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T }, 102 { PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X550EM_A_1G_T_L } 103}; 104 105/********************************************************************* 106 * Function prototypes 107 *********************************************************************/ 108int ixgbe_probe(struct device *, void *, void *); 109void ixgbe_attach(struct device *, struct device *, void *); 110int ixgbe_detach(struct device *, int); 111int ixgbe_activate(struct device *, int); 112void ixgbe_start(struct ifqueue *); 113int ixgbe_ioctl(struct ifnet *, u_long, caddr_t); 114int ixgbe_rxrinfo(struct ix_softc *, struct if_rxrinfo *); 115int ixgbe_get_sffpage(struct ix_softc *, struct if_sffpage *); 116void ixgbe_watchdog(struct ifnet *); 117void ixgbe_init(void *); 118void ixgbe_stop(void *); 119void ixgbe_media_status(struct ifnet *, struct ifmediareq *); 120int ixgbe_media_change(struct ifnet *); 121void ixgbe_identify_hardware(struct ix_softc *); 122int ixgbe_allocate_pci_resources(struct ix_softc *); 123int ixgbe_allocate_legacy(struct ix_softc *); 124int ixgbe_allocate_msix(struct ix_softc *); 125void ixgbe_setup_msix(struct ix_softc *); 126int ixgbe_allocate_queues(struct ix_softc *); 127void ixgbe_free_pci_resources(struct ix_softc *); 128void ixgbe_setup_interface(struct ix_softc *); 129void ixgbe_config_gpie(struct ix_softc *); 130void ixgbe_config_delay_values(struct ix_softc *); 131void ixgbe_add_media_types(struct ix_softc *); 132void ixgbe_config_link(struct ix_softc *); 133 134int ixgbe_allocate_transmit_buffers(struct ix_txring *); 135int ixgbe_setup_transmit_structures(struct ix_softc *); 136int ixgbe_setup_transmit_ring(struct ix_txring *); 137void ixgbe_initialize_transmit_units(struct ix_softc *); 138void ixgbe_free_transmit_structures(struct ix_softc *); 139void ixgbe_free_transmit_buffers(struct ix_txring *); 140 141int ixgbe_allocate_receive_buffers(struct ix_rxring *); 142int ixgbe_setup_receive_structures(struct ix_softc *); 143int ixgbe_setup_receive_ring(struct ix_rxring *); 144void ixgbe_initialize_receive_units(struct ix_softc *); 145void ixgbe_free_receive_structures(struct ix_softc *); 146void ixgbe_free_receive_buffers(struct ix_rxring *); 147void ixgbe_initialize_rss_mapping(struct ix_softc *); 148int ixgbe_rxfill(struct ix_rxring *); 149void ixgbe_rxrefill(void *); 150 151int ixgbe_intr(struct ix_softc *sc); 152void ixgbe_enable_intr(struct ix_softc *); 153void ixgbe_disable_intr(struct ix_softc *); 154int ixgbe_txeof(struct ix_txring *); 155int ixgbe_rxeof(struct ix_rxring *); 156void ixgbe_rx_offload(uint32_t, uint16_t, struct mbuf *); 157void ixgbe_iff(struct ix_softc *); 158void ixgbe_map_queue_statistics(struct ix_softc *); 159void ixgbe_update_link_status(struct ix_softc *); 160int ixgbe_get_buf(struct ix_rxring *, int); 161int ixgbe_encap(struct ix_txring *, struct mbuf *); 162int ixgbe_dma_malloc(struct ix_softc *, bus_size_t, 163 struct ixgbe_dma_alloc *, int); 164void ixgbe_dma_free(struct ix_softc *, struct ixgbe_dma_alloc *); 165static int 166 ixgbe_tx_ctx_setup(struct ix_txring *, struct mbuf *, uint32_t *, 167 uint32_t *); 168void ixgbe_set_ivar(struct ix_softc *, uint8_t, uint8_t, int8_t); 169void ixgbe_configure_ivars(struct ix_softc *); 170uint8_t *ixgbe_mc_array_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 171 172void ixgbe_setup_vlan_hw_support(struct ix_softc *); 173 174/* Support for pluggable optic modules */ 175void ixgbe_handle_mod(struct ix_softc *); 176void ixgbe_handle_msf(struct ix_softc *); 177void ixgbe_handle_phy(struct ix_softc *); 178 179/* Legacy (single vector interrupt handler */ 180int ixgbe_legacy_intr(void *); 181void ixgbe_enable_queue(struct ix_softc *, uint32_t); 182void ixgbe_enable_queues(struct ix_softc *); 183void ixgbe_disable_queue(struct ix_softc *, uint32_t); 184 185/* MSI-X (multiple vectors interrupt handlers) */ 186int ixgbe_link_intr(void *); 187int ixgbe_queue_intr(void *); 188 189#if NKSTAT > 0 190static void ix_kstats(struct ix_softc *); 191static void ix_rxq_kstats(struct ix_softc *, struct ix_rxring *); 192static void ix_txq_kstats(struct ix_softc *, struct ix_txring *); 193static void ix_kstats_tick(void *); 194#endif 195 196/********************************************************************* 197 * OpenBSD Device Interface Entry Points 198 *********************************************************************/ 199 200struct cfdriver ix_cd = { 201 NULL, "ix", DV_IFNET 202}; 203 204const struct cfattach ix_ca = { 205 sizeof(struct ix_softc), ixgbe_probe, ixgbe_attach, ixgbe_detach, 206 ixgbe_activate 207}; 208 209int ixgbe_smart_speed = ixgbe_smart_speed_on; 210int ixgbe_enable_msix = 1; 211 212/********************************************************************* 213 * Device identification routine 214 * 215 * ixgbe_probe determines if the driver should be loaded on 216 * adapter based on PCI vendor/device id of the adapter. 217 * 218 * return 0 on success, positive on failure 219 *********************************************************************/ 220 221int 222ixgbe_probe(struct device *parent, void *match, void *aux) 223{ 224 INIT_DEBUGOUT("ixgbe_probe: begin"); 225 226 return (pci_matchbyid((struct pci_attach_args *)aux, ixgbe_devices, 227 nitems(ixgbe_devices))); 228} 229 230/********************************************************************* 231 * Device initialization routine 232 * 233 * The attach entry point is called when the driver is being loaded. 234 * This routine identifies the type of hardware, allocates all resources 235 * and initializes the hardware. 236 * 237 * return 0 on success, positive on failure 238 *********************************************************************/ 239 240void 241ixgbe_attach(struct device *parent, struct device *self, void *aux) 242{ 243 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 244 struct ix_softc *sc = (struct ix_softc *)self; 245 int error = 0; 246 uint16_t csum; 247 uint32_t ctrl_ext; 248 struct ixgbe_hw *hw = &sc->hw; 249 250 INIT_DEBUGOUT("ixgbe_attach: begin"); 251 252 sc->osdep.os_sc = sc; 253 sc->osdep.os_pa = *pa; 254 255 rw_init(&sc->sfflock, "ixsff"); 256 257#if NKSTAT > 0 258 ix_kstats(sc); 259#endif 260 261 /* Determine hardware revision */ 262 ixgbe_identify_hardware(sc); 263 264 /* Indicate to RX setup to use Jumbo Clusters */ 265 sc->num_tx_desc = DEFAULT_TXD; 266 sc->num_rx_desc = DEFAULT_RXD; 267 268 /* Do base PCI setup - map BAR0 */ 269 if (ixgbe_allocate_pci_resources(sc)) 270 goto err_out; 271 272 /* Allocate our TX/RX Queues */ 273 if (ixgbe_allocate_queues(sc)) 274 goto err_out; 275 276 /* Allocate multicast array memory. */ 277 sc->mta = mallocarray(IXGBE_ETH_LENGTH_OF_ADDRESS, 278 MAX_NUM_MULTICAST_ADDRESSES, M_DEVBUF, M_NOWAIT); 279 if (sc->mta == NULL) { 280 printf(": Can not allocate multicast setup array\n"); 281 goto err_late; 282 } 283 284 /* Initialize the shared code */ 285 error = ixgbe_init_shared_code(hw); 286 if (error) { 287 printf(": Unable to initialize the shared code\n"); 288 goto err_late; 289 } 290 291 /* Make sure we have a good EEPROM before we read from it */ 292 if (sc->hw.eeprom.ops.validate_checksum(&sc->hw, &csum) < 0) { 293 printf(": The EEPROM Checksum Is Not Valid\n"); 294 goto err_late; 295 } 296 297 error = ixgbe_init_hw(hw); 298 if (error == IXGBE_ERR_EEPROM_VERSION) { 299 printf(": This device is a pre-production adapter/" 300 "LOM. Please be aware there may be issues associated " 301 "with your hardware.\nIf you are experiencing problems " 302 "please contact your Intel or hardware representative " 303 "who provided you with this hardware.\n"); 304 } else if (error && (error != IXGBE_ERR_SFP_NOT_PRESENT && 305 error != IXGBE_ERR_SFP_NOT_SUPPORTED)) { 306 printf(": Hardware Initialization Failure\n"); 307 goto err_late; 308 } 309 310 bcopy(sc->hw.mac.addr, sc->arpcom.ac_enaddr, 311 IXGBE_ETH_LENGTH_OF_ADDRESS); 312 313 if (sc->sc_intrmap) 314 error = ixgbe_allocate_msix(sc); 315 else 316 error = ixgbe_allocate_legacy(sc); 317 if (error) 318 goto err_late; 319 320 /* Enable the optics for 82599 SFP+ fiber */ 321 if (sc->hw.mac.ops.enable_tx_laser) 322 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 323 324 /* Enable power to the phy */ 325 if (hw->phy.ops.set_phy_power) 326 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 327 328 /* Setup OS specific network interface */ 329 ixgbe_setup_interface(sc); 330 331 /* Get the PCI-E bus info and determine LAN ID */ 332 hw->mac.ops.get_bus_info(hw); 333 334 /* Set an initial default flow control value */ 335 sc->fc = ixgbe_fc_full; 336 337 /* let hardware know driver is loaded */ 338 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 339 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 340 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 341 342 printf(", address %s\n", ether_sprintf(sc->hw.mac.addr)); 343 344 INIT_DEBUGOUT("ixgbe_attach: end"); 345 return; 346 347err_late: 348 ixgbe_free_transmit_structures(sc); 349 ixgbe_free_receive_structures(sc); 350err_out: 351 ixgbe_free_pci_resources(sc); 352 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 353 MAX_NUM_MULTICAST_ADDRESSES); 354} 355 356/********************************************************************* 357 * Device removal routine 358 * 359 * The detach entry point is called when the driver is being removed. 360 * This routine stops the adapter and deallocates all the resources 361 * that were allocated for driver operation. 362 * 363 * return 0 on success, positive on failure 364 *********************************************************************/ 365 366int 367ixgbe_detach(struct device *self, int flags) 368{ 369 struct ix_softc *sc = (struct ix_softc *)self; 370 struct ifnet *ifp = &sc->arpcom.ac_if; 371 uint32_t ctrl_ext; 372 373 INIT_DEBUGOUT("ixgbe_detach: begin"); 374 375 ixgbe_stop(sc); 376 377 /* let hardware know driver is unloading */ 378 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 379 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 380 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 381 382 ether_ifdetach(ifp); 383 if_detach(ifp); 384 385 ixgbe_free_pci_resources(sc); 386 387 ixgbe_free_transmit_structures(sc); 388 ixgbe_free_receive_structures(sc); 389 free(sc->mta, M_DEVBUF, IXGBE_ETH_LENGTH_OF_ADDRESS * 390 MAX_NUM_MULTICAST_ADDRESSES); 391 392 /* XXX kstat */ 393 394 return (0); 395} 396 397int 398ixgbe_activate(struct device *self, int act) 399{ 400 struct ix_softc *sc = (struct ix_softc *)self; 401 struct ifnet *ifp = &sc->arpcom.ac_if; 402 struct ixgbe_hw *hw = &sc->hw; 403 uint32_t ctrl_ext; 404 int rv = 0; 405 406 switch (act) { 407 case DVACT_QUIESCE: 408 if (ifp->if_flags & IFF_RUNNING) 409 ixgbe_stop(sc); 410 break; 411 case DVACT_RESUME: 412 ixgbe_init_hw(hw); 413 414 /* Enable the optics for 82599 SFP+ fiber */ 415 if (sc->hw.mac.ops.enable_tx_laser) 416 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 417 418 /* Enable power to the phy */ 419 if (hw->phy.ops.set_phy_power) 420 hw->phy.ops.set_phy_power(&sc->hw, TRUE); 421 422 /* Get the PCI-E bus info and determine LAN ID */ 423 hw->mac.ops.get_bus_info(hw); 424 425 /* let hardware know driver is loaded */ 426 ctrl_ext = IXGBE_READ_REG(&sc->hw, IXGBE_CTRL_EXT); 427 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 428 IXGBE_WRITE_REG(&sc->hw, IXGBE_CTRL_EXT, ctrl_ext); 429 430 if (ifp->if_flags & IFF_UP) 431 ixgbe_init(sc); 432 break; 433 default: 434 break; 435 } 436 return (rv); 437} 438 439/********************************************************************* 440 * Transmit entry point 441 * 442 * ixgbe_start is called by the stack to initiate a transmit. 443 * The driver will remain in this routine as long as there are 444 * packets to transmit and transmit resources are available. 445 * In case resources are not available stack is notified and 446 * the packet is requeued. 447 **********************************************************************/ 448 449void 450ixgbe_start(struct ifqueue *ifq) 451{ 452 struct ifnet *ifp = ifq->ifq_if; 453 struct ix_softc *sc = ifp->if_softc; 454 struct ix_txring *txr = ifq->ifq_softc; 455 struct mbuf *m_head; 456 unsigned int head, free, used; 457 int post = 0; 458 459 if (!sc->link_up) 460 return; 461 462 head = txr->next_avail_desc; 463 free = txr->next_to_clean; 464 if (free <= head) 465 free += sc->num_tx_desc; 466 free -= head; 467 468 membar_consumer(); 469 470 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 471 0, txr->txdma.dma_map->dm_mapsize, 472 BUS_DMASYNC_POSTWRITE); 473 474 for (;;) { 475 /* Check that we have the minimal number of TX descriptors. */ 476 if (free <= IXGBE_TX_OP_THRESHOLD) { 477 ifq_set_oactive(ifq); 478 break; 479 } 480 481 m_head = ifq_dequeue(ifq); 482 if (m_head == NULL) 483 break; 484 485 used = ixgbe_encap(txr, m_head); 486 if (used == 0) { 487 m_freem(m_head); 488 continue; 489 } 490 491 free -= used; 492 493#if NBPFILTER > 0 494 if (ifp->if_bpf) 495 bpf_mtap_ether(ifp->if_bpf, m_head, BPF_DIRECTION_OUT); 496#endif 497 498 /* Set timeout in case hardware has problems transmitting */ 499 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 500 ifp->if_timer = IXGBE_TX_TIMEOUT; 501 502 post = 1; 503 } 504 505 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 506 0, txr->txdma.dma_map->dm_mapsize, 507 BUS_DMASYNC_PREWRITE); 508 509 /* 510 * Advance the Transmit Descriptor Tail (Tdt), this tells the 511 * hardware that this frame is available to transmit. 512 */ 513 if (post) 514 IXGBE_WRITE_REG(&sc->hw, IXGBE_TDT(txr->me), 515 txr->next_avail_desc); 516} 517 518/********************************************************************* 519 * Ioctl entry point 520 * 521 * ixgbe_ioctl is called when the user wants to configure the 522 * interface. 523 * 524 * return 0 on success, positive on failure 525 **********************************************************************/ 526 527int 528ixgbe_ioctl(struct ifnet * ifp, u_long command, caddr_t data) 529{ 530 struct ix_softc *sc = ifp->if_softc; 531 struct ifreq *ifr = (struct ifreq *) data; 532 int s, error = 0; 533 534 s = splnet(); 535 536 switch (command) { 537 case SIOCSIFADDR: 538 IOCTL_DEBUGOUT("ioctl: SIOCxIFADDR (Get/Set Interface Addr)"); 539 ifp->if_flags |= IFF_UP; 540 if (!(ifp->if_flags & IFF_RUNNING)) 541 ixgbe_init(sc); 542 break; 543 544 case SIOCSIFFLAGS: 545 IOCTL_DEBUGOUT("ioctl: SIOCSIFFLAGS (Set Interface Flags)"); 546 if (ifp->if_flags & IFF_UP) { 547 if (ifp->if_flags & IFF_RUNNING) 548 error = ENETRESET; 549 else 550 ixgbe_init(sc); 551 } else { 552 if (ifp->if_flags & IFF_RUNNING) 553 ixgbe_stop(sc); 554 } 555 break; 556 557 case SIOCSIFMEDIA: 558 case SIOCGIFMEDIA: 559 IOCTL_DEBUGOUT("ioctl: SIOCxIFMEDIA (Get/Set Interface Media)"); 560 error = ifmedia_ioctl(ifp, ifr, &sc->media, command); 561 break; 562 563 case SIOCGIFRXR: 564 error = ixgbe_rxrinfo(sc, (struct if_rxrinfo *)ifr->ifr_data); 565 break; 566 567 case SIOCGIFSFFPAGE: 568 error = rw_enter(&sc->sfflock, RW_WRITE|RW_INTR); 569 if (error != 0) 570 break; 571 572 error = ixgbe_get_sffpage(sc, (struct if_sffpage *)data); 573 rw_exit(&sc->sfflock); 574 break; 575 576 default: 577 error = ether_ioctl(ifp, &sc->arpcom, command, data); 578 } 579 580 if (error == ENETRESET) { 581 if (ifp->if_flags & IFF_RUNNING) { 582 ixgbe_disable_intr(sc); 583 ixgbe_iff(sc); 584 ixgbe_enable_intr(sc); 585 ixgbe_enable_queues(sc); 586 } 587 error = 0; 588 } 589 590 splx(s); 591 return (error); 592} 593 594int 595ixgbe_get_sffpage(struct ix_softc *sc, struct if_sffpage *sff) 596{ 597 struct ixgbe_hw *hw = &sc->hw; 598 uint32_t swfw_mask = hw->phy.phy_semaphore_mask; 599 uint8_t page; 600 size_t i; 601 int error = EIO; 602 603 if (hw->phy.type == ixgbe_phy_fw) 604 return (ENODEV); 605 606 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)) 607 return (EBUSY); /* XXX */ 608 609 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 610 if (hw->phy.ops.read_i2c_byte_unlocked(hw, 127, 611 IFSFF_ADDR_EEPROM, &page)) 612 goto error; 613 if (page != sff->sff_page && 614 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 615 IFSFF_ADDR_EEPROM, sff->sff_page)) 616 goto error; 617 } 618 619 for (i = 0; i < sizeof(sff->sff_data); i++) { 620 if (hw->phy.ops.read_i2c_byte_unlocked(hw, i, 621 sff->sff_addr, &sff->sff_data[i])) 622 goto error; 623 } 624 625 if (sff->sff_addr == IFSFF_ADDR_EEPROM) { 626 if (page != sff->sff_page && 627 hw->phy.ops.write_i2c_byte_unlocked(hw, 127, 628 IFSFF_ADDR_EEPROM, page)) 629 goto error; 630 } 631 632 error = 0; 633error: 634 hw->mac.ops.release_swfw_sync(hw, swfw_mask); 635 return (error); 636} 637 638int 639ixgbe_rxrinfo(struct ix_softc *sc, struct if_rxrinfo *ifri) 640{ 641 struct if_rxring_info *ifr, ifr1; 642 struct ix_rxring *rxr; 643 int error, i; 644 u_int n = 0; 645 646 if (sc->num_queues > 1) { 647 ifr = mallocarray(sc->num_queues, sizeof(*ifr), M_DEVBUF, 648 M_WAITOK | M_ZERO); 649 } else 650 ifr = &ifr1; 651 652 for (i = 0; i < sc->num_queues; i++) { 653 rxr = &sc->rx_rings[i]; 654 ifr[n].ifr_size = MCLBYTES; 655 snprintf(ifr[n].ifr_name, sizeof(ifr[n].ifr_name), "%d", i); 656 ifr[n].ifr_info = rxr->rx_ring; 657 n++; 658 } 659 660 error = if_rxr_info_ioctl(ifri, sc->num_queues, ifr); 661 662 if (sc->num_queues > 1) 663 free(ifr, M_DEVBUF, sc->num_queues * sizeof(*ifr)); 664 return (error); 665} 666 667/********************************************************************* 668 * Watchdog entry point 669 * 670 **********************************************************************/ 671 672void 673ixgbe_watchdog(struct ifnet * ifp) 674{ 675 struct ix_softc *sc = (struct ix_softc *)ifp->if_softc; 676 struct ix_txring *txr = sc->tx_rings; 677 struct ixgbe_hw *hw = &sc->hw; 678 int tx_hang = FALSE; 679 int i; 680 681 /* 682 * The timer is set to 5 every time ixgbe_start() queues a packet. 683 * Anytime all descriptors are clean the timer is set to 0. 684 */ 685 for (i = 0; i < sc->num_queues; i++, txr++) { 686 if (txr->watchdog_timer == 0 || --txr->watchdog_timer) 687 continue; 688 else { 689 tx_hang = TRUE; 690 break; 691 } 692 } 693 if (tx_hang == FALSE) 694 return; 695 696 /* 697 * If we are in this routine because of pause frames, then don't 698 * reset the hardware. 699 */ 700 if (!(IXGBE_READ_REG(hw, IXGBE_TFCS) & IXGBE_TFCS_TXON)) { 701 for (i = 0; i < sc->num_queues; i++, txr++) 702 txr->watchdog_timer = IXGBE_TX_TIMEOUT; 703 ifp->if_timer = IXGBE_TX_TIMEOUT; 704 return; 705 } 706 707 708 printf("%s: Watchdog timeout -- resetting\n", ifp->if_xname); 709 for (i = 0; i < sc->num_queues; i++, txr++) { 710 printf("%s: Queue(%d) tdh = %d, hw tdt = %d\n", ifp->if_xname, i, 711 IXGBE_READ_REG(hw, IXGBE_TDH(i)), 712 IXGBE_READ_REG(hw, IXGBE_TDT(i))); 713 printf("%s: TX(%d) Next TX to Clean = %d\n", ifp->if_xname, 714 i, txr->next_to_clean); 715 } 716 ifp->if_flags &= ~IFF_RUNNING; 717 718 ixgbe_init(sc); 719} 720 721/********************************************************************* 722 * Init entry point 723 * 724 * This routine is used in two ways. It is used by the stack as 725 * init entry point in network interface structure. It is also used 726 * by the driver as a hw/sw initialization routine to get to a 727 * consistent state. 728 * 729 * return 0 on success, positive on failure 730 **********************************************************************/ 731#define IXGBE_MHADD_MFS_SHIFT 16 732 733void 734ixgbe_init(void *arg) 735{ 736 struct ix_softc *sc = (struct ix_softc *)arg; 737 struct ifnet *ifp = &sc->arpcom.ac_if; 738 struct ix_rxring *rxr = sc->rx_rings; 739 uint32_t k, txdctl, rxdctl, rxctrl, mhadd, itr; 740 int i, s, err; 741 742 INIT_DEBUGOUT("ixgbe_init: begin"); 743 744 s = splnet(); 745 746 ixgbe_stop(sc); 747 748 /* reprogram the RAR[0] in case user changed it. */ 749 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 750 751 /* Get the latest mac address, User can use a LAA */ 752 bcopy(sc->arpcom.ac_enaddr, sc->hw.mac.addr, 753 IXGBE_ETH_LENGTH_OF_ADDRESS); 754 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, 1); 755 sc->hw.addr_ctrl.rar_used_count = 1; 756 757 /* Prepare transmit descriptors and buffers */ 758 if (ixgbe_setup_transmit_structures(sc)) { 759 printf("%s: Could not setup transmit structures\n", 760 ifp->if_xname); 761 ixgbe_stop(sc); 762 splx(s); 763 return; 764 } 765 766 ixgbe_init_hw(&sc->hw); 767 ixgbe_initialize_transmit_units(sc); 768 769 /* Use 2k clusters, even for jumbo frames */ 770 sc->rx_mbuf_sz = MCLBYTES + ETHER_ALIGN; 771 772 /* Prepare receive descriptors and buffers */ 773 if (ixgbe_setup_receive_structures(sc)) { 774 printf("%s: Could not setup receive structures\n", 775 ifp->if_xname); 776 ixgbe_stop(sc); 777 splx(s); 778 return; 779 } 780 781 /* Configure RX settings */ 782 ixgbe_initialize_receive_units(sc); 783 784 /* Enable SDP & MSIX interrupts based on adapter */ 785 ixgbe_config_gpie(sc); 786 787 /* Program promiscuous mode and multicast filters. */ 788 ixgbe_iff(sc); 789 790 /* Set MRU size */ 791 mhadd = IXGBE_READ_REG(&sc->hw, IXGBE_MHADD); 792 mhadd &= ~IXGBE_MHADD_MFS_MASK; 793 mhadd |= sc->max_frame_size << IXGBE_MHADD_MFS_SHIFT; 794 IXGBE_WRITE_REG(&sc->hw, IXGBE_MHADD, mhadd); 795 796 /* Now enable all the queues */ 797 for (i = 0; i < sc->num_queues; i++) { 798 txdctl = IXGBE_READ_REG(&sc->hw, IXGBE_TXDCTL(i)); 799 txdctl |= IXGBE_TXDCTL_ENABLE; 800 /* Set WTHRESH to 8, burst writeback */ 801 txdctl |= (8 << 16); 802 /* 803 * When the internal queue falls below PTHRESH (16), 804 * start prefetching as long as there are at least 805 * HTHRESH (1) buffers ready. 806 */ 807 txdctl |= (16 << 0) | (1 << 8); 808 IXGBE_WRITE_REG(&sc->hw, IXGBE_TXDCTL(i), txdctl); 809 } 810 811 for (i = 0; i < sc->num_queues; i++) { 812 rxdctl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 813 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 814 /* 815 * PTHRESH = 21 816 * HTHRESH = 4 817 * WTHRESH = 8 818 */ 819 rxdctl &= ~0x3FFFFF; 820 rxdctl |= 0x080420; 821 } 822 rxdctl |= IXGBE_RXDCTL_ENABLE; 823 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), rxdctl); 824 for (k = 0; k < 10; k++) { 825 if (IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)) & 826 IXGBE_RXDCTL_ENABLE) 827 break; 828 else 829 msec_delay(1); 830 } 831 IXGBE_WRITE_FLUSH(&sc->hw); 832 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(i), rxr->last_desc_filled); 833 } 834 835 /* Set up VLAN support and filter */ 836 ixgbe_setup_vlan_hw_support(sc); 837 838 /* Enable Receive engine */ 839 rxctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXCTRL); 840 if (sc->hw.mac.type == ixgbe_mac_82598EB) 841 rxctrl |= IXGBE_RXCTRL_DMBYPS; 842 rxctrl |= IXGBE_RXCTRL_RXEN; 843 sc->hw.mac.ops.enable_rx_dma(&sc->hw, rxctrl); 844 845 /* Set up MSI/X routing */ 846 if (sc->sc_intrmap) { 847 ixgbe_configure_ivars(sc); 848 /* Set up auto-mask */ 849 if (sc->hw.mac.type == ixgbe_mac_82598EB) 850 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 851 else { 852 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF); 853 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF); 854 } 855 } else { /* Simple settings for Legacy/MSI */ 856 ixgbe_set_ivar(sc, 0, 0, 0); 857 ixgbe_set_ivar(sc, 0, 0, 1); 858 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE); 859 } 860 861 /* Check on any SFP devices that need to be kick-started */ 862 if (sc->hw.phy.type == ixgbe_phy_none) { 863 err = sc->hw.phy.ops.identify(&sc->hw); 864 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 865 printf("Unsupported SFP+ module type was detected.\n"); 866 splx(s); 867 return; 868 } 869 } 870 871 /* Setup interrupt moderation */ 872 itr = (4000000 / IXGBE_INTS_PER_SEC) & 0xff8; 873 if (sc->hw.mac.type != ixgbe_mac_82598EB) 874 itr |= IXGBE_EITR_LLI_MOD | IXGBE_EITR_CNT_WDIS; 875 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(0), itr); 876 877 if (sc->sc_intrmap) { 878 /* Set moderation on the Link interrupt */ 879 IXGBE_WRITE_REG(&sc->hw, IXGBE_EITR(sc->linkvec), 880 IXGBE_LINK_ITR); 881 } 882 883 /* Enable power to the phy */ 884 if (sc->hw.phy.ops.set_phy_power) 885 sc->hw.phy.ops.set_phy_power(&sc->hw, TRUE); 886 887 /* Config/Enable Link */ 888 ixgbe_config_link(sc); 889 890 /* Hardware Packet Buffer & Flow Control setup */ 891 ixgbe_config_delay_values(sc); 892 893 /* Initialize the FC settings */ 894 sc->hw.mac.ops.start_hw(&sc->hw); 895 896 /* And now turn on interrupts */ 897 ixgbe_enable_intr(sc); 898 ixgbe_enable_queues(sc); 899 900 /* Now inform the stack we're ready */ 901 ifp->if_flags |= IFF_RUNNING; 902 for (i = 0; i < sc->num_queues; i++) 903 ifq_clr_oactive(ifp->if_ifqs[i]); 904 905#if NKSTAT > 0 906 ix_kstats_tick(sc); 907#endif 908 909 splx(s); 910} 911 912void 913ixgbe_config_gpie(struct ix_softc *sc) 914{ 915 struct ixgbe_hw *hw = &sc->hw; 916 uint32_t gpie; 917 918 gpie = IXGBE_READ_REG(&sc->hw, IXGBE_GPIE); 919 920 /* Fan Failure Interrupt */ 921 if (hw->device_id == IXGBE_DEV_ID_82598AT) 922 gpie |= IXGBE_SDP1_GPIEN; 923 924 if (sc->hw.mac.type == ixgbe_mac_82599EB) { 925 /* Add for Module detection */ 926 gpie |= IXGBE_SDP2_GPIEN; 927 928 /* Media ready */ 929 if (hw->device_id != IXGBE_DEV_ID_82599_QSFP_SF_QP) 930 gpie |= IXGBE_SDP1_GPIEN; 931 932 /* 933 * Set LL interval to max to reduce the number of low latency 934 * interrupts hitting the card when the ring is getting full. 935 */ 936 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 937 } 938 939 if (sc->hw.mac.type == ixgbe_mac_X540 || 940 sc->hw.mac.type == ixgbe_mac_X550EM_x || 941 sc->hw.mac.type == ixgbe_mac_X550EM_a) { 942 /* 943 * Thermal Failure Detection (X540) 944 * Link Detection (X552 SFP+, X552/X557-AT) 945 */ 946 gpie |= IXGBE_SDP0_GPIEN_X540; 947 948 /* 949 * Set LL interval to max to reduce the number of low latency 950 * interrupts hitting the card when the ring is getting full. 951 */ 952 gpie |= 0xf << IXGBE_GPIE_LLI_DELAY_SHIFT; 953 } 954 955 if (sc->sc_intrmap) { 956 /* Enable Enhanced MSIX mode */ 957 gpie |= IXGBE_GPIE_MSIX_MODE; 958 gpie |= IXGBE_GPIE_EIAME | IXGBE_GPIE_PBA_SUPPORT | 959 IXGBE_GPIE_OCD; 960 } 961 962 IXGBE_WRITE_REG(&sc->hw, IXGBE_GPIE, gpie); 963} 964 965/* 966 * Requires sc->max_frame_size to be set. 967 */ 968void 969ixgbe_config_delay_values(struct ix_softc *sc) 970{ 971 struct ixgbe_hw *hw = &sc->hw; 972 uint32_t rxpb, frame, size, tmp; 973 974 frame = sc->max_frame_size; 975 976 /* Calculate High Water */ 977 switch (hw->mac.type) { 978 case ixgbe_mac_X540: 979 case ixgbe_mac_X550: 980 case ixgbe_mac_X550EM_x: 981 case ixgbe_mac_X550EM_a: 982 tmp = IXGBE_DV_X540(frame, frame); 983 break; 984 default: 985 tmp = IXGBE_DV(frame, frame); 986 break; 987 } 988 size = IXGBE_BT2KB(tmp); 989 rxpb = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0)) >> 10; 990 hw->fc.high_water[0] = rxpb - size; 991 992 /* Now calculate Low Water */ 993 switch (hw->mac.type) { 994 case ixgbe_mac_X540: 995 case ixgbe_mac_X550: 996 case ixgbe_mac_X550EM_x: 997 case ixgbe_mac_X550EM_a: 998 tmp = IXGBE_LOW_DV_X540(frame); 999 break; 1000 default: 1001 tmp = IXGBE_LOW_DV(frame); 1002 break; 1003 } 1004 hw->fc.low_water[0] = IXGBE_BT2KB(tmp); 1005 1006 hw->fc.requested_mode = sc->fc; 1007 hw->fc.pause_time = IXGBE_FC_PAUSE; 1008 hw->fc.send_xon = TRUE; 1009} 1010 1011/* 1012 * MSIX Interrupt Handlers 1013 */ 1014void 1015ixgbe_enable_queue(struct ix_softc *sc, uint32_t vector) 1016{ 1017 uint64_t queue = 1ULL << vector; 1018 uint32_t mask; 1019 1020 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1021 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1022 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS, mask); 1023 } else { 1024 mask = (queue & 0xFFFFFFFF); 1025 if (mask) 1026 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(0), mask); 1027 mask = (queue >> 32); 1028 if (mask) 1029 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMS_EX(1), mask); 1030 } 1031} 1032 1033void 1034ixgbe_enable_queues(struct ix_softc *sc) 1035{ 1036 struct ix_queue *que; 1037 int i; 1038 1039 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) 1040 ixgbe_enable_queue(sc, que->msix); 1041} 1042 1043void 1044ixgbe_disable_queue(struct ix_softc *sc, uint32_t vector) 1045{ 1046 uint64_t queue = 1ULL << vector; 1047 uint32_t mask; 1048 1049 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 1050 mask = (IXGBE_EIMS_RTX_QUEUE & queue); 1051 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, mask); 1052 } else { 1053 mask = (queue & 0xFFFFFFFF); 1054 if (mask) 1055 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), mask); 1056 mask = (queue >> 32); 1057 if (mask) 1058 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), mask); 1059 } 1060} 1061 1062/* 1063 * MSIX Interrupt Handlers 1064 */ 1065int 1066ixgbe_link_intr(void *vsc) 1067{ 1068 struct ix_softc *sc = (struct ix_softc *)vsc; 1069 1070 return ixgbe_intr(sc); 1071} 1072 1073int 1074ixgbe_queue_intr(void *vque) 1075{ 1076 struct ix_queue *que = vque; 1077 struct ix_softc *sc = que->sc; 1078 struct ifnet *ifp = &sc->arpcom.ac_if; 1079 struct ix_rxring *rxr = que->rxr; 1080 struct ix_txring *txr = que->txr; 1081 1082 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1083 ixgbe_rxeof(rxr); 1084 ixgbe_txeof(txr); 1085 ixgbe_rxrefill(rxr); 1086 } 1087 1088 ixgbe_enable_queue(sc, que->msix); 1089 1090 return (1); 1091} 1092 1093/********************************************************************* 1094 * 1095 * Legacy Interrupt Service routine 1096 * 1097 **********************************************************************/ 1098 1099int 1100ixgbe_legacy_intr(void *arg) 1101{ 1102 struct ix_softc *sc = (struct ix_softc *)arg; 1103 struct ifnet *ifp = &sc->arpcom.ac_if; 1104 struct ix_rxring *rxr = sc->rx_rings; 1105 struct ix_txring *txr = sc->tx_rings; 1106 int rv; 1107 1108 rv = ixgbe_intr(sc); 1109 if (rv == 0) { 1110 return (0); 1111 } 1112 1113 if (ISSET(ifp->if_flags, IFF_RUNNING)) { 1114 ixgbe_rxeof(rxr); 1115 ixgbe_txeof(txr); 1116 ixgbe_rxrefill(rxr); 1117 } 1118 1119 ixgbe_enable_queues(sc); 1120 return (rv); 1121} 1122 1123int 1124ixgbe_intr(struct ix_softc *sc) 1125{ 1126 struct ifnet *ifp = &sc->arpcom.ac_if; 1127 struct ixgbe_hw *hw = &sc->hw; 1128 uint32_t reg_eicr, mod_mask, msf_mask; 1129 1130 if (sc->sc_intrmap) { 1131 /* Pause other interrupts */ 1132 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_OTHER); 1133 /* First get the cause */ 1134 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICS); 1135 /* Be sure the queue bits are not cleared */ 1136 reg_eicr &= ~IXGBE_EICR_RTX_QUEUE; 1137 /* Clear interrupt with write */ 1138 IXGBE_WRITE_REG(hw, IXGBE_EICR, reg_eicr); 1139 } else { 1140 reg_eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 1141 if (reg_eicr == 0) { 1142 ixgbe_enable_intr(sc); 1143 ixgbe_enable_queues(sc); 1144 return (0); 1145 } 1146 } 1147 1148 /* Link status change */ 1149 if (reg_eicr & IXGBE_EICR_LSC) { 1150 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); 1151 KERNEL_LOCK(); 1152 ixgbe_update_link_status(sc); 1153 KERNEL_UNLOCK(); 1154 } 1155 1156 if (hw->mac.type != ixgbe_mac_82598EB) { 1157 if (reg_eicr & IXGBE_EICR_ECC) { 1158 printf("%s: CRITICAL: ECC ERROR!! " 1159 "Please Reboot!!\n", sc->dev.dv_xname); 1160 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC); 1161 } 1162 /* Check for over temp condition */ 1163 if (reg_eicr & IXGBE_EICR_TS) { 1164 printf("%s: CRITICAL: OVER TEMP!! " 1165 "PHY IS SHUT DOWN!!\n", ifp->if_xname); 1166 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_TS); 1167 } 1168 } 1169 1170 /* Pluggable optics-related interrupt */ 1171 if (ixgbe_is_sfp(hw)) { 1172 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP) { 1173 mod_mask = IXGBE_EICR_GPI_SDP0_X540; 1174 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1175 } else if (hw->mac.type == ixgbe_mac_X540 || 1176 hw->mac.type == ixgbe_mac_X550 || 1177 hw->mac.type == ixgbe_mac_X550EM_x) { 1178 mod_mask = IXGBE_EICR_GPI_SDP2_X540; 1179 msf_mask = IXGBE_EICR_GPI_SDP1_X540; 1180 } else { 1181 mod_mask = IXGBE_EICR_GPI_SDP2; 1182 msf_mask = IXGBE_EICR_GPI_SDP1; 1183 } 1184 if (reg_eicr & mod_mask) { 1185 /* Clear the interrupt */ 1186 IXGBE_WRITE_REG(hw, IXGBE_EICR, mod_mask); 1187 KERNEL_LOCK(); 1188 ixgbe_handle_mod(sc); 1189 KERNEL_UNLOCK(); 1190 } else if ((hw->phy.media_type != ixgbe_media_type_copper) && 1191 (reg_eicr & msf_mask)) { 1192 /* Clear the interrupt */ 1193 IXGBE_WRITE_REG(hw, IXGBE_EICR, msf_mask); 1194 KERNEL_LOCK(); 1195 ixgbe_handle_msf(sc); 1196 KERNEL_UNLOCK(); 1197 } 1198 } 1199 1200 /* Check for fan failure */ 1201 if ((hw->device_id == IXGBE_DEV_ID_82598AT) && 1202 (reg_eicr & IXGBE_EICR_GPI_SDP1)) { 1203 printf("%s: CRITICAL: FAN FAILURE!! " 1204 "REPLACE IMMEDIATELY!!\n", ifp->if_xname); 1205 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 1206 } 1207 1208 /* External PHY interrupt */ 1209 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 1210 (reg_eicr & IXGBE_EICR_GPI_SDP0_X540)) { 1211 /* Clear the interrupt */ 1212 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP0_X540); 1213 KERNEL_LOCK(); 1214 ixgbe_handle_phy(sc); 1215 KERNEL_UNLOCK(); 1216 } 1217 1218 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); 1219 1220 return (1); 1221} 1222 1223/********************************************************************* 1224 * 1225 * Media Ioctl callback 1226 * 1227 * This routine is called whenever the user queries the status of 1228 * the interface using ifconfig. 1229 * 1230 **********************************************************************/ 1231void 1232ixgbe_media_status(struct ifnet * ifp, struct ifmediareq *ifmr) 1233{ 1234 struct ix_softc *sc = ifp->if_softc; 1235 uint64_t layer; 1236 1237 ifmr->ifm_active = IFM_ETHER; 1238 ifmr->ifm_status = IFM_AVALID; 1239 1240 INIT_DEBUGOUT("ixgbe_media_status: begin"); 1241 ixgbe_update_link_status(sc); 1242 1243 if (!LINK_STATE_IS_UP(ifp->if_link_state)) 1244 return; 1245 1246 ifmr->ifm_status |= IFM_ACTIVE; 1247 layer = sc->phy_layer; 1248 1249 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T || 1250 layer & IXGBE_PHYSICAL_LAYER_1000BASE_T || 1251 layer & IXGBE_PHYSICAL_LAYER_100BASE_TX || 1252 layer & IXGBE_PHYSICAL_LAYER_10BASE_T) { 1253 switch (sc->link_speed) { 1254 case IXGBE_LINK_SPEED_10GB_FULL: 1255 ifmr->ifm_active |= IFM_10G_T | IFM_FDX; 1256 break; 1257 case IXGBE_LINK_SPEED_1GB_FULL: 1258 ifmr->ifm_active |= IFM_1000_T | IFM_FDX; 1259 break; 1260 case IXGBE_LINK_SPEED_100_FULL: 1261 ifmr->ifm_active |= IFM_100_TX | IFM_FDX; 1262 break; 1263 case IXGBE_LINK_SPEED_10_FULL: 1264 ifmr->ifm_active |= IFM_10_T | IFM_FDX; 1265 break; 1266 } 1267 } 1268 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1269 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) { 1270 switch (sc->link_speed) { 1271 case IXGBE_LINK_SPEED_10GB_FULL: 1272 ifmr->ifm_active |= IFM_10G_SFP_CU | IFM_FDX; 1273 break; 1274 } 1275 } 1276 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1277 switch (sc->link_speed) { 1278 case IXGBE_LINK_SPEED_10GB_FULL: 1279 ifmr->ifm_active |= IFM_10G_LR | IFM_FDX; 1280 break; 1281 case IXGBE_LINK_SPEED_1GB_FULL: 1282 ifmr->ifm_active |= IFM_1000_LX | IFM_FDX; 1283 break; 1284 } 1285 } 1286 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR || 1287 layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) { 1288 switch (sc->link_speed) { 1289 case IXGBE_LINK_SPEED_10GB_FULL: 1290 ifmr->ifm_active |= IFM_10G_SR | IFM_FDX; 1291 break; 1292 case IXGBE_LINK_SPEED_1GB_FULL: 1293 ifmr->ifm_active |= IFM_1000_SX | IFM_FDX; 1294 break; 1295 } 1296 } 1297 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) { 1298 switch (sc->link_speed) { 1299 case IXGBE_LINK_SPEED_10GB_FULL: 1300 ifmr->ifm_active |= IFM_10G_CX4 | IFM_FDX; 1301 break; 1302 } 1303 } 1304 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) { 1305 switch (sc->link_speed) { 1306 case IXGBE_LINK_SPEED_10GB_FULL: 1307 ifmr->ifm_active |= IFM_10G_KR | IFM_FDX; 1308 break; 1309 case IXGBE_LINK_SPEED_2_5GB_FULL: 1310 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1311 break; 1312 case IXGBE_LINK_SPEED_1GB_FULL: 1313 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1314 break; 1315 } 1316 } else if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4 || 1317 layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX || 1318 layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) { 1319 switch (sc->link_speed) { 1320 case IXGBE_LINK_SPEED_10GB_FULL: 1321 ifmr->ifm_active |= IFM_10G_KX4 | IFM_FDX; 1322 break; 1323 case IXGBE_LINK_SPEED_2_5GB_FULL: 1324 ifmr->ifm_active |= IFM_2500_KX | IFM_FDX; 1325 break; 1326 case IXGBE_LINK_SPEED_1GB_FULL: 1327 ifmr->ifm_active |= IFM_1000_KX | IFM_FDX; 1328 break; 1329 } 1330 } 1331 1332 switch (sc->hw.fc.current_mode) { 1333 case ixgbe_fc_tx_pause: 1334 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_TXPAUSE; 1335 break; 1336 case ixgbe_fc_rx_pause: 1337 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE; 1338 break; 1339 case ixgbe_fc_full: 1340 ifmr->ifm_active |= IFM_FLOW | IFM_ETH_RXPAUSE | 1341 IFM_ETH_TXPAUSE; 1342 break; 1343 default: 1344 ifmr->ifm_active &= ~(IFM_FLOW | IFM_ETH_RXPAUSE | 1345 IFM_ETH_TXPAUSE); 1346 break; 1347 } 1348} 1349 1350/********************************************************************* 1351 * 1352 * Media Ioctl callback 1353 * 1354 * This routine is called when the user changes speed/duplex using 1355 * media/mediopt option with ifconfig. 1356 * 1357 **********************************************************************/ 1358int 1359ixgbe_media_change(struct ifnet *ifp) 1360{ 1361 struct ix_softc *sc = ifp->if_softc; 1362 struct ixgbe_hw *hw = &sc->hw; 1363 struct ifmedia *ifm = &sc->media; 1364 ixgbe_link_speed speed = 0; 1365 1366 if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER) 1367 return (EINVAL); 1368 1369 if (hw->phy.media_type == ixgbe_media_type_backplane) 1370 return (ENODEV); 1371 1372 switch (IFM_SUBTYPE(ifm->ifm_media)) { 1373 case IFM_AUTO: 1374 case IFM_10G_T: 1375 speed |= IXGBE_LINK_SPEED_100_FULL; 1376 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1377 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1378 break; 1379 case IFM_10G_SR: 1380 case IFM_10G_KR: 1381 case IFM_10G_LR: 1382 case IFM_10G_LRM: 1383 case IFM_10G_CX4: 1384 case IFM_10G_KX4: 1385 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1386 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1387 break; 1388 case IFM_10G_SFP_CU: 1389 speed |= IXGBE_LINK_SPEED_10GB_FULL; 1390 break; 1391 case IFM_1000_T: 1392 speed |= IXGBE_LINK_SPEED_100_FULL; 1393 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1394 break; 1395 case IFM_1000_LX: 1396 case IFM_1000_SX: 1397 case IFM_1000_CX: 1398 case IFM_1000_KX: 1399 speed |= IXGBE_LINK_SPEED_1GB_FULL; 1400 break; 1401 case IFM_100_TX: 1402 speed |= IXGBE_LINK_SPEED_100_FULL; 1403 break; 1404 case IFM_10_T: 1405 speed |= IXGBE_LINK_SPEED_10_FULL; 1406 break; 1407 default: 1408 return (EINVAL); 1409 } 1410 1411 hw->mac.autotry_restart = TRUE; 1412 hw->mac.ops.setup_link(hw, speed, TRUE); 1413 1414 return (0); 1415} 1416 1417/********************************************************************* 1418 * 1419 * This routine maps the mbufs to tx descriptors, allowing the 1420 * TX engine to transmit the packets. 1421 * - return 0 on success, positive on failure 1422 * 1423 **********************************************************************/ 1424 1425int 1426ixgbe_encap(struct ix_txring *txr, struct mbuf *m_head) 1427{ 1428 struct ix_softc *sc = txr->sc; 1429 uint32_t olinfo_status = 0, cmd_type_len; 1430 int i, j, ntxc; 1431 int first, last = 0; 1432 bus_dmamap_t map; 1433 struct ixgbe_tx_buf *txbuf; 1434 union ixgbe_adv_tx_desc *txd = NULL; 1435 1436 /* Basic descriptor defines */ 1437 cmd_type_len = (IXGBE_ADVTXD_DTYP_DATA | 1438 IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT); 1439 1440 /* 1441 * Important to capture the first descriptor 1442 * used because it will contain the index of 1443 * the one we tell the hardware to report back 1444 */ 1445 first = txr->next_avail_desc; 1446 txbuf = &txr->tx_buffers[first]; 1447 map = txbuf->map; 1448 1449 /* 1450 * Set the appropriate offload context 1451 * this will becomes the first descriptor. 1452 */ 1453 ntxc = ixgbe_tx_ctx_setup(txr, m_head, &cmd_type_len, &olinfo_status); 1454 if (ntxc == -1) 1455 goto xmit_fail; 1456 1457 /* 1458 * Map the packet for DMA. 1459 */ 1460 switch (bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1461 m_head, BUS_DMA_NOWAIT)) { 1462 case 0: 1463 break; 1464 case EFBIG: 1465 if (m_defrag(m_head, M_NOWAIT) == 0 && 1466 bus_dmamap_load_mbuf(txr->txdma.dma_tag, map, 1467 m_head, BUS_DMA_NOWAIT) == 0) 1468 break; 1469 /* FALLTHROUGH */ 1470 default: 1471 return (0); 1472 } 1473 1474 i = txr->next_avail_desc + ntxc; 1475 if (i >= sc->num_tx_desc) 1476 i -= sc->num_tx_desc; 1477 1478 for (j = 0; j < map->dm_nsegs; j++) { 1479 txd = &txr->tx_base[i]; 1480 1481 txd->read.buffer_addr = htole64(map->dm_segs[j].ds_addr); 1482 txd->read.cmd_type_len = htole32(txr->txd_cmd | 1483 cmd_type_len | map->dm_segs[j].ds_len); 1484 txd->read.olinfo_status = htole32(olinfo_status); 1485 last = i; /* descriptor that will get completion IRQ */ 1486 1487 if (++i == sc->num_tx_desc) 1488 i = 0; 1489 } 1490 1491 txd->read.cmd_type_len |= 1492 htole32(IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS); 1493 1494 bus_dmamap_sync(txr->txdma.dma_tag, map, 0, map->dm_mapsize, 1495 BUS_DMASYNC_PREWRITE); 1496 1497 /* Set the index of the descriptor that will be marked done */ 1498 txbuf->m_head = m_head; 1499 txbuf->eop_index = last; 1500 1501 membar_producer(); 1502 1503 txr->next_avail_desc = i; 1504 1505 return (ntxc + j); 1506 1507xmit_fail: 1508 bus_dmamap_unload(txr->txdma.dma_tag, txbuf->map); 1509 return (0); 1510} 1511 1512void 1513ixgbe_iff(struct ix_softc *sc) 1514{ 1515 struct ifnet *ifp = &sc->arpcom.ac_if; 1516 struct arpcom *ac = &sc->arpcom; 1517 uint32_t fctrl; 1518 uint8_t *mta; 1519 uint8_t *update_ptr; 1520 struct ether_multi *enm; 1521 struct ether_multistep step; 1522 int mcnt = 0; 1523 1524 IOCTL_DEBUGOUT("ixgbe_iff: begin"); 1525 1526 mta = sc->mta; 1527 bzero(mta, sizeof(uint8_t) * IXGBE_ETH_LENGTH_OF_ADDRESS * 1528 MAX_NUM_MULTICAST_ADDRESSES); 1529 1530 fctrl = IXGBE_READ_REG(&sc->hw, IXGBE_FCTRL); 1531 fctrl &= ~(IXGBE_FCTRL_MPE | IXGBE_FCTRL_UPE); 1532 ifp->if_flags &= ~IFF_ALLMULTI; 1533 1534 if (ifp->if_flags & IFF_PROMISC || ac->ac_multirangecnt > 0 || 1535 ac->ac_multicnt > MAX_NUM_MULTICAST_ADDRESSES) { 1536 ifp->if_flags |= IFF_ALLMULTI; 1537 fctrl |= IXGBE_FCTRL_MPE; 1538 if (ifp->if_flags & IFF_PROMISC) 1539 fctrl |= IXGBE_FCTRL_UPE; 1540 } else { 1541 ETHER_FIRST_MULTI(step, &sc->arpcom, enm); 1542 while (enm != NULL) { 1543 bcopy(enm->enm_addrlo, 1544 &mta[mcnt * IXGBE_ETH_LENGTH_OF_ADDRESS], 1545 IXGBE_ETH_LENGTH_OF_ADDRESS); 1546 mcnt++; 1547 1548 ETHER_NEXT_MULTI(step, enm); 1549 } 1550 1551 update_ptr = mta; 1552 sc->hw.mac.ops.update_mc_addr_list(&sc->hw, update_ptr, mcnt, 1553 ixgbe_mc_array_itr, TRUE); 1554 } 1555 1556 IXGBE_WRITE_REG(&sc->hw, IXGBE_FCTRL, fctrl); 1557} 1558 1559/* 1560 * This is an iterator function now needed by the multicast 1561 * shared code. It simply feeds the shared code routine the 1562 * addresses in the array of ixgbe_iff() one by one. 1563 */ 1564uint8_t * 1565ixgbe_mc_array_itr(struct ixgbe_hw *hw, uint8_t **update_ptr, uint32_t *vmdq) 1566{ 1567 uint8_t *addr = *update_ptr; 1568 uint8_t *newptr; 1569 *vmdq = 0; 1570 1571 newptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 1572 *update_ptr = newptr; 1573 return addr; 1574} 1575 1576void 1577ixgbe_update_link_status(struct ix_softc *sc) 1578{ 1579 struct ifnet *ifp = &sc->arpcom.ac_if; 1580 int link_state = LINK_STATE_DOWN; 1581 1582 splassert(IPL_NET); 1583 KERNEL_ASSERT_LOCKED(); 1584 1585 ixgbe_check_link(&sc->hw, &sc->link_speed, &sc->link_up, 0); 1586 1587 ifp->if_baudrate = 0; 1588 if (sc->link_up) { 1589 link_state = LINK_STATE_FULL_DUPLEX; 1590 1591 switch (sc->link_speed) { 1592 case IXGBE_LINK_SPEED_UNKNOWN: 1593 ifp->if_baudrate = 0; 1594 break; 1595 case IXGBE_LINK_SPEED_100_FULL: 1596 ifp->if_baudrate = IF_Mbps(100); 1597 break; 1598 case IXGBE_LINK_SPEED_1GB_FULL: 1599 ifp->if_baudrate = IF_Gbps(1); 1600 break; 1601 case IXGBE_LINK_SPEED_10GB_FULL: 1602 ifp->if_baudrate = IF_Gbps(10); 1603 break; 1604 } 1605 1606 /* Update any Flow Control changes */ 1607 sc->hw.mac.ops.fc_enable(&sc->hw); 1608 } 1609 if (ifp->if_link_state != link_state) { 1610 ifp->if_link_state = link_state; 1611 if_link_state_change(ifp); 1612 } 1613} 1614 1615 1616/********************************************************************* 1617 * 1618 * This routine disables all traffic on the adapter by issuing a 1619 * global reset on the MAC and deallocates TX/RX buffers. 1620 * 1621 **********************************************************************/ 1622 1623void 1624ixgbe_stop(void *arg) 1625{ 1626 struct ix_softc *sc = arg; 1627 struct ifnet *ifp = &sc->arpcom.ac_if; 1628 int i; 1629 1630 /* Tell the stack that the interface is no longer active */ 1631 ifp->if_flags &= ~IFF_RUNNING; 1632 1633#if NKSTAT > 0 1634 timeout_del(&sc->sc_kstat_tmo); 1635#endif 1636 ifp->if_timer = 0; 1637 1638 INIT_DEBUGOUT("ixgbe_stop: begin\n"); 1639 ixgbe_disable_intr(sc); 1640 1641 sc->hw.mac.ops.reset_hw(&sc->hw); 1642 sc->hw.adapter_stopped = FALSE; 1643 sc->hw.mac.ops.stop_adapter(&sc->hw); 1644 if (sc->hw.mac.type == ixgbe_mac_82599EB) 1645 sc->hw.mac.ops.stop_mac_link_on_d3(&sc->hw); 1646 /* Turn off the laser */ 1647 if (sc->hw.mac.ops.disable_tx_laser) 1648 sc->hw.mac.ops.disable_tx_laser(&sc->hw); 1649 1650 /* reprogram the RAR[0] in case user changed it. */ 1651 ixgbe_set_rar(&sc->hw, 0, sc->hw.mac.addr, 0, IXGBE_RAH_AV); 1652 1653 intr_barrier(sc->tag); 1654 for (i = 0; i < sc->num_queues; i++) { 1655 struct ifqueue *ifq = ifp->if_ifqs[i]; 1656 ifq_barrier(ifq); 1657 ifq_clr_oactive(ifq); 1658 1659 if (sc->queues[i].tag != NULL) 1660 intr_barrier(sc->queues[i].tag); 1661 timeout_del(&sc->rx_rings[i].rx_refill); 1662 } 1663 1664 KASSERT((ifp->if_flags & IFF_RUNNING) == 0); 1665 1666 /* Should we really clear all structures on stop? */ 1667 ixgbe_free_transmit_structures(sc); 1668 ixgbe_free_receive_structures(sc); 1669 1670 ixgbe_update_link_status(sc); 1671} 1672 1673 1674/********************************************************************* 1675 * 1676 * Determine hardware revision. 1677 * 1678 **********************************************************************/ 1679void 1680ixgbe_identify_hardware(struct ix_softc *sc) 1681{ 1682 struct ixgbe_osdep *os = &sc->osdep; 1683 struct pci_attach_args *pa = &os->os_pa; 1684 uint32_t reg; 1685 1686 /* Save off the information about this board */ 1687 sc->hw.vendor_id = PCI_VENDOR(pa->pa_id); 1688 sc->hw.device_id = PCI_PRODUCT(pa->pa_id); 1689 1690 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_CLASS_REG); 1691 sc->hw.revision_id = PCI_REVISION(reg); 1692 1693 reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_SUBSYS_ID_REG); 1694 sc->hw.subsystem_vendor_id = PCI_VENDOR(reg); 1695 sc->hw.subsystem_device_id = PCI_PRODUCT(reg); 1696 1697 /* We need this here to set the num_segs below */ 1698 ixgbe_set_mac_type(&sc->hw); 1699 1700 /* Pick up the 82599 and VF settings */ 1701 if (sc->hw.mac.type != ixgbe_mac_82598EB) 1702 sc->hw.phy.smart_speed = ixgbe_smart_speed; 1703 sc->num_segs = IXGBE_82599_SCATTER; 1704} 1705 1706/********************************************************************* 1707 * 1708 * Setup the Legacy or MSI Interrupt handler 1709 * 1710 **********************************************************************/ 1711int 1712ixgbe_allocate_legacy(struct ix_softc *sc) 1713{ 1714 struct ixgbe_osdep *os = &sc->osdep; 1715 struct pci_attach_args *pa = &os->os_pa; 1716 const char *intrstr = NULL; 1717 pci_chipset_tag_t pc = pa->pa_pc; 1718 pci_intr_handle_t ih; 1719 1720 /* We allocate a single interrupt resource */ 1721 if (pci_intr_map_msi(pa, &ih) != 0 && 1722 pci_intr_map(pa, &ih) != 0) { 1723 printf(": couldn't map interrupt\n"); 1724 return (ENXIO); 1725 } 1726 1727#if 0 1728 /* XXX */ 1729 /* Tasklets for Link, SFP and Multispeed Fiber */ 1730 TASK_INIT(&sc->link_task, 0, ixgbe_handle_link, sc); 1731 TASK_INIT(&sc->mod_task, 0, ixgbe_handle_mod, sc); 1732 TASK_INIT(&sc->msf_task, 0, ixgbe_handle_msf, sc); 1733#endif 1734 1735 intrstr = pci_intr_string(pc, ih); 1736 sc->tag = pci_intr_establish(pc, ih, IPL_NET | IPL_MPSAFE, 1737 ixgbe_legacy_intr, sc, sc->dev.dv_xname); 1738 if (sc->tag == NULL) { 1739 printf(": couldn't establish interrupt"); 1740 if (intrstr != NULL) 1741 printf(" at %s", intrstr); 1742 printf("\n"); 1743 return (ENXIO); 1744 } 1745 printf(": %s", intrstr); 1746 1747 /* For simplicity in the handlers */ 1748 sc->que_mask = IXGBE_EIMS_ENABLE_MASK; 1749 1750 return (0); 1751} 1752 1753/********************************************************************* 1754 * 1755 * Setup the MSI-X Interrupt handlers 1756 * 1757 **********************************************************************/ 1758int 1759ixgbe_allocate_msix(struct ix_softc *sc) 1760{ 1761 struct ixgbe_osdep *os = &sc->osdep; 1762 struct pci_attach_args *pa = &os->os_pa; 1763 int i = 0, error = 0; 1764 struct ix_queue *que; 1765 pci_intr_handle_t ih; 1766 1767 for (i = 0, que = sc->queues; i < sc->num_queues; i++, que++) { 1768 if (pci_intr_map_msix(pa, i, &ih)) { 1769 printf("ixgbe_allocate_msix: " 1770 "pci_intr_map_msix vec %d failed\n", i); 1771 error = ENOMEM; 1772 goto fail; 1773 } 1774 1775 que->tag = pci_intr_establish_cpu(pa->pa_pc, ih, 1776 IPL_NET | IPL_MPSAFE, intrmap_cpu(sc->sc_intrmap, i), 1777 ixgbe_queue_intr, que, que->name); 1778 if (que->tag == NULL) { 1779 printf("ixgbe_allocate_msix: " 1780 "pci_intr_establish vec %d failed\n", i); 1781 error = ENOMEM; 1782 goto fail; 1783 } 1784 1785 que->msix = i; 1786 } 1787 1788 /* Now the link status/control last MSI-X vector */ 1789 if (pci_intr_map_msix(pa, i, &ih)) { 1790 printf("ixgbe_allocate_msix: " 1791 "pci_intr_map_msix link vector failed\n"); 1792 error = ENOMEM; 1793 goto fail; 1794 } 1795 1796 sc->tag = pci_intr_establish(pa->pa_pc, ih, IPL_NET | IPL_MPSAFE, 1797 ixgbe_link_intr, sc, sc->dev.dv_xname); 1798 if (sc->tag == NULL) { 1799 printf("ixgbe_allocate_msix: " 1800 "pci_intr_establish link vector failed\n"); 1801 error = ENOMEM; 1802 goto fail; 1803 } 1804 sc->linkvec = i; 1805 printf(", %s, %d queue%s", pci_intr_string(pa->pa_pc, ih), 1806 i, (i > 1) ? "s" : ""); 1807 1808 return (0); 1809fail: 1810 for (que = sc->queues; i > 0; i--, que++) { 1811 if (que->tag == NULL) 1812 continue; 1813 pci_intr_disestablish(pa->pa_pc, que->tag); 1814 que->tag = NULL; 1815 } 1816 1817 return (error); 1818} 1819 1820void 1821ixgbe_setup_msix(struct ix_softc *sc) 1822{ 1823 struct ixgbe_osdep *os = &sc->osdep; 1824 struct pci_attach_args *pa = &os->os_pa; 1825 int nmsix; 1826 unsigned int maxq; 1827 1828 if (!ixgbe_enable_msix) 1829 return; 1830 1831 nmsix = pci_intr_msix_count(pa); 1832 if (nmsix <= 1) 1833 return; 1834 1835 /* give one vector to events */ 1836 nmsix--; 1837 1838 /* XXX the number of queues is limited to what we can keep stats on */ 1839 maxq = (sc->hw.mac.type == ixgbe_mac_82598EB) ? 8 : 16; 1840 1841 sc->sc_intrmap = intrmap_create(&sc->dev, nmsix, maxq, 0); 1842 sc->num_queues = intrmap_count(sc->sc_intrmap); 1843} 1844 1845int 1846ixgbe_allocate_pci_resources(struct ix_softc *sc) 1847{ 1848 struct ixgbe_osdep *os = &sc->osdep; 1849 struct pci_attach_args *pa = &os->os_pa; 1850 int val; 1851 1852 val = pci_conf_read(pa->pa_pc, pa->pa_tag, PCIR_BAR(0)); 1853 if (PCI_MAPREG_TYPE(val) != PCI_MAPREG_TYPE_MEM) { 1854 printf(": mmba is not mem space\n"); 1855 return (ENXIO); 1856 } 1857 1858 if (pci_mapreg_map(pa, PCIR_BAR(0), PCI_MAPREG_MEM_TYPE(val), 0, 1859 &os->os_memt, &os->os_memh, &os->os_membase, &os->os_memsize, 0)) { 1860 printf(": cannot find mem space\n"); 1861 return (ENXIO); 1862 } 1863 sc->hw.hw_addr = (uint8_t *)os->os_membase; 1864 1865 /* Legacy defaults */ 1866 sc->num_queues = 1; 1867 sc->hw.back = os; 1868 1869 /* Now setup MSI or MSI/X, return us the number of supported vectors. */ 1870 ixgbe_setup_msix(sc); 1871 1872 return (0); 1873} 1874 1875void 1876ixgbe_free_pci_resources(struct ix_softc * sc) 1877{ 1878 struct ixgbe_osdep *os = &sc->osdep; 1879 struct pci_attach_args *pa = &os->os_pa; 1880 struct ix_queue *que = sc->queues; 1881 int i; 1882 1883 /* Release all msix queue resources: */ 1884 for (i = 0; i < sc->num_queues; i++, que++) { 1885 if (que->tag) 1886 pci_intr_disestablish(pa->pa_pc, que->tag); 1887 que->tag = NULL; 1888 } 1889 1890 if (sc->tag) 1891 pci_intr_disestablish(pa->pa_pc, sc->tag); 1892 sc->tag = NULL; 1893 if (os->os_membase != 0) 1894 bus_space_unmap(os->os_memt, os->os_memh, os->os_memsize); 1895 os->os_membase = 0; 1896} 1897 1898/********************************************************************* 1899 * 1900 * Setup networking device structure and register an interface. 1901 * 1902 **********************************************************************/ 1903void 1904ixgbe_setup_interface(struct ix_softc *sc) 1905{ 1906 struct ifnet *ifp = &sc->arpcom.ac_if; 1907 int i; 1908 1909 strlcpy(ifp->if_xname, sc->dev.dv_xname, IFNAMSIZ); 1910 ifp->if_softc = sc; 1911 ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 1912 ifp->if_xflags = IFXF_MPSAFE; 1913 ifp->if_ioctl = ixgbe_ioctl; 1914 ifp->if_qstart = ixgbe_start; 1915 ifp->if_timer = 0; 1916 ifp->if_watchdog = ixgbe_watchdog; 1917 ifp->if_hardmtu = IXGBE_MAX_FRAME_SIZE - 1918 ETHER_HDR_LEN - ETHER_CRC_LEN; 1919 ifq_init_maxlen(&ifp->if_snd, sc->num_tx_desc - 1); 1920 1921 ifp->if_capabilities = IFCAP_VLAN_MTU; 1922 1923#if NVLAN > 0 1924 ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING; 1925#endif 1926 1927 ifp->if_capabilities |= IFCAP_CSUM_TCPv4 | IFCAP_CSUM_UDPv4; 1928 ifp->if_capabilities |= IFCAP_CSUM_TCPv6 | IFCAP_CSUM_UDPv6; 1929 ifp->if_capabilities |= IFCAP_CSUM_IPv4; 1930 1931 ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6; 1932 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 1933#ifndef __sparc64__ 1934 ifp->if_xflags |= IFXF_LRO; 1935#endif 1936 ifp->if_capabilities |= IFCAP_LRO; 1937 } 1938 1939 /* 1940 * Specify the media types supported by this sc and register 1941 * callbacks to update media and link information 1942 */ 1943 ifmedia_init(&sc->media, IFM_IMASK, ixgbe_media_change, 1944 ixgbe_media_status); 1945 ixgbe_add_media_types(sc); 1946 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 1947 1948 if_attach(ifp); 1949 ether_ifattach(ifp); 1950 1951 if_attach_queues(ifp, sc->num_queues); 1952 if_attach_iqueues(ifp, sc->num_queues); 1953 for (i = 0; i < sc->num_queues; i++) { 1954 struct ifqueue *ifq = ifp->if_ifqs[i]; 1955 struct ifiqueue *ifiq = ifp->if_iqs[i]; 1956 struct ix_txring *txr = &sc->tx_rings[i]; 1957 struct ix_rxring *rxr = &sc->rx_rings[i]; 1958 1959 ifq->ifq_softc = txr; 1960 txr->ifq = ifq; 1961 1962 ifiq->ifiq_softc = rxr; 1963 rxr->ifiq = ifiq; 1964 1965#if NKSTAT > 0 1966 ix_txq_kstats(sc, txr); 1967 ix_rxq_kstats(sc, rxr); 1968#endif 1969 } 1970 1971 sc->max_frame_size = IXGBE_MAX_FRAME_SIZE; 1972} 1973 1974void 1975ixgbe_add_media_types(struct ix_softc *sc) 1976{ 1977 struct ixgbe_hw *hw = &sc->hw; 1978 uint64_t layer; 1979 1980 sc->phy_layer = hw->mac.ops.get_supported_physical_layer(hw); 1981 layer = sc->phy_layer; 1982 1983 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_T) 1984 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_T, 0, NULL); 1985 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_T) 1986 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 1987 if (layer & IXGBE_PHYSICAL_LAYER_100BASE_TX) 1988 ifmedia_add(&sc->media, IFM_ETHER | IFM_100_TX, 0, NULL); 1989 if (layer & IXGBE_PHYSICAL_LAYER_SFP_PLUS_CU || 1990 layer & IXGBE_PHYSICAL_LAYER_SFP_ACTIVE_DA) 1991 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SFP_CU, 0, NULL); 1992 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_LR) { 1993 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_LR, 0, NULL); 1994 if (hw->phy.multispeed_fiber) 1995 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_LX, 0, 1996 NULL); 1997 } 1998 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_SR) { 1999 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_SR, 0, NULL); 2000 if (hw->phy.multispeed_fiber) 2001 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, 2002 NULL); 2003 } else if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_SX) 2004 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_SX, 0, NULL); 2005 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_CX4) 2006 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_CX4, 0, NULL); 2007 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KR) 2008 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KR, 0, NULL); 2009 if (layer & IXGBE_PHYSICAL_LAYER_10GBASE_KX4) 2010 ifmedia_add(&sc->media, IFM_ETHER | IFM_10G_KX4, 0, NULL); 2011 if (layer & IXGBE_PHYSICAL_LAYER_1000BASE_KX) 2012 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_KX, 0, NULL); 2013 if (layer & IXGBE_PHYSICAL_LAYER_2500BASE_KX) 2014 ifmedia_add(&sc->media, IFM_ETHER | IFM_2500_KX, 0, NULL); 2015 2016 if (hw->device_id == IXGBE_DEV_ID_82598AT) { 2017 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T | IFM_FDX, 0, 2018 NULL); 2019 ifmedia_add(&sc->media, IFM_ETHER | IFM_1000_T, 0, NULL); 2020 } 2021 2022 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2023} 2024 2025void 2026ixgbe_config_link(struct ix_softc *sc) 2027{ 2028 uint32_t autoneg, err = 0; 2029 bool negotiate; 2030 2031 if (ixgbe_is_sfp(&sc->hw)) { 2032 if (sc->hw.phy.multispeed_fiber) { 2033 sc->hw.mac.ops.setup_sfp(&sc->hw); 2034 if (sc->hw.mac.ops.enable_tx_laser) 2035 sc->hw.mac.ops.enable_tx_laser(&sc->hw); 2036 ixgbe_handle_msf(sc); 2037 } else 2038 ixgbe_handle_mod(sc); 2039 } else { 2040 if (sc->hw.mac.ops.check_link) 2041 err = sc->hw.mac.ops.check_link(&sc->hw, &autoneg, 2042 &sc->link_up, FALSE); 2043 if (err) 2044 return; 2045 autoneg = sc->hw.phy.autoneg_advertised; 2046 if ((!autoneg) && (sc->hw.mac.ops.get_link_capabilities)) 2047 err = sc->hw.mac.ops.get_link_capabilities(&sc->hw, 2048 &autoneg, &negotiate); 2049 if (err) 2050 return; 2051 if (sc->hw.mac.ops.setup_link) 2052 sc->hw.mac.ops.setup_link(&sc->hw, 2053 autoneg, sc->link_up); 2054 } 2055} 2056 2057/******************************************************************** 2058 * Manage DMA'able memory. 2059 *******************************************************************/ 2060int 2061ixgbe_dma_malloc(struct ix_softc *sc, bus_size_t size, 2062 struct ixgbe_dma_alloc *dma, int mapflags) 2063{ 2064 struct ifnet *ifp = &sc->arpcom.ac_if; 2065 struct ixgbe_osdep *os = &sc->osdep; 2066 int r; 2067 2068 dma->dma_tag = os->os_pa.pa_dmat; 2069 r = bus_dmamap_create(dma->dma_tag, size, 1, 2070 size, 0, BUS_DMA_NOWAIT, &dma->dma_map); 2071 if (r != 0) { 2072 printf("%s: ixgbe_dma_malloc: bus_dmamap_create failed; " 2073 "error %u\n", ifp->if_xname, r); 2074 goto fail_0; 2075 } 2076 2077 r = bus_dmamem_alloc(dma->dma_tag, size, PAGE_SIZE, 0, &dma->dma_seg, 2078 1, &dma->dma_nseg, BUS_DMA_NOWAIT); 2079 if (r != 0) { 2080 printf("%s: ixgbe_dma_malloc: bus_dmamem_alloc failed; " 2081 "error %u\n", ifp->if_xname, r); 2082 goto fail_1; 2083 } 2084 2085 r = bus_dmamem_map(dma->dma_tag, &dma->dma_seg, dma->dma_nseg, size, 2086 &dma->dma_vaddr, BUS_DMA_NOWAIT); 2087 if (r != 0) { 2088 printf("%s: ixgbe_dma_malloc: bus_dmamem_map failed; " 2089 "error %u\n", ifp->if_xname, r); 2090 goto fail_2; 2091 } 2092 2093 r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, 2094 size, NULL, mapflags | BUS_DMA_NOWAIT); 2095 if (r != 0) { 2096 printf("%s: ixgbe_dma_malloc: bus_dmamap_load failed; " 2097 "error %u\n", ifp->if_xname, r); 2098 goto fail_3; 2099 } 2100 2101 dma->dma_size = size; 2102 return (0); 2103fail_3: 2104 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, size); 2105fail_2: 2106 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2107fail_1: 2108 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2109fail_0: 2110 dma->dma_map = NULL; 2111 dma->dma_tag = NULL; 2112 return (r); 2113} 2114 2115void 2116ixgbe_dma_free(struct ix_softc *sc, struct ixgbe_dma_alloc *dma) 2117{ 2118 if (dma->dma_tag == NULL) 2119 return; 2120 2121 if (dma->dma_map != NULL) { 2122 bus_dmamap_sync(dma->dma_tag, dma->dma_map, 0, 2123 dma->dma_map->dm_mapsize, 2124 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 2125 bus_dmamap_unload(dma->dma_tag, dma->dma_map); 2126 bus_dmamem_unmap(dma->dma_tag, dma->dma_vaddr, dma->dma_size); 2127 bus_dmamem_free(dma->dma_tag, &dma->dma_seg, dma->dma_nseg); 2128 bus_dmamap_destroy(dma->dma_tag, dma->dma_map); 2129 dma->dma_map = NULL; 2130 } 2131} 2132 2133 2134/********************************************************************* 2135 * 2136 * Allocate memory for the transmit and receive rings, and then 2137 * the descriptors associated with each, called only once at attach. 2138 * 2139 **********************************************************************/ 2140int 2141ixgbe_allocate_queues(struct ix_softc *sc) 2142{ 2143 struct ifnet *ifp = &sc->arpcom.ac_if; 2144 struct ix_queue *que; 2145 struct ix_txring *txr; 2146 struct ix_rxring *rxr; 2147 int rsize, tsize; 2148 int txconf = 0, rxconf = 0, i; 2149 2150 /* First allocate the top level queue structs */ 2151 if (!(sc->queues = mallocarray(sc->num_queues, 2152 sizeof(struct ix_queue), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2153 printf("%s: Unable to allocate queue memory\n", ifp->if_xname); 2154 goto fail; 2155 } 2156 2157 /* Then allocate the TX ring struct memory */ 2158 if (!(sc->tx_rings = mallocarray(sc->num_queues, 2159 sizeof(struct ix_txring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2160 printf("%s: Unable to allocate TX ring memory\n", ifp->if_xname); 2161 goto fail; 2162 } 2163 2164 /* Next allocate the RX */ 2165 if (!(sc->rx_rings = mallocarray(sc->num_queues, 2166 sizeof(struct ix_rxring), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2167 printf("%s: Unable to allocate RX ring memory\n", ifp->if_xname); 2168 goto rx_fail; 2169 } 2170 2171 /* For the ring itself */ 2172 tsize = roundup2(sc->num_tx_desc * 2173 sizeof(union ixgbe_adv_tx_desc), DBA_ALIGN); 2174 2175 /* 2176 * Now set up the TX queues, txconf is needed to handle the 2177 * possibility that things fail midcourse and we need to 2178 * undo memory gracefully 2179 */ 2180 for (i = 0; i < sc->num_queues; i++, txconf++) { 2181 /* Set up some basics */ 2182 txr = &sc->tx_rings[i]; 2183 txr->sc = sc; 2184 txr->me = i; 2185 2186 if (ixgbe_dma_malloc(sc, tsize, 2187 &txr->txdma, BUS_DMA_NOWAIT)) { 2188 printf("%s: Unable to allocate TX Descriptor memory\n", 2189 ifp->if_xname); 2190 goto err_tx_desc; 2191 } 2192 txr->tx_base = (union ixgbe_adv_tx_desc *)txr->txdma.dma_vaddr; 2193 bzero((void *)txr->tx_base, tsize); 2194 } 2195 2196 /* 2197 * Next the RX queues... 2198 */ 2199 rsize = roundup2(sc->num_rx_desc * 2200 sizeof(union ixgbe_adv_rx_desc), 4096); 2201 for (i = 0; i < sc->num_queues; i++, rxconf++) { 2202 rxr = &sc->rx_rings[i]; 2203 /* Set up some basics */ 2204 rxr->sc = sc; 2205 rxr->me = i; 2206 timeout_set(&rxr->rx_refill, ixgbe_rxrefill, rxr); 2207 2208 if (ixgbe_dma_malloc(sc, rsize, 2209 &rxr->rxdma, BUS_DMA_NOWAIT)) { 2210 printf("%s: Unable to allocate RxDescriptor memory\n", 2211 ifp->if_xname); 2212 goto err_rx_desc; 2213 } 2214 rxr->rx_base = (union ixgbe_adv_rx_desc *)rxr->rxdma.dma_vaddr; 2215 bzero((void *)rxr->rx_base, rsize); 2216 } 2217 2218 /* 2219 * Finally set up the queue holding structs 2220 */ 2221 for (i = 0; i < sc->num_queues; i++) { 2222 que = &sc->queues[i]; 2223 que->sc = sc; 2224 que->txr = &sc->tx_rings[i]; 2225 que->rxr = &sc->rx_rings[i]; 2226 snprintf(que->name, sizeof(que->name), "%s:%d", 2227 sc->dev.dv_xname, i); 2228 } 2229 2230 return (0); 2231 2232err_rx_desc: 2233 for (rxr = sc->rx_rings; rxconf > 0; rxr++, rxconf--) 2234 ixgbe_dma_free(sc, &rxr->rxdma); 2235err_tx_desc: 2236 for (txr = sc->tx_rings; txconf > 0; txr++, txconf--) 2237 ixgbe_dma_free(sc, &txr->txdma); 2238 free(sc->rx_rings, M_DEVBUF, sc->num_queues * sizeof(struct ix_rxring)); 2239 sc->rx_rings = NULL; 2240rx_fail: 2241 free(sc->tx_rings, M_DEVBUF, sc->num_queues * sizeof(struct ix_txring)); 2242 sc->tx_rings = NULL; 2243fail: 2244 return (ENOMEM); 2245} 2246 2247/********************************************************************* 2248 * 2249 * Allocate memory for tx_buffer structures. The tx_buffer stores all 2250 * the information needed to transmit a packet on the wire. This is 2251 * called only once at attach, setup is done every reset. 2252 * 2253 **********************************************************************/ 2254int 2255ixgbe_allocate_transmit_buffers(struct ix_txring *txr) 2256{ 2257 struct ix_softc *sc = txr->sc; 2258 struct ifnet *ifp = &sc->arpcom.ac_if; 2259 struct ixgbe_tx_buf *txbuf; 2260 int error, i; 2261 2262 if (!(txr->tx_buffers = mallocarray(sc->num_tx_desc, 2263 sizeof(struct ixgbe_tx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2264 printf("%s: Unable to allocate tx_buffer memory\n", 2265 ifp->if_xname); 2266 error = ENOMEM; 2267 goto fail; 2268 } 2269 txr->txtag = txr->txdma.dma_tag; 2270 2271 /* Create the descriptor buffer dma maps */ 2272 for (i = 0; i < sc->num_tx_desc; i++) { 2273 txbuf = &txr->tx_buffers[i]; 2274 error = bus_dmamap_create(txr->txdma.dma_tag, MAXMCLBYTES, 2275 sc->num_segs, PAGE_SIZE, 0, 2276 BUS_DMA_NOWAIT, &txbuf->map); 2277 2278 if (error != 0) { 2279 printf("%s: Unable to create TX DMA map\n", 2280 ifp->if_xname); 2281 goto fail; 2282 } 2283 } 2284 2285 return 0; 2286fail: 2287 return (error); 2288} 2289 2290/********************************************************************* 2291 * 2292 * Initialize a transmit ring. 2293 * 2294 **********************************************************************/ 2295int 2296ixgbe_setup_transmit_ring(struct ix_txring *txr) 2297{ 2298 struct ix_softc *sc = txr->sc; 2299 int error; 2300 2301 /* Now allocate transmit buffers for the ring */ 2302 if ((error = ixgbe_allocate_transmit_buffers(txr)) != 0) 2303 return (error); 2304 2305 /* Clear the old ring contents */ 2306 bzero((void *)txr->tx_base, 2307 (sizeof(union ixgbe_adv_tx_desc)) * sc->num_tx_desc); 2308 2309 /* Reset indices */ 2310 txr->next_avail_desc = 0; 2311 txr->next_to_clean = 0; 2312 2313 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2314 0, txr->txdma.dma_map->dm_mapsize, 2315 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2316 2317 return (0); 2318} 2319 2320/********************************************************************* 2321 * 2322 * Initialize all transmit rings. 2323 * 2324 **********************************************************************/ 2325int 2326ixgbe_setup_transmit_structures(struct ix_softc *sc) 2327{ 2328 struct ix_txring *txr = sc->tx_rings; 2329 int i, error; 2330 2331 for (i = 0; i < sc->num_queues; i++, txr++) { 2332 if ((error = ixgbe_setup_transmit_ring(txr)) != 0) 2333 goto fail; 2334 } 2335 2336 return (0); 2337fail: 2338 ixgbe_free_transmit_structures(sc); 2339 return (error); 2340} 2341 2342/********************************************************************* 2343 * 2344 * Enable transmit unit. 2345 * 2346 **********************************************************************/ 2347void 2348ixgbe_initialize_transmit_units(struct ix_softc *sc) 2349{ 2350 struct ifnet *ifp = &sc->arpcom.ac_if; 2351 struct ix_txring *txr; 2352 struct ixgbe_hw *hw = &sc->hw; 2353 int i; 2354 uint64_t tdba; 2355 uint32_t txctrl; 2356 uint32_t hlreg; 2357 2358 /* Setup the Base and Length of the Tx Descriptor Ring */ 2359 2360 for (i = 0; i < sc->num_queues; i++) { 2361 txr = &sc->tx_rings[i]; 2362 2363 /* Setup descriptor base address */ 2364 tdba = txr->txdma.dma_map->dm_segs[0].ds_addr; 2365 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(i), 2366 (tdba & 0x00000000ffffffffULL)); 2367 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(i), (tdba >> 32)); 2368 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(i), 2369 sc->num_tx_desc * sizeof(struct ixgbe_legacy_tx_desc)); 2370 2371 /* Setup the HW Tx Head and Tail descriptor pointers */ 2372 IXGBE_WRITE_REG(hw, IXGBE_TDH(i), 0); 2373 IXGBE_WRITE_REG(hw, IXGBE_TDT(i), 0); 2374 2375 /* Setup Transmit Descriptor Cmd Settings */ 2376 txr->txd_cmd = IXGBE_TXD_CMD_IFCS; 2377 txr->queue_status = IXGBE_QUEUE_IDLE; 2378 txr->watchdog_timer = 0; 2379 2380 /* Disable Head Writeback */ 2381 switch (hw->mac.type) { 2382 case ixgbe_mac_82598EB: 2383 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(i)); 2384 break; 2385 case ixgbe_mac_82599EB: 2386 case ixgbe_mac_X540: 2387 default: 2388 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(i)); 2389 break; 2390 } 2391 txctrl &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2392 switch (hw->mac.type) { 2393 case ixgbe_mac_82598EB: 2394 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(i), txctrl); 2395 break; 2396 case ixgbe_mac_82599EB: 2397 case ixgbe_mac_X540: 2398 default: 2399 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(i), txctrl); 2400 break; 2401 } 2402 } 2403 ifp->if_timer = 0; 2404 2405 if (hw->mac.type != ixgbe_mac_82598EB) { 2406 uint32_t dmatxctl, rttdcs; 2407 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2408 dmatxctl |= IXGBE_DMATXCTL_TE; 2409 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl); 2410 /* Disable arbiter to set MTQC */ 2411 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2412 rttdcs |= IXGBE_RTTDCS_ARBDIS; 2413 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2414 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2415 rttdcs &= ~IXGBE_RTTDCS_ARBDIS; 2416 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs); 2417 } 2418 2419 /* Enable TCP/UDP padding when using TSO */ 2420 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2421 hlreg |= IXGBE_HLREG0_TXPADEN; 2422 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2423} 2424 2425/********************************************************************* 2426 * 2427 * Free all transmit rings. 2428 * 2429 **********************************************************************/ 2430void 2431ixgbe_free_transmit_structures(struct ix_softc *sc) 2432{ 2433 struct ix_txring *txr = sc->tx_rings; 2434 int i; 2435 2436 for (i = 0; i < sc->num_queues; i++, txr++) 2437 ixgbe_free_transmit_buffers(txr); 2438} 2439 2440/********************************************************************* 2441 * 2442 * Free transmit ring related data structures. 2443 * 2444 **********************************************************************/ 2445void 2446ixgbe_free_transmit_buffers(struct ix_txring *txr) 2447{ 2448 struct ix_softc *sc = txr->sc; 2449 struct ixgbe_tx_buf *tx_buffer; 2450 int i; 2451 2452 INIT_DEBUGOUT("free_transmit_ring: begin"); 2453 2454 if (txr->tx_buffers == NULL) 2455 return; 2456 2457 tx_buffer = txr->tx_buffers; 2458 for (i = 0; i < sc->num_tx_desc; i++, tx_buffer++) { 2459 if (tx_buffer->map != NULL && tx_buffer->map->dm_nsegs > 0) { 2460 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2461 0, tx_buffer->map->dm_mapsize, 2462 BUS_DMASYNC_POSTWRITE); 2463 bus_dmamap_unload(txr->txdma.dma_tag, 2464 tx_buffer->map); 2465 } 2466 if (tx_buffer->m_head != NULL) { 2467 m_freem(tx_buffer->m_head); 2468 tx_buffer->m_head = NULL; 2469 } 2470 if (tx_buffer->map != NULL) { 2471 bus_dmamap_destroy(txr->txdma.dma_tag, 2472 tx_buffer->map); 2473 tx_buffer->map = NULL; 2474 } 2475 } 2476 2477 if (txr->tx_buffers != NULL) 2478 free(txr->tx_buffers, M_DEVBUF, 2479 sc->num_tx_desc * sizeof(struct ixgbe_tx_buf)); 2480 txr->tx_buffers = NULL; 2481 txr->txtag = NULL; 2482} 2483 2484/********************************************************************* 2485 * 2486 * Advanced Context Descriptor setup for VLAN or CSUM 2487 * 2488 **********************************************************************/ 2489 2490static inline int 2491ixgbe_tx_offload(struct mbuf *mp, uint32_t *vlan_macip_lens, 2492 uint32_t *type_tucmd_mlhl, uint32_t *olinfo_status, uint32_t *cmd_type_len, 2493 uint32_t *mss_l4len_idx) 2494{ 2495 struct ether_extracted ext; 2496 int offload = 0; 2497 2498 ether_extract_headers(mp, &ext); 2499 2500 *vlan_macip_lens |= (sizeof(*ext.eh) << IXGBE_ADVTXD_MACLEN_SHIFT); 2501 2502 if (ext.ip4) { 2503 if (ISSET(mp->m_pkthdr.csum_flags, M_IPV4_CSUM_OUT)) { 2504 *olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8; 2505 offload = 1; 2506 } 2507 2508 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4; 2509#ifdef INET6 2510 } else if (ext.ip6) { 2511 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6; 2512#endif 2513 } else { 2514 if (mp->m_pkthdr.csum_flags & M_TCP_TSO) 2515 tcpstat_inc(tcps_outbadtso); 2516 return offload; 2517 } 2518 2519 *vlan_macip_lens |= ext.iphlen; 2520 2521 if (ext.tcp) { 2522 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP; 2523 if (ISSET(mp->m_pkthdr.csum_flags, M_TCP_CSUM_OUT)) { 2524 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2525 offload = 1; 2526 } 2527 } else if (ext.udp) { 2528 *type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP; 2529 if (ISSET(mp->m_pkthdr.csum_flags, M_UDP_CSUM_OUT)) { 2530 *olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8; 2531 offload = 1; 2532 } 2533 } 2534 2535 if (mp->m_pkthdr.csum_flags & M_TCP_TSO) { 2536 if (ext.tcp && mp->m_pkthdr.ph_mss > 0) { 2537 uint32_t hdrlen, thlen, paylen, outlen; 2538 2539 thlen = ext.tcphlen; 2540 2541 outlen = mp->m_pkthdr.ph_mss; 2542 *mss_l4len_idx |= outlen << IXGBE_ADVTXD_MSS_SHIFT; 2543 *mss_l4len_idx |= thlen << IXGBE_ADVTXD_L4LEN_SHIFT; 2544 2545 hdrlen = sizeof(*ext.eh) + ext.iphlen + thlen; 2546 paylen = mp->m_pkthdr.len - hdrlen; 2547 CLR(*olinfo_status, IXGBE_ADVTXD_PAYLEN_MASK 2548 << IXGBE_ADVTXD_PAYLEN_SHIFT); 2549 *olinfo_status |= paylen << IXGBE_ADVTXD_PAYLEN_SHIFT; 2550 2551 *cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; 2552 offload = 1; 2553 2554 tcpstat_add(tcps_outpkttso, 2555 (paylen + outlen - 1) / outlen); 2556 } else 2557 tcpstat_inc(tcps_outbadtso); 2558 } 2559 2560 return offload; 2561} 2562 2563static int 2564ixgbe_tx_ctx_setup(struct ix_txring *txr, struct mbuf *mp, 2565 uint32_t *cmd_type_len, uint32_t *olinfo_status) 2566{ 2567 struct ixgbe_adv_tx_context_desc *TXD; 2568 struct ixgbe_tx_buf *tx_buffer; 2569 uint32_t vlan_macip_lens = 0, type_tucmd_mlhl = 0; 2570 uint32_t mss_l4len_idx = 0; 2571 int ctxd = txr->next_avail_desc; 2572 int offload = 0; 2573 2574 /* Indicate the whole packet as payload when not doing TSO */ 2575 *olinfo_status |= mp->m_pkthdr.len << IXGBE_ADVTXD_PAYLEN_SHIFT; 2576 2577#if NVLAN > 0 2578 if (ISSET(mp->m_flags, M_VLANTAG)) { 2579 uint32_t vtag = mp->m_pkthdr.ether_vtag; 2580 vlan_macip_lens |= (vtag << IXGBE_ADVTXD_VLAN_SHIFT); 2581 *cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; 2582 offload |= 1; 2583 } 2584#endif 2585 2586 offload |= ixgbe_tx_offload(mp, &vlan_macip_lens, &type_tucmd_mlhl, 2587 olinfo_status, cmd_type_len, &mss_l4len_idx); 2588 2589 if (!offload) 2590 return (0); 2591 2592 TXD = (struct ixgbe_adv_tx_context_desc *)&txr->tx_base[ctxd]; 2593 tx_buffer = &txr->tx_buffers[ctxd]; 2594 2595 type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 2596 2597 /* Now copy bits into descriptor */ 2598 TXD->vlan_macip_lens = htole32(vlan_macip_lens); 2599 TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 2600 TXD->seqnum_seed = htole32(0); 2601 TXD->mss_l4len_idx = htole32(mss_l4len_idx); 2602 2603 tx_buffer->m_head = NULL; 2604 tx_buffer->eop_index = -1; 2605 2606 return (1); 2607} 2608 2609/********************************************************************** 2610 * 2611 * Examine each tx_buffer in the used queue. If the hardware is done 2612 * processing the packet then free associated resources. The 2613 * tx_buffer is put back on the free queue. 2614 * 2615 **********************************************************************/ 2616int 2617ixgbe_txeof(struct ix_txring *txr) 2618{ 2619 struct ix_softc *sc = txr->sc; 2620 struct ifqueue *ifq = txr->ifq; 2621 struct ifnet *ifp = &sc->arpcom.ac_if; 2622 unsigned int head, tail, last; 2623 struct ixgbe_tx_buf *tx_buffer; 2624 struct ixgbe_legacy_tx_desc *tx_desc; 2625 2626 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 2627 return FALSE; 2628 2629 head = txr->next_avail_desc; 2630 tail = txr->next_to_clean; 2631 2632 membar_consumer(); 2633 2634 if (head == tail) 2635 return (FALSE); 2636 2637 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2638 0, txr->txdma.dma_map->dm_mapsize, 2639 BUS_DMASYNC_POSTREAD); 2640 2641 for (;;) { 2642 tx_buffer = &txr->tx_buffers[tail]; 2643 last = tx_buffer->eop_index; 2644 tx_desc = (struct ixgbe_legacy_tx_desc *)&txr->tx_base[last]; 2645 2646 if (!ISSET(tx_desc->upper.fields.status, IXGBE_TXD_STAT_DD)) 2647 break; 2648 2649 bus_dmamap_sync(txr->txdma.dma_tag, tx_buffer->map, 2650 0, tx_buffer->map->dm_mapsize, BUS_DMASYNC_POSTWRITE); 2651 bus_dmamap_unload(txr->txdma.dma_tag, tx_buffer->map); 2652 m_freem(tx_buffer->m_head); 2653 2654 tx_buffer->m_head = NULL; 2655 tx_buffer->eop_index = -1; 2656 2657 tail = last + 1; 2658 if (tail == sc->num_tx_desc) 2659 tail = 0; 2660 if (head == tail) { 2661 /* All clean, turn off the timer */ 2662 ifp->if_timer = 0; 2663 break; 2664 } 2665 } 2666 2667 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 2668 0, txr->txdma.dma_map->dm_mapsize, 2669 BUS_DMASYNC_PREREAD); 2670 2671 membar_producer(); 2672 2673 txr->next_to_clean = tail; 2674 2675 if (ifq_is_oactive(ifq)) 2676 ifq_restart(ifq); 2677 2678 return TRUE; 2679} 2680 2681/********************************************************************* 2682 * 2683 * Get a buffer from system mbuf buffer pool. 2684 * 2685 **********************************************************************/ 2686int 2687ixgbe_get_buf(struct ix_rxring *rxr, int i) 2688{ 2689 struct ix_softc *sc = rxr->sc; 2690 struct ixgbe_rx_buf *rxbuf; 2691 struct mbuf *mp; 2692 int error; 2693 union ixgbe_adv_rx_desc *rxdesc; 2694 2695 rxbuf = &rxr->rx_buffers[i]; 2696 rxdesc = &rxr->rx_base[i]; 2697 if (rxbuf->buf) { 2698 printf("%s: ixgbe_get_buf: slot %d already has an mbuf\n", 2699 sc->dev.dv_xname, i); 2700 return (ENOBUFS); 2701 } 2702 2703 /* needed in any case so prealocate since this one will fail for sure */ 2704 mp = MCLGETL(NULL, M_DONTWAIT, sc->rx_mbuf_sz); 2705 if (!mp) 2706 return (ENOBUFS); 2707 2708 mp->m_data += (mp->m_ext.ext_size - sc->rx_mbuf_sz); 2709 mp->m_len = mp->m_pkthdr.len = sc->rx_mbuf_sz; 2710 2711 error = bus_dmamap_load_mbuf(rxr->rxdma.dma_tag, rxbuf->map, 2712 mp, BUS_DMA_NOWAIT); 2713 if (error) { 2714 m_freem(mp); 2715 return (error); 2716 } 2717 2718 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 2719 0, rxbuf->map->dm_mapsize, BUS_DMASYNC_PREREAD); 2720 rxbuf->buf = mp; 2721 2722 rxdesc->read.pkt_addr = htole64(rxbuf->map->dm_segs[0].ds_addr); 2723 2724 return (0); 2725} 2726 2727/********************************************************************* 2728 * 2729 * Allocate memory for rx_buffer structures. Since we use one 2730 * rx_buffer per received packet, the maximum number of rx_buffer's 2731 * that we'll need is equal to the number of receive descriptors 2732 * that we've allocated. 2733 * 2734 **********************************************************************/ 2735int 2736ixgbe_allocate_receive_buffers(struct ix_rxring *rxr) 2737{ 2738 struct ix_softc *sc = rxr->sc; 2739 struct ifnet *ifp = &sc->arpcom.ac_if; 2740 struct ixgbe_rx_buf *rxbuf; 2741 int i, error; 2742 2743 if (!(rxr->rx_buffers = mallocarray(sc->num_rx_desc, 2744 sizeof(struct ixgbe_rx_buf), M_DEVBUF, M_NOWAIT | M_ZERO))) { 2745 printf("%s: Unable to allocate rx_buffer memory\n", 2746 ifp->if_xname); 2747 error = ENOMEM; 2748 goto fail; 2749 } 2750 2751 rxbuf = rxr->rx_buffers; 2752 for (i = 0; i < sc->num_rx_desc; i++, rxbuf++) { 2753 error = bus_dmamap_create(rxr->rxdma.dma_tag, 16 * 1024, 1, 2754 16 * 1024, 0, BUS_DMA_NOWAIT, &rxbuf->map); 2755 if (error) { 2756 printf("%s: Unable to create Pack DMA map\n", 2757 ifp->if_xname); 2758 goto fail; 2759 } 2760 } 2761 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 0, 2762 rxr->rxdma.dma_map->dm_mapsize, 2763 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2764 2765 return (0); 2766 2767fail: 2768 return (error); 2769} 2770 2771/********************************************************************* 2772 * 2773 * Initialize a receive ring and its buffers. 2774 * 2775 **********************************************************************/ 2776int 2777ixgbe_setup_receive_ring(struct ix_rxring *rxr) 2778{ 2779 struct ix_softc *sc = rxr->sc; 2780 struct ifnet *ifp = &sc->arpcom.ac_if; 2781 int rsize, error; 2782 2783 rsize = roundup2(sc->num_rx_desc * 2784 sizeof(union ixgbe_adv_rx_desc), 4096); 2785 /* Clear the ring contents */ 2786 bzero((void *)rxr->rx_base, rsize); 2787 2788 if ((error = ixgbe_allocate_receive_buffers(rxr)) != 0) 2789 return (error); 2790 2791 /* Setup our descriptor indices */ 2792 rxr->next_to_check = 0; 2793 rxr->last_desc_filled = sc->num_rx_desc - 1; 2794 2795 if_rxr_init(&rxr->rx_ring, 2 * ((ifp->if_hardmtu / MCLBYTES) + 1), 2796 sc->num_rx_desc - 1); 2797 2798 ixgbe_rxfill(rxr); 2799 if (if_rxr_inuse(&rxr->rx_ring) == 0) { 2800 printf("%s: unable to fill any rx descriptors\n", 2801 sc->dev.dv_xname); 2802 return (ENOBUFS); 2803 } 2804 2805 return (0); 2806} 2807 2808int 2809ixgbe_rxfill(struct ix_rxring *rxr) 2810{ 2811 struct ix_softc *sc = rxr->sc; 2812 int post = 0; 2813 u_int slots; 2814 int i; 2815 2816 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2817 0, rxr->rxdma.dma_map->dm_mapsize, 2818 BUS_DMASYNC_POSTWRITE); 2819 2820 i = rxr->last_desc_filled; 2821 for (slots = if_rxr_get(&rxr->rx_ring, sc->num_rx_desc); 2822 slots > 0; slots--) { 2823 if (++i == sc->num_rx_desc) 2824 i = 0; 2825 2826 if (ixgbe_get_buf(rxr, i) != 0) 2827 break; 2828 2829 rxr->last_desc_filled = i; 2830 post = 1; 2831 } 2832 2833 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 2834 0, rxr->rxdma.dma_map->dm_mapsize, 2835 BUS_DMASYNC_PREWRITE); 2836 2837 if_rxr_put(&rxr->rx_ring, slots); 2838 2839 return (post); 2840} 2841 2842void 2843ixgbe_rxrefill(void *xrxr) 2844{ 2845 struct ix_rxring *rxr = xrxr; 2846 struct ix_softc *sc = rxr->sc; 2847 2848 if (ixgbe_rxfill(rxr)) { 2849 /* Advance the Rx Queue "Tail Pointer" */ 2850 IXGBE_WRITE_REG(&sc->hw, IXGBE_RDT(rxr->me), 2851 rxr->last_desc_filled); 2852 } else if (if_rxr_inuse(&rxr->rx_ring) == 0) 2853 timeout_add(&rxr->rx_refill, 1); 2854 2855} 2856 2857/********************************************************************* 2858 * 2859 * Initialize all receive rings. 2860 * 2861 **********************************************************************/ 2862int 2863ixgbe_setup_receive_structures(struct ix_softc *sc) 2864{ 2865 struct ix_rxring *rxr = sc->rx_rings; 2866 int i; 2867 2868 for (i = 0; i < sc->num_queues; i++, rxr++) 2869 if (ixgbe_setup_receive_ring(rxr)) 2870 goto fail; 2871 2872 return (0); 2873fail: 2874 ixgbe_free_receive_structures(sc); 2875 return (ENOBUFS); 2876} 2877 2878/********************************************************************* 2879 * 2880 * Setup receive registers and features. 2881 * 2882 **********************************************************************/ 2883#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 2884 2885void 2886ixgbe_initialize_receive_units(struct ix_softc *sc) 2887{ 2888 struct ifnet *ifp = &sc->arpcom.ac_if; 2889 struct ix_rxring *rxr = sc->rx_rings; 2890 struct ixgbe_hw *hw = &sc->hw; 2891 uint32_t bufsz, fctrl, srrctl, rxcsum, rdrxctl; 2892 uint32_t hlreg; 2893 int i; 2894 2895 /* 2896 * Make sure receives are disabled while 2897 * setting up the descriptor ring 2898 */ 2899 ixgbe_disable_rx(hw); 2900 2901 /* Enable broadcasts */ 2902 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2903 fctrl |= IXGBE_FCTRL_BAM; 2904 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 2905 fctrl |= IXGBE_FCTRL_DPF; 2906 fctrl |= IXGBE_FCTRL_PMCF; 2907 } 2908 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl); 2909 2910 hlreg = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2911 /* Always enable jumbo frame reception */ 2912 hlreg |= IXGBE_HLREG0_JUMBOEN; 2913 /* Always enable CRC stripping */ 2914 hlreg |= IXGBE_HLREG0_RXCRCSTRP; 2915 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg); 2916 2917 if (ISSET(ifp->if_xflags, IFXF_LRO)) { 2918 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2919 2920 /* This field has to be set to zero. */ 2921 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2922 2923 /* RSC Coalescing on ACK Change */ 2924 rdrxctl |= IXGBE_RDRXCTL_RSCACKC; 2925 rdrxctl |= IXGBE_RDRXCTL_FCOE_WRFIX; 2926 2927 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl); 2928 } 2929 2930 bufsz = (sc->rx_mbuf_sz - ETHER_ALIGN) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 2931 2932 for (i = 0; i < sc->num_queues; i++, rxr++) { 2933 uint64_t rdba = rxr->rxdma.dma_map->dm_segs[0].ds_addr; 2934 2935 /* Setup the Base and Length of the Rx Descriptor Ring */ 2936 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(i), 2937 (rdba & 0x00000000ffffffffULL)); 2938 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(i), (rdba >> 32)); 2939 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(i), 2940 sc->num_rx_desc * sizeof(union ixgbe_adv_rx_desc)); 2941 2942 /* Set up the SRRCTL register */ 2943 srrctl = bufsz | IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2944 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(i), srrctl); 2945 2946 if (ISSET(ifp->if_xflags, IFXF_LRO)) { 2947 rdrxctl = IXGBE_READ_REG(&sc->hw, IXGBE_RSCCTL(i)); 2948 2949 /* Enable Receive Side Coalescing */ 2950 rdrxctl |= IXGBE_RSCCTL_RSCEN; 2951 rdrxctl |= IXGBE_RSCCTL_MAXDESC_16; 2952 2953 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), rdrxctl); 2954 } 2955 2956 /* Setup the HW Rx Head and Tail Descriptor Pointers */ 2957 IXGBE_WRITE_REG(hw, IXGBE_RDH(i), 0); 2958 IXGBE_WRITE_REG(hw, IXGBE_RDT(i), 0); 2959 } 2960 2961 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 2962 uint32_t psrtype = IXGBE_PSRTYPE_TCPHDR | 2963 IXGBE_PSRTYPE_UDPHDR | 2964 IXGBE_PSRTYPE_IPV4HDR | 2965 IXGBE_PSRTYPE_IPV6HDR; 2966 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), psrtype); 2967 } 2968 2969 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2970 rxcsum &= ~IXGBE_RXCSUM_PCSD; 2971 2972 ixgbe_initialize_rss_mapping(sc); 2973 2974 /* Setup RSS */ 2975 if (sc->num_queues > 1) { 2976 /* RSS and RX IPP Checksum are mutually exclusive */ 2977 rxcsum |= IXGBE_RXCSUM_PCSD; 2978 } 2979 2980 /* Map QPRC/QPRDC/QPTC on a per queue basis */ 2981 ixgbe_map_queue_statistics(sc); 2982 2983 /* This is useful for calculating UDP/IP fragment checksums */ 2984 if (!(rxcsum & IXGBE_RXCSUM_PCSD)) 2985 rxcsum |= IXGBE_RXCSUM_IPPCSE; 2986 2987 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2988} 2989 2990void 2991ixgbe_initialize_rss_mapping(struct ix_softc *sc) 2992{ 2993 struct ixgbe_hw *hw = &sc->hw; 2994 uint32_t reta = 0, mrqc, rss_key[10]; 2995 int i, j, queue_id, table_size, index_mult; 2996 2997 /* set up random bits */ 2998 stoeplitz_to_key(&rss_key, sizeof(rss_key)); 2999 3000 /* Set multiplier for RETA setup and table size based on MAC */ 3001 index_mult = 0x1; 3002 table_size = 128; 3003 switch (sc->hw.mac.type) { 3004 case ixgbe_mac_82598EB: 3005 index_mult = 0x11; 3006 break; 3007 case ixgbe_mac_X550: 3008 case ixgbe_mac_X550EM_x: 3009 case ixgbe_mac_X550EM_a: 3010 table_size = 512; 3011 break; 3012 default: 3013 break; 3014 } 3015 3016 /* Set up the redirection table */ 3017 for (i = 0, j = 0; i < table_size; i++, j++) { 3018 if (j == sc->num_queues) j = 0; 3019 queue_id = (j * index_mult); 3020 /* 3021 * The low 8 bits are for hash value (n+0); 3022 * The next 8 bits are for hash value (n+1), etc. 3023 */ 3024 reta = reta >> 8; 3025 reta = reta | ( ((uint32_t) queue_id) << 24); 3026 if ((i & 3) == 3) { 3027 if (i < 128) 3028 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 3029 else 3030 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 3031 reta); 3032 reta = 0; 3033 } 3034 } 3035 3036 /* Now fill our hash function seeds */ 3037 for (i = 0; i < 10; i++) 3038 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), rss_key[i]); 3039 3040 /* 3041 * Disable UDP - IP fragments aren't currently being handled 3042 * and so we end up with a mix of 2-tuple and 4-tuple 3043 * traffic. 3044 */ 3045 mrqc = IXGBE_MRQC_RSSEN 3046 | IXGBE_MRQC_RSS_FIELD_IPV4 3047 | IXGBE_MRQC_RSS_FIELD_IPV4_TCP 3048 | IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP 3049 | IXGBE_MRQC_RSS_FIELD_IPV6_EX 3050 | IXGBE_MRQC_RSS_FIELD_IPV6 3051 | IXGBE_MRQC_RSS_FIELD_IPV6_TCP 3052 ; 3053 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 3054} 3055 3056/********************************************************************* 3057 * 3058 * Free all receive rings. 3059 * 3060 **********************************************************************/ 3061void 3062ixgbe_free_receive_structures(struct ix_softc *sc) 3063{ 3064 struct ix_rxring *rxr; 3065 int i; 3066 3067 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3068 if_rxr_init(&rxr->rx_ring, 0, 0); 3069 3070 for (i = 0, rxr = sc->rx_rings; i < sc->num_queues; i++, rxr++) 3071 ixgbe_free_receive_buffers(rxr); 3072} 3073 3074/********************************************************************* 3075 * 3076 * Free receive ring data structures 3077 * 3078 **********************************************************************/ 3079void 3080ixgbe_free_receive_buffers(struct ix_rxring *rxr) 3081{ 3082 struct ix_softc *sc; 3083 struct ixgbe_rx_buf *rxbuf; 3084 int i; 3085 3086 sc = rxr->sc; 3087 if (rxr->rx_buffers != NULL) { 3088 for (i = 0; i < sc->num_rx_desc; i++) { 3089 rxbuf = &rxr->rx_buffers[i]; 3090 if (rxbuf->buf != NULL) { 3091 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 3092 0, rxbuf->map->dm_mapsize, 3093 BUS_DMASYNC_POSTREAD); 3094 bus_dmamap_unload(rxr->rxdma.dma_tag, 3095 rxbuf->map); 3096 m_freem(rxbuf->buf); 3097 rxbuf->buf = NULL; 3098 } 3099 if (rxbuf->map != NULL) { 3100 bus_dmamap_destroy(rxr->rxdma.dma_tag, 3101 rxbuf->map); 3102 rxbuf->map = NULL; 3103 } 3104 } 3105 free(rxr->rx_buffers, M_DEVBUF, 3106 sc->num_rx_desc * sizeof(struct ixgbe_rx_buf)); 3107 rxr->rx_buffers = NULL; 3108 } 3109} 3110 3111/********************************************************************* 3112 * 3113 * This routine executes in interrupt context. It replenishes 3114 * the mbufs in the descriptor and sends data which has been 3115 * dma'ed into host memory to upper layer. 3116 * 3117 *********************************************************************/ 3118int 3119ixgbe_rxeof(struct ix_rxring *rxr) 3120{ 3121 struct ix_softc *sc = rxr->sc; 3122 struct ifnet *ifp = &sc->arpcom.ac_if; 3123 struct mbuf_list ml = MBUF_LIST_INITIALIZER(); 3124 struct mbuf *mp, *sendmp; 3125 uint8_t eop = 0; 3126 uint16_t len, vtag; 3127 uint32_t staterr = 0; 3128 struct ixgbe_rx_buf *rxbuf, *nxbuf; 3129 union ixgbe_adv_rx_desc *rxdesc; 3130 size_t dsize = sizeof(union ixgbe_adv_rx_desc); 3131 int i, nextp, rsccnt; 3132 3133 if (!ISSET(ifp->if_flags, IFF_RUNNING)) 3134 return FALSE; 3135 3136 i = rxr->next_to_check; 3137 while (if_rxr_inuse(&rxr->rx_ring) > 0) { 3138 uint32_t hash; 3139 uint16_t hashtype; 3140 3141 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3142 dsize * i, dsize, BUS_DMASYNC_POSTREAD); 3143 3144 rxdesc = &rxr->rx_base[i]; 3145 staterr = letoh32(rxdesc->wb.upper.status_error); 3146 if (!ISSET(staterr, IXGBE_RXD_STAT_DD)) { 3147 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3148 dsize * i, dsize, 3149 BUS_DMASYNC_PREREAD); 3150 break; 3151 } 3152 3153 /* Zero out the receive descriptors status */ 3154 rxdesc->wb.upper.status_error = 0; 3155 rxbuf = &rxr->rx_buffers[i]; 3156 3157 /* pull the mbuf off the ring */ 3158 bus_dmamap_sync(rxr->rxdma.dma_tag, rxbuf->map, 0, 3159 rxbuf->map->dm_mapsize, BUS_DMASYNC_POSTREAD); 3160 bus_dmamap_unload(rxr->rxdma.dma_tag, rxbuf->map); 3161 3162 mp = rxbuf->buf; 3163 len = letoh16(rxdesc->wb.upper.length); 3164 vtag = letoh16(rxdesc->wb.upper.vlan); 3165 eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0); 3166 hash = lemtoh32(&rxdesc->wb.lower.hi_dword.rss); 3167 hashtype = 3168 lemtoh16(&rxdesc->wb.lower.lo_dword.hs_rss.pkt_info) & 3169 IXGBE_RXDADV_RSSTYPE_MASK; 3170 rsccnt = lemtoh32(&rxdesc->wb.lower.lo_dword.data) & 3171 IXGBE_RXDADV_RSCCNT_MASK; 3172 rsccnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; 3173 3174 if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) { 3175 if (rxbuf->fmp) { 3176 m_freem(rxbuf->fmp); 3177 } else { 3178 m_freem(mp); 3179 } 3180 rxbuf->fmp = NULL; 3181 rxbuf->buf = NULL; 3182 goto next_desc; 3183 } 3184 3185 if (mp == NULL) { 3186 panic("%s: ixgbe_rxeof: NULL mbuf in slot %d " 3187 "(nrx %d, filled %d)", sc->dev.dv_xname, 3188 i, if_rxr_inuse(&rxr->rx_ring), 3189 rxr->last_desc_filled); 3190 } 3191 3192 if (!eop) { 3193 /* 3194 * Figure out the next descriptor of this frame. 3195 */ 3196 if (rsccnt) { 3197 nextp = staterr & IXGBE_RXDADV_NEXTP_MASK; 3198 nextp >>= IXGBE_RXDADV_NEXTP_SHIFT; 3199 } else { 3200 nextp = i + 1; 3201 } 3202 if (nextp == sc->num_rx_desc) 3203 nextp = 0; 3204 nxbuf = &rxr->rx_buffers[nextp]; 3205 /* prefetch(nxbuf); */ 3206 } 3207 3208 /* 3209 * Rather than using the fmp/lmp global pointers 3210 * we now keep the head of a packet chain in the 3211 * buffer struct and pass this along from one 3212 * descriptor to the next, until we get EOP. 3213 */ 3214 mp->m_len = len; 3215 /* 3216 * See if there is a stored head 3217 * that determines what we are 3218 */ 3219 sendmp = rxbuf->fmp; 3220 rxbuf->buf = rxbuf->fmp = NULL; 3221 3222 if (sendmp == NULL) { 3223 /* first desc of a non-ps chain */ 3224 sendmp = mp; 3225 sendmp->m_pkthdr.len = 0; 3226 sendmp->m_pkthdr.ph_mss = 0; 3227 } else { 3228 mp->m_flags &= ~M_PKTHDR; 3229 } 3230 sendmp->m_pkthdr.len += mp->m_len; 3231 /* 3232 * This function iterates over interleaved descriptors. 3233 * Thus, we reuse ph_mss as global segment counter per 3234 * TCP connection, instead of introducing a new variable 3235 * in m_pkthdr. 3236 */ 3237 if (rsccnt) 3238 sendmp->m_pkthdr.ph_mss += rsccnt - 1; 3239 3240 /* Pass the head pointer on */ 3241 if (eop == 0) { 3242 nxbuf->fmp = sendmp; 3243 sendmp = NULL; 3244 mp->m_next = nxbuf->buf; 3245 } else { /* Sending this frame? */ 3246 ixgbe_rx_offload(staterr, vtag, sendmp); 3247 3248 if (hashtype != IXGBE_RXDADV_RSSTYPE_NONE) { 3249 sendmp->m_pkthdr.ph_flowid = hash; 3250 SET(sendmp->m_pkthdr.csum_flags, M_FLOWID); 3251 } 3252 3253 ml_enqueue(&ml, sendmp); 3254 } 3255next_desc: 3256 if_rxr_put(&rxr->rx_ring, 1); 3257 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 3258 dsize * i, dsize, 3259 BUS_DMASYNC_PREREAD); 3260 3261 /* Advance our pointers to the next descriptor. */ 3262 if (++i == sc->num_rx_desc) 3263 i = 0; 3264 } 3265 rxr->next_to_check = i; 3266 3267 if (ifiq_input(rxr->ifiq, &ml)) 3268 if_rxr_livelocked(&rxr->rx_ring); 3269 3270 if (!(staterr & IXGBE_RXD_STAT_DD)) 3271 return FALSE; 3272 3273 return TRUE; 3274} 3275 3276/********************************************************************* 3277 * 3278 * Check VLAN indication from hardware and inform the stack about the 3279 * annotated TAG. 3280 * 3281 * Verify that the hardware indicated that the checksum is valid. 3282 * Inform the stack about the status of checksum so that stack 3283 * doesn't spend time verifying the checksum. 3284 * 3285 * Propagate TCP LRO packet from hardware to the stack with MSS annotation. 3286 * 3287 *********************************************************************/ 3288void 3289ixgbe_rx_offload(uint32_t staterr, uint16_t vtag, struct mbuf *m) 3290{ 3291 uint16_t status = (uint16_t) staterr; 3292 uint8_t errors = (uint8_t) (staterr >> 24); 3293 int16_t pkts; 3294 3295 /* 3296 * VLAN Offload 3297 */ 3298 3299#if NVLAN > 0 3300 if (ISSET(staterr, IXGBE_RXD_STAT_VP)) { 3301 m->m_pkthdr.ether_vtag = vtag; 3302 SET(m->m_flags, M_VLANTAG); 3303 } 3304#endif 3305 3306 /* 3307 * Checksum Offload 3308 */ 3309 3310 if (ISSET(status, IXGBE_RXD_STAT_IPCS)) { 3311 if (ISSET(errors, IXGBE_RXD_ERR_IPE)) 3312 SET(m->m_pkthdr.csum_flags, M_IPV4_CSUM_IN_BAD); 3313 else 3314 SET(m->m_pkthdr.csum_flags, M_IPV4_CSUM_IN_OK); 3315 } 3316 if (ISSET(status, IXGBE_RXD_STAT_L4CS) && 3317 !ISSET(status, IXGBE_RXD_STAT_UDPCS)) { 3318 if (ISSET(errors, IXGBE_RXD_ERR_TCPE)) { 3319 /* on some hardware IPv6 + TCP + Bad is broken */ 3320 if (ISSET(status, IXGBE_RXD_STAT_IPCS)) 3321 SET(m->m_pkthdr.csum_flags, M_TCP_CSUM_IN_BAD); 3322 } else 3323 SET(m->m_pkthdr.csum_flags, M_TCP_CSUM_IN_OK); 3324 } 3325 if (ISSET(status, IXGBE_RXD_STAT_L4CS) && 3326 ISSET(status, IXGBE_RXD_STAT_UDPCS)) { 3327 if (ISSET(errors, IXGBE_RXD_ERR_TCPE)) 3328 SET(m->m_pkthdr.csum_flags, M_UDP_CSUM_IN_BAD); 3329 else 3330 SET(m->m_pkthdr.csum_flags, M_UDP_CSUM_IN_OK); 3331 } 3332 3333 /* 3334 * TCP Large Receive Offload 3335 */ 3336 3337 pkts = m->m_pkthdr.ph_mss; 3338 m->m_pkthdr.ph_mss = 0; 3339 3340 if (pkts > 1) { 3341 struct ether_extracted ext; 3342 uint32_t paylen; 3343 3344 /* 3345 * Calculate the payload size: 3346 * 3347 * The packet length returned by the NIC (m->m_pkthdr.len) 3348 * can contain padding, which we don't want to count in to the 3349 * payload size. Therefore, we calculate the real payload size 3350 * based on the total ip length field (ext.iplen). 3351 */ 3352 ether_extract_headers(m, &ext); 3353 paylen = ext.iplen; 3354 if (ext.ip4 || ext.ip6) 3355 paylen -= ext.iphlen; 3356 if (ext.tcp) { 3357 paylen -= ext.tcphlen; 3358 tcpstat_inc(tcps_inhwlro); 3359 tcpstat_add(tcps_inpktlro, pkts); 3360 } else { 3361 tcpstat_inc(tcps_inbadlro); 3362 } 3363 3364 /* 3365 * If we gonna forward this packet, we have to mark it as TSO, 3366 * set a correct mss, and recalculate the TCP checksum. 3367 */ 3368 if (ext.tcp && paylen >= pkts) { 3369 SET(m->m_pkthdr.csum_flags, M_TCP_TSO); 3370 m->m_pkthdr.ph_mss = paylen / pkts; 3371 } 3372 if (ext.tcp && ISSET(m->m_pkthdr.csum_flags, M_TCP_CSUM_IN_OK)) 3373 SET(m->m_pkthdr.csum_flags, M_TCP_CSUM_OUT); 3374 } 3375} 3376 3377void 3378ixgbe_setup_vlan_hw_support(struct ix_softc *sc) 3379{ 3380 uint32_t ctrl; 3381 int i; 3382 3383 /* 3384 * A soft reset zero's out the VFTA, so 3385 * we need to repopulate it now. 3386 */ 3387 for (i = 0; i < IXGBE_VFTA_SIZE; i++) { 3388 if (sc->shadow_vfta[i] != 0) 3389 IXGBE_WRITE_REG(&sc->hw, IXGBE_VFTA(i), 3390 sc->shadow_vfta[i]); 3391 } 3392 3393 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_VLNCTRL); 3394#if 0 3395 /* Enable the Filter Table if enabled */ 3396 if (ifp->if_capenable & IFCAP_VLAN_HWFILTER) { 3397 ctrl &= ~IXGBE_VLNCTRL_CFIEN; 3398 ctrl |= IXGBE_VLNCTRL_VFE; 3399 } 3400#endif 3401 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3402 ctrl |= IXGBE_VLNCTRL_VME; 3403 IXGBE_WRITE_REG(&sc->hw, IXGBE_VLNCTRL, ctrl); 3404 3405 /* On 82599 the VLAN enable is per/queue in RXDCTL */ 3406 if (sc->hw.mac.type != ixgbe_mac_82598EB) { 3407 for (i = 0; i < sc->num_queues; i++) { 3408 ctrl = IXGBE_READ_REG(&sc->hw, IXGBE_RXDCTL(i)); 3409 ctrl |= IXGBE_RXDCTL_VME; 3410 IXGBE_WRITE_REG(&sc->hw, IXGBE_RXDCTL(i), ctrl); 3411 } 3412 } 3413} 3414 3415void 3416ixgbe_enable_intr(struct ix_softc *sc) 3417{ 3418 struct ixgbe_hw *hw = &sc->hw; 3419 uint32_t mask, fwsm; 3420 3421 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); 3422 /* Enable Fan Failure detection */ 3423 if (hw->device_id == IXGBE_DEV_ID_82598AT) 3424 mask |= IXGBE_EIMS_GPI_SDP1; 3425 3426 switch (sc->hw.mac.type) { 3427 case ixgbe_mac_82599EB: 3428 mask |= IXGBE_EIMS_ECC; 3429 /* Temperature sensor on some adapters */ 3430 mask |= IXGBE_EIMS_GPI_SDP0; 3431 /* SFP+ (RX_LOS_N & MOD_ABS_N) */ 3432 mask |= IXGBE_EIMS_GPI_SDP1; 3433 mask |= IXGBE_EIMS_GPI_SDP2; 3434 break; 3435 case ixgbe_mac_X540: 3436 mask |= IXGBE_EIMS_ECC; 3437 /* Detect if Thermal Sensor is enabled */ 3438 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM); 3439 if (fwsm & IXGBE_FWSM_TS_ENABLED) 3440 mask |= IXGBE_EIMS_TS; 3441 break; 3442 case ixgbe_mac_X550: 3443 case ixgbe_mac_X550EM_x: 3444 case ixgbe_mac_X550EM_a: 3445 mask |= IXGBE_EIMS_ECC; 3446 /* MAC thermal sensor is automatically enabled */ 3447 mask |= IXGBE_EIMS_TS; 3448 /* Some devices use SDP0 for important information */ 3449 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 3450 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) 3451 mask |= IXGBE_EIMS_GPI_SDP0_X540; 3452 default: 3453 break; 3454 } 3455 3456 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask); 3457 3458 /* With MSI-X we use auto clear */ 3459 if (sc->sc_intrmap) { 3460 mask = IXGBE_EIMS_ENABLE_MASK; 3461 /* Don't autoclear Link */ 3462 mask &= ~IXGBE_EIMS_OTHER; 3463 mask &= ~IXGBE_EIMS_LSC; 3464 IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask); 3465 } 3466 3467 IXGBE_WRITE_FLUSH(hw); 3468} 3469 3470void 3471ixgbe_disable_intr(struct ix_softc *sc) 3472{ 3473 if (sc->sc_intrmap) 3474 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIAC, 0); 3475 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3476 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, ~0); 3477 } else { 3478 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC, 0xFFFF0000); 3479 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(0), ~0); 3480 IXGBE_WRITE_REG(&sc->hw, IXGBE_EIMC_EX(1), ~0); 3481 } 3482 IXGBE_WRITE_FLUSH(&sc->hw); 3483} 3484 3485uint16_t 3486ixgbe_read_pci_cfg(struct ixgbe_hw *hw, uint32_t reg) 3487{ 3488 struct pci_attach_args *pa; 3489 uint32_t value; 3490 int high = 0; 3491 3492 if (reg & 0x2) { 3493 high = 1; 3494 reg &= ~0x2; 3495 } 3496 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3497 value = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3498 3499 if (high) 3500 value >>= 16; 3501 3502 return (value & 0xffff); 3503} 3504 3505void 3506ixgbe_write_pci_cfg(struct ixgbe_hw *hw, uint32_t reg, uint16_t value) 3507{ 3508 struct pci_attach_args *pa; 3509 uint32_t rv; 3510 int high = 0; 3511 3512 /* Need to do read/mask/write... because 16 vs 32 bit!!! */ 3513 if (reg & 0x2) { 3514 high = 1; 3515 reg &= ~0x2; 3516 } 3517 pa = &((struct ixgbe_osdep *)hw->back)->os_pa; 3518 rv = pci_conf_read(pa->pa_pc, pa->pa_tag, reg); 3519 if (!high) 3520 rv = (rv & 0xffff0000) | value; 3521 else 3522 rv = (rv & 0xffff) | ((uint32_t)value << 16); 3523 pci_conf_write(pa->pa_pc, pa->pa_tag, reg, rv); 3524} 3525 3526/* 3527 * Setup the correct IVAR register for a particular MSIX interrupt 3528 * (yes this is all very magic and confusing :) 3529 * - entry is the register array entry 3530 * - vector is the MSIX vector for this queue 3531 * - type is RX/TX/MISC 3532 */ 3533void 3534ixgbe_set_ivar(struct ix_softc *sc, uint8_t entry, uint8_t vector, int8_t type) 3535{ 3536 struct ixgbe_hw *hw = &sc->hw; 3537 uint32_t ivar, index; 3538 3539 vector |= IXGBE_IVAR_ALLOC_VAL; 3540 3541 switch (hw->mac.type) { 3542 3543 case ixgbe_mac_82598EB: 3544 if (type == -1) 3545 entry = IXGBE_IVAR_OTHER_CAUSES_INDEX; 3546 else 3547 entry += (type * 64); 3548 index = (entry >> 2) & 0x1F; 3549 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3550 ivar &= ~((uint32_t)0xFF << (8 * (entry & 0x3))); 3551 ivar |= ((uint32_t)vector << (8 * (entry & 0x3))); 3552 IXGBE_WRITE_REG(&sc->hw, IXGBE_IVAR(index), ivar); 3553 break; 3554 3555 case ixgbe_mac_82599EB: 3556 case ixgbe_mac_X540: 3557 case ixgbe_mac_X550: 3558 case ixgbe_mac_X550EM_x: 3559 case ixgbe_mac_X550EM_a: 3560 if (type == -1) { /* MISC IVAR */ 3561 index = (entry & 1) * 8; 3562 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3563 ivar &= ~((uint32_t)0xFF << index); 3564 ivar |= ((uint32_t)vector << index); 3565 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3566 } else { /* RX/TX IVARS */ 3567 index = (16 * (entry & 1)) + (8 * type); 3568 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(entry >> 1)); 3569 ivar &= ~((uint32_t)0xFF << index); 3570 ivar |= ((uint32_t)vector << index); 3571 IXGBE_WRITE_REG(hw, IXGBE_IVAR(entry >> 1), ivar); 3572 } 3573 3574 default: 3575 break; 3576 } 3577} 3578 3579void 3580ixgbe_configure_ivars(struct ix_softc *sc) 3581{ 3582 struct ix_queue *que = sc->queues; 3583 uint32_t newitr; 3584 int i; 3585 3586 newitr = (4000000 / IXGBE_INTS_PER_SEC) & 0x0FF8; 3587 3588 for (i = 0; i < sc->num_queues; i++, que++) { 3589 /* First the RX queue entry */ 3590 ixgbe_set_ivar(sc, i, que->msix, 0); 3591 /* ... and the TX */ 3592 ixgbe_set_ivar(sc, i, que->msix, 1); 3593 /* Set an Initial EITR value */ 3594 IXGBE_WRITE_REG(&sc->hw, 3595 IXGBE_EITR(que->msix), newitr); 3596 } 3597 3598 /* For the Link interrupt */ 3599 ixgbe_set_ivar(sc, 1, sc->linkvec, -1); 3600} 3601 3602/* 3603 * SFP module interrupts handler 3604 */ 3605void 3606ixgbe_handle_mod(struct ix_softc *sc) 3607{ 3608 struct ixgbe_hw *hw = &sc->hw; 3609 uint32_t err; 3610 3611 err = hw->phy.ops.identify_sfp(hw); 3612 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3613 printf("%s: Unsupported SFP+ module type was detected!\n", 3614 sc->dev.dv_xname); 3615 return; 3616 } 3617 err = hw->mac.ops.setup_sfp(hw); 3618 if (err == IXGBE_ERR_SFP_NOT_SUPPORTED) { 3619 printf("%s: Setup failure - unsupported SFP+ module type!\n", 3620 sc->dev.dv_xname); 3621 return; 3622 } 3623 3624 ixgbe_handle_msf(sc); 3625} 3626 3627 3628/* 3629 * MSF (multispeed fiber) interrupts handler 3630 */ 3631void 3632ixgbe_handle_msf(struct ix_softc *sc) 3633{ 3634 struct ixgbe_hw *hw = &sc->hw; 3635 uint32_t autoneg; 3636 bool negotiate; 3637 3638 autoneg = hw->phy.autoneg_advertised; 3639 if ((!autoneg) && (hw->mac.ops.get_link_capabilities)) { 3640 if (hw->mac.ops.get_link_capabilities(hw, &autoneg, &negotiate)) 3641 return; 3642 } 3643 if (hw->mac.ops.setup_link) 3644 hw->mac.ops.setup_link(hw, autoneg, TRUE); 3645 3646 ifmedia_delete_instance(&sc->media, IFM_INST_ANY); 3647 ixgbe_add_media_types(sc); 3648 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 3649} 3650 3651/* 3652 * External PHY interrupts handler 3653 */ 3654void 3655ixgbe_handle_phy(struct ix_softc *sc) 3656{ 3657 struct ixgbe_hw *hw = &sc->hw; 3658 int error; 3659 3660 error = hw->phy.ops.handle_lasi(hw); 3661 if (error == IXGBE_ERR_OVERTEMP) 3662 printf("%s: CRITICAL: EXTERNAL PHY OVER TEMP!! " 3663 " PHY will downshift to lower power state!\n", 3664 sc->dev.dv_xname); 3665 else if (error) 3666 printf("%s: Error handling LASI interrupt: %d\n", 3667 sc->dev.dv_xname, error); 3668 3669} 3670 3671#if NKSTAT > 0 3672enum ix_counter_idx { 3673 ix_counter_crcerrs, 3674 ix_counter_lxontxc, 3675 ix_counter_lxonrxc, 3676 ix_counter_lxofftxc, 3677 ix_counter_lxoffrxc, 3678 ix_counter_prc64, 3679 ix_counter_prc127, 3680 ix_counter_prc255, 3681 ix_counter_prc511, 3682 ix_counter_prc1023, 3683 ix_counter_prc1522, 3684 ix_counter_gptc, 3685 ix_counter_gorc, 3686 ix_counter_gotc, 3687 ix_counter_ruc, 3688 ix_counter_rfc, 3689 ix_counter_roc, 3690 ix_counter_rjc, 3691 ix_counter_tor, 3692 ix_counter_tpr, 3693 ix_counter_tpt, 3694 ix_counter_gprc, 3695 ix_counter_bprc, 3696 ix_counter_mprc, 3697 ix_counter_ptc64, 3698 ix_counter_ptc127, 3699 ix_counter_ptc255, 3700 ix_counter_ptc511, 3701 ix_counter_ptc1023, 3702 ix_counter_ptc1522, 3703 ix_counter_mptc, 3704 ix_counter_bptc, 3705 3706 ix_counter_num, 3707}; 3708 3709CTASSERT(KSTAT_KV_U_PACKETS <= 0xff); 3710CTASSERT(KSTAT_KV_U_BYTES <= 0xff); 3711 3712struct ix_counter { 3713 char name[KSTAT_KV_NAMELEN]; 3714 uint32_t reg; 3715 uint8_t width; 3716 uint8_t unit; 3717}; 3718 3719static const struct ix_counter ix_counters[ix_counter_num] = { 3720 [ix_counter_crcerrs] = { "crc errs", IXGBE_CRCERRS, 32, 3721 KSTAT_KV_U_PACKETS }, 3722 [ix_counter_lxontxc] = { "tx link xon", IXGBE_LXONTXC, 32, 3723 KSTAT_KV_U_PACKETS }, 3724 [ix_counter_lxonrxc] = { "rx link xon", 0, 32, 3725 KSTAT_KV_U_PACKETS }, 3726 [ix_counter_lxofftxc] = { "tx link xoff", IXGBE_LXOFFTXC, 32, 3727 KSTAT_KV_U_PACKETS }, 3728 [ix_counter_lxoffrxc] = { "rx link xoff", 0, 32, 3729 KSTAT_KV_U_PACKETS }, 3730 [ix_counter_prc64] = { "rx 64B", IXGBE_PRC64, 32, 3731 KSTAT_KV_U_PACKETS }, 3732 [ix_counter_prc127] = { "rx 65-127B", IXGBE_PRC127, 32, 3733 KSTAT_KV_U_PACKETS }, 3734 [ix_counter_prc255] = { "rx 128-255B", IXGBE_PRC255, 32, 3735 KSTAT_KV_U_PACKETS }, 3736 [ix_counter_prc511] = { "rx 256-511B", IXGBE_PRC511, 32, 3737 KSTAT_KV_U_PACKETS }, 3738 [ix_counter_prc1023] = { "rx 512-1023B", IXGBE_PRC1023, 32, 3739 KSTAT_KV_U_PACKETS }, 3740 [ix_counter_prc1522] = { "rx 1024-maxB", IXGBE_PRC1522, 32, 3741 KSTAT_KV_U_PACKETS }, 3742 [ix_counter_gptc] = { "tx good", IXGBE_GPTC, 32, 3743 KSTAT_KV_U_PACKETS }, 3744 [ix_counter_gorc] = { "rx good", IXGBE_GORCL, 36, 3745 KSTAT_KV_U_BYTES }, 3746 [ix_counter_gotc] = { "tx good", IXGBE_GOTCL, 36, 3747 KSTAT_KV_U_BYTES }, 3748 [ix_counter_ruc] = { "rx undersize", IXGBE_RUC, 32, 3749 KSTAT_KV_U_PACKETS }, 3750 [ix_counter_rfc] = { "rx fragment", IXGBE_RFC, 32, 3751 KSTAT_KV_U_PACKETS }, 3752 [ix_counter_roc] = { "rx oversize", IXGBE_ROC, 32, 3753 KSTAT_KV_U_PACKETS }, 3754 [ix_counter_rjc] = { "rx jabber", IXGBE_RJC, 32, 3755 KSTAT_KV_U_PACKETS }, 3756 [ix_counter_tor] = { "rx total", IXGBE_TORL, 36, 3757 KSTAT_KV_U_BYTES }, 3758 [ix_counter_tpr] = { "rx total", IXGBE_TPR, 32, 3759 KSTAT_KV_U_PACKETS }, 3760 [ix_counter_tpt] = { "tx total", IXGBE_TPT, 32, 3761 KSTAT_KV_U_PACKETS }, 3762 [ix_counter_gprc] = { "rx good", IXGBE_GPRC, 32, 3763 KSTAT_KV_U_PACKETS }, 3764 [ix_counter_bprc] = { "rx bcast", IXGBE_BPRC, 32, 3765 KSTAT_KV_U_PACKETS }, 3766 [ix_counter_mprc] = { "rx mcast", IXGBE_MPRC, 32, 3767 KSTAT_KV_U_PACKETS }, 3768 [ix_counter_ptc64] = { "tx 64B", IXGBE_PTC64, 32, 3769 KSTAT_KV_U_PACKETS }, 3770 [ix_counter_ptc127] = { "tx 65-127B", IXGBE_PTC127, 32, 3771 KSTAT_KV_U_PACKETS }, 3772 [ix_counter_ptc255] = { "tx 128-255B", IXGBE_PTC255, 32, 3773 KSTAT_KV_U_PACKETS }, 3774 [ix_counter_ptc511] = { "tx 256-511B", IXGBE_PTC511, 32, 3775 KSTAT_KV_U_PACKETS }, 3776 [ix_counter_ptc1023] = { "tx 512-1023B", IXGBE_PTC1023, 32, 3777 KSTAT_KV_U_PACKETS }, 3778 [ix_counter_ptc1522] = { "tx 1024-maxB", IXGBE_PTC1522, 32, 3779 KSTAT_KV_U_PACKETS }, 3780 [ix_counter_mptc] = { "tx mcast", IXGBE_MPTC, 32, 3781 KSTAT_KV_U_PACKETS }, 3782 [ix_counter_bptc] = { "tx bcast", IXGBE_BPTC, 32, 3783 KSTAT_KV_U_PACKETS }, 3784}; 3785 3786struct ix_rxq_kstats { 3787 struct kstat_kv qprc; 3788 struct kstat_kv qbrc; 3789 struct kstat_kv qprdc; 3790}; 3791 3792static const struct ix_rxq_kstats ix_rxq_kstats_tpl = { 3793 KSTAT_KV_UNIT_INITIALIZER("packets", 3794 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3795 KSTAT_KV_UNIT_INITIALIZER("bytes", 3796 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3797 KSTAT_KV_UNIT_INITIALIZER("qdrops", 3798 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3799}; 3800 3801struct ix_txq_kstats { 3802 struct kstat_kv qptc; 3803 struct kstat_kv qbtc; 3804}; 3805 3806static const struct ix_txq_kstats ix_txq_kstats_tpl = { 3807 KSTAT_KV_UNIT_INITIALIZER("packets", 3808 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_PACKETS), 3809 KSTAT_KV_UNIT_INITIALIZER("bytes", 3810 KSTAT_KV_T_COUNTER64, KSTAT_KV_U_BYTES), 3811}; 3812 3813static int ix_kstats_read(struct kstat *ks); 3814static int ix_rxq_kstats_read(struct kstat *ks); 3815static int ix_txq_kstats_read(struct kstat *ks); 3816 3817static void 3818ix_kstats(struct ix_softc *sc) 3819{ 3820 struct kstat *ks; 3821 struct kstat_kv *kvs; 3822 unsigned int i; 3823 3824 mtx_init(&sc->sc_kstat_mtx, IPL_SOFTCLOCK); 3825 timeout_set(&sc->sc_kstat_tmo, ix_kstats_tick, sc); 3826 3827 ks = kstat_create(sc->dev.dv_xname, 0, "ix-stats", 0, 3828 KSTAT_T_KV, 0); 3829 if (ks == NULL) 3830 return; 3831 3832 kvs = mallocarray(nitems(ix_counters), sizeof(*kvs), 3833 M_DEVBUF, M_WAITOK|M_ZERO); 3834 3835 for (i = 0; i < nitems(ix_counters); i++) { 3836 const struct ix_counter *ixc = &ix_counters[i]; 3837 3838 kstat_kv_unit_init(&kvs[i], ixc->name, 3839 KSTAT_KV_T_COUNTER64, ixc->unit); 3840 } 3841 3842 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3843 ks->ks_softc = sc; 3844 ks->ks_data = kvs; 3845 ks->ks_datalen = nitems(ix_counters) * sizeof(*kvs); 3846 ks->ks_read = ix_kstats_read; 3847 3848 sc->sc_kstat = ks; 3849 kstat_install(ks); 3850} 3851 3852static void 3853ix_rxq_kstats(struct ix_softc *sc, struct ix_rxring *rxr) 3854{ 3855 struct ix_rxq_kstats *stats; 3856 struct kstat *ks; 3857 3858 ks = kstat_create(sc->dev.dv_xname, 0, "ix-rxq", rxr->me, 3859 KSTAT_T_KV, 0); 3860 if (ks == NULL) 3861 return; 3862 3863 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3864 *stats = ix_rxq_kstats_tpl; 3865 3866 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3867 ks->ks_softc = rxr; 3868 ks->ks_data = stats; 3869 ks->ks_datalen = sizeof(*stats); 3870 ks->ks_read = ix_rxq_kstats_read; 3871 3872 rxr->kstat = ks; 3873 kstat_install(ks); 3874} 3875 3876static void 3877ix_txq_kstats(struct ix_softc *sc, struct ix_txring *txr) 3878{ 3879 struct ix_txq_kstats *stats; 3880 struct kstat *ks; 3881 3882 ks = kstat_create(sc->dev.dv_xname, 0, "ix-txq", txr->me, 3883 KSTAT_T_KV, 0); 3884 if (ks == NULL) 3885 return; 3886 3887 stats = malloc(sizeof(*stats), M_DEVBUF, M_WAITOK|M_ZERO); 3888 *stats = ix_txq_kstats_tpl; 3889 3890 kstat_set_mutex(ks, &sc->sc_kstat_mtx); 3891 ks->ks_softc = txr; 3892 ks->ks_data = stats; 3893 ks->ks_datalen = sizeof(*stats); 3894 ks->ks_read = ix_txq_kstats_read; 3895 3896 txr->kstat = ks; 3897 kstat_install(ks); 3898} 3899 3900/********************************************************************** 3901 * 3902 * Update the board statistics counters. 3903 * 3904 **********************************************************************/ 3905 3906static void 3907ix_kstats_tick(void *arg) 3908{ 3909 struct ix_softc *sc = arg; 3910 int i; 3911 3912 timeout_add_sec(&sc->sc_kstat_tmo, 1); 3913 3914 mtx_enter(&sc->sc_kstat_mtx); 3915 ix_kstats_read(sc->sc_kstat); 3916 for (i = 0; i < sc->num_queues; i++) { 3917 ix_rxq_kstats_read(sc->rx_rings[i].kstat); 3918 ix_txq_kstats_read(sc->tx_rings[i].kstat); 3919 } 3920 mtx_leave(&sc->sc_kstat_mtx); 3921} 3922 3923static uint64_t 3924ix_read36(struct ixgbe_hw *hw, bus_size_t loreg, bus_size_t hireg) 3925{ 3926 uint64_t lo, hi; 3927 3928 lo = IXGBE_READ_REG(hw, loreg); 3929 hi = IXGBE_READ_REG(hw, hireg); 3930 3931 return (((hi & 0xf) << 32) | lo); 3932} 3933 3934static int 3935ix_kstats_read(struct kstat *ks) 3936{ 3937 struct ix_softc *sc = ks->ks_softc; 3938 struct kstat_kv *kvs = ks->ks_data; 3939 struct ixgbe_hw *hw = &sc->hw; 3940 unsigned int i; 3941 3942 for (i = 0; i < nitems(ix_counters); i++) { 3943 const struct ix_counter *ixc = &ix_counters[i]; 3944 uint32_t reg = ixc->reg; 3945 uint64_t v; 3946 3947 if (reg == 0) 3948 continue; 3949 3950 if (ixc->width > 32) { 3951 if (sc->hw.mac.type == ixgbe_mac_82598EB) 3952 v = IXGBE_READ_REG(hw, reg + 4); 3953 else 3954 v = ix_read36(hw, reg, reg + 4); 3955 } else 3956 v = IXGBE_READ_REG(hw, reg); 3957 3958 kstat_kv_u64(&kvs[i]) += v; 3959 } 3960 3961 /* handle the exceptions */ 3962 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3963 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3964 IXGBE_READ_REG(hw, IXGBE_LXONRXC); 3965 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3966 IXGBE_READ_REG(hw, IXGBE_LXOFFRXC); 3967 } else { 3968 kstat_kv_u64(&kvs[ix_counter_lxonrxc]) += 3969 IXGBE_READ_REG(hw, IXGBE_LXONRXCNT); 3970 kstat_kv_u64(&kvs[ix_counter_lxoffrxc]) += 3971 IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT); 3972 } 3973 3974 getnanouptime(&ks->ks_updated); 3975 3976 return (0); 3977} 3978 3979int 3980ix_rxq_kstats_read(struct kstat *ks) 3981{ 3982 struct ix_rxq_kstats *stats = ks->ks_data; 3983 struct ix_rxring *rxr = ks->ks_softc; 3984 struct ix_softc *sc = rxr->sc; 3985 struct ixgbe_hw *hw = &sc->hw; 3986 uint32_t i = rxr->me; 3987 3988 kstat_kv_u64(&stats->qprc) += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); 3989 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 3990 kstat_kv_u64(&stats->qprdc) += 3991 IXGBE_READ_REG(hw, IXGBE_RNBC(i)); 3992 kstat_kv_u64(&stats->qbrc) += 3993 IXGBE_READ_REG(hw, IXGBE_QBRC(i)); 3994 } else { 3995 kstat_kv_u64(&stats->qprdc) += 3996 IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); 3997 kstat_kv_u64(&stats->qbrc) += 3998 ix_read36(hw, IXGBE_QBRC_L(i), IXGBE_QBRC_H(i)); 3999 } 4000 4001 getnanouptime(&ks->ks_updated); 4002 4003 return (0); 4004} 4005 4006int 4007ix_txq_kstats_read(struct kstat *ks) 4008{ 4009 struct ix_txq_kstats *stats = ks->ks_data; 4010 struct ix_txring *txr = ks->ks_softc; 4011 struct ix_softc *sc = txr->sc; 4012 struct ixgbe_hw *hw = &sc->hw; 4013 uint32_t i = txr->me; 4014 4015 kstat_kv_u64(&stats->qptc) += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); 4016 if (sc->hw.mac.type == ixgbe_mac_82598EB) { 4017 kstat_kv_u64(&stats->qbtc) += 4018 IXGBE_READ_REG(hw, IXGBE_QBTC(i)); 4019 } else { 4020 kstat_kv_u64(&stats->qbtc) += 4021 ix_read36(hw, IXGBE_QBTC_L(i), IXGBE_QBTC_H(i)); 4022 } 4023 4024 getnanouptime(&ks->ks_updated); 4025 4026 return (0); 4027} 4028#endif /* NKVSTAT > 0 */ 4029 4030void 4031ixgbe_map_queue_statistics(struct ix_softc *sc) 4032{ 4033 int i; 4034 uint32_t r; 4035 4036 for (i = 0; i < 32; i++) { 4037 /* 4038 * Queues 0-15 are mapped 1:1 4039 * Queue 0 -> Counter 0 4040 * Queue 1 -> Counter 1 4041 * Queue 2 -> Counter 2.... 4042 * Queues 16-127 are mapped to Counter 0 4043 */ 4044 if (i < 4) { 4045 r = (i * 4 + 0); 4046 r |= (i * 4 + 1) << 8; 4047 r |= (i * 4 + 2) << 16; 4048 r |= (i * 4 + 3) << 24; 4049 } else 4050 r = 0; 4051 4052 IXGBE_WRITE_REG(&sc->hw, IXGBE_RQSMR(i), r); 4053 IXGBE_WRITE_REG(&sc->hw, IXGBE_TQSM(i), r); 4054 } 4055} 4056