1/*- 2 * Copyright (C) 2013 Emulex 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the Emulex Corporation nor the names of its 16 * contributors may be used to endorse or promote products derived from 17 * this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Contact Information: 32 * freebsd-drivers@emulex.com 33 * 34 * Emulex 35 * 3333 Susan Street 36 * Costa Mesa, CA 92626 37 */ 38 39/* $FreeBSD: stable/11/sys/dev/oce/oce_if.c 369531 2021-03-30 15:05:49Z freqlabs $ */ 40 41#include "opt_inet6.h" 42#include "opt_inet.h" 43 44#include "oce_if.h" 45#include "oce_user.h" 46 47#define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO) 48 49/* UE Status Low CSR */ 50static char *ue_status_low_desc[] = { 51 "CEV", 52 "CTX", 53 "DBUF", 54 "ERX", 55 "Host", 56 "MPU", 57 "NDMA", 58 "PTC ", 59 "RDMA ", 60 "RXF ", 61 "RXIPS ", 62 "RXULP0 ", 63 "RXULP1 ", 64 "RXULP2 ", 65 "TIM ", 66 "TPOST ", 67 "TPRE ", 68 "TXIPS ", 69 "TXULP0 ", 70 "TXULP1 ", 71 "UC ", 72 "WDMA ", 73 "TXULP2 ", 74 "HOST1 ", 75 "P0_OB_LINK ", 76 "P1_OB_LINK ", 77 "HOST_GPIO ", 78 "MBOX ", 79 "AXGMAC0", 80 "AXGMAC1", 81 "JTAG", 82 "MPU_INTPEND" 83}; 84 85/* UE Status High CSR */ 86static char *ue_status_hi_desc[] = { 87 "LPCMEMHOST", 88 "MGMT_MAC", 89 "PCS0ONLINE", 90 "MPU_IRAM", 91 "PCS1ONLINE", 92 "PCTL0", 93 "PCTL1", 94 "PMEM", 95 "RR", 96 "TXPB", 97 "RXPP", 98 "XAUI", 99 "TXP", 100 "ARM", 101 "IPC", 102 "HOST2", 103 "HOST3", 104 "HOST4", 105 "HOST5", 106 "HOST6", 107 "HOST7", 108 "HOST8", 109 "HOST9", 110 "NETC", 111 "Unknown", 112 "Unknown", 113 "Unknown", 114 "Unknown", 115 "Unknown", 116 "Unknown", 117 "Unknown", 118 "Unknown" 119}; 120 121struct oce_common_cqe_info{ 122 uint8_t vtp:1; 123 uint8_t l4_cksum_pass:1; 124 uint8_t ip_cksum_pass:1; 125 uint8_t ipv6_frame:1; 126 uint8_t qnq:1; 127 uint8_t rsvd:3; 128 uint8_t num_frags; 129 uint16_t pkt_size; 130 uint16_t vtag; 131}; 132 133 134/* Driver entry points prototypes */ 135static int oce_probe(device_t dev); 136static int oce_attach(device_t dev); 137static int oce_detach(device_t dev); 138static int oce_shutdown(device_t dev); 139static int oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data); 140static void oce_init(void *xsc); 141static int oce_multiq_start(struct ifnet *ifp, struct mbuf *m); 142static void oce_multiq_flush(struct ifnet *ifp); 143 144/* Driver interrupt routines protypes */ 145static void oce_intr(void *arg, int pending); 146static int oce_setup_intr(POCE_SOFTC sc); 147static int oce_fast_isr(void *arg); 148static int oce_alloc_intr(POCE_SOFTC sc, int vector, 149 void (*isr) (void *arg, int pending)); 150 151/* Media callbacks prototypes */ 152static void oce_media_status(struct ifnet *ifp, struct ifmediareq *req); 153static int oce_media_change(struct ifnet *ifp); 154 155/* Transmit routines prototypes */ 156static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index); 157static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq); 158static void oce_process_tx_completion(struct oce_wq *wq); 159static int oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, 160 struct oce_wq *wq); 161 162/* Receive routines prototypes */ 163static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 164static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe); 165static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe); 166static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq); 167static uint16_t oce_rq_handler_lro(void *arg); 168static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2); 169static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2); 170static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m); 171 172/* Helper function prototypes in this file */ 173static int oce_attach_ifp(POCE_SOFTC sc); 174static void oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 175static void oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag); 176static int oce_vid_config(POCE_SOFTC sc); 177static void oce_mac_addr_set(POCE_SOFTC sc); 178static int oce_handle_passthrough(struct ifnet *ifp, caddr_t data); 179static void oce_local_timer(void *arg); 180static void oce_if_deactivate(POCE_SOFTC sc); 181static void oce_if_activate(POCE_SOFTC sc); 182static void setup_max_queues_want(POCE_SOFTC sc); 183static void update_queues_got(POCE_SOFTC sc); 184static void process_link_state(POCE_SOFTC sc, 185 struct oce_async_cqe_link_state *acqe); 186static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m); 187static void oce_get_config(POCE_SOFTC sc); 188static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete); 189static void oce_read_env_variables(POCE_SOFTC sc); 190 191 192/* IP specific */ 193#if defined(INET6) || defined(INET) 194static int oce_init_lro(POCE_SOFTC sc); 195static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp); 196#endif 197 198static device_method_t oce_dispatch[] = { 199 DEVMETHOD(device_probe, oce_probe), 200 DEVMETHOD(device_attach, oce_attach), 201 DEVMETHOD(device_detach, oce_detach), 202 DEVMETHOD(device_shutdown, oce_shutdown), 203 204 DEVMETHOD_END 205}; 206 207static driver_t oce_driver = { 208 "oce", 209 oce_dispatch, 210 sizeof(OCE_SOFTC) 211}; 212static devclass_t oce_devclass; 213 214 215DRIVER_MODULE(oce, pci, oce_driver, oce_devclass, 0, 0); 216MODULE_DEPEND(oce, pci, 1, 1, 1); 217MODULE_DEPEND(oce, ether, 1, 1, 1); 218MODULE_VERSION(oce, 1); 219 220 221/* global vars */ 222const char component_revision[32] = {"///" COMPONENT_REVISION "///"}; 223 224/* Module capabilites and parameters */ 225uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED; 226uint32_t oce_enable_rss = OCE_MODCAP_RSS; 227uint32_t oce_rq_buf_size = 2048; 228 229TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled); 230TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss); 231 232 233/* Supported devices table */ 234static uint32_t supportedDevices[] = { 235 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2, 236 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3, 237 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3, 238 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201, 239 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF, 240 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH 241}; 242 243POCE_SOFTC softc_head = NULL; 244POCE_SOFTC softc_tail = NULL; 245 246struct oce_rdma_if *oce_rdma_if = NULL; 247 248/***************************************************************************** 249 * Driver entry points functions * 250 *****************************************************************************/ 251 252static int 253oce_probe(device_t dev) 254{ 255 uint16_t vendor = 0; 256 uint16_t device = 0; 257 int i = 0; 258 char str[256] = {0}; 259 POCE_SOFTC sc; 260 261 sc = device_get_softc(dev); 262 bzero(sc, sizeof(OCE_SOFTC)); 263 sc->dev = dev; 264 265 vendor = pci_get_vendor(dev); 266 device = pci_get_device(dev); 267 268 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) { 269 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) { 270 if (device == (supportedDevices[i] & 0xffff)) { 271 sprintf(str, "%s:%s", "Emulex CNA NIC function", 272 component_revision); 273 device_set_desc_copy(dev, str); 274 275 switch (device) { 276 case PCI_PRODUCT_BE2: 277 sc->flags |= OCE_FLAGS_BE2; 278 break; 279 case PCI_PRODUCT_BE3: 280 sc->flags |= OCE_FLAGS_BE3; 281 break; 282 case PCI_PRODUCT_XE201: 283 case PCI_PRODUCT_XE201_VF: 284 sc->flags |= OCE_FLAGS_XE201; 285 break; 286 case PCI_PRODUCT_SH: 287 sc->flags |= OCE_FLAGS_SH; 288 break; 289 default: 290 return ENXIO; 291 } 292 return BUS_PROBE_DEFAULT; 293 } 294 } 295 } 296 297 return ENXIO; 298} 299 300 301static int 302oce_attach(device_t dev) 303{ 304 POCE_SOFTC sc; 305 int rc = 0; 306 307 sc = device_get_softc(dev); 308 309 rc = oce_hw_pci_alloc(sc); 310 if (rc) 311 return rc; 312 313 sc->tx_ring_size = OCE_TX_RING_SIZE; 314 sc->rx_ring_size = OCE_RX_RING_SIZE; 315 /* receive fragment size should be multiple of 2K */ 316 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048); 317 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL; 318 sc->promisc = OCE_DEFAULT_PROMISCUOUS; 319 320 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock"); 321 LOCK_CREATE(&sc->dev_lock, "Device_lock"); 322 323 /* initialise the hardware */ 324 rc = oce_hw_init(sc); 325 if (rc) 326 goto pci_res_free; 327 328 oce_read_env_variables(sc); 329 330 oce_get_config(sc); 331 332 setup_max_queues_want(sc); 333 334 rc = oce_setup_intr(sc); 335 if (rc) 336 goto mbox_free; 337 338 rc = oce_queue_init_all(sc); 339 if (rc) 340 goto intr_free; 341 342 rc = oce_attach_ifp(sc); 343 if (rc) 344 goto queues_free; 345 346#if defined(INET6) || defined(INET) 347 rc = oce_init_lro(sc); 348 if (rc) 349 goto ifp_free; 350#endif 351 352 rc = oce_hw_start(sc); 353 if (rc) 354 goto lro_free; 355 356 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config, 357 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST); 358 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig, 359 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST); 360 361 rc = oce_stats_init(sc); 362 if (rc) 363 goto vlan_free; 364 365 oce_add_sysctls(sc); 366 367 callout_init(&sc->timer, CALLOUT_MPSAFE); 368 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc); 369 if (rc) 370 goto stats_free; 371 372 sc->next =NULL; 373 if (softc_tail != NULL) { 374 softc_tail->next = sc; 375 } else { 376 softc_head = sc; 377 } 378 softc_tail = sc; 379 380 return 0; 381 382stats_free: 383 callout_drain(&sc->timer); 384 oce_stats_free(sc); 385vlan_free: 386 if (sc->vlan_attach) 387 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 388 if (sc->vlan_detach) 389 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 390 oce_hw_intr_disable(sc); 391lro_free: 392#if defined(INET6) || defined(INET) 393 oce_free_lro(sc); 394ifp_free: 395#endif 396 ether_ifdetach(sc->ifp); 397 if_free(sc->ifp); 398queues_free: 399 oce_queue_release_all(sc); 400intr_free: 401 oce_intr_free(sc); 402mbox_free: 403 oce_dma_free(sc, &sc->bsmbx); 404pci_res_free: 405 oce_hw_pci_free(sc); 406 LOCK_DESTROY(&sc->dev_lock); 407 LOCK_DESTROY(&sc->bmbx_lock); 408 return rc; 409 410} 411 412 413static int 414oce_detach(device_t dev) 415{ 416 POCE_SOFTC sc = device_get_softc(dev); 417 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL; 418 419 poce_sc_tmp = softc_head; 420 ppoce_sc_tmp1 = &softc_head; 421 while (poce_sc_tmp != NULL) { 422 if (poce_sc_tmp == sc) { 423 *ppoce_sc_tmp1 = sc->next; 424 if (sc->next == NULL) { 425 softc_tail = poce_sc_tmp2; 426 } 427 break; 428 } 429 poce_sc_tmp2 = poce_sc_tmp; 430 ppoce_sc_tmp1 = &poce_sc_tmp->next; 431 poce_sc_tmp = poce_sc_tmp->next; 432 } 433 434 LOCK(&sc->dev_lock); 435 oce_if_deactivate(sc); 436 UNLOCK(&sc->dev_lock); 437 438 callout_drain(&sc->timer); 439 440 if (sc->vlan_attach != NULL) 441 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach); 442 if (sc->vlan_detach != NULL) 443 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach); 444 445 ether_ifdetach(sc->ifp); 446 447 if_free(sc->ifp); 448 449 oce_hw_shutdown(sc); 450 451 bus_generic_detach(dev); 452 453 return 0; 454} 455 456 457static int 458oce_shutdown(device_t dev) 459{ 460 int rc; 461 462 rc = oce_detach(dev); 463 464 return rc; 465} 466 467 468static int 469oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data) 470{ 471 struct ifreq *ifr = (struct ifreq *)data; 472 POCE_SOFTC sc = ifp->if_softc; 473 struct ifi2creq i2c; 474 uint8_t offset = 0; 475 int rc = 0; 476 uint32_t u; 477 478 switch (command) { 479 480 case SIOCGIFMEDIA: 481 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command); 482 break; 483 484 case SIOCSIFMTU: 485 if (ifr->ifr_mtu > OCE_MAX_MTU) 486 rc = EINVAL; 487 else 488 ifp->if_mtu = ifr->ifr_mtu; 489 break; 490 491 case SIOCSIFFLAGS: 492 if (ifp->if_flags & IFF_UP) { 493 if (!(ifp->if_drv_flags & IFF_DRV_RUNNING)) { 494 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 495 oce_init(sc); 496 } 497 device_printf(sc->dev, "Interface Up\n"); 498 } else { 499 LOCK(&sc->dev_lock); 500 501 sc->ifp->if_drv_flags &= 502 ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 503 oce_if_deactivate(sc); 504 505 UNLOCK(&sc->dev_lock); 506 507 device_printf(sc->dev, "Interface Down\n"); 508 } 509 510 if ((ifp->if_flags & IFF_PROMISC) && !sc->promisc) { 511 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1)))) 512 sc->promisc = TRUE; 513 } else if (!(ifp->if_flags & IFF_PROMISC) && sc->promisc) { 514 if (!oce_rxf_set_promiscuous(sc, 0)) 515 sc->promisc = FALSE; 516 } 517 518 break; 519 520 case SIOCADDMULTI: 521 case SIOCDELMULTI: 522 rc = oce_hw_update_multicast(sc); 523 if (rc) 524 device_printf(sc->dev, 525 "Update multicast address failed\n"); 526 break; 527 528 case SIOCSIFCAP: 529 u = ifr->ifr_reqcap ^ ifp->if_capenable; 530 531 if (u & IFCAP_TXCSUM) { 532 ifp->if_capenable ^= IFCAP_TXCSUM; 533 ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP); 534 535 if (IFCAP_TSO & ifp->if_capenable && 536 !(IFCAP_TXCSUM & ifp->if_capenable)) { 537 u &= ~IFCAP_TSO; 538 ifp->if_capenable &= ~IFCAP_TSO; 539 ifp->if_hwassist &= ~CSUM_TSO; 540 if_printf(ifp, 541 "TSO disabled due to -txcsum.\n"); 542 } 543 } 544 545 if (u & IFCAP_RXCSUM) 546 ifp->if_capenable ^= IFCAP_RXCSUM; 547 548 if (u & IFCAP_TSO4) { 549 ifp->if_capenable ^= IFCAP_TSO4; 550 551 if (IFCAP_TSO & ifp->if_capenable) { 552 if (IFCAP_TXCSUM & ifp->if_capenable) 553 ifp->if_hwassist |= CSUM_TSO; 554 else { 555 ifp->if_capenable &= ~IFCAP_TSO; 556 ifp->if_hwassist &= ~CSUM_TSO; 557 if_printf(ifp, 558 "Enable txcsum first.\n"); 559 rc = EAGAIN; 560 } 561 } else 562 ifp->if_hwassist &= ~CSUM_TSO; 563 } 564 565 if (u & IFCAP_VLAN_HWTAGGING) 566 ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING; 567 568 if (u & IFCAP_VLAN_HWFILTER) { 569 ifp->if_capenable ^= IFCAP_VLAN_HWFILTER; 570 oce_vid_config(sc); 571 } 572#if defined(INET6) || defined(INET) 573 if (u & IFCAP_LRO) { 574 ifp->if_capenable ^= IFCAP_LRO; 575 if(sc->enable_hwlro) { 576 if(ifp->if_capenable & IFCAP_LRO) { 577 rc = oce_mbox_nic_set_iface_lro_config(sc, 1); 578 }else { 579 rc = oce_mbox_nic_set_iface_lro_config(sc, 0); 580 } 581 } 582 } 583#endif 584 585 break; 586 587 case SIOCGI2C: 588 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c)); 589 if (rc) 590 break; 591 592 if (i2c.dev_addr != PAGE_NUM_A0 && 593 i2c.dev_addr != PAGE_NUM_A2) { 594 rc = EINVAL; 595 break; 596 } 597 598 if (i2c.len > sizeof(i2c.data)) { 599 rc = EINVAL; 600 break; 601 } 602 603 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr); 604 if(rc) { 605 rc = -rc; 606 break; 607 } 608 609 if (i2c.dev_addr == PAGE_NUM_A0) 610 offset = i2c.offset; 611 else 612 offset = TRANSCEIVER_A0_SIZE + i2c.offset; 613 614 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len); 615 616 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c)); 617 break; 618 619 case SIOCGPRIVATE_0: 620 rc = priv_check(curthread, PRIV_DRIVER); 621 if (rc != 0) 622 break; 623 rc = oce_handle_passthrough(ifp, data); 624 break; 625 default: 626 rc = ether_ioctl(ifp, command, data); 627 break; 628 } 629 630 return rc; 631} 632 633 634static void 635oce_init(void *arg) 636{ 637 POCE_SOFTC sc = arg; 638 639 LOCK(&sc->dev_lock); 640 641 if (sc->ifp->if_flags & IFF_UP) { 642 oce_if_deactivate(sc); 643 oce_if_activate(sc); 644 } 645 646 UNLOCK(&sc->dev_lock); 647 648} 649 650 651static int 652oce_multiq_start(struct ifnet *ifp, struct mbuf *m) 653{ 654 POCE_SOFTC sc = ifp->if_softc; 655 struct oce_wq *wq = NULL; 656 int queue_index = 0; 657 int status = 0; 658 659 if (!sc->link_status) 660 return ENXIO; 661 662 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) 663 queue_index = m->m_pkthdr.flowid % sc->nwqs; 664 665 wq = sc->wq[queue_index]; 666 667 LOCK(&wq->tx_lock); 668 status = oce_multiq_transmit(ifp, m, wq); 669 UNLOCK(&wq->tx_lock); 670 671 return status; 672 673} 674 675 676static void 677oce_multiq_flush(struct ifnet *ifp) 678{ 679 POCE_SOFTC sc = ifp->if_softc; 680 struct mbuf *m; 681 int i = 0; 682 683 for (i = 0; i < sc->nwqs; i++) { 684 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL) 685 m_freem(m); 686 } 687 if_qflush(ifp); 688} 689 690 691 692/***************************************************************************** 693 * Driver interrupt routines functions * 694 *****************************************************************************/ 695 696static void 697oce_intr(void *arg, int pending) 698{ 699 700 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 701 POCE_SOFTC sc = ii->sc; 702 struct oce_eq *eq = ii->eq; 703 struct oce_eqe *eqe; 704 struct oce_cq *cq = NULL; 705 int i, num_eqes = 0; 706 707 708 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 709 BUS_DMASYNC_POSTWRITE); 710 do { 711 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); 712 if (eqe->evnt == 0) 713 break; 714 eqe->evnt = 0; 715 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 716 BUS_DMASYNC_POSTWRITE); 717 RING_GET(eq->ring, 1); 718 num_eqes++; 719 720 } while (TRUE); 721 722 if (!num_eqes) 723 goto eq_arm; /* Spurious */ 724 725 /* Clear EQ entries, but dont arm */ 726 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE); 727 728 /* Process TX, RX and MCC. But dont arm CQ*/ 729 for (i = 0; i < eq->cq_valid; i++) { 730 cq = eq->cq[i]; 731 (*cq->cq_handler)(cq->cb_arg); 732 } 733 734 /* Arm all cqs connected to this EQ */ 735 for (i = 0; i < eq->cq_valid; i++) { 736 cq = eq->cq[i]; 737 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 738 } 739 740eq_arm: 741 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 742 743 return; 744} 745 746 747static int 748oce_setup_intr(POCE_SOFTC sc) 749{ 750 int rc = 0, use_intx = 0; 751 int vector = 0, req_vectors = 0; 752 int tot_req_vectors, tot_vectors; 753 754 if (is_rss_enabled(sc)) 755 req_vectors = MAX((sc->nrqs - 1), sc->nwqs); 756 else 757 req_vectors = 1; 758 759 tot_req_vectors = req_vectors; 760 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 761 if (req_vectors > 1) { 762 tot_req_vectors += OCE_RDMA_VECTORS; 763 sc->roce_intr_count = OCE_RDMA_VECTORS; 764 } 765 } 766 767 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) { 768 sc->intr_count = req_vectors; 769 tot_vectors = tot_req_vectors; 770 rc = pci_alloc_msix(sc->dev, &tot_vectors); 771 if (rc != 0) { 772 use_intx = 1; 773 pci_release_msi(sc->dev); 774 } else { 775 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 776 if (tot_vectors < tot_req_vectors) { 777 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) { 778 sc->roce_intr_count = (tot_vectors / 2); 779 } 780 sc->intr_count = tot_vectors - sc->roce_intr_count; 781 } 782 } else { 783 sc->intr_count = tot_vectors; 784 } 785 sc->flags |= OCE_FLAGS_USING_MSIX; 786 } 787 } else 788 use_intx = 1; 789 790 if (use_intx) 791 sc->intr_count = 1; 792 793 /* Scale number of queues based on intr we got */ 794 update_queues_got(sc); 795 796 if (use_intx) { 797 device_printf(sc->dev, "Using legacy interrupt\n"); 798 rc = oce_alloc_intr(sc, vector, oce_intr); 799 if (rc) 800 goto error; 801 } else { 802 for (; vector < sc->intr_count; vector++) { 803 rc = oce_alloc_intr(sc, vector, oce_intr); 804 if (rc) 805 goto error; 806 } 807 } 808 809 return 0; 810error: 811 oce_intr_free(sc); 812 return rc; 813} 814 815 816static int 817oce_fast_isr(void *arg) 818{ 819 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg; 820 POCE_SOFTC sc = ii->sc; 821 822 if (ii->eq == NULL) 823 return FILTER_STRAY; 824 825 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE); 826 827 taskqueue_enqueue(ii->tq, &ii->task); 828 829 ii->eq->intr++; 830 831 return FILTER_HANDLED; 832} 833 834 835static int 836oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending)) 837{ 838 POCE_INTR_INFO ii; 839 int rc = 0, rr; 840 841 if (vector >= OCE_MAX_EQ) 842 return (EINVAL); 843 844 ii = &sc->intrs[vector]; 845 846 /* Set the resource id for the interrupt. 847 * MSIx is vector + 1 for the resource id, 848 * INTx is 0 for the resource id. 849 */ 850 if (sc->flags & OCE_FLAGS_USING_MSIX) 851 rr = vector + 1; 852 else 853 rr = 0; 854 ii->intr_res = bus_alloc_resource_any(sc->dev, 855 SYS_RES_IRQ, 856 &rr, RF_ACTIVE|RF_SHAREABLE); 857 ii->irq_rr = rr; 858 if (ii->intr_res == NULL) { 859 device_printf(sc->dev, 860 "Could not allocate interrupt\n"); 861 rc = ENXIO; 862 return rc; 863 } 864 865 TASK_INIT(&ii->task, 0, isr, ii); 866 ii->vector = vector; 867 sprintf(ii->task_name, "oce_task[%d]", ii->vector); 868 ii->tq = taskqueue_create_fast(ii->task_name, 869 M_NOWAIT, 870 taskqueue_thread_enqueue, 871 &ii->tq); 872 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq", 873 device_get_nameunit(sc->dev)); 874 875 ii->sc = sc; 876 rc = bus_setup_intr(sc->dev, 877 ii->intr_res, 878 INTR_TYPE_NET, 879 oce_fast_isr, NULL, ii, &ii->tag); 880 return rc; 881 882} 883 884 885void 886oce_intr_free(POCE_SOFTC sc) 887{ 888 int i = 0; 889 890 for (i = 0; i < sc->intr_count; i++) { 891 892 if (sc->intrs[i].tag != NULL) 893 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res, 894 sc->intrs[i].tag); 895 if (sc->intrs[i].tq != NULL) 896 taskqueue_free(sc->intrs[i].tq); 897 898 if (sc->intrs[i].intr_res != NULL) 899 bus_release_resource(sc->dev, SYS_RES_IRQ, 900 sc->intrs[i].irq_rr, 901 sc->intrs[i].intr_res); 902 sc->intrs[i].tag = NULL; 903 sc->intrs[i].intr_res = NULL; 904 } 905 906 if (sc->flags & OCE_FLAGS_USING_MSIX) 907 pci_release_msi(sc->dev); 908 909} 910 911 912 913/****************************************************************************** 914* Media callbacks functions * 915******************************************************************************/ 916 917static void 918oce_media_status(struct ifnet *ifp, struct ifmediareq *req) 919{ 920 POCE_SOFTC sc = (POCE_SOFTC) ifp->if_softc; 921 922 923 req->ifm_status = IFM_AVALID; 924 req->ifm_active = IFM_ETHER; 925 926 if (sc->link_status == 1) 927 req->ifm_status |= IFM_ACTIVE; 928 else 929 return; 930 931 switch (sc->link_speed) { 932 case 1: /* 10 Mbps */ 933 req->ifm_active |= IFM_10_T | IFM_FDX; 934 sc->speed = 10; 935 break; 936 case 2: /* 100 Mbps */ 937 req->ifm_active |= IFM_100_TX | IFM_FDX; 938 sc->speed = 100; 939 break; 940 case 3: /* 1 Gbps */ 941 req->ifm_active |= IFM_1000_T | IFM_FDX; 942 sc->speed = 1000; 943 break; 944 case 4: /* 10 Gbps */ 945 req->ifm_active |= IFM_10G_SR | IFM_FDX; 946 sc->speed = 10000; 947 break; 948 case 5: /* 20 Gbps */ 949 req->ifm_active |= IFM_10G_SR | IFM_FDX; 950 sc->speed = 20000; 951 break; 952 case 6: /* 25 Gbps */ 953 req->ifm_active |= IFM_10G_SR | IFM_FDX; 954 sc->speed = 25000; 955 break; 956 case 7: /* 40 Gbps */ 957 req->ifm_active |= IFM_40G_SR4 | IFM_FDX; 958 sc->speed = 40000; 959 break; 960 default: 961 sc->speed = 0; 962 break; 963 } 964 965 return; 966} 967 968 969int 970oce_media_change(struct ifnet *ifp) 971{ 972 return 0; 973} 974 975 976static void oce_is_pkt_dest_bmc(POCE_SOFTC sc, 977 struct mbuf *m, boolean_t *os2bmc, 978 struct mbuf **m_new) 979{ 980 struct ether_header *eh = NULL; 981 982 eh = mtod(m, struct ether_header *); 983 984 if (!is_os2bmc_enabled(sc) || *os2bmc) { 985 *os2bmc = FALSE; 986 goto done; 987 } 988 if (!ETHER_IS_MULTICAST(eh->ether_dhost)) 989 goto done; 990 991 if (is_mc_allowed_on_bmc(sc, eh) || 992 is_bc_allowed_on_bmc(sc, eh) || 993 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) { 994 *os2bmc = TRUE; 995 goto done; 996 } 997 998 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) { 999 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 1000 uint8_t nexthdr = ip6->ip6_nxt; 1001 if (nexthdr == IPPROTO_ICMPV6) { 1002 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1); 1003 switch (icmp6->icmp6_type) { 1004 case ND_ROUTER_ADVERT: 1005 *os2bmc = is_ipv6_ra_filt_enabled(sc); 1006 goto done; 1007 case ND_NEIGHBOR_ADVERT: 1008 *os2bmc = is_ipv6_na_filt_enabled(sc); 1009 goto done; 1010 default: 1011 break; 1012 } 1013 } 1014 } 1015 1016 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) { 1017 struct ip *ip = mtod(m, struct ip *); 1018 int iphlen = ip->ip_hl << 2; 1019 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen); 1020 switch (uh->uh_dport) { 1021 case DHCP_CLIENT_PORT: 1022 *os2bmc = is_dhcp_client_filt_enabled(sc); 1023 goto done; 1024 case DHCP_SERVER_PORT: 1025 *os2bmc = is_dhcp_srvr_filt_enabled(sc); 1026 goto done; 1027 case NET_BIOS_PORT1: 1028 case NET_BIOS_PORT2: 1029 *os2bmc = is_nbios_filt_enabled(sc); 1030 goto done; 1031 case DHCPV6_RAS_PORT: 1032 *os2bmc = is_ipv6_ras_filt_enabled(sc); 1033 goto done; 1034 default: 1035 break; 1036 } 1037 } 1038done: 1039 if (*os2bmc) { 1040 *m_new = m_dup(m, M_NOWAIT); 1041 if (!*m_new) { 1042 *os2bmc = FALSE; 1043 return; 1044 } 1045 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL); 1046 } 1047} 1048 1049 1050 1051/***************************************************************************** 1052 * Transmit routines functions * 1053 *****************************************************************************/ 1054 1055static int 1056oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index) 1057{ 1058 int rc = 0, i, retry_cnt = 0; 1059 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS]; 1060 struct mbuf *m, *m_temp, *m_new = NULL; 1061 struct oce_wq *wq = sc->wq[wq_index]; 1062 struct oce_packet_desc *pd; 1063 struct oce_nic_hdr_wqe *nichdr; 1064 struct oce_nic_frag_wqe *nicfrag; 1065 struct ether_header *eh = NULL; 1066 int num_wqes; 1067 uint32_t reg_value; 1068 boolean_t complete = TRUE; 1069 boolean_t os2bmc = FALSE; 1070 1071 m = *mpp; 1072 if (!m) 1073 return EINVAL; 1074 1075 if (!(m->m_flags & M_PKTHDR)) { 1076 rc = ENXIO; 1077 goto free_ret; 1078 } 1079 1080 /* Don't allow non-TSO packets longer than MTU */ 1081 if (!is_tso_pkt(m)) { 1082 eh = mtod(m, struct ether_header *); 1083 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE)) 1084 goto free_ret; 1085 } 1086 1087 if(oce_tx_asic_stall_verify(sc, m)) { 1088 m = oce_insert_vlan_tag(sc, m, &complete); 1089 if(!m) { 1090 device_printf(sc->dev, "Insertion unsuccessful\n"); 1091 return 0; 1092 } 1093 1094 } 1095 1096 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less 1097 * may cause a transmit stall on that port. So the work-around is to 1098 * pad short packets (<= 32 bytes) to a 36-byte length. 1099 */ 1100 if(IS_SH(sc) || IS_XE201(sc) ) { 1101 if(m->m_pkthdr.len <= 32) { 1102 char buf[36]; 1103 bzero((void *)buf, 36); 1104 m_append(m, (36 - m->m_pkthdr.len), buf); 1105 } 1106 } 1107 1108tx_start: 1109 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1110 /* consolidate packet buffers for TSO/LSO segment offload */ 1111#if defined(INET6) || defined(INET) 1112 m = oce_tso_setup(sc, mpp); 1113#else 1114 m = NULL; 1115#endif 1116 if (m == NULL) { 1117 rc = ENXIO; 1118 goto free_ret; 1119 } 1120 } 1121 1122 1123 pd = &wq->pckts[wq->pkt_desc_head]; 1124 1125retry: 1126 rc = bus_dmamap_load_mbuf_sg(wq->tag, 1127 pd->map, 1128 m, segs, &pd->nsegs, BUS_DMA_NOWAIT); 1129 if (rc == 0) { 1130 num_wqes = pd->nsegs + 1; 1131 if (IS_BE(sc) || IS_SH(sc)) { 1132 /*Dummy required only for BE3.*/ 1133 if (num_wqes & 1) 1134 num_wqes++; 1135 } 1136 if (num_wqes >= RING_NUM_FREE(wq->ring)) { 1137 bus_dmamap_unload(wq->tag, pd->map); 1138 return EBUSY; 1139 } 1140 atomic_store_rel_int(&wq->pkt_desc_head, 1141 (wq->pkt_desc_head + 1) % \ 1142 OCE_WQ_PACKET_ARRAY_SIZE); 1143 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE); 1144 pd->mbuf = m; 1145 1146 nichdr = 1147 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe); 1148 nichdr->u0.dw[0] = 0; 1149 nichdr->u0.dw[1] = 0; 1150 nichdr->u0.dw[2] = 0; 1151 nichdr->u0.dw[3] = 0; 1152 1153 nichdr->u0.s.complete = complete; 1154 nichdr->u0.s.mgmt = os2bmc; 1155 nichdr->u0.s.event = 1; 1156 nichdr->u0.s.crc = 1; 1157 nichdr->u0.s.forward = 0; 1158 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0; 1159 nichdr->u0.s.udpcs = 1160 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0; 1161 nichdr->u0.s.tcpcs = 1162 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0; 1163 nichdr->u0.s.num_wqe = num_wqes; 1164 nichdr->u0.s.total_length = m->m_pkthdr.len; 1165 1166 if (m->m_flags & M_VLANTAG) { 1167 nichdr->u0.s.vlan = 1; /*Vlan present*/ 1168 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag; 1169 } 1170 1171 if (m->m_pkthdr.csum_flags & CSUM_TSO) { 1172 if (m->m_pkthdr.tso_segsz) { 1173 nichdr->u0.s.lso = 1; 1174 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz; 1175 } 1176 if (!IS_BE(sc) || !IS_SH(sc)) 1177 nichdr->u0.s.ipcs = 1; 1178 } 1179 1180 RING_PUT(wq->ring, 1); 1181 atomic_add_int(&wq->ring->num_used, 1); 1182 1183 for (i = 0; i < pd->nsegs; i++) { 1184 nicfrag = 1185 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1186 struct oce_nic_frag_wqe); 1187 nicfrag->u0.s.rsvd0 = 0; 1188 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr); 1189 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr); 1190 nicfrag->u0.s.frag_len = segs[i].ds_len; 1191 pd->wqe_idx = wq->ring->pidx; 1192 RING_PUT(wq->ring, 1); 1193 atomic_add_int(&wq->ring->num_used, 1); 1194 } 1195 if (num_wqes > (pd->nsegs + 1)) { 1196 nicfrag = 1197 RING_GET_PRODUCER_ITEM_VA(wq->ring, 1198 struct oce_nic_frag_wqe); 1199 nicfrag->u0.dw[0] = 0; 1200 nicfrag->u0.dw[1] = 0; 1201 nicfrag->u0.dw[2] = 0; 1202 nicfrag->u0.dw[3] = 0; 1203 pd->wqe_idx = wq->ring->pidx; 1204 RING_PUT(wq->ring, 1); 1205 atomic_add_int(&wq->ring->num_used, 1); 1206 pd->nsegs++; 1207 } 1208 1209 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1); 1210 wq->tx_stats.tx_reqs++; 1211 wq->tx_stats.tx_wrbs += num_wqes; 1212 wq->tx_stats.tx_bytes += m->m_pkthdr.len; 1213 wq->tx_stats.tx_pkts++; 1214 1215 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map, 1216 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1217 reg_value = (num_wqes << 16) | wq->wq_id; 1218 1219 /* if os2bmc is not enabled or if the pkt is already tagged as 1220 bmc, do nothing 1221 */ 1222 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new); 1223 1224 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value); 1225 1226 } else if (rc == EFBIG) { 1227 if (retry_cnt == 0) { 1228 m_temp = m_defrag(m, M_NOWAIT); 1229 if (m_temp == NULL) 1230 goto free_ret; 1231 m = m_temp; 1232 *mpp = m_temp; 1233 retry_cnt = retry_cnt + 1; 1234 goto retry; 1235 } else 1236 goto free_ret; 1237 } else if (rc == ENOMEM) 1238 return rc; 1239 else 1240 goto free_ret; 1241 1242 if (os2bmc) { 1243 m = m_new; 1244 goto tx_start; 1245 } 1246 1247 return 0; 1248 1249free_ret: 1250 m_freem(*mpp); 1251 *mpp = NULL; 1252 return rc; 1253} 1254 1255 1256static void 1257oce_process_tx_completion(struct oce_wq *wq) 1258{ 1259 struct oce_packet_desc *pd; 1260 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 1261 struct mbuf *m; 1262 1263 pd = &wq->pckts[wq->pkt_desc_tail]; 1264 atomic_store_rel_int(&wq->pkt_desc_tail, 1265 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE); 1266 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1); 1267 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1268 bus_dmamap_unload(wq->tag, pd->map); 1269 1270 m = pd->mbuf; 1271 m_freem(m); 1272 pd->mbuf = NULL; 1273 1274 1275 if (sc->ifp->if_drv_flags & IFF_DRV_OACTIVE) { 1276 if (wq->ring->num_used < (wq->ring->num_items / 2)) { 1277 sc->ifp->if_drv_flags &= ~(IFF_DRV_OACTIVE); 1278 oce_tx_restart(sc, wq); 1279 } 1280 } 1281} 1282 1283 1284static void 1285oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq) 1286{ 1287 1288 if ((sc->ifp->if_drv_flags & IFF_DRV_RUNNING) != IFF_DRV_RUNNING) 1289 return; 1290 1291#if __FreeBSD_version >= 800000 1292 if (!drbr_empty(sc->ifp, wq->br)) 1293#else 1294 if (!IFQ_DRV_IS_EMPTY(&sc->ifp->if_snd)) 1295#endif 1296 taskqueue_enqueue(taskqueue_swi, &wq->txtask); 1297 1298} 1299 1300 1301#if defined(INET6) || defined(INET) 1302static struct mbuf * 1303oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp) 1304{ 1305 struct mbuf *m; 1306#ifdef INET 1307 struct ip *ip; 1308#endif 1309#ifdef INET6 1310 struct ip6_hdr *ip6; 1311#endif 1312 struct ether_vlan_header *eh; 1313 struct tcphdr *th; 1314 uint16_t etype; 1315 int total_len = 0, ehdrlen = 0; 1316 1317 m = *mpp; 1318 1319 if (M_WRITABLE(m) == 0) { 1320 m = m_dup(*mpp, M_NOWAIT); 1321 if (!m) 1322 return NULL; 1323 m_freem(*mpp); 1324 *mpp = m; 1325 } 1326 1327 eh = mtod(m, struct ether_vlan_header *); 1328 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1329 etype = ntohs(eh->evl_proto); 1330 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1331 } else { 1332 etype = ntohs(eh->evl_encap_proto); 1333 ehdrlen = ETHER_HDR_LEN; 1334 } 1335 1336 switch (etype) { 1337#ifdef INET 1338 case ETHERTYPE_IP: 1339 ip = (struct ip *)(m->m_data + ehdrlen); 1340 if (ip->ip_p != IPPROTO_TCP) 1341 return NULL; 1342 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2)); 1343 1344 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2); 1345 break; 1346#endif 1347#ifdef INET6 1348 case ETHERTYPE_IPV6: 1349 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen); 1350 if (ip6->ip6_nxt != IPPROTO_TCP) 1351 return NULL; 1352 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr)); 1353 1354 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2); 1355 break; 1356#endif 1357 default: 1358 return NULL; 1359 } 1360 1361 m = m_pullup(m, total_len); 1362 if (!m) 1363 return NULL; 1364 *mpp = m; 1365 return m; 1366 1367} 1368#endif /* INET6 || INET */ 1369 1370void 1371oce_tx_task(void *arg, int npending) 1372{ 1373 struct oce_wq *wq = arg; 1374 POCE_SOFTC sc = wq->parent; 1375 struct ifnet *ifp = sc->ifp; 1376 int rc = 0; 1377 1378#if __FreeBSD_version >= 800000 1379 LOCK(&wq->tx_lock); 1380 rc = oce_multiq_transmit(ifp, NULL, wq); 1381 if (rc) { 1382 device_printf(sc->dev, 1383 "TX[%d] restart failed\n", wq->queue_index); 1384 } 1385 UNLOCK(&wq->tx_lock); 1386#else 1387 oce_start(ifp); 1388#endif 1389 1390} 1391 1392 1393void 1394oce_start(struct ifnet *ifp) 1395{ 1396 POCE_SOFTC sc = ifp->if_softc; 1397 struct mbuf *m; 1398 int rc = 0; 1399 int def_q = 0; /* Defualt tx queue is 0*/ 1400 1401 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1402 IFF_DRV_RUNNING) 1403 return; 1404 1405 if (!sc->link_status) 1406 return; 1407 1408 do { 1409 IF_DEQUEUE(&sc->ifp->if_snd, m); 1410 if (m == NULL) 1411 break; 1412 1413 LOCK(&sc->wq[def_q]->tx_lock); 1414 rc = oce_tx(sc, &m, def_q); 1415 UNLOCK(&sc->wq[def_q]->tx_lock); 1416 if (rc) { 1417 if (m != NULL) { 1418 sc->wq[def_q]->tx_stats.tx_stops ++; 1419 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1420 IFQ_DRV_PREPEND(&ifp->if_snd, m); 1421 m = NULL; 1422 } 1423 break; 1424 } 1425 if (m != NULL) 1426 ETHER_BPF_MTAP(ifp, m); 1427 1428 } while (TRUE); 1429 1430 return; 1431} 1432 1433 1434/* Handle the Completion Queue for transmit */ 1435uint16_t 1436oce_wq_handler(void *arg) 1437{ 1438 struct oce_wq *wq = (struct oce_wq *)arg; 1439 POCE_SOFTC sc = wq->parent; 1440 struct oce_cq *cq = wq->cq; 1441 struct oce_nic_tx_cqe *cqe; 1442 int num_cqes = 0; 1443 1444 LOCK(&wq->tx_compl_lock); 1445 bus_dmamap_sync(cq->ring->dma.tag, 1446 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1447 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1448 while (cqe->u0.dw[3]) { 1449 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe)); 1450 1451 wq->ring->cidx = cqe->u0.s.wqe_index + 1; 1452 if (wq->ring->cidx >= wq->ring->num_items) 1453 wq->ring->cidx -= wq->ring->num_items; 1454 1455 oce_process_tx_completion(wq); 1456 wq->tx_stats.tx_compl++; 1457 cqe->u0.dw[3] = 0; 1458 RING_GET(cq->ring, 1); 1459 bus_dmamap_sync(cq->ring->dma.tag, 1460 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1461 cqe = 1462 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1463 num_cqes++; 1464 } 1465 1466 if (num_cqes) 1467 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 1468 1469 UNLOCK(&wq->tx_compl_lock); 1470 return num_cqes; 1471} 1472 1473 1474static int 1475oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m, struct oce_wq *wq) 1476{ 1477 POCE_SOFTC sc = ifp->if_softc; 1478 int status = 0, queue_index = 0; 1479 struct mbuf *next = NULL; 1480 struct buf_ring *br = NULL; 1481 1482 br = wq->br; 1483 queue_index = wq->queue_index; 1484 1485 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != 1486 IFF_DRV_RUNNING) { 1487 if (m != NULL) 1488 status = drbr_enqueue(ifp, br, m); 1489 return status; 1490 } 1491 1492 if (m != NULL) { 1493 if ((status = drbr_enqueue(ifp, br, m)) != 0) 1494 return status; 1495 } 1496 while ((next = drbr_peek(ifp, br)) != NULL) { 1497 if (oce_tx(sc, &next, queue_index)) { 1498 if (next == NULL) { 1499 drbr_advance(ifp, br); 1500 } else { 1501 drbr_putback(ifp, br, next); 1502 wq->tx_stats.tx_stops ++; 1503 ifp->if_drv_flags |= IFF_DRV_OACTIVE; 1504 } 1505 break; 1506 } 1507 drbr_advance(ifp, br); 1508 if_inc_counter(ifp, IFCOUNTER_OBYTES, next->m_pkthdr.len); 1509 if (next->m_flags & M_MCAST) 1510 if_inc_counter(ifp, IFCOUNTER_OMCASTS, 1); 1511 ETHER_BPF_MTAP(ifp, next); 1512 } 1513 1514 return 0; 1515} 1516 1517 1518 1519 1520/***************************************************************************** 1521 * Receive routines functions * 1522 *****************************************************************************/ 1523 1524static void 1525oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2) 1526{ 1527 uint32_t *p; 1528 struct ether_header *eh = NULL; 1529 struct tcphdr *tcp_hdr = NULL; 1530 struct ip *ip4_hdr = NULL; 1531 struct ip6_hdr *ip6 = NULL; 1532 uint32_t payload_len = 0; 1533 1534 eh = mtod(m, struct ether_header *); 1535 /* correct IP header */ 1536 if(!cqe2->ipv6_frame) { 1537 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header)); 1538 ip4_hdr->ip_ttl = cqe2->frame_lifespan; 1539 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header)); 1540 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip)); 1541 }else { 1542 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header)); 1543 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan; 1544 payload_len = cqe2->coalesced_size - sizeof(struct ether_header) 1545 - sizeof(struct ip6_hdr); 1546 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len); 1547 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr)); 1548 } 1549 1550 /* correct tcp header */ 1551 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num); 1552 if(cqe2->push) { 1553 tcp_hdr->th_flags |= TH_PUSH; 1554 } 1555 tcp_hdr->th_win = htons(cqe2->tcp_window); 1556 tcp_hdr->th_sum = 0xffff; 1557 if(cqe2->ts_opt) { 1558 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2); 1559 *p = cqe1->tcp_timestamp_val; 1560 *(p+1) = cqe1->tcp_timestamp_ecr; 1561 } 1562 1563 return; 1564} 1565 1566static void 1567oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m) 1568{ 1569 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1570 uint32_t i = 0, frag_len = 0; 1571 uint32_t len = cqe_info->pkt_size; 1572 struct oce_packet_desc *pd; 1573 struct mbuf *tail = NULL; 1574 1575 for (i = 0; i < cqe_info->num_frags; i++) { 1576 if (rq->ring->cidx == rq->ring->pidx) { 1577 device_printf(sc->dev, 1578 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n"); 1579 return; 1580 } 1581 pd = &rq->pckts[rq->ring->cidx]; 1582 1583 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1584 bus_dmamap_unload(rq->tag, pd->map); 1585 RING_GET(rq->ring, 1); 1586 rq->pending--; 1587 1588 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len; 1589 pd->mbuf->m_len = frag_len; 1590 1591 if (tail != NULL) { 1592 /* additional fragments */ 1593 pd->mbuf->m_flags &= ~M_PKTHDR; 1594 tail->m_next = pd->mbuf; 1595 if(rq->islro) 1596 tail->m_nextpkt = NULL; 1597 tail = pd->mbuf; 1598 } else { 1599 /* first fragment, fill out much of the packet header */ 1600 pd->mbuf->m_pkthdr.len = len; 1601 if(rq->islro) 1602 pd->mbuf->m_nextpkt = NULL; 1603 pd->mbuf->m_pkthdr.csum_flags = 0; 1604 if (IF_CSUM_ENABLED(sc)) { 1605 if (cqe_info->l4_cksum_pass) { 1606 if(!cqe_info->ipv6_frame) { /* IPV4 */ 1607 pd->mbuf->m_pkthdr.csum_flags |= 1608 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1609 }else { /* IPV6 frame */ 1610 if(rq->islro) { 1611 pd->mbuf->m_pkthdr.csum_flags |= 1612 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 1613 } 1614 } 1615 pd->mbuf->m_pkthdr.csum_data = 0xffff; 1616 } 1617 if (cqe_info->ip_cksum_pass) { 1618 pd->mbuf->m_pkthdr.csum_flags |= 1619 (CSUM_IP_CHECKED|CSUM_IP_VALID); 1620 } 1621 } 1622 *m = tail = pd->mbuf; 1623 } 1624 pd->mbuf = NULL; 1625 len -= frag_len; 1626 } 1627 1628 return; 1629} 1630 1631static void 1632oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2) 1633{ 1634 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1635 struct nic_hwlro_cqe_part1 *cqe1 = NULL; 1636 struct mbuf *m = NULL; 1637 struct oce_common_cqe_info cq_info; 1638 1639 /* parse cqe */ 1640 if(cqe2 == NULL) { 1641 cq_info.pkt_size = cqe->pkt_size; 1642 cq_info.vtag = cqe->vlan_tag; 1643 cq_info.l4_cksum_pass = cqe->l4_cksum_pass; 1644 cq_info.ip_cksum_pass = cqe->ip_cksum_pass; 1645 cq_info.ipv6_frame = cqe->ipv6_frame; 1646 cq_info.vtp = cqe->vtp; 1647 cq_info.qnq = cqe->qnq; 1648 }else { 1649 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe; 1650 cq_info.pkt_size = cqe2->coalesced_size; 1651 cq_info.vtag = cqe2->vlan_tag; 1652 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass; 1653 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass; 1654 cq_info.ipv6_frame = cqe2->ipv6_frame; 1655 cq_info.vtp = cqe2->vtp; 1656 cq_info.qnq = cqe1->qnq; 1657 } 1658 1659 cq_info.vtag = BSWAP_16(cq_info.vtag); 1660 1661 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size; 1662 if(cq_info.pkt_size % rq->cfg.frag_size) 1663 cq_info.num_frags++; 1664 1665 oce_rx_mbuf_chain(rq, &cq_info, &m); 1666 1667 if (m) { 1668 if(cqe2) { 1669 //assert(cqe2->valid != 0); 1670 1671 //assert(cqe2->cqe_type != 2); 1672 oce_correct_header(m, cqe1, cqe2); 1673 } 1674 1675 m->m_pkthdr.rcvif = sc->ifp; 1676#if __FreeBSD_version >= 800000 1677 if (rq->queue_index) 1678 m->m_pkthdr.flowid = (rq->queue_index - 1); 1679 else 1680 m->m_pkthdr.flowid = rq->queue_index; 1681 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1682#endif 1683 /* This deternies if vlan tag is Valid */ 1684 if (cq_info.vtp) { 1685 if (sc->function_mode & FNM_FLEX10_MODE) { 1686 /* FLEX10. If QnQ is not set, neglect VLAN */ 1687 if (cq_info.qnq) { 1688 m->m_pkthdr.ether_vtag = cq_info.vtag; 1689 m->m_flags |= M_VLANTAG; 1690 } 1691 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) { 1692 /* In UMC mode generally pvid will be striped by 1693 hw. But in some cases we have seen it comes 1694 with pvid. So if pvid == vlan, neglect vlan. 1695 */ 1696 m->m_pkthdr.ether_vtag = cq_info.vtag; 1697 m->m_flags |= M_VLANTAG; 1698 } 1699 } 1700 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1701 1702 (*sc->ifp->if_input) (sc->ifp, m); 1703 1704 /* Update rx stats per queue */ 1705 rq->rx_stats.rx_pkts++; 1706 rq->rx_stats.rx_bytes += cq_info.pkt_size; 1707 rq->rx_stats.rx_frags += cq_info.num_frags; 1708 rq->rx_stats.rx_ucast_pkts++; 1709 } 1710 return; 1711} 1712 1713static void 1714oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe) 1715{ 1716 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1717 int len; 1718 struct mbuf *m = NULL; 1719 struct oce_common_cqe_info cq_info; 1720 uint16_t vtag = 0; 1721 1722 /* Is it a flush compl that has no data */ 1723 if(!cqe->u0.s.num_fragments) 1724 goto exit; 1725 1726 len = cqe->u0.s.pkt_size; 1727 if (!len) { 1728 /*partial DMA workaround for Lancer*/ 1729 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1730 goto exit; 1731 } 1732 1733 if (!oce_cqe_portid_valid(sc, cqe)) { 1734 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1735 goto exit; 1736 } 1737 1738 /* Get vlan_tag value */ 1739 if(IS_BE(sc) || IS_SH(sc)) 1740 vtag = BSWAP_16(cqe->u0.s.vlan_tag); 1741 else 1742 vtag = cqe->u0.s.vlan_tag; 1743 1744 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass; 1745 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass; 1746 cq_info.ipv6_frame = cqe->u0.s.ip_ver; 1747 cq_info.num_frags = cqe->u0.s.num_fragments; 1748 cq_info.pkt_size = cqe->u0.s.pkt_size; 1749 1750 oce_rx_mbuf_chain(rq, &cq_info, &m); 1751 1752 if (m) { 1753 m->m_pkthdr.rcvif = sc->ifp; 1754#if __FreeBSD_version >= 800000 1755 if (rq->queue_index) 1756 m->m_pkthdr.flowid = (rq->queue_index - 1); 1757 else 1758 m->m_pkthdr.flowid = rq->queue_index; 1759 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); 1760#endif 1761 /* This deternies if vlan tag is Valid */ 1762 if (oce_cqe_vtp_valid(sc, cqe)) { 1763 if (sc->function_mode & FNM_FLEX10_MODE) { 1764 /* FLEX10. If QnQ is not set, neglect VLAN */ 1765 if (cqe->u0.s.qnq) { 1766 m->m_pkthdr.ether_vtag = vtag; 1767 m->m_flags |= M_VLANTAG; 1768 } 1769 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) { 1770 /* In UMC mode generally pvid will be striped by 1771 hw. But in some cases we have seen it comes 1772 with pvid. So if pvid == vlan, neglect vlan. 1773 */ 1774 m->m_pkthdr.ether_vtag = vtag; 1775 m->m_flags |= M_VLANTAG; 1776 } 1777 } 1778 1779 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1); 1780#if defined(INET6) || defined(INET) 1781 /* Try to queue to LRO */ 1782 if (IF_LRO_ENABLED(sc) && 1783 (cqe->u0.s.ip_cksum_pass) && 1784 (cqe->u0.s.l4_cksum_pass) && 1785 (!cqe->u0.s.ip_ver) && 1786 (rq->lro.lro_cnt != 0)) { 1787 1788 if (tcp_lro_rx(&rq->lro, m, 0) == 0) { 1789 rq->lro_pkts_queued ++; 1790 goto post_done; 1791 } 1792 /* If LRO posting fails then try to post to STACK */ 1793 } 1794#endif 1795 1796 (*sc->ifp->if_input) (sc->ifp, m); 1797#if defined(INET6) || defined(INET) 1798post_done: 1799#endif 1800 /* Update rx stats per queue */ 1801 rq->rx_stats.rx_pkts++; 1802 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size; 1803 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments; 1804 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET) 1805 rq->rx_stats.rx_mcast_pkts++; 1806 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET) 1807 rq->rx_stats.rx_ucast_pkts++; 1808 } 1809exit: 1810 return; 1811} 1812 1813 1814void 1815oce_discard_rx_comp(struct oce_rq *rq, int num_frags) 1816{ 1817 uint32_t i = 0; 1818 struct oce_packet_desc *pd; 1819 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1820 1821 for (i = 0; i < num_frags; i++) { 1822 if (rq->ring->cidx == rq->ring->pidx) { 1823 device_printf(sc->dev, 1824 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n"); 1825 return; 1826 } 1827 pd = &rq->pckts[rq->ring->cidx]; 1828 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1829 bus_dmamap_unload(rq->tag, pd->map); 1830 if (pd->mbuf != NULL) { 1831 m_freem(pd->mbuf); 1832 pd->mbuf = NULL; 1833 } 1834 1835 RING_GET(rq->ring, 1); 1836 rq->pending--; 1837 } 1838} 1839 1840 1841static int 1842oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1843{ 1844 struct oce_nic_rx_cqe_v1 *cqe_v1; 1845 int vtp = 0; 1846 1847 if (sc->be3_native) { 1848 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1849 vtp = cqe_v1->u0.s.vlan_tag_present; 1850 } else 1851 vtp = cqe->u0.s.vlan_tag_present; 1852 1853 return vtp; 1854 1855} 1856 1857 1858static int 1859oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe) 1860{ 1861 struct oce_nic_rx_cqe_v1 *cqe_v1; 1862 int port_id = 0; 1863 1864 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) { 1865 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe; 1866 port_id = cqe_v1->u0.s.port; 1867 if (sc->port_id != port_id) 1868 return 0; 1869 } else 1870 ;/* For BE3 legacy and Lancer this is dummy */ 1871 1872 return 1; 1873 1874} 1875 1876#if defined(INET6) || defined(INET) 1877void 1878oce_rx_flush_lro(struct oce_rq *rq) 1879{ 1880 struct lro_ctrl *lro = &rq->lro; 1881 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1882 1883 if (!IF_LRO_ENABLED(sc)) 1884 return; 1885 1886 tcp_lro_flush_all(lro); 1887 rq->lro_pkts_queued = 0; 1888 1889 return; 1890} 1891 1892 1893static int 1894oce_init_lro(POCE_SOFTC sc) 1895{ 1896 struct lro_ctrl *lro = NULL; 1897 int i = 0, rc = 0; 1898 1899 for (i = 0; i < sc->nrqs; i++) { 1900 lro = &sc->rq[i]->lro; 1901 rc = tcp_lro_init(lro); 1902 if (rc != 0) { 1903 device_printf(sc->dev, "LRO init failed\n"); 1904 return rc; 1905 } 1906 lro->ifp = sc->ifp; 1907 } 1908 1909 return rc; 1910} 1911 1912 1913void 1914oce_free_lro(POCE_SOFTC sc) 1915{ 1916 struct lro_ctrl *lro = NULL; 1917 int i = 0; 1918 1919 for (i = 0; i < sc->nrqs; i++) { 1920 lro = &sc->rq[i]->lro; 1921 if (lro) 1922 tcp_lro_free(lro); 1923 } 1924} 1925#endif 1926 1927int 1928oce_alloc_rx_bufs(struct oce_rq *rq, int count) 1929{ 1930 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 1931 int i, in, rc; 1932 struct oce_packet_desc *pd; 1933 bus_dma_segment_t segs[6]; 1934 int nsegs, added = 0; 1935 struct oce_nic_rqe *rqe; 1936 pd_rxulp_db_t rxdb_reg; 1937 uint32_t val = 0; 1938 uint32_t oce_max_rq_posts = 64; 1939 1940 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t)); 1941 for (i = 0; i < count; i++) { 1942 in = (rq->ring->pidx + 1) % OCE_RQ_PACKET_ARRAY_SIZE; 1943 1944 pd = &rq->pckts[rq->ring->pidx]; 1945 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size); 1946 if (pd->mbuf == NULL) { 1947 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size); 1948 break; 1949 } 1950 pd->mbuf->m_nextpkt = NULL; 1951 1952 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size; 1953 1954 rc = bus_dmamap_load_mbuf_sg(rq->tag, 1955 pd->map, 1956 pd->mbuf, 1957 segs, &nsegs, BUS_DMA_NOWAIT); 1958 if (rc) { 1959 m_free(pd->mbuf); 1960 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc); 1961 break; 1962 } 1963 1964 if (nsegs != 1) { 1965 i--; 1966 continue; 1967 } 1968 1969 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD); 1970 1971 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe); 1972 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr); 1973 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr); 1974 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe)); 1975 RING_PUT(rq->ring, 1); 1976 added++; 1977 rq->pending++; 1978 } 1979 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS; 1980 if (added != 0) { 1981 for (i = added / oce_max_rq_posts; i > 0; i--) { 1982 rxdb_reg.bits.num_posted = oce_max_rq_posts; 1983 rxdb_reg.bits.qid = rq->rq_id; 1984 if(rq->islro) { 1985 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1986 val |= oce_max_rq_posts << 16; 1987 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 1988 }else { 1989 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 1990 } 1991 added -= oce_max_rq_posts; 1992 } 1993 if (added > 0) { 1994 rxdb_reg.bits.qid = rq->rq_id; 1995 rxdb_reg.bits.num_posted = added; 1996 if(rq->islro) { 1997 val |= rq->rq_id & DB_LRO_RQ_ID_MASK; 1998 val |= added << 16; 1999 OCE_WRITE_REG32(sc, db, DB_OFFSET, val); 2000 }else { 2001 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0); 2002 } 2003 } 2004 } 2005 2006 return 0; 2007} 2008 2009static void 2010oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq) 2011{ 2012 if (num_cqes) { 2013 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE); 2014 if(!sc->enable_hwlro) { 2015 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1) 2016 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1)); 2017 }else { 2018 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64) 2019 oce_alloc_rx_bufs(rq, 64); 2020 } 2021 } 2022 2023 return; 2024} 2025 2026uint16_t 2027oce_rq_handler_lro(void *arg) 2028{ 2029 struct oce_rq *rq = (struct oce_rq *)arg; 2030 struct oce_cq *cq = rq->cq; 2031 POCE_SOFTC sc = rq->parent; 2032 struct nic_hwlro_singleton_cqe *cqe; 2033 struct nic_hwlro_cqe_part2 *cqe2; 2034 int num_cqes = 0; 2035 2036 LOCK(&rq->rx_lock); 2037 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2038 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2039 while (cqe->valid) { 2040 if(cqe->cqe_type == 0) { /* singleton cqe */ 2041 /* we should not get singleton cqe after cqe1 on same rq */ 2042 if(rq->cqe_firstpart != NULL) { 2043 device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); 2044 goto exit_rq_handler_lro; 2045 } 2046 if(cqe->error != 0) { 2047 rq->rx_stats.rxcp_err++; 2048 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2049 } 2050 oce_rx_lro(rq, cqe, NULL); 2051 rq->rx_stats.rx_compl++; 2052 cqe->valid = 0; 2053 RING_GET(cq->ring, 1); 2054 num_cqes++; 2055 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2056 break; 2057 }else if(cqe->cqe_type == 0x1) { /* first part */ 2058 /* we should not get cqe1 after cqe1 on same rq */ 2059 if(rq->cqe_firstpart != NULL) { 2060 device_printf(sc->dev, "Got cqe1 after cqe1 \n"); 2061 goto exit_rq_handler_lro; 2062 } 2063 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; 2064 RING_GET(cq->ring, 1); 2065 }else if(cqe->cqe_type == 0x2) { /* second part */ 2066 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; 2067 if(cqe2->error != 0) { 2068 rq->rx_stats.rxcp_err++; 2069 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2070 } 2071 /* We should not get cqe2 without cqe1 */ 2072 if(rq->cqe_firstpart == NULL) { 2073 device_printf(sc->dev, "Got cqe2 without cqe1 \n"); 2074 goto exit_rq_handler_lro; 2075 } 2076 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2); 2077 2078 rq->rx_stats.rx_compl++; 2079 rq->cqe_firstpart->valid = 0; 2080 cqe2->valid = 0; 2081 rq->cqe_firstpart = NULL; 2082 2083 RING_GET(cq->ring, 1); 2084 num_cqes += 2; 2085 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2086 break; 2087 } 2088 2089 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2090 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 2091 } 2092 oce_check_rx_bufs(sc, num_cqes, rq); 2093exit_rq_handler_lro: 2094 UNLOCK(&rq->rx_lock); 2095 return 0; 2096} 2097 2098/* Handle the Completion Queue for receive */ 2099uint16_t 2100oce_rq_handler(void *arg) 2101{ 2102 struct oce_rq *rq = (struct oce_rq *)arg; 2103 struct oce_cq *cq = rq->cq; 2104 POCE_SOFTC sc = rq->parent; 2105 struct oce_nic_rx_cqe *cqe; 2106 int num_cqes = 0; 2107 2108 if(rq->islro) { 2109 oce_rq_handler_lro(arg); 2110 return 0; 2111 } 2112 LOCK(&rq->rx_lock); 2113 bus_dmamap_sync(cq->ring->dma.tag, 2114 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2115 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2116 while (cqe->u0.dw[2]) { 2117 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); 2118 2119 if (cqe->u0.s.error == 0) { 2120 oce_rx(rq, cqe); 2121 } else { 2122 rq->rx_stats.rxcp_err++; 2123 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1); 2124 /* Post L3/L4 errors to stack.*/ 2125 oce_rx(rq, cqe); 2126 } 2127 rq->rx_stats.rx_compl++; 2128 cqe->u0.dw[2] = 0; 2129 2130#if defined(INET6) || defined(INET) 2131 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) { 2132 oce_rx_flush_lro(rq); 2133 } 2134#endif 2135 2136 RING_GET(cq->ring, 1); 2137 bus_dmamap_sync(cq->ring->dma.tag, 2138 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2139 cqe = 2140 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 2141 num_cqes++; 2142 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled)) 2143 break; 2144 } 2145 2146#if defined(INET6) || defined(INET) 2147 if (IF_LRO_ENABLED(sc)) 2148 oce_rx_flush_lro(rq); 2149#endif 2150 2151 oce_check_rx_bufs(sc, num_cqes, rq); 2152 UNLOCK(&rq->rx_lock); 2153 return 0; 2154 2155} 2156 2157 2158 2159 2160/***************************************************************************** 2161 * Helper function prototypes in this file * 2162 *****************************************************************************/ 2163 2164static int 2165oce_attach_ifp(POCE_SOFTC sc) 2166{ 2167 2168 sc->ifp = if_alloc(IFT_ETHER); 2169 if (!sc->ifp) 2170 return ENOMEM; 2171 2172 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status); 2173 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2174 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO); 2175 2176 sc->ifp->if_flags = IFF_BROADCAST | IFF_MULTICAST; 2177 sc->ifp->if_ioctl = oce_ioctl; 2178 sc->ifp->if_start = oce_start; 2179 sc->ifp->if_init = oce_init; 2180 sc->ifp->if_mtu = ETHERMTU; 2181 sc->ifp->if_softc = sc; 2182#if __FreeBSD_version >= 800000 2183 sc->ifp->if_transmit = oce_multiq_start; 2184 sc->ifp->if_qflush = oce_multiq_flush; 2185#endif 2186 2187 if_initname(sc->ifp, 2188 device_get_name(sc->dev), device_get_unit(sc->dev)); 2189 2190 sc->ifp->if_snd.ifq_drv_maxlen = OCE_MAX_TX_DESC - 1; 2191 IFQ_SET_MAXLEN(&sc->ifp->if_snd, sc->ifp->if_snd.ifq_drv_maxlen); 2192 IFQ_SET_READY(&sc->ifp->if_snd); 2193 2194 sc->ifp->if_hwassist = OCE_IF_HWASSIST; 2195 sc->ifp->if_hwassist |= CSUM_TSO; 2196 sc->ifp->if_hwassist |= (CSUM_IP | CSUM_TCP | CSUM_UDP); 2197 2198 sc->ifp->if_capabilities = OCE_IF_CAPABILITIES; 2199 sc->ifp->if_capabilities |= IFCAP_HWCSUM; 2200 sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER; 2201 2202#if defined(INET6) || defined(INET) 2203 sc->ifp->if_capabilities |= IFCAP_TSO; 2204 sc->ifp->if_capabilities |= IFCAP_LRO; 2205 sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO; 2206#endif 2207 2208 sc->ifp->if_capenable = sc->ifp->if_capabilities; 2209 sc->ifp->if_baudrate = IF_Gbps(10); 2210 2211#if __FreeBSD_version >= 1000000 2212 sc->ifp->if_hw_tsomax = 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2213 sc->ifp->if_hw_tsomaxsegcount = OCE_MAX_TX_ELEMENTS; 2214 sc->ifp->if_hw_tsomaxsegsize = 4096; 2215#endif 2216 2217 ether_ifattach(sc->ifp, sc->macaddr.mac_addr); 2218 2219 return 0; 2220} 2221 2222 2223static void 2224oce_add_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2225{ 2226 POCE_SOFTC sc = ifp->if_softc; 2227 2228 if (ifp->if_softc != arg) 2229 return; 2230 if ((vtag == 0) || (vtag > 4095)) 2231 return; 2232 2233 sc->vlan_tag[vtag] = 1; 2234 sc->vlans_added++; 2235 if (sc->vlans_added <= (sc->max_vlans + 1)) 2236 oce_vid_config(sc); 2237} 2238 2239 2240static void 2241oce_del_vlan(void *arg, struct ifnet *ifp, uint16_t vtag) 2242{ 2243 POCE_SOFTC sc = ifp->if_softc; 2244 2245 if (ifp->if_softc != arg) 2246 return; 2247 if ((vtag == 0) || (vtag > 4095)) 2248 return; 2249 2250 sc->vlan_tag[vtag] = 0; 2251 sc->vlans_added--; 2252 oce_vid_config(sc); 2253} 2254 2255 2256/* 2257 * A max of 64 vlans can be configured in BE. If the user configures 2258 * more, place the card in vlan promiscuous mode. 2259 */ 2260static int 2261oce_vid_config(POCE_SOFTC sc) 2262{ 2263 struct normal_vlan vtags[MAX_VLANFILTER_SIZE]; 2264 uint16_t ntags = 0, i; 2265 int status = 0; 2266 2267 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) && 2268 (sc->ifp->if_capenable & IFCAP_VLAN_HWFILTER)) { 2269 for (i = 0; i < MAX_VLANS; i++) { 2270 if (sc->vlan_tag[i]) { 2271 vtags[ntags].vtag = i; 2272 ntags++; 2273 } 2274 } 2275 if (ntags) 2276 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2277 vtags, ntags, 1, 0); 2278 } else 2279 status = oce_config_vlan(sc, (uint8_t) sc->if_id, 2280 NULL, 0, 1, 1); 2281 return status; 2282} 2283 2284 2285static void 2286oce_mac_addr_set(POCE_SOFTC sc) 2287{ 2288 uint32_t old_pmac_id = sc->pmac_id; 2289 int status = 0; 2290 2291 2292 status = bcmp((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2293 sc->macaddr.size_of_struct); 2294 if (!status) 2295 return; 2296 2297 status = oce_mbox_macaddr_add(sc, (uint8_t *)(IF_LLADDR(sc->ifp)), 2298 sc->if_id, &sc->pmac_id); 2299 if (!status) { 2300 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id); 2301 bcopy((IF_LLADDR(sc->ifp)), sc->macaddr.mac_addr, 2302 sc->macaddr.size_of_struct); 2303 } 2304 if (status) 2305 device_printf(sc->dev, "Failed update macaddress\n"); 2306 2307} 2308 2309 2310static int 2311oce_handle_passthrough(struct ifnet *ifp, caddr_t data) 2312{ 2313 POCE_SOFTC sc = ifp->if_softc; 2314 struct ifreq *ifr = (struct ifreq *)data; 2315 int rc = ENXIO; 2316 char cookie[32] = {0}; 2317 void *priv_data = ifr_data_get_ptr(ifr); 2318 void *ioctl_ptr; 2319 uint32_t req_size; 2320 struct mbx_hdr req; 2321 OCE_DMA_MEM dma_mem; 2322 struct mbx_common_get_cntl_attr *fw_cmd; 2323 2324 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE))) 2325 return EFAULT; 2326 2327 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE))) 2328 return EINVAL; 2329 2330 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE); 2331 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr))) 2332 return EFAULT; 2333 2334 req_size = le32toh(req.u0.req.request_length); 2335 if (req_size > 65536) 2336 return EINVAL; 2337 2338 req_size += sizeof(struct mbx_hdr); 2339 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0); 2340 if (rc) 2341 return ENOMEM; 2342 2343 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) { 2344 rc = EFAULT; 2345 goto dma_free; 2346 } 2347 2348 rc = oce_pass_through_mbox(sc, &dma_mem, req_size); 2349 if (rc) { 2350 rc = EIO; 2351 goto dma_free; 2352 } 2353 2354 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) 2355 rc = EFAULT; 2356 2357 /* 2358 firmware is filling all the attributes for this ioctl except 2359 the driver version..so fill it 2360 */ 2361 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) { 2362 fw_cmd = (struct mbx_common_get_cntl_attr *) ioctl_ptr; 2363 strncpy(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str, 2364 COMPONENT_REVISION, strlen(COMPONENT_REVISION)); 2365 } 2366 2367dma_free: 2368 oce_dma_free(sc, &dma_mem); 2369 return rc; 2370 2371} 2372 2373static void 2374oce_eqd_set_periodic(POCE_SOFTC sc) 2375{ 2376 struct oce_set_eqd set_eqd[OCE_MAX_EQ]; 2377 struct oce_aic_obj *aic; 2378 struct oce_eq *eqo; 2379 uint64_t now = 0, delta; 2380 int eqd, i, num = 0; 2381 uint32_t tx_reqs = 0, rxpkts = 0, pps; 2382 struct oce_wq *wq; 2383 struct oce_rq *rq; 2384 2385 #define ticks_to_msecs(t) (1000 * (t) / hz) 2386 2387 for (i = 0 ; i < sc->neqs; i++) { 2388 eqo = sc->eq[i]; 2389 aic = &sc->aic_obj[i]; 2390 /* When setting the static eq delay from the user space */ 2391 if (!aic->enable) { 2392 if (aic->ticks) 2393 aic->ticks = 0; 2394 eqd = aic->et_eqd; 2395 goto modify_eqd; 2396 } 2397 2398 if (i == 0) { 2399 rq = sc->rq[0]; 2400 rxpkts = rq->rx_stats.rx_pkts; 2401 } else 2402 rxpkts = 0; 2403 if (i + 1 < sc->nrqs) { 2404 rq = sc->rq[i + 1]; 2405 rxpkts += rq->rx_stats.rx_pkts; 2406 } 2407 if (i < sc->nwqs) { 2408 wq = sc->wq[i]; 2409 tx_reqs = wq->tx_stats.tx_reqs; 2410 } else 2411 tx_reqs = 0; 2412 now = ticks; 2413 2414 if (!aic->ticks || now < aic->ticks || 2415 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) { 2416 aic->prev_rxpkts = rxpkts; 2417 aic->prev_txreqs = tx_reqs; 2418 aic->ticks = now; 2419 continue; 2420 } 2421 2422 delta = ticks_to_msecs(now - aic->ticks); 2423 2424 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) + 2425 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta); 2426 eqd = (pps / 15000) << 2; 2427 if (eqd < 8) 2428 eqd = 0; 2429 2430 /* Make sure that the eq delay is in the known range */ 2431 eqd = min(eqd, aic->max_eqd); 2432 eqd = max(eqd, aic->min_eqd); 2433 2434 aic->prev_rxpkts = rxpkts; 2435 aic->prev_txreqs = tx_reqs; 2436 aic->ticks = now; 2437 2438modify_eqd: 2439 if (eqd != aic->cur_eqd) { 2440 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2441 set_eqd[num].eq_id = eqo->eq_id; 2442 aic->cur_eqd = eqd; 2443 num++; 2444 } 2445 } 2446 2447 /* Is there atleast one eq that needs to be modified? */ 2448 for(i = 0; i < num; i += 8) { 2449 if((num - i) >=8 ) 2450 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8); 2451 else 2452 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i)); 2453 } 2454 2455} 2456 2457static void oce_detect_hw_error(POCE_SOFTC sc) 2458{ 2459 2460 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0; 2461 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2462 uint32_t i; 2463 2464 if (sc->hw_error) 2465 return; 2466 2467 if (IS_XE201(sc)) { 2468 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET); 2469 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2470 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET); 2471 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET); 2472 } 2473 } else { 2474 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW); 2475 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH); 2476 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK); 2477 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK); 2478 2479 ue_low = (ue_low & ~ue_low_mask); 2480 ue_high = (ue_high & ~ue_high_mask); 2481 } 2482 2483 /* On certain platforms BE hardware can indicate spurious UEs. 2484 * Allow the h/w to stop working completely in case of a real UE. 2485 * Hence not setting the hw_error for UE detection. 2486 */ 2487 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2488 sc->hw_error = TRUE; 2489 device_printf(sc->dev, "Error detected in the card\n"); 2490 } 2491 2492 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2493 device_printf(sc->dev, 2494 "ERR: sliport status 0x%x\n", sliport_status); 2495 device_printf(sc->dev, 2496 "ERR: sliport error1 0x%x\n", sliport_err1); 2497 device_printf(sc->dev, 2498 "ERR: sliport error2 0x%x\n", sliport_err2); 2499 } 2500 2501 if (ue_low) { 2502 for (i = 0; ue_low; ue_low >>= 1, i++) { 2503 if (ue_low & 1) 2504 device_printf(sc->dev, "UE: %s bit set\n", 2505 ue_status_low_desc[i]); 2506 } 2507 } 2508 2509 if (ue_high) { 2510 for (i = 0; ue_high; ue_high >>= 1, i++) { 2511 if (ue_high & 1) 2512 device_printf(sc->dev, "UE: %s bit set\n", 2513 ue_status_hi_desc[i]); 2514 } 2515 } 2516 2517} 2518 2519 2520static void 2521oce_local_timer(void *arg) 2522{ 2523 POCE_SOFTC sc = arg; 2524 int i = 0; 2525 2526 oce_detect_hw_error(sc); 2527 oce_refresh_nic_stats(sc); 2528 oce_refresh_queue_stats(sc); 2529 oce_mac_addr_set(sc); 2530 2531 /* TX Watch Dog*/ 2532 for (i = 0; i < sc->nwqs; i++) 2533 oce_tx_restart(sc, sc->wq[i]); 2534 2535 /* calculate and set the eq delay for optimal interrupt rate */ 2536 if (IS_BE(sc) || IS_SH(sc)) 2537 oce_eqd_set_periodic(sc); 2538 2539 callout_reset(&sc->timer, hz, oce_local_timer, sc); 2540} 2541 2542static void 2543oce_tx_compl_clean(POCE_SOFTC sc) 2544{ 2545 struct oce_wq *wq; 2546 int i = 0, timeo = 0, num_wqes = 0; 2547 int pending_txqs = sc->nwqs; 2548 2549 /* Stop polling for compls when HW has been silent for 10ms or 2550 * hw_error or no outstanding completions expected 2551 */ 2552 do { 2553 pending_txqs = sc->nwqs; 2554 2555 for_all_wq_queues(sc, wq, i) { 2556 num_wqes = oce_wq_handler(wq); 2557 2558 if(num_wqes) 2559 timeo = 0; 2560 2561 if(!wq->ring->num_used) 2562 pending_txqs--; 2563 } 2564 2565 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error) 2566 break; 2567 2568 DELAY(1000); 2569 } while (TRUE); 2570 2571 for_all_wq_queues(sc, wq, i) { 2572 while(wq->ring->num_used) { 2573 LOCK(&wq->tx_compl_lock); 2574 oce_process_tx_completion(wq); 2575 UNLOCK(&wq->tx_compl_lock); 2576 } 2577 } 2578 2579} 2580 2581/* NOTE : This should only be called holding 2582 * DEVICE_LOCK. 2583 */ 2584static void 2585oce_if_deactivate(POCE_SOFTC sc) 2586{ 2587 int i; 2588 struct oce_rq *rq; 2589 struct oce_wq *wq; 2590 struct oce_eq *eq; 2591 2592 sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 2593 2594 oce_tx_compl_clean(sc); 2595 2596 /* Stop intrs and finish any bottom halves pending */ 2597 oce_hw_intr_disable(sc); 2598 2599 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire 2600 any other lock. So unlock device lock and require after 2601 completing taskqueue_drain. 2602 */ 2603 UNLOCK(&sc->dev_lock); 2604 for (i = 0; i < sc->intr_count; i++) { 2605 if (sc->intrs[i].tq != NULL) { 2606 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task); 2607 } 2608 } 2609 LOCK(&sc->dev_lock); 2610 2611 /* Delete RX queue in card with flush param */ 2612 oce_stop_rx(sc); 2613 2614 /* Invalidate any pending cq and eq entries*/ 2615 for_all_evnt_queues(sc, eq, i) 2616 oce_drain_eq(eq); 2617 for_all_rq_queues(sc, rq, i) 2618 oce_drain_rq_cq(rq); 2619 for_all_wq_queues(sc, wq, i) 2620 oce_drain_wq_cq(wq); 2621 2622 /* But still we need to get MCC aync events. 2623 So enable intrs and also arm first EQ 2624 */ 2625 oce_hw_intr_enable(sc); 2626 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE); 2627 2628 DELAY(10); 2629} 2630 2631 2632static void 2633oce_if_activate(POCE_SOFTC sc) 2634{ 2635 struct oce_eq *eq; 2636 struct oce_rq *rq; 2637 struct oce_wq *wq; 2638 int i, rc = 0; 2639 2640 sc->ifp->if_drv_flags |= IFF_DRV_RUNNING; 2641 2642 oce_hw_intr_disable(sc); 2643 2644 oce_start_rx(sc); 2645 2646 for_all_rq_queues(sc, rq, i) { 2647 rc = oce_start_rq(rq); 2648 if (rc) 2649 device_printf(sc->dev, "Unable to start RX\n"); 2650 } 2651 2652 for_all_wq_queues(sc, wq, i) { 2653 rc = oce_start_wq(wq); 2654 if (rc) 2655 device_printf(sc->dev, "Unable to start TX\n"); 2656 } 2657 2658 2659 for_all_evnt_queues(sc, eq, i) 2660 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE); 2661 2662 oce_hw_intr_enable(sc); 2663 2664} 2665 2666static void 2667process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe) 2668{ 2669 /* Update Link status */ 2670 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) == 2671 ASYNC_EVENT_LINK_UP) { 2672 sc->link_status = ASYNC_EVENT_LINK_UP; 2673 if_link_state_change(sc->ifp, LINK_STATE_UP); 2674 } else { 2675 sc->link_status = ASYNC_EVENT_LINK_DOWN; 2676 if_link_state_change(sc->ifp, LINK_STATE_DOWN); 2677 } 2678} 2679 2680 2681static void oce_async_grp5_osbmc_process(POCE_SOFTC sc, 2682 struct oce_async_evt_grp5_os2bmc *evt) 2683{ 2684 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc)); 2685 if (evt->u.s.mgmt_enable) 2686 sc->flags |= OCE_FLAGS_OS2BMC; 2687 else 2688 return; 2689 2690 sc->bmc_filt_mask = evt->u.s.arp_filter; 2691 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1); 2692 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2); 2693 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3); 2694 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4); 2695 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5); 2696 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6); 2697 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7); 2698 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8); 2699} 2700 2701 2702static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe) 2703{ 2704 struct oce_async_event_grp5_pvid_state *gcqe; 2705 struct oce_async_evt_grp5_os2bmc *bmccqe; 2706 2707 switch (cqe->u0.s.async_type) { 2708 case ASYNC_EVENT_PVID_STATE: 2709 /* GRP5 PVID */ 2710 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe; 2711 if (gcqe->enabled) 2712 sc->pvid = gcqe->tag & VLAN_VID_MASK; 2713 else 2714 sc->pvid = 0; 2715 break; 2716 case ASYNC_EVENT_OS2BMC: 2717 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe; 2718 oce_async_grp5_osbmc_process(sc, bmccqe); 2719 break; 2720 default: 2721 break; 2722 } 2723} 2724 2725/* Handle the Completion Queue for the Mailbox/Async notifications */ 2726uint16_t 2727oce_mq_handler(void *arg) 2728{ 2729 struct oce_mq *mq = (struct oce_mq *)arg; 2730 POCE_SOFTC sc = mq->parent; 2731 struct oce_cq *cq = mq->cq; 2732 int num_cqes = 0, evt_type = 0, optype = 0; 2733 struct oce_mq_cqe *cqe; 2734 struct oce_async_cqe_link_state *acqe; 2735 struct oce_async_event_qnq *dbgcqe; 2736 2737 2738 bus_dmamap_sync(cq->ring->dma.tag, 2739 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2740 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2741 2742 while (cqe->u0.dw[3]) { 2743 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe)); 2744 if (cqe->u0.s.async_event) { 2745 evt_type = cqe->u0.s.event_type; 2746 optype = cqe->u0.s.async_type; 2747 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) { 2748 /* Link status evt */ 2749 acqe = (struct oce_async_cqe_link_state *)cqe; 2750 process_link_state(sc, acqe); 2751 } else if (evt_type == ASYNC_EVENT_GRP5) { 2752 oce_process_grp5_events(sc, cqe); 2753 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG && 2754 optype == ASYNC_EVENT_DEBUG_QNQ) { 2755 dbgcqe = (struct oce_async_event_qnq *)cqe; 2756 if(dbgcqe->valid) 2757 sc->qnqid = dbgcqe->vlan_tag; 2758 sc->qnq_debug_event = TRUE; 2759 } 2760 } 2761 cqe->u0.dw[3] = 0; 2762 RING_GET(cq->ring, 1); 2763 bus_dmamap_sync(cq->ring->dma.tag, 2764 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 2765 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe); 2766 num_cqes++; 2767 } 2768 2769 if (num_cqes) 2770 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 2771 2772 return 0; 2773} 2774 2775 2776static void 2777setup_max_queues_want(POCE_SOFTC sc) 2778{ 2779 /* Check if it is FLEX machine. Is so dont use RSS */ 2780 if ((sc->function_mode & FNM_FLEX10_MODE) || 2781 (sc->function_mode & FNM_UMC_MODE) || 2782 (sc->function_mode & FNM_VNIC_MODE) || 2783 (!is_rss_enabled(sc)) || 2784 IS_BE2(sc)) { 2785 sc->nrqs = 1; 2786 sc->nwqs = 1; 2787 } else { 2788 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2789 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs); 2790 } 2791 2792 if (IS_BE2(sc) && is_rss_enabled(sc)) 2793 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1; 2794} 2795 2796 2797static void 2798update_queues_got(POCE_SOFTC sc) 2799{ 2800 if (is_rss_enabled(sc)) { 2801 sc->nrqs = sc->intr_count + 1; 2802 sc->nwqs = sc->intr_count; 2803 } else { 2804 sc->nrqs = 1; 2805 sc->nwqs = 1; 2806 } 2807 2808 if (IS_BE2(sc)) 2809 sc->nwqs = 1; 2810} 2811 2812static int 2813oce_check_ipv6_ext_hdr(struct mbuf *m) 2814{ 2815 struct ether_header *eh = mtod(m, struct ether_header *); 2816 caddr_t m_datatemp = m->m_data; 2817 2818 if (eh->ether_type == htons(ETHERTYPE_IPV6)) { 2819 m->m_data += sizeof(struct ether_header); 2820 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *); 2821 2822 if((ip6->ip6_nxt != IPPROTO_TCP) && \ 2823 (ip6->ip6_nxt != IPPROTO_UDP)){ 2824 struct ip6_ext *ip6e = NULL; 2825 m->m_data += sizeof(struct ip6_hdr); 2826 2827 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *); 2828 if(ip6e->ip6e_len == 0xff) { 2829 m->m_data = m_datatemp; 2830 return TRUE; 2831 } 2832 } 2833 m->m_data = m_datatemp; 2834 } 2835 return FALSE; 2836} 2837 2838static int 2839is_be3_a1(POCE_SOFTC sc) 2840{ 2841 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) { 2842 return TRUE; 2843 } 2844 return FALSE; 2845} 2846 2847static struct mbuf * 2848oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete) 2849{ 2850 uint16_t vlan_tag = 0; 2851 2852 if(!M_WRITABLE(m)) 2853 return NULL; 2854 2855 /* Embed vlan tag in the packet if it is not part of it */ 2856 if(m->m_flags & M_VLANTAG) { 2857 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag); 2858 m->m_flags &= ~M_VLANTAG; 2859 } 2860 2861 /* if UMC, ignore vlan tag insertion and instead insert pvid */ 2862 if(sc->pvid) { 2863 if(!vlan_tag) 2864 vlan_tag = sc->pvid; 2865 if (complete) 2866 *complete = FALSE; 2867 } 2868 2869 if(vlan_tag) { 2870 m = ether_vlanencap(m, vlan_tag); 2871 } 2872 2873 if(sc->qnqid) { 2874 m = ether_vlanencap(m, sc->qnqid); 2875 2876 if (complete) 2877 *complete = FALSE; 2878 } 2879 return m; 2880} 2881 2882static int 2883oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m) 2884{ 2885 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \ 2886 oce_check_ipv6_ext_hdr(m)) { 2887 return TRUE; 2888 } 2889 return FALSE; 2890} 2891 2892static void 2893oce_get_config(POCE_SOFTC sc) 2894{ 2895 int rc = 0; 2896 uint32_t max_rss = 0; 2897 2898 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native)) 2899 max_rss = OCE_LEGACY_MODE_RSS; 2900 else 2901 max_rss = OCE_MAX_RSS; 2902 2903 if (!IS_BE(sc)) { 2904 rc = oce_get_profile_config(sc, max_rss); 2905 if (rc) { 2906 sc->nwqs = OCE_MAX_WQ; 2907 sc->nrssqs = max_rss; 2908 sc->nrqs = sc->nrssqs + 1; 2909 } 2910 } 2911 else { /* For BE3 don't rely on fw for determining the resources */ 2912 sc->nrssqs = max_rss; 2913 sc->nrqs = sc->nrssqs + 1; 2914 sc->nwqs = OCE_MAX_WQ; 2915 sc->max_vlans = MAX_VLANFILTER_SIZE; 2916 } 2917} 2918 2919static void 2920oce_rdma_close(void) 2921{ 2922 if (oce_rdma_if != NULL) { 2923 oce_rdma_if = NULL; 2924 } 2925} 2926 2927static void 2928oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr) 2929{ 2930 memcpy(macaddr, sc->macaddr.mac_addr, 6); 2931} 2932 2933int 2934oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if) 2935{ 2936 POCE_SOFTC sc; 2937 struct oce_dev_info di; 2938 int i; 2939 2940 if ((rdma_info == NULL) || (rdma_if == NULL)) { 2941 return -EINVAL; 2942 } 2943 2944 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) || 2945 (rdma_if->size != OCE_RDMA_IF_SIZE)) { 2946 return -ENXIO; 2947 } 2948 2949 rdma_info->close = oce_rdma_close; 2950 rdma_info->mbox_post = oce_mbox_post; 2951 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init; 2952 rdma_info->get_mac_addr = oce_get_mac_addr; 2953 2954 oce_rdma_if = rdma_if; 2955 2956 sc = softc_head; 2957 while (sc != NULL) { 2958 if (oce_rdma_if->announce != NULL) { 2959 memset(&di, 0, sizeof(di)); 2960 di.dev = sc->dev; 2961 di.softc = sc; 2962 di.ifp = sc->ifp; 2963 di.db_bhandle = sc->db_bhandle; 2964 di.db_btag = sc->db_btag; 2965 di.db_page_size = 4096; 2966 if (sc->flags & OCE_FLAGS_USING_MSIX) { 2967 di.intr_mode = OCE_INTERRUPT_MODE_MSIX; 2968 } else if (sc->flags & OCE_FLAGS_USING_MSI) { 2969 di.intr_mode = OCE_INTERRUPT_MODE_MSI; 2970 } else { 2971 di.intr_mode = OCE_INTERRUPT_MODE_INTX; 2972 } 2973 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk 2974 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) { 2975 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count; 2976 di.msix.start_vector = sc->intr_count; 2977 for (i=0; i<di.msix.num_vectors; i++) { 2978 di.msix.vector_list[i] = sc->intrs[i].vector; 2979 } 2980 } else { 2981 } 2982 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6); 2983 di.vendor_id = pci_get_vendor(sc->dev); 2984 di.dev_id = pci_get_device(sc->dev); 2985 2986 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) { 2987 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED; 2988 } 2989 2990 rdma_if->announce(&di); 2991 sc = sc->next; 2992 } 2993 } 2994 2995 return 0; 2996} 2997 2998static void 2999oce_read_env_variables( POCE_SOFTC sc ) 3000{ 3001 char *value = NULL; 3002 int rc = 0; 3003 3004 /* read if user wants to enable hwlro or swlro */ 3005 //value = getenv("oce_enable_hwlro"); 3006 if(value && IS_SH(sc)) { 3007 sc->enable_hwlro = strtol(value, NULL, 10); 3008 if(sc->enable_hwlro) { 3009 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL); 3010 if(rc) { 3011 device_printf(sc->dev, "no hardware lro support\n"); 3012 device_printf(sc->dev, "software lro enabled\n"); 3013 sc->enable_hwlro = 0; 3014 }else { 3015 device_printf(sc->dev, "hardware lro enabled\n"); 3016 oce_max_rsp_handled = 32; 3017 } 3018 }else { 3019 device_printf(sc->dev, "software lro enabled\n"); 3020 } 3021 }else { 3022 sc->enable_hwlro = 0; 3023 } 3024 3025 /* read mbuf size */ 3026 //value = getenv("oce_rq_buf_size"); 3027 if(value && IS_SH(sc)) { 3028 oce_rq_buf_size = strtol(value, NULL, 10); 3029 switch(oce_rq_buf_size) { 3030 case 2048: 3031 case 4096: 3032 case 9216: 3033 case 16384: 3034 break; 3035 3036 default: 3037 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n"); 3038 oce_rq_buf_size = 2048; 3039 } 3040 } 3041 3042 return; 3043} 3044