ena.c revision 361468
11553Srgrimes/*- 21553Srgrimes * BSD LICENSE 31553Srgrimes * 41553Srgrimes * Copyright (c) 2015-2019 Amazon.com, Inc. or its affiliates. 51553Srgrimes * All rights reserved. 61553Srgrimes * 71553Srgrimes * Redistribution and use in source and binary forms, with or without 81553Srgrimes * modification, are permitted provided that the following conditions 91553Srgrimes * are met: 101553Srgrimes * 111553Srgrimes * 1. Redistributions of source code must retain the above copyright 121553Srgrimes * notice, this list of conditions and the following disclaimer. 131553Srgrimes * 141553Srgrimes * 2. Redistributions in binary form must reproduce the above copyright 151553Srgrimes * notice, this list of conditions and the following disclaimer in the 161553Srgrimes * documentation and/or other materials provided with the distribution. 171553Srgrimes * 181553Srgrimes * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 191553Srgrimes * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 201553Srgrimes * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 211553Srgrimes * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 221553Srgrimes * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 231553Srgrimes * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 241553Srgrimes * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 251553Srgrimes * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 261553Srgrimes * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 271553Srgrimes * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 281553Srgrimes * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 291553Srgrimes */ 301553Srgrimes#include <sys/cdefs.h> 311553Srgrimes__FBSDID("$FreeBSD: stable/11/sys/dev/ena/ena.c 361468 2020-05-25 17:47:31Z mw $"); 321553Srgrimes 331553Srgrimes#include <sys/param.h> 341553Srgrimes#include <sys/systm.h> 351553Srgrimes#include <sys/bus.h> 361553Srgrimes#include <sys/endian.h> 371553Srgrimes#include <sys/kernel.h> 381553Srgrimes#include <sys/kthread.h> 391553Srgrimes#include <sys/malloc.h> 401553Srgrimes#include <sys/mbuf.h> 411553Srgrimes#include <sys/module.h> 421553Srgrimes#include <sys/rman.h> 431553Srgrimes#include <sys/smp.h> 441553Srgrimes#include <sys/socket.h> 451553Srgrimes#include <sys/sockio.h> 461553Srgrimes#include <sys/sysctl.h> 471553Srgrimes#include <sys/taskqueue.h> 481553Srgrimes#include <sys/time.h> 491553Srgrimes#include <sys/eventhandler.h> 501553Srgrimes 511553Srgrimes#include <machine/bus.h> 521553Srgrimes#include <machine/resource.h> 531553Srgrimes#include <machine/in_cksum.h> 541553Srgrimes 551553Srgrimes#include <net/bpf.h> 561553Srgrimes#include <net/ethernet.h> 571553Srgrimes#include <net/if.h> 581553Srgrimes#include <net/if_var.h> 591553Srgrimes#include <net/if_arp.h> 601553Srgrimes#include <net/if_dl.h> 611553Srgrimes#include <net/if_media.h> 621553Srgrimes#include <net/if_types.h> 631553Srgrimes#include <net/if_vlan_var.h> 641553Srgrimes 651553Srgrimes#include <netinet/in_systm.h> 661553Srgrimes#include <netinet/in.h> 671553Srgrimes#include <netinet/if_ether.h> 681553Srgrimes#include <netinet/ip.h> 691553Srgrimes#include <netinet/ip6.h> 701553Srgrimes#include <netinet/tcp.h> 711553Srgrimes#include <netinet/udp.h> 721553Srgrimes 731553Srgrimes#include <dev/pci/pcivar.h> 741553Srgrimes#include <dev/pci/pcireg.h> 751553Srgrimes 761553Srgrimes#include <vm/vm.h> 771553Srgrimes#include <vm/pmap.h> 781553Srgrimes 791553Srgrimes#include "ena_datapath.h" 801553Srgrimes#include "ena.h" 811553Srgrimes#include "ena_sysctl.h" 821553Srgrimes 831553Srgrimes#ifdef DEV_NETMAP 841553Srgrimes#include "ena_netmap.h" 851553Srgrimes#endif /* DEV_NETMAP */ 861553Srgrimes 871553Srgrimes/********************************************************* 881553Srgrimes * Function prototypes 891553Srgrimes *********************************************************/ 901553Srgrimesstatic int ena_probe(device_t); 911553Srgrimesstatic void ena_intr_msix_mgmnt(void *); 921553Srgrimesstatic void ena_free_pci_resources(struct ena_adapter *); 931553Srgrimesstatic int ena_change_mtu(if_t, int); 941553Srgrimesstatic inline void ena_alloc_counters(counter_u64_t *, int); 951553Srgrimesstatic inline void ena_free_counters(counter_u64_t *, int); 961553Srgrimesstatic inline void ena_reset_counters(counter_u64_t *, int); 971553Srgrimesstatic void ena_init_io_rings_common(struct ena_adapter *, 981553Srgrimes struct ena_ring *, uint16_t); 991553Srgrimesstatic void ena_init_io_rings(struct ena_adapter *); 1001553Srgrimesstatic void ena_free_io_ring_resources(struct ena_adapter *, unsigned int); 1011553Srgrimesstatic void ena_free_all_io_rings_resources(struct ena_adapter *); 1021553Srgrimesstatic int ena_setup_tx_dma_tag(struct ena_adapter *); 1031553Srgrimesstatic int ena_free_tx_dma_tag(struct ena_adapter *); 1041553Srgrimesstatic int ena_setup_rx_dma_tag(struct ena_adapter *); 1051553Srgrimesstatic int ena_free_rx_dma_tag(struct ena_adapter *); 1061553Srgrimesstatic void ena_release_all_tx_dmamap(struct ena_ring *); 1071553Srgrimesstatic int ena_setup_tx_resources(struct ena_adapter *, int); 1081553Srgrimesstatic void ena_free_tx_resources(struct ena_adapter *, int); 1091553Srgrimesstatic int ena_setup_all_tx_resources(struct ena_adapter *); 1101553Srgrimesstatic void ena_free_all_tx_resources(struct ena_adapter *); 1111553Srgrimesstatic int ena_setup_rx_resources(struct ena_adapter *, unsigned int); 1121553Srgrimesstatic void ena_free_rx_resources(struct ena_adapter *, unsigned int); 1131553Srgrimesstatic int ena_setup_all_rx_resources(struct ena_adapter *); 1141553Srgrimesstatic void ena_free_all_rx_resources(struct ena_adapter *); 1151553Srgrimesstatic inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *, 1161553Srgrimes struct ena_rx_buffer *); 1171553Srgrimesstatic void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *, 1181553Srgrimes struct ena_rx_buffer *); 1191553Srgrimesstatic void ena_free_rx_bufs(struct ena_adapter *, unsigned int); 1201553Srgrimesstatic void ena_refill_all_rx_bufs(struct ena_adapter *); 1211553Srgrimesstatic void ena_free_all_rx_bufs(struct ena_adapter *); 1221553Srgrimesstatic void ena_free_tx_bufs(struct ena_adapter *, unsigned int); 1231553Srgrimesstatic void ena_free_all_tx_bufs(struct ena_adapter *); 1241553Srgrimesstatic void ena_destroy_all_tx_queues(struct ena_adapter *); 1251553Srgrimesstatic void ena_destroy_all_rx_queues(struct ena_adapter *); 1261553Srgrimesstatic void ena_destroy_all_io_queues(struct ena_adapter *); 1271553Srgrimesstatic int ena_create_io_queues(struct ena_adapter *); 1281553Srgrimesstatic int ena_handle_msix(void *); 1291553Srgrimesstatic int ena_enable_msix(struct ena_adapter *); 1301553Srgrimesstatic void ena_setup_mgmnt_intr(struct ena_adapter *); 1311553Srgrimesstatic int ena_setup_io_intr(struct ena_adapter *); 1321553Srgrimesstatic int ena_request_mgmnt_irq(struct ena_adapter *); 1331553Srgrimesstatic int ena_request_io_irq(struct ena_adapter *); 1341553Srgrimesstatic void ena_free_mgmnt_irq(struct ena_adapter *); 1351553Srgrimesstatic void ena_free_io_irq(struct ena_adapter *); 1361553Srgrimesstatic void ena_free_irqs(struct ena_adapter*); 1371553Srgrimesstatic void ena_disable_msix(struct ena_adapter *); 1381553Srgrimesstatic void ena_unmask_all_io_irqs(struct ena_adapter *); 1391553Srgrimesstatic int ena_rss_configure(struct ena_adapter *); 1401553Srgrimesstatic int ena_up_complete(struct ena_adapter *); 1411553Srgrimesstatic uint64_t ena_get_counter(if_t, ift_counter); 1421553Srgrimesstatic int ena_media_change(if_t); 1431553Srgrimesstatic void ena_media_status(if_t, struct ifmediareq *); 1441553Srgrimesstatic void ena_init(void *); 1451553Srgrimesstatic int ena_ioctl(if_t, u_long, caddr_t); 1461553Srgrimesstatic int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *); 1471553Srgrimesstatic void ena_update_host_info(struct ena_admin_host_info *, if_t); 1481553Srgrimesstatic void ena_update_hwassist(struct ena_adapter *); 1491553Srgrimesstatic int ena_setup_ifnet(device_t, struct ena_adapter *, 1501553Srgrimes struct ena_com_dev_get_features_ctx *); 1511553Srgrimesstatic int ena_enable_wc(struct resource *); 1521553Srgrimesstatic int ena_set_queues_placement_policy(device_t, struct ena_com_dev *, 1531553Srgrimes struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *); 1541553Srgrimesstatic int ena_calc_io_queue_num(struct ena_adapter *, 1551553Srgrimes struct ena_com_dev_get_features_ctx *); 1561553Srgrimesstatic int ena_calc_queue_size(struct ena_adapter *, 1571553Srgrimes struct ena_calc_queue_size_ctx *); 1581553Srgrimesstatic int ena_handle_updated_queues(struct ena_adapter *, 1591553Srgrimes struct ena_com_dev_get_features_ctx *); 1601553Srgrimesstatic int ena_rss_init_default(struct ena_adapter *); 1611553Srgrimesstatic void ena_rss_init_default_deferred(void *); 1621553Srgrimesstatic void ena_config_host_info(struct ena_com_dev *, device_t); 1631553Srgrimesstatic int ena_attach(device_t); 1641553Srgrimesstatic int ena_detach(device_t); 1651553Srgrimesstatic int ena_device_init(struct ena_adapter *, device_t, 1661553Srgrimes struct ena_com_dev_get_features_ctx *, int *); 1671553Srgrimesstatic int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *, 1681553Srgrimes int); 1691553Srgrimesstatic void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *); 1701553Srgrimesstatic void unimplemented_aenq_handler(void *, 1711553Srgrimes struct ena_admin_aenq_entry *); 1721553Srgrimesstatic void ena_timer_service(void *); 1731553Srgrimes 1741553Srgrimesstatic char ena_version[] = DEVICE_NAME DRV_MODULE_NAME " v" DRV_MODULE_VERSION; 1751553Srgrimes 1761553Srgrimesstatic ena_vendor_info_t ena_vendor_info_array[] = { 1771553Srgrimes { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0}, 1781553Srgrimes { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_PF, 0}, 1791553Srgrimes { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0}, 1801553Srgrimes { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_VF, 0}, 1811553Srgrimes /* Last entry */ 1821553Srgrimes { 0, 0, 0 } 1831553Srgrimes}; 1841553Srgrimes 1851553Srgrimes/* 1861553Srgrimes * Contains pointers to event handlers, e.g. link state chage. 1871553Srgrimes */ 1881553Srgrimesstatic struct ena_aenq_handlers aenq_handlers; 1891553Srgrimes 1901553Srgrimesvoid 1911553Srgrimesena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) 1921553Srgrimes{ 1931553Srgrimes if (error != 0) 1941553Srgrimes return; 1951553Srgrimes *(bus_addr_t *) arg = segs[0].ds_addr; 1961553Srgrimes} 1971553Srgrimes 1981553Srgrimesint 1991553Srgrimesena_dma_alloc(device_t dmadev, bus_size_t size, 2001553Srgrimes ena_mem_handle_t *dma , int mapflags) 2011553Srgrimes{ 2021553Srgrimes struct ena_adapter* adapter = device_get_softc(dmadev); 2031553Srgrimes uint32_t maxsize; 2041553Srgrimes uint64_t dma_space_addr; 2051553Srgrimes int error; 2061553Srgrimes 2071553Srgrimes maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE; 2081553Srgrimes 2091553Srgrimes dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width); 2101553Srgrimes if (unlikely(dma_space_addr == 0)) 2111553Srgrimes dma_space_addr = BUS_SPACE_MAXADDR; 2121553Srgrimes 2131553Srgrimes error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */ 2141553Srgrimes 8, 0, /* alignment, bounds */ 2151553Srgrimes dma_space_addr, /* lowaddr of exclusion window */ 2161553Srgrimes BUS_SPACE_MAXADDR,/* highaddr of exclusion window */ 2171553Srgrimes NULL, NULL, /* filter, filterarg */ 2181553Srgrimes maxsize, /* maxsize */ 2191553Srgrimes 1, /* nsegments */ 2201553Srgrimes maxsize, /* maxsegsize */ 2211553Srgrimes BUS_DMA_ALLOCNOW, /* flags */ 2221553Srgrimes NULL, /* lockfunc */ 2231553Srgrimes NULL, /* lockarg */ 2241553Srgrimes &dma->tag); 2251553Srgrimes if (unlikely(error != 0)) { 2261553Srgrimes ena_trace(ENA_ALERT, "bus_dma_tag_create failed: %d\n", error); 2271553Srgrimes goto fail_tag; 2281553Srgrimes } 2291553Srgrimes 2301553Srgrimes error = bus_dmamem_alloc(dma->tag, (void**) &dma->vaddr, 2311553Srgrimes BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map); 2321553Srgrimes if (unlikely(error != 0)) { 2331553Srgrimes ena_trace(ENA_ALERT, "bus_dmamem_alloc(%ju) failed: %d\n", 2341553Srgrimes (uintmax_t)size, error); 2351553Srgrimes goto fail_map_create; 2361553Srgrimes } 2371553Srgrimes 2381553Srgrimes dma->paddr = 0; 2391553Srgrimes error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, 2401553Srgrimes size, ena_dmamap_callback, &dma->paddr, mapflags); 2411553Srgrimes if (unlikely((error != 0) || (dma->paddr == 0))) { 2421553Srgrimes ena_trace(ENA_ALERT, ": bus_dmamap_load failed: %d\n", error); 2431553Srgrimes goto fail_map_load; 2441553Srgrimes } 2451553Srgrimes 2461553Srgrimes bus_dmamap_sync(dma->tag, dma->map, 2471553Srgrimes BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 2481553Srgrimes 2491553Srgrimes return (0); 2501553Srgrimes 2511553Srgrimesfail_map_load: 2521553Srgrimes bus_dmamem_free(dma->tag, dma->vaddr, dma->map); 2531553Srgrimesfail_map_create: 2541553Srgrimes bus_dma_tag_destroy(dma->tag); 2551553Srgrimesfail_tag: 2561553Srgrimes dma->tag = NULL; 2571553Srgrimes dma->vaddr = NULL; 2581553Srgrimes dma->paddr = 0; 2591553Srgrimes 2601553Srgrimes return (error); 2611553Srgrimes} 2621553Srgrimes 2631553Srgrimesstatic void 2641553Srgrimesena_free_pci_resources(struct ena_adapter *adapter) 2651553Srgrimes{ 2661553Srgrimes device_t pdev = adapter->pdev; 2671553Srgrimes 2681553Srgrimes if (adapter->memory != NULL) { 2691553Srgrimes bus_release_resource(pdev, SYS_RES_MEMORY, 2701553Srgrimes PCIR_BAR(ENA_MEM_BAR), adapter->memory); 2711553Srgrimes } 2721553Srgrimes 2731553Srgrimes if (adapter->registers != NULL) { 2741553Srgrimes bus_release_resource(pdev, SYS_RES_MEMORY, 2751553Srgrimes PCIR_BAR(ENA_REG_BAR), adapter->registers); 2761553Srgrimes } 2771553Srgrimes} 2781553Srgrimes 2791553Srgrimesstatic int 2801553Srgrimesena_probe(device_t dev) 2811553Srgrimes{ 2821553Srgrimes ena_vendor_info_t *ent; 2831553Srgrimes char adapter_name[60]; 2841553Srgrimes uint16_t pci_vendor_id = 0; 2851553Srgrimes uint16_t pci_device_id = 0; 2861553Srgrimes 2871553Srgrimes pci_vendor_id = pci_get_vendor(dev); 2881553Srgrimes pci_device_id = pci_get_device(dev); 2891553Srgrimes 2901553Srgrimes ent = ena_vendor_info_array; 2911553Srgrimes while (ent->vendor_id != 0) { 2921553Srgrimes if ((pci_vendor_id == ent->vendor_id) && 2931553Srgrimes (pci_device_id == ent->device_id)) { 2941553Srgrimes ena_trace(ENA_DBG, "vendor=%x device=%x\n", 2951553Srgrimes pci_vendor_id, pci_device_id); 2961553Srgrimes 2971553Srgrimes sprintf(adapter_name, DEVICE_DESC); 2981553Srgrimes device_set_desc_copy(dev, adapter_name); 2991553Srgrimes return (BUS_PROBE_DEFAULT); 3001553Srgrimes } 3011553Srgrimes 3021553Srgrimes ent++; 3031553Srgrimes 3041553Srgrimes } 3051553Srgrimes 3061553Srgrimes return (ENXIO); 3071553Srgrimes} 3081553Srgrimes 3091553Srgrimesstatic int 3101553Srgrimesena_change_mtu(if_t ifp, int new_mtu) 3111553Srgrimes{ 3121553Srgrimes struct ena_adapter *adapter = if_getsoftc(ifp); 3131553Srgrimes int rc; 3141553Srgrimes 3151553Srgrimes if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) { 3161553Srgrimes device_printf(adapter->pdev, "Invalid MTU setting. " 3171553Srgrimes "new_mtu: %d max mtu: %d min mtu: %d\n", 3181553Srgrimes new_mtu, adapter->max_mtu, ENA_MIN_MTU); 3191553Srgrimes return (EINVAL); 3201553Srgrimes } 3211553Srgrimes 3221553Srgrimes rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); 3231553Srgrimes if (likely(rc == 0)) { 3241553Srgrimes ena_trace(ENA_DBG, "set MTU to %d\n", new_mtu); 3251553Srgrimes if_setmtu(ifp, new_mtu); 3261553Srgrimes } else { 3271553Srgrimes device_printf(adapter->pdev, "Failed to set MTU to %d\n", 3281553Srgrimes new_mtu); 3291553Srgrimes } 3301553Srgrimes 3311553Srgrimes return (rc); 3321553Srgrimes} 3331553Srgrimes 3341553Srgrimesstatic inline void 3351553Srgrimesena_alloc_counters(counter_u64_t *begin, int size) 3361553Srgrimes{ 3371553Srgrimes counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 3381553Srgrimes 3391553Srgrimes for (; begin < end; ++begin) 3401553Srgrimes *begin = counter_u64_alloc(M_WAITOK); 3411553Srgrimes} 3421553Srgrimes 3431553Srgrimesstatic inline void 3441553Srgrimesena_free_counters(counter_u64_t *begin, int size) 3451553Srgrimes{ 3461553Srgrimes counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 3471553Srgrimes 3481553Srgrimes for (; begin < end; ++begin) 3491553Srgrimes counter_u64_free(*begin); 3501553Srgrimes} 3511553Srgrimes 3521553Srgrimesstatic inline void 3531553Srgrimesena_reset_counters(counter_u64_t *begin, int size) 3541553Srgrimes{ 3551553Srgrimes counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 3561553Srgrimes 3571553Srgrimes for (; begin < end; ++begin) 3581553Srgrimes counter_u64_zero(*begin); 3591553Srgrimes} 3601553Srgrimes 3611553Srgrimesstatic void 3621553Srgrimesena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring, 3631553Srgrimes uint16_t qid) 3641553Srgrimes{ 3651553Srgrimes 3661553Srgrimes ring->qid = qid; 3671553Srgrimes ring->adapter = adapter; 3681553Srgrimes ring->ena_dev = adapter->ena_dev; 3691553Srgrimes ring->first_interrupt = false; 3701553Srgrimes ring->no_interrupt_event_cnt = 0; 3711553Srgrimes ring->rx_mbuf_sz = ena_mbuf_sz; 3721553Srgrimes} 3731553Srgrimes 3741553Srgrimesstatic void 3751553Srgrimesena_init_io_rings(struct ena_adapter *adapter) 3761553Srgrimes{ 3771553Srgrimes struct ena_com_dev *ena_dev; 3781553Srgrimes struct ena_ring *txr, *rxr; 3791553Srgrimes struct ena_que *que; 3801553Srgrimes int i; 3811553Srgrimes 3821553Srgrimes ena_dev = adapter->ena_dev; 3831553Srgrimes 3841553Srgrimes for (i = 0; i < adapter->num_queues; i++) { 3851553Srgrimes txr = &adapter->tx_ring[i]; 3861553Srgrimes rxr = &adapter->rx_ring[i]; 3871553Srgrimes 3881553Srgrimes /* TX/RX common ring state */ 3891553Srgrimes ena_init_io_rings_common(adapter, txr, i); 3901553Srgrimes ena_init_io_rings_common(adapter, rxr, i); 3911553Srgrimes 3921553Srgrimes /* TX specific ring state */ 3931553Srgrimes txr->ring_size = adapter->tx_ring_size; 3941553Srgrimes txr->tx_max_header_size = ena_dev->tx_max_header_size; 3951553Srgrimes txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type; 3961553Srgrimes txr->smoothed_interval = 3971553Srgrimes ena_com_get_nonadaptive_moderation_interval_tx(ena_dev); 3981553Srgrimes 3991553Srgrimes /* Allocate a buf ring */ 4001553Srgrimes txr->buf_ring_size = adapter->buf_ring_size; 4011553Srgrimes txr->br = buf_ring_alloc(txr->buf_ring_size, M_DEVBUF, 4021553Srgrimes M_WAITOK, &txr->ring_mtx); 4031553Srgrimes 4041553Srgrimes /* Alloc TX statistics. */ 4051553Srgrimes ena_alloc_counters((counter_u64_t *)&txr->tx_stats, 4061553Srgrimes sizeof(txr->tx_stats)); 4071553Srgrimes 4081553Srgrimes /* RX specific ring state */ 4091553Srgrimes rxr->ring_size = adapter->rx_ring_size; 4101553Srgrimes rxr->smoothed_interval = 4111553Srgrimes ena_com_get_nonadaptive_moderation_interval_rx(ena_dev); 4121553Srgrimes 4131553Srgrimes /* Alloc RX statistics. */ 4141553Srgrimes ena_alloc_counters((counter_u64_t *)&rxr->rx_stats, 4151553Srgrimes sizeof(rxr->rx_stats)); 4161553Srgrimes 4171553Srgrimes /* Initialize locks */ 4181553Srgrimes snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)", 4191553Srgrimes device_get_nameunit(adapter->pdev), i); 4201553Srgrimes snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)", 4211553Srgrimes device_get_nameunit(adapter->pdev), i); 4221553Srgrimes 4231553Srgrimes mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF); 4241553Srgrimes 4251553Srgrimes que = &adapter->que[i]; 4261553Srgrimes que->adapter = adapter; 4271553Srgrimes que->id = i; 4281553Srgrimes que->tx_ring = txr; 4291553Srgrimes que->rx_ring = rxr; 4301553Srgrimes 4311553Srgrimes txr->que = que; 4321553Srgrimes rxr->que = que; 4331553Srgrimes 4341553Srgrimes rxr->empty_rx_queue = 0; 4351553Srgrimes } 4361553Srgrimes} 4371553Srgrimes 4381553Srgrimesstatic void 4391553Srgrimesena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid) 4401553Srgrimes{ 4411553Srgrimes struct ena_ring *txr = &adapter->tx_ring[qid]; 4421553Srgrimes struct ena_ring *rxr = &adapter->rx_ring[qid]; 4431553Srgrimes 4441553Srgrimes ena_free_counters((counter_u64_t *)&txr->tx_stats, 4451553Srgrimes sizeof(txr->tx_stats)); 4461553Srgrimes ena_free_counters((counter_u64_t *)&rxr->rx_stats, 4471553Srgrimes sizeof(rxr->rx_stats)); 4481553Srgrimes 4491553Srgrimes ENA_RING_MTX_LOCK(txr); 4501553Srgrimes drbr_free(txr->br, M_DEVBUF); 4511553Srgrimes ENA_RING_MTX_UNLOCK(txr); 4521553Srgrimes 4531553Srgrimes mtx_destroy(&txr->ring_mtx); 4541553Srgrimes} 4551553Srgrimes 4561553Srgrimesstatic void 4571553Srgrimesena_free_all_io_rings_resources(struct ena_adapter *adapter) 4581553Srgrimes{ 4591553Srgrimes int i; 4601553Srgrimes 4611553Srgrimes for (i = 0; i < adapter->num_queues; i++) 4621553Srgrimes ena_free_io_ring_resources(adapter, i); 4631553Srgrimes 4641553Srgrimes} 4651553Srgrimes 4661553Srgrimesstatic int 4671553Srgrimesena_setup_tx_dma_tag(struct ena_adapter *adapter) 4681553Srgrimes{ 4691553Srgrimes int ret; 4701553Srgrimes 4711553Srgrimes /* Create DMA tag for Tx buffers */ 4721553Srgrimes ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), 4731553Srgrimes 1, 0, /* alignment, bounds */ 4741553Srgrimes ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ 4751553Srgrimes BUS_SPACE_MAXADDR, /* highaddr of excl window */ 4761553Srgrimes NULL, NULL, /* filter, filterarg */ 4771553Srgrimes ENA_TSO_MAXSIZE, /* maxsize */ 4781553Srgrimes adapter->max_tx_sgl_size - 1, /* nsegments */ 4791553Srgrimes ENA_TSO_MAXSIZE, /* maxsegsize */ 4801553Srgrimes 0, /* flags */ 4811553Srgrimes NULL, /* lockfunc */ 4821553Srgrimes NULL, /* lockfuncarg */ 4831553Srgrimes &adapter->tx_buf_tag); 4841553Srgrimes 4851553Srgrimes return (ret); 4861553Srgrimes} 4871553Srgrimes 4881553Srgrimesstatic int 4891553Srgrimesena_free_tx_dma_tag(struct ena_adapter *adapter) 4901553Srgrimes{ 4911553Srgrimes int ret; 4921553Srgrimes 4931553Srgrimes ret = bus_dma_tag_destroy(adapter->tx_buf_tag); 4941553Srgrimes 4951553Srgrimes if (likely(ret == 0)) 4961553Srgrimes adapter->tx_buf_tag = NULL; 4971553Srgrimes 4981553Srgrimes return (ret); 4991553Srgrimes} 5001553Srgrimes 5011553Srgrimesstatic int 5021553Srgrimesena_setup_rx_dma_tag(struct ena_adapter *adapter) 5031553Srgrimes{ 5041553Srgrimes int ret; 5051553Srgrimes 5061553Srgrimes /* Create DMA tag for Rx buffers*/ 5071553Srgrimes ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */ 5081553Srgrimes 1, 0, /* alignment, bounds */ 5091553Srgrimes ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ 5101553Srgrimes BUS_SPACE_MAXADDR, /* highaddr of excl window */ 5111553Srgrimes NULL, NULL, /* filter, filterarg */ 5121553Srgrimes ena_mbuf_sz, /* maxsize */ 5131553Srgrimes adapter->max_rx_sgl_size, /* nsegments */ 5141553Srgrimes ena_mbuf_sz, /* maxsegsize */ 5151553Srgrimes 0, /* flags */ 5161553Srgrimes NULL, /* lockfunc */ 5171553Srgrimes NULL, /* lockarg */ 5181553Srgrimes &adapter->rx_buf_tag); 5191553Srgrimes 5201553Srgrimes return (ret); 5211553Srgrimes} 5221553Srgrimes 5231553Srgrimesstatic int 5241553Srgrimesena_free_rx_dma_tag(struct ena_adapter *adapter) 5251553Srgrimes{ 5261553Srgrimes int ret; 5271553Srgrimes 5281553Srgrimes ret = bus_dma_tag_destroy(adapter->rx_buf_tag); 5291553Srgrimes 5301553Srgrimes if (likely(ret == 0)) 5311553Srgrimes adapter->rx_buf_tag = NULL; 5321553Srgrimes 5331553Srgrimes return (ret); 5341553Srgrimes} 5351553Srgrimes 5361553Srgrimesstatic void 5371553Srgrimesena_release_all_tx_dmamap(struct ena_ring *tx_ring) 5381553Srgrimes{ 5391553Srgrimes struct ena_adapter *adapter = tx_ring->adapter; 5401553Srgrimes struct ena_tx_buffer *tx_info; 5411553Srgrimes bus_dma_tag_t tx_tag = adapter->tx_buf_tag;; 5421553Srgrimes int i; 5431553Srgrimes#ifdef DEV_NETMAP 5441553Srgrimes struct ena_netmap_tx_info *nm_info; 5451553Srgrimes int j; 5461553Srgrimes#endif /* DEV_NETMAP */ 5471553Srgrimes 5481553Srgrimes for (i = 0; i < tx_ring->ring_size; ++i) { 5491553Srgrimes tx_info = &tx_ring->tx_buffer_info[i]; 5501553Srgrimes#ifdef DEV_NETMAP 5511553Srgrimes if (adapter->ifp->if_capenable & IFCAP_NETMAP) { 5521553Srgrimes nm_info = &tx_info->nm_info; 5531553Srgrimes for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) { 5541553Srgrimes if (nm_info->map_seg[j] != NULL) { 5551553Srgrimes bus_dmamap_destroy(tx_tag, 5561553Srgrimes nm_info->map_seg[j]); 5571553Srgrimes nm_info->map_seg[j] = NULL; 5581553Srgrimes } 5591553Srgrimes } 5601553Srgrimes } 5611553Srgrimes#endif /* DEV_NETMAP */ 5621553Srgrimes if (tx_info->dmamap != NULL) { 5631553Srgrimes bus_dmamap_destroy(tx_tag, tx_info->dmamap); 5641553Srgrimes tx_info->dmamap = NULL; 5651553Srgrimes } 5661553Srgrimes } 5671553Srgrimes} 5681553Srgrimes 5691553Srgrimes/** 5701553Srgrimes * ena_setup_tx_resources - allocate Tx resources (Descriptors) 5711553Srgrimes * @adapter: network interface device structure 5721553Srgrimes * @qid: queue index 5731553Srgrimes * 5741553Srgrimes * Returns 0 on success, otherwise on failure. 5751553Srgrimes **/ 5761553Srgrimesstatic int 5771553Srgrimesena_setup_tx_resources(struct ena_adapter *adapter, int qid) 5781553Srgrimes{ 5791553Srgrimes struct ena_que *que = &adapter->que[qid]; 5801553Srgrimes struct ena_ring *tx_ring = que->tx_ring; 5811553Srgrimes int size, i, err; 5821553Srgrimes#ifdef DEV_NETMAP 5831553Srgrimes bus_dmamap_t *map; 5841553Srgrimes int j; 5851553Srgrimes 5861553Srgrimes ena_netmap_reset_tx_ring(adapter, qid); 5871553Srgrimes#endif /* DEV_NETMAP */ 5881553Srgrimes 5891553Srgrimes size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size; 5901553Srgrimes 5911553Srgrimes tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 5921553Srgrimes if (unlikely(tx_ring->tx_buffer_info == NULL)) 5931553Srgrimes return (ENOMEM); 5941553Srgrimes 5951553Srgrimes size = sizeof(uint16_t) * tx_ring->ring_size; 5961553Srgrimes tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 5971553Srgrimes if (unlikely(tx_ring->free_tx_ids == NULL)) 5981553Srgrimes goto err_buf_info_free; 5991553Srgrimes 6001553Srgrimes size = tx_ring->tx_max_header_size; 6011553Srgrimes tx_ring->push_buf_intermediate_buf = malloc(size, M_DEVBUF, 6021553Srgrimes M_NOWAIT | M_ZERO); 6031553Srgrimes if (unlikely(tx_ring->push_buf_intermediate_buf == NULL)) 6041553Srgrimes goto err_tx_ids_free; 6051553Srgrimes 6061553Srgrimes /* Req id stack for TX OOO completions */ 6071553Srgrimes for (i = 0; i < tx_ring->ring_size; i++) 6081553Srgrimes tx_ring->free_tx_ids[i] = i; 6091553Srgrimes 6101553Srgrimes /* Reset TX statistics. */ 6111553Srgrimes ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats, 6121553Srgrimes sizeof(tx_ring->tx_stats)); 6131553Srgrimes 6141553Srgrimes tx_ring->next_to_use = 0; 6151553Srgrimes tx_ring->next_to_clean = 0; 6161553Srgrimes tx_ring->acum_pkts = 0; 6171553Srgrimes 6181553Srgrimes /* Make sure that drbr is empty */ 6191553Srgrimes ENA_RING_MTX_LOCK(tx_ring); 6201553Srgrimes drbr_flush(adapter->ifp, tx_ring->br); 6211553Srgrimes ENA_RING_MTX_UNLOCK(tx_ring); 6221553Srgrimes 6231553Srgrimes /* ... and create the buffer DMA maps */ 6241553Srgrimes for (i = 0; i < tx_ring->ring_size; i++) { 6251553Srgrimes err = bus_dmamap_create(adapter->tx_buf_tag, 0, 6261553Srgrimes &tx_ring->tx_buffer_info[i].dmamap); 6271553Srgrimes if (unlikely(err != 0)) { 6281553Srgrimes ena_trace(ENA_ALERT, 6291553Srgrimes "Unable to create Tx DMA map for buffer %d\n", 6301553Srgrimes i); 6311553Srgrimes goto err_map_release; 6321553Srgrimes } 6331553Srgrimes 6341553Srgrimes#ifdef DEV_NETMAP 6351553Srgrimes if (adapter->ifp->if_capenable & IFCAP_NETMAP) { 6361553Srgrimes map = tx_ring->tx_buffer_info[i].nm_info.map_seg; 6371553Srgrimes for (j = 0; j < ENA_PKT_MAX_BUFS; j++) { 6381553Srgrimes err = bus_dmamap_create(adapter->tx_buf_tag, 0, 6391553Srgrimes &map[j]); 6401553Srgrimes if (unlikely(err != 0)) { 6411553Srgrimes ena_trace(ENA_ALERT, "Unable to create " 6421553Srgrimes "Tx DMA for buffer %d %d\n", i, j); 6431553Srgrimes goto err_map_release; 6441553Srgrimes } 6451553Srgrimes } 6461553Srgrimes } 6471553Srgrimes#endif /* DEV_NETMAP */ 6481553Srgrimes } 6491553Srgrimes 6501553Srgrimes /* Allocate taskqueues */ 6511553Srgrimes TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring); 6521553Srgrimes tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT, 6531553Srgrimes taskqueue_thread_enqueue, &tx_ring->enqueue_tq); 6541553Srgrimes if (unlikely(tx_ring->enqueue_tq == NULL)) { 6551553Srgrimes ena_trace(ENA_ALERT, 6561553Srgrimes "Unable to create taskqueue for enqueue task\n"); 6571553Srgrimes i = tx_ring->ring_size; 6581553Srgrimes goto err_map_release; 6591553Srgrimes } 6601553Srgrimes 6611553Srgrimes tx_ring->running = true; 6621553Srgrimes 6631553Srgrimes taskqueue_start_threads(&tx_ring->enqueue_tq, 1, PI_NET, 6641553Srgrimes "%s txeq %d", device_get_nameunit(adapter->pdev), que->cpu); 6651553Srgrimes 6661553Srgrimes return (0); 6671553Srgrimes 6681553Srgrimeserr_map_release: 6691553Srgrimes ena_release_all_tx_dmamap(tx_ring); 6701553Srgrimeserr_tx_ids_free: 6711553Srgrimes free(tx_ring->free_tx_ids, M_DEVBUF); 6721553Srgrimes tx_ring->free_tx_ids = NULL; 6731553Srgrimeserr_buf_info_free: 6741553Srgrimes free(tx_ring->tx_buffer_info, M_DEVBUF); 6751553Srgrimes tx_ring->tx_buffer_info = NULL; 6761553Srgrimes 6771553Srgrimes return (ENOMEM); 6781553Srgrimes} 6791553Srgrimes 6801553Srgrimes/** 6811553Srgrimes * ena_free_tx_resources - Free Tx Resources per Queue 6821553Srgrimes * @adapter: network interface device structure 6831553Srgrimes * @qid: queue index 6841553Srgrimes * 6851553Srgrimes * Free all transmit software resources 6861553Srgrimes **/ 6871553Srgrimesstatic void 6881553Srgrimesena_free_tx_resources(struct ena_adapter *adapter, int qid) 6891553Srgrimes{ 6901553Srgrimes struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 6911553Srgrimes#ifdef DEV_NETMAP 6921553Srgrimes struct ena_netmap_tx_info *nm_info; 6931553Srgrimes int j; 6941553Srgrimes#endif /* DEV_NETMAP */ 6951553Srgrimes 6961553Srgrimes while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task, 6971553Srgrimes NULL)) 6981553Srgrimes taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 6991553Srgrimes 7001553Srgrimes taskqueue_free(tx_ring->enqueue_tq); 7011553Srgrimes 7021553Srgrimes ENA_RING_MTX_LOCK(tx_ring); 7031553Srgrimes /* Flush buffer ring, */ 7041553Srgrimes drbr_flush(adapter->ifp, tx_ring->br); 7051553Srgrimes 7061553Srgrimes /* Free buffer DMA maps, */ 7071553Srgrimes for (int i = 0; i < tx_ring->ring_size; i++) { 7081553Srgrimes bus_dmamap_sync(adapter->tx_buf_tag, 7091553Srgrimes tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE); 7101553Srgrimes bus_dmamap_unload(adapter->tx_buf_tag, 7111553Srgrimes tx_ring->tx_buffer_info[i].dmamap); 7121553Srgrimes bus_dmamap_destroy(adapter->tx_buf_tag, 7131553Srgrimes tx_ring->tx_buffer_info[i].dmamap); 7141553Srgrimes 7151553Srgrimes#ifdef DEV_NETMAP 7161553Srgrimes if (adapter->ifp->if_capenable & IFCAP_NETMAP) { 7171553Srgrimes nm_info = &tx_ring->tx_buffer_info[i].nm_info; 7181553Srgrimes for (j = 0; j < ENA_PKT_MAX_BUFS; j++) { 7191553Srgrimes if (nm_info->socket_buf_idx[j] != 0) { 7201553Srgrimes bus_dmamap_sync(adapter->tx_buf_tag, 7211553Srgrimes nm_info->map_seg[j], 7221553Srgrimes BUS_DMASYNC_POSTWRITE); 7231553Srgrimes ena_netmap_unload(adapter, 7241553Srgrimes nm_info->map_seg[j]); 7251553Srgrimes } 7261553Srgrimes bus_dmamap_destroy(adapter->tx_buf_tag, 7271553Srgrimes nm_info->map_seg[j]); 7281553Srgrimes nm_info->socket_buf_idx[j] = 0; 7291553Srgrimes } 7301553Srgrimes } 7311553Srgrimes#endif /* DEV_NETMAP */ 7321553Srgrimes 7331553Srgrimes m_freem(tx_ring->tx_buffer_info[i].mbuf); 7341553Srgrimes tx_ring->tx_buffer_info[i].mbuf = NULL; 7351553Srgrimes } 7361553Srgrimes ENA_RING_MTX_UNLOCK(tx_ring); 7371553Srgrimes 7381553Srgrimes /* And free allocated memory. */ 7391553Srgrimes free(tx_ring->tx_buffer_info, M_DEVBUF); 7401553Srgrimes tx_ring->tx_buffer_info = NULL; 7411553Srgrimes 7421553Srgrimes free(tx_ring->free_tx_ids, M_DEVBUF); 7431553Srgrimes tx_ring->free_tx_ids = NULL; 7441553Srgrimes 7451553Srgrimes ENA_MEM_FREE(adapter->ena_dev->dmadev, 7461553Srgrimes tx_ring->push_buf_intermediate_buf); 7471553Srgrimes tx_ring->push_buf_intermediate_buf = NULL; 7481553Srgrimes} 7491553Srgrimes 7501553Srgrimes/** 7511553Srgrimes * ena_setup_all_tx_resources - allocate all queues Tx resources 7521553Srgrimes * @adapter: network interface device structure 7531553Srgrimes * 7541553Srgrimes * Returns 0 on success, otherwise on failure. 7551553Srgrimes **/ 7561553Srgrimesstatic int 7571553Srgrimesena_setup_all_tx_resources(struct ena_adapter *adapter) 7581553Srgrimes{ 7591553Srgrimes int i, rc; 7601553Srgrimes 7611553Srgrimes for (i = 0; i < adapter->num_queues; i++) { 7621553Srgrimes rc = ena_setup_tx_resources(adapter, i); 7631553Srgrimes if (rc != 0) { 7641553Srgrimes device_printf(adapter->pdev, 7651553Srgrimes "Allocation for Tx Queue %u failed\n", i); 7661553Srgrimes goto err_setup_tx; 7671553Srgrimes } 7681553Srgrimes } 7691553Srgrimes 7701553Srgrimes return (0); 7711553Srgrimes 7721553Srgrimeserr_setup_tx: 7731553Srgrimes /* Rewind the index freeing the rings as we go */ 7741553Srgrimes while (i--) 7751553Srgrimes ena_free_tx_resources(adapter, i); 7761553Srgrimes return (rc); 7771553Srgrimes} 7781553Srgrimes 7791553Srgrimes/** 7801553Srgrimes * ena_free_all_tx_resources - Free Tx Resources for All Queues 7811553Srgrimes * @adapter: network interface device structure 7821553Srgrimes * 7831553Srgrimes * Free all transmit software resources 7841553Srgrimes **/ 7851553Srgrimesstatic void 7861553Srgrimesena_free_all_tx_resources(struct ena_adapter *adapter) 7871553Srgrimes{ 7881553Srgrimes int i; 7891553Srgrimes 7901553Srgrimes for (i = 0; i < adapter->num_queues; i++) 7911553Srgrimes ena_free_tx_resources(adapter, i); 7921553Srgrimes} 7931553Srgrimes 7941553Srgrimes/** 7951553Srgrimes * ena_setup_rx_resources - allocate Rx resources (Descriptors) 7961553Srgrimes * @adapter: network interface device structure 7971553Srgrimes * @qid: queue index 7981553Srgrimes * 7991553Srgrimes * Returns 0 on success, otherwise on failure. 8001553Srgrimes **/ 8011553Srgrimesstatic int 8021553Srgrimesena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid) 8031553Srgrimes{ 8041553Srgrimes struct ena_que *que = &adapter->que[qid]; 8051553Srgrimes struct ena_ring *rx_ring = que->rx_ring; 8061553Srgrimes int size, err, i; 8071553Srgrimes 8081553Srgrimes size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size; 8091553Srgrimes 8101553Srgrimes#ifdef DEV_NETMAP 8111553Srgrimes ena_netmap_reset_rx_ring(adapter, qid); 8121553Srgrimes rx_ring->initialized = false; 8131553Srgrimes#endif /* DEV_NETMAP */ 8141553Srgrimes 8151553Srgrimes /* 8161553Srgrimes * Alloc extra element so in rx path 8171553Srgrimes * we can always prefetch rx_info + 1 8181553Srgrimes */ 8191553Srgrimes size += sizeof(struct ena_rx_buffer); 8201553Srgrimes 8211553Srgrimes rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO); 8221553Srgrimes 8231553Srgrimes size = sizeof(uint16_t) * rx_ring->ring_size; 8241553Srgrimes rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK); 8251553Srgrimes 8261553Srgrimes for (i = 0; i < rx_ring->ring_size; i++) 8271553Srgrimes rx_ring->free_rx_ids[i] = i; 8281553Srgrimes 8291553Srgrimes /* Reset RX statistics. */ 8301553Srgrimes ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats, 8311553Srgrimes sizeof(rx_ring->rx_stats)); 8321553Srgrimes 8331553Srgrimes rx_ring->next_to_clean = 0; 8341553Srgrimes rx_ring->next_to_use = 0; 8351553Srgrimes 8361553Srgrimes /* ... and create the buffer DMA maps */ 8371553Srgrimes for (i = 0; i < rx_ring->ring_size; i++) { 8381553Srgrimes err = bus_dmamap_create(adapter->rx_buf_tag, 0, 8391553Srgrimes &(rx_ring->rx_buffer_info[i].map)); 8401553Srgrimes if (err != 0) { 8411553Srgrimes ena_trace(ENA_ALERT, 8421553Srgrimes "Unable to create Rx DMA map for buffer %d\n", i); 8431553Srgrimes goto err_buf_info_unmap; 8441553Srgrimes } 8451553Srgrimes } 8461553Srgrimes 8471553Srgrimes /* Create LRO for the ring */ 8481553Srgrimes if ((adapter->ifp->if_capenable & IFCAP_LRO) != 0) { 8491553Srgrimes int err = tcp_lro_init(&rx_ring->lro); 8501553Srgrimes if (err != 0) { 8511553Srgrimes device_printf(adapter->pdev, 8521553Srgrimes "LRO[%d] Initialization failed!\n", qid); 8531553Srgrimes } else { 8541553Srgrimes ena_trace(ENA_INFO, 8551553Srgrimes "RX Soft LRO[%d] Initialized\n", qid); 8561553Srgrimes rx_ring->lro.ifp = adapter->ifp; 8571553Srgrimes } 8581553Srgrimes } 8591553Srgrimes 8601553Srgrimes return (0); 8611553Srgrimes 8621553Srgrimeserr_buf_info_unmap: 8631553Srgrimes while (i--) { 8641553Srgrimes bus_dmamap_destroy(adapter->rx_buf_tag, 8651553Srgrimes rx_ring->rx_buffer_info[i].map); 8661553Srgrimes } 8671553Srgrimes 8681553Srgrimes free(rx_ring->free_rx_ids, M_DEVBUF); 8691553Srgrimes rx_ring->free_rx_ids = NULL; 8701553Srgrimes free(rx_ring->rx_buffer_info, M_DEVBUF); 8711553Srgrimes rx_ring->rx_buffer_info = NULL; 8721553Srgrimes return (ENOMEM); 8731553Srgrimes} 8741553Srgrimes 8751553Srgrimes/** 8761553Srgrimes * ena_free_rx_resources - Free Rx Resources 8771553Srgrimes * @adapter: network interface device structure 8781553Srgrimes * @qid: queue index 8791553Srgrimes * 8801553Srgrimes * Free all receive software resources 8811553Srgrimes **/ 8821553Srgrimesstatic void 8831553Srgrimesena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid) 8841553Srgrimes{ 8851553Srgrimes struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 8861553Srgrimes 8871553Srgrimes /* Free buffer DMA maps, */ 8881553Srgrimes for (int i = 0; i < rx_ring->ring_size; i++) { 8891553Srgrimes bus_dmamap_sync(adapter->rx_buf_tag, 8901553Srgrimes rx_ring->rx_buffer_info[i].map, BUS_DMASYNC_POSTREAD); 8911553Srgrimes m_freem(rx_ring->rx_buffer_info[i].mbuf); 8921553Srgrimes rx_ring->rx_buffer_info[i].mbuf = NULL; 8931553Srgrimes bus_dmamap_unload(adapter->rx_buf_tag, 8941553Srgrimes rx_ring->rx_buffer_info[i].map); 8951553Srgrimes bus_dmamap_destroy(adapter->rx_buf_tag, 8961553Srgrimes rx_ring->rx_buffer_info[i].map); 8971553Srgrimes } 8981553Srgrimes 8991553Srgrimes /* free LRO resources, */ 9001553Srgrimes tcp_lro_free(&rx_ring->lro); 9011553Srgrimes 9021553Srgrimes /* free allocated memory */ 9031553Srgrimes free(rx_ring->rx_buffer_info, M_DEVBUF); 9041553Srgrimes rx_ring->rx_buffer_info = NULL; 9051553Srgrimes 9061553Srgrimes free(rx_ring->free_rx_ids, M_DEVBUF); 9071553Srgrimes rx_ring->free_rx_ids = NULL; 9081553Srgrimes} 9091553Srgrimes 9101553Srgrimes/** 9111553Srgrimes * ena_setup_all_rx_resources - allocate all queues Rx resources 9121553Srgrimes * @adapter: network interface device structure 9131553Srgrimes * 9141553Srgrimes * Returns 0 on success, otherwise on failure. 9151553Srgrimes **/ 9161553Srgrimesstatic int 9171553Srgrimesena_setup_all_rx_resources(struct ena_adapter *adapter) 9181553Srgrimes{ 9191553Srgrimes int i, rc = 0; 9201553Srgrimes 9211553Srgrimes for (i = 0; i < adapter->num_queues; i++) { 9221553Srgrimes rc = ena_setup_rx_resources(adapter, i); 9231553Srgrimes if (rc != 0) { 9241553Srgrimes device_printf(adapter->pdev, 9251553Srgrimes "Allocation for Rx Queue %u failed\n", i); 9261553Srgrimes goto err_setup_rx; 9271553Srgrimes } 9281553Srgrimes } 9291553Srgrimes return (0); 9301553Srgrimes 9311553Srgrimeserr_setup_rx: 9321553Srgrimes /* rewind the index freeing the rings as we go */ 9331553Srgrimes while (i--) 9341553Srgrimes ena_free_rx_resources(adapter, i); 9351553Srgrimes return (rc); 9361553Srgrimes} 9371553Srgrimes 9381553Srgrimes/** 9391553Srgrimes * ena_free_all_rx_resources - Free Rx resources for all queues 9401553Srgrimes * @adapter: network interface device structure 9411553Srgrimes * 9421553Srgrimes * Free all receive software resources 9431553Srgrimes **/ 9441553Srgrimesstatic void 9451553Srgrimesena_free_all_rx_resources(struct ena_adapter *adapter) 9461553Srgrimes{ 9471553Srgrimes int i; 9481553Srgrimes 9491553Srgrimes for (i = 0; i < adapter->num_queues; i++) 9501553Srgrimes ena_free_rx_resources(adapter, i); 9511553Srgrimes} 9521553Srgrimes 9531553Srgrimesstatic inline int 9541553Srgrimesena_alloc_rx_mbuf(struct ena_adapter *adapter, 9551553Srgrimes struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info) 9561553Srgrimes{ 9571553Srgrimes struct ena_com_buf *ena_buf; 9581553Srgrimes bus_dma_segment_t segs[1]; 9591553Srgrimes int nsegs, error; 9601553Srgrimes int mlen; 9611553Srgrimes 9621553Srgrimes /* if previous allocated frag is not used */ 9631553Srgrimes if (unlikely(rx_info->mbuf != NULL)) 9641553Srgrimes return (0); 9651553Srgrimes 9661553Srgrimes /* Get mbuf using UMA allocator */ 9671553Srgrimes rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, 9681553Srgrimes rx_ring->rx_mbuf_sz); 9691553Srgrimes 9701553Srgrimes if (unlikely(rx_info->mbuf == NULL)) { 9711553Srgrimes counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1); 9721553Srgrimes rx_info->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 9731553Srgrimes if (unlikely(rx_info->mbuf == NULL)) { 9741553Srgrimes counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); 9751553Srgrimes return (ENOMEM); 9761553Srgrimes } 9771553Srgrimes mlen = MCLBYTES; 9781553Srgrimes } else { 9791553Srgrimes mlen = rx_ring->rx_mbuf_sz; 9801553Srgrimes } 9811553Srgrimes /* Set mbuf length*/ 9821553Srgrimes rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen; 9831553Srgrimes 9841553Srgrimes /* Map packets for DMA */ 9851553Srgrimes ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH, 9861553Srgrimes "Using tag %p for buffers' DMA mapping, mbuf %p len: %d\n", 9871553Srgrimes adapter->rx_buf_tag,rx_info->mbuf, rx_info->mbuf->m_len); 9881553Srgrimes error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map, 9891553Srgrimes rx_info->mbuf, segs, &nsegs, BUS_DMA_NOWAIT); 9901553Srgrimes if (unlikely((error != 0) || (nsegs != 1))) { 9911553Srgrimes ena_trace(ENA_WARNING, "failed to map mbuf, error: %d, " 9921553Srgrimes "nsegs: %d\n", error, nsegs); 9931553Srgrimes counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1); 9941553Srgrimes goto exit; 9951553Srgrimes 9961553Srgrimes } 9971553Srgrimes 9981553Srgrimes bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD); 9991553Srgrimes 10001553Srgrimes ena_buf = &rx_info->ena_buf; 10011553Srgrimes ena_buf->paddr = segs[0].ds_addr; 10021553Srgrimes ena_buf->len = mlen; 10031553Srgrimes 10041553Srgrimes ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH, 10051553Srgrimes "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n", 10061553Srgrimes rx_info->mbuf, rx_info,ena_buf->len, (uintmax_t)ena_buf->paddr); 10071553Srgrimes 10081553Srgrimes return (0); 10091553Srgrimes 10101553Srgrimesexit: 10111553Srgrimes m_freem(rx_info->mbuf); 10121553Srgrimes rx_info->mbuf = NULL; 10131553Srgrimes return (EFAULT); 10141553Srgrimes} 10151553Srgrimes 10161553Srgrimesstatic void 10171553Srgrimesena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring, 10181553Srgrimes struct ena_rx_buffer *rx_info) 10191553Srgrimes{ 10201553Srgrimes 10211553Srgrimes if (rx_info->mbuf == NULL) { 10221553Srgrimes ena_trace(ENA_WARNING, "Trying to free unallocated buffer\n"); 10231553Srgrimes return; 10241553Srgrimes } 10251553Srgrimes 10261553Srgrimes bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 10271553Srgrimes BUS_DMASYNC_POSTREAD); 10281553Srgrimes bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map); 10291553Srgrimes m_freem(rx_info->mbuf); 10301553Srgrimes rx_info->mbuf = NULL; 10311553Srgrimes} 10321553Srgrimes 10331553Srgrimes/** 10341553Srgrimes * ena_refill_rx_bufs - Refills ring with descriptors 10351553Srgrimes * @rx_ring: the ring which we want to feed with free descriptors 10361553Srgrimes * @num: number of descriptors to refill 10371553Srgrimes * Refills the ring with newly allocated DMA-mapped mbufs for receiving 10381553Srgrimes **/ 10391553Srgrimesint 10401553Srgrimesena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num) 10411553Srgrimes{ 10421553Srgrimes struct ena_adapter *adapter = rx_ring->adapter; 10431553Srgrimes uint16_t next_to_use, req_id; 10441553Srgrimes uint32_t i; 10451553Srgrimes int rc; 10461553Srgrimes 10471553Srgrimes ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, "refill qid: %d\n", 10481553Srgrimes rx_ring->qid); 10491553Srgrimes 10501553Srgrimes next_to_use = rx_ring->next_to_use; 10511553Srgrimes 10521553Srgrimes for (i = 0; i < num; i++) { 10531553Srgrimes struct ena_rx_buffer *rx_info; 10541553Srgrimes 10551553Srgrimes ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, 10561553Srgrimes "RX buffer - next to use: %d\n", next_to_use); 10571553Srgrimes 10581553Srgrimes req_id = rx_ring->free_rx_ids[next_to_use]; 10591553Srgrimes rx_info = &rx_ring->rx_buffer_info[req_id]; 10601553Srgrimes#ifdef DEV_NETMAP 10611553Srgrimes if (ena_rx_ring_in_netmap(adapter, rx_ring->qid)) 10621553Srgrimes rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, rx_info); 10631553Srgrimes else 10641553Srgrimes#endif /* DEV_NETMAP */ 10651553Srgrimes rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info); 10661553Srgrimes if (unlikely(rc != 0)) { 10671553Srgrimes ena_trace(ENA_WARNING, 10681553Srgrimes "failed to alloc buffer for rx queue %d\n", 10691553Srgrimes rx_ring->qid); 10701553Srgrimes break; 10711553Srgrimes } 10721553Srgrimes rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq, 10731553Srgrimes &rx_info->ena_buf, req_id); 10741553Srgrimes if (unlikely(rc != 0)) { 10751553Srgrimes ena_trace(ENA_WARNING, 10761553Srgrimes "failed to add buffer for rx queue %d\n", 10771553Srgrimes rx_ring->qid); 10781553Srgrimes break; 10791553Srgrimes } 10801553Srgrimes next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, 10811553Srgrimes rx_ring->ring_size); 10821553Srgrimes } 10831553Srgrimes 10841553Srgrimes if (unlikely(i < num)) { 10851553Srgrimes counter_u64_add(rx_ring->rx_stats.refil_partial, 1); 10861553Srgrimes ena_trace(ENA_WARNING, 10871553Srgrimes "refilled rx qid %d with only %d mbufs (from %d)\n", 10881553Srgrimes rx_ring->qid, i, num); 10891553Srgrimes } 10901553Srgrimes 10911553Srgrimes if (likely(i != 0)) { 10921553Srgrimes wmb(); 10931553Srgrimes ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq); 10941553Srgrimes } 10951553Srgrimes rx_ring->next_to_use = next_to_use; 10961553Srgrimes return (i); 10971553Srgrimes} 10981553Srgrimes 10991553Srgrimesstatic void 11001553Srgrimesena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid) 11011553Srgrimes{ 11021553Srgrimes struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 11031553Srgrimes unsigned int i; 11041553Srgrimes 11051553Srgrimes for (i = 0; i < rx_ring->ring_size; i++) { 11061553Srgrimes struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i]; 11071553Srgrimes 1108 if (rx_info->mbuf != NULL) 1109 ena_free_rx_mbuf(adapter, rx_ring, rx_info); 1110#ifdef DEV_NETMAP 1111 if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) && 1112 (adapter->ifp->if_capenable & IFCAP_NETMAP)) { 1113 if (rx_info->netmap_buf_idx != 0) 1114 ena_netmap_free_rx_slot(adapter, rx_ring, 1115 rx_info); 1116 } 1117#endif /* DEV_NETMAP */ 1118 } 1119} 1120 1121/** 1122 * ena_refill_all_rx_bufs - allocate all queues Rx buffers 1123 * @adapter: network interface device structure 1124 * 1125 */ 1126static void 1127ena_refill_all_rx_bufs(struct ena_adapter *adapter) 1128{ 1129 struct ena_ring *rx_ring; 1130 int i, rc, bufs_num; 1131 1132 for (i = 0; i < adapter->num_queues; i++) { 1133 rx_ring = &adapter->rx_ring[i]; 1134 bufs_num = rx_ring->ring_size - 1; 1135 rc = ena_refill_rx_bufs(rx_ring, bufs_num); 1136 if (unlikely(rc != bufs_num)) 1137 ena_trace(ENA_WARNING, "refilling Queue %d failed. " 1138 "Allocated %d buffers from: %d\n", i, rc, bufs_num); 1139#ifdef DEV_NETMAP 1140 rx_ring->initialized = true; 1141#endif /* DEV_NETMAP */ 1142 } 1143} 1144 1145static void 1146ena_free_all_rx_bufs(struct ena_adapter *adapter) 1147{ 1148 int i; 1149 1150 for (i = 0; i < adapter->num_queues; i++) 1151 ena_free_rx_bufs(adapter, i); 1152} 1153 1154/** 1155 * ena_free_tx_bufs - Free Tx Buffers per Queue 1156 * @adapter: network interface device structure 1157 * @qid: queue index 1158 **/ 1159static void 1160ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid) 1161{ 1162 bool print_once = true; 1163 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 1164 1165 ENA_RING_MTX_LOCK(tx_ring); 1166 for (int i = 0; i < tx_ring->ring_size; i++) { 1167 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i]; 1168 1169 if (tx_info->mbuf == NULL) 1170 continue; 1171 1172 if (print_once) { 1173 device_printf(adapter->pdev, 1174 "free uncompleted tx mbuf qid %d idx 0x%x\n", 1175 qid, i); 1176 print_once = false; 1177 } else { 1178 ena_trace(ENA_DBG, 1179 "free uncompleted tx mbuf qid %d idx 0x%x\n", 1180 qid, i); 1181 } 1182 1183 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, 1184 BUS_DMASYNC_POSTWRITE); 1185 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); 1186 1187 m_free(tx_info->mbuf); 1188 tx_info->mbuf = NULL; 1189 } 1190 ENA_RING_MTX_UNLOCK(tx_ring); 1191} 1192 1193static void 1194ena_free_all_tx_bufs(struct ena_adapter *adapter) 1195{ 1196 1197 for (int i = 0; i < adapter->num_queues; i++) 1198 ena_free_tx_bufs(adapter, i); 1199} 1200 1201static void 1202ena_destroy_all_tx_queues(struct ena_adapter *adapter) 1203{ 1204 uint16_t ena_qid; 1205 int i; 1206 1207 for (i = 0; i < adapter->num_queues; i++) { 1208 ena_qid = ENA_IO_TXQ_IDX(i); 1209 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1210 } 1211} 1212 1213static void 1214ena_destroy_all_rx_queues(struct ena_adapter *adapter) 1215{ 1216 uint16_t ena_qid; 1217 int i; 1218 1219 for (i = 0; i < adapter->num_queues; i++) { 1220 ena_qid = ENA_IO_RXQ_IDX(i); 1221 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1222 } 1223} 1224 1225static void 1226ena_destroy_all_io_queues(struct ena_adapter *adapter) 1227{ 1228 struct ena_que *queue; 1229 int i; 1230 1231 for (i = 0; i < adapter->num_queues; i++) { 1232 queue = &adapter->que[i]; 1233 while (taskqueue_cancel(queue->cleanup_tq, 1234 &queue->cleanup_task, NULL)) 1235 taskqueue_drain(queue->cleanup_tq, 1236 &queue->cleanup_task); 1237 taskqueue_free(queue->cleanup_tq); 1238 } 1239 1240 ena_destroy_all_tx_queues(adapter); 1241 ena_destroy_all_rx_queues(adapter); 1242} 1243 1244static int 1245ena_create_io_queues(struct ena_adapter *adapter) 1246{ 1247 struct ena_com_dev *ena_dev = adapter->ena_dev; 1248 struct ena_com_create_io_ctx ctx; 1249 struct ena_ring *ring; 1250 struct ena_que *queue; 1251 uint16_t ena_qid; 1252 uint32_t msix_vector; 1253 int rc, i; 1254 1255 /* Create TX queues */ 1256 for (i = 0; i < adapter->num_queues; i++) { 1257 msix_vector = ENA_IO_IRQ_IDX(i); 1258 ena_qid = ENA_IO_TXQ_IDX(i); 1259 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 1260 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 1261 ctx.queue_size = adapter->tx_ring_size; 1262 ctx.msix_vector = msix_vector; 1263 ctx.qid = ena_qid; 1264 rc = ena_com_create_io_queue(ena_dev, &ctx); 1265 if (rc != 0) { 1266 device_printf(adapter->pdev, 1267 "Failed to create io TX queue #%d rc: %d\n", i, rc); 1268 goto err_tx; 1269 } 1270 ring = &adapter->tx_ring[i]; 1271 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1272 &ring->ena_com_io_sq, 1273 &ring->ena_com_io_cq); 1274 if (rc != 0) { 1275 device_printf(adapter->pdev, 1276 "Failed to get TX queue handlers. TX queue num" 1277 " %d rc: %d\n", i, rc); 1278 ena_com_destroy_io_queue(ena_dev, ena_qid); 1279 goto err_tx; 1280 } 1281 } 1282 1283 /* Create RX queues */ 1284 for (i = 0; i < adapter->num_queues; i++) { 1285 msix_vector = ENA_IO_IRQ_IDX(i); 1286 ena_qid = ENA_IO_RXQ_IDX(i); 1287 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1288 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1289 ctx.queue_size = adapter->rx_ring_size; 1290 ctx.msix_vector = msix_vector; 1291 ctx.qid = ena_qid; 1292 rc = ena_com_create_io_queue(ena_dev, &ctx); 1293 if (unlikely(rc != 0)) { 1294 device_printf(adapter->pdev, 1295 "Failed to create io RX queue[%d] rc: %d\n", i, rc); 1296 goto err_rx; 1297 } 1298 1299 ring = &adapter->rx_ring[i]; 1300 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1301 &ring->ena_com_io_sq, 1302 &ring->ena_com_io_cq); 1303 if (unlikely(rc != 0)) { 1304 device_printf(adapter->pdev, 1305 "Failed to get RX queue handlers. RX queue num" 1306 " %d rc: %d\n", i, rc); 1307 ena_com_destroy_io_queue(ena_dev, ena_qid); 1308 goto err_rx; 1309 } 1310 } 1311 1312 for (i = 0; i < adapter->num_queues; i++) { 1313 queue = &adapter->que[i]; 1314 1315 TASK_INIT(&queue->cleanup_task, 0, ena_cleanup, queue); 1316 queue->cleanup_tq = taskqueue_create_fast("ena cleanup", 1317 M_WAITOK, taskqueue_thread_enqueue, &queue->cleanup_tq); 1318 1319 taskqueue_start_threads(&queue->cleanup_tq, 1, PI_NET, 1320 "%s queue %d cleanup", 1321 device_get_nameunit(adapter->pdev), i); 1322 } 1323 1324 return (0); 1325 1326err_rx: 1327 while (i--) 1328 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i)); 1329 i = adapter->num_queues; 1330err_tx: 1331 while (i--) 1332 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i)); 1333 1334 return (ENXIO); 1335} 1336 1337/********************************************************************* 1338 * 1339 * MSIX & Interrupt Service routine 1340 * 1341 **********************************************************************/ 1342 1343/** 1344 * ena_handle_msix - MSIX Interrupt Handler for admin/async queue 1345 * @arg: interrupt number 1346 **/ 1347static void 1348ena_intr_msix_mgmnt(void *arg) 1349{ 1350 struct ena_adapter *adapter = (struct ena_adapter *)arg; 1351 1352 ena_com_admin_q_comp_intr_handler(adapter->ena_dev); 1353 if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))) 1354 ena_com_aenq_intr_handler(adapter->ena_dev, arg); 1355} 1356 1357/** 1358 * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx 1359 * @arg: queue 1360 **/ 1361static int 1362ena_handle_msix(void *arg) 1363{ 1364 struct ena_que *queue = arg; 1365 struct ena_adapter *adapter = queue->adapter; 1366 if_t ifp = adapter->ifp; 1367 1368 if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) 1369 return (FILTER_STRAY); 1370 1371 taskqueue_enqueue(queue->cleanup_tq, &queue->cleanup_task); 1372 1373 return (FILTER_HANDLED); 1374} 1375 1376static int 1377ena_enable_msix(struct ena_adapter *adapter) 1378{ 1379 device_t dev = adapter->pdev; 1380 int msix_vecs, msix_req; 1381 int i, rc = 0; 1382 1383 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { 1384 device_printf(dev, "Error, MSI-X is already enabled\n"); 1385 return (EINVAL); 1386 } 1387 1388 /* Reserved the max msix vectors we might need */ 1389 msix_vecs = ENA_MAX_MSIX_VEC(adapter->num_queues); 1390 1391 adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry), 1392 M_DEVBUF, M_WAITOK | M_ZERO); 1393 1394 ena_trace(ENA_DBG, "trying to enable MSI-X, vectors: %d\n", msix_vecs); 1395 1396 for (i = 0; i < msix_vecs; i++) { 1397 adapter->msix_entries[i].entry = i; 1398 /* Vectors must start from 1 */ 1399 adapter->msix_entries[i].vector = i + 1; 1400 } 1401 1402 msix_req = msix_vecs; 1403 rc = pci_alloc_msix(dev, &msix_vecs); 1404 if (unlikely(rc != 0)) { 1405 device_printf(dev, 1406 "Failed to enable MSIX, vectors %d rc %d\n", msix_vecs, rc); 1407 1408 rc = ENOSPC; 1409 goto err_msix_free; 1410 } 1411 1412 if (msix_vecs != msix_req) { 1413 if (msix_vecs == ENA_ADMIN_MSIX_VEC) { 1414 device_printf(dev, 1415 "Not enough number of MSI-x allocated: %d\n", 1416 msix_vecs); 1417 pci_release_msi(dev); 1418 rc = ENOSPC; 1419 goto err_msix_free; 1420 } 1421 device_printf(dev, "Enable only %d MSI-x (out of %d), reduce " 1422 "the number of queues\n", msix_vecs, msix_req); 1423 adapter->num_queues = msix_vecs - ENA_ADMIN_MSIX_VEC; 1424 } 1425 1426 adapter->msix_vecs = msix_vecs; 1427 ENA_FLAG_SET_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); 1428 1429 return (0); 1430 1431err_msix_free: 1432 free(adapter->msix_entries, M_DEVBUF); 1433 adapter->msix_entries = NULL; 1434 1435 return (rc); 1436} 1437 1438static void 1439ena_setup_mgmnt_intr(struct ena_adapter *adapter) 1440{ 1441 1442 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, 1443 ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s", 1444 device_get_nameunit(adapter->pdev)); 1445 /* 1446 * Handler is NULL on purpose, it will be set 1447 * when mgmnt interrupt is acquired 1448 */ 1449 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL; 1450 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter; 1451 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector = 1452 adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector; 1453} 1454 1455static int 1456ena_setup_io_intr(struct ena_adapter *adapter) 1457{ 1458 static int last_bind_cpu = -1; 1459 int irq_idx; 1460 1461 if (adapter->msix_entries == NULL) 1462 return (EINVAL); 1463 1464 for (int i = 0; i < adapter->num_queues; i++) { 1465 irq_idx = ENA_IO_IRQ_IDX(i); 1466 1467 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE, 1468 "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i); 1469 adapter->irq_tbl[irq_idx].handler = ena_handle_msix; 1470 adapter->irq_tbl[irq_idx].data = &adapter->que[i]; 1471 adapter->irq_tbl[irq_idx].vector = 1472 adapter->msix_entries[irq_idx].vector; 1473 ena_trace(ENA_INFO | ENA_IOQ, "ena_setup_io_intr vector: %d\n", 1474 adapter->msix_entries[irq_idx].vector); 1475 1476 /* 1477 * We want to bind rings to the corresponding cpu 1478 * using something similar to the RSS round-robin technique. 1479 */ 1480 if (unlikely(last_bind_cpu < 0)) 1481 last_bind_cpu = CPU_FIRST(); 1482 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = 1483 last_bind_cpu; 1484 last_bind_cpu = CPU_NEXT(last_bind_cpu); 1485 } 1486 1487 return (0); 1488} 1489 1490static int 1491ena_request_mgmnt_irq(struct ena_adapter *adapter) 1492{ 1493 struct ena_irq *irq; 1494 unsigned long flags; 1495 int rc, rcc; 1496 1497 flags = RF_ACTIVE | RF_SHAREABLE; 1498 1499 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; 1500 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, 1501 &irq->vector, flags); 1502 1503 if (unlikely(irq->res == NULL)) { 1504 device_printf(adapter->pdev, "could not allocate " 1505 "irq vector: %d\n", irq->vector); 1506 return (ENXIO); 1507 } 1508 1509 rc = bus_setup_intr(adapter->pdev, irq->res, 1510 INTR_TYPE_NET | INTR_MPSAFE, NULL, ena_intr_msix_mgmnt, 1511 irq->data, &irq->cookie); 1512 if (unlikely(rc != 0)) { 1513 device_printf(adapter->pdev, "failed to register " 1514 "interrupt handler for irq %ju: %d\n", 1515 rman_get_start(irq->res), rc); 1516 goto err_res_free; 1517 } 1518 irq->requested = true; 1519 1520 return (rc); 1521 1522err_res_free: 1523 ena_trace(ENA_INFO | ENA_ADMQ, "releasing resource for irq %d\n", 1524 irq->vector); 1525 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1526 irq->vector, irq->res); 1527 if (unlikely(rcc != 0)) 1528 device_printf(adapter->pdev, "dev has no parent while " 1529 "releasing res for irq: %d\n", irq->vector); 1530 irq->res = NULL; 1531 1532 return (rc); 1533} 1534 1535static int 1536ena_request_io_irq(struct ena_adapter *adapter) 1537{ 1538 struct ena_irq *irq; 1539 unsigned long flags = 0; 1540 int rc = 0, i, rcc; 1541 1542 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter))) { 1543 device_printf(adapter->pdev, 1544 "failed to request I/O IRQ: MSI-X is not enabled\n"); 1545 return (EINVAL); 1546 } else { 1547 flags = RF_ACTIVE | RF_SHAREABLE; 1548 } 1549 1550 for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { 1551 irq = &adapter->irq_tbl[i]; 1552 1553 if (unlikely(irq->requested)) 1554 continue; 1555 1556 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, 1557 &irq->vector, flags); 1558 if (unlikely(irq->res == NULL)) { 1559 rc = ENOMEM; 1560 device_printf(adapter->pdev, "could not allocate " 1561 "irq vector: %d\n", irq->vector); 1562 goto err; 1563 } 1564 1565 rc = bus_setup_intr(adapter->pdev, irq->res, 1566 INTR_TYPE_NET | INTR_MPSAFE, irq->handler, NULL, 1567 irq->data, &irq->cookie); 1568 if (unlikely(rc != 0)) { 1569 device_printf(adapter->pdev, "failed to register " 1570 "interrupt handler for irq %ju: %d\n", 1571 rman_get_start(irq->res), rc); 1572 goto err; 1573 } 1574 irq->requested = true; 1575 1576 ena_trace(ENA_INFO, "queue %d - cpu %d\n", 1577 i - ENA_IO_IRQ_FIRST_IDX, irq->cpu); 1578 } 1579 1580 return (rc); 1581 1582err: 1583 1584 for (; i >= ENA_IO_IRQ_FIRST_IDX; i--) { 1585 irq = &adapter->irq_tbl[i]; 1586 rcc = 0; 1587 1588 /* Once we entered err: section and irq->requested is true we 1589 free both intr and resources */ 1590 if (irq->requested) 1591 rcc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); 1592 if (unlikely(rcc != 0)) 1593 device_printf(adapter->pdev, "could not release" 1594 " irq: %d, error: %d\n", irq->vector, rcc); 1595 1596 /* If we entred err: section without irq->requested set we know 1597 it was bus_alloc_resource_any() that needs cleanup, provided 1598 res is not NULL. In case res is NULL no work in needed in 1599 this iteration */ 1600 rcc = 0; 1601 if (irq->res != NULL) { 1602 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1603 irq->vector, irq->res); 1604 } 1605 if (unlikely(rcc != 0)) 1606 device_printf(adapter->pdev, "dev has no parent while " 1607 "releasing res for irq: %d\n", irq->vector); 1608 irq->requested = false; 1609 irq->res = NULL; 1610 } 1611 1612 return (rc); 1613} 1614 1615static void 1616ena_free_mgmnt_irq(struct ena_adapter *adapter) 1617{ 1618 struct ena_irq *irq; 1619 int rc; 1620 1621 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; 1622 if (irq->requested) { 1623 ena_trace(ENA_INFO | ENA_ADMQ, "tear down irq: %d\n", 1624 irq->vector); 1625 rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); 1626 if (unlikely(rc != 0)) 1627 device_printf(adapter->pdev, "failed to tear " 1628 "down irq: %d\n", irq->vector); 1629 irq->requested = 0; 1630 } 1631 1632 if (irq->res != NULL) { 1633 ena_trace(ENA_INFO | ENA_ADMQ, "release resource irq: %d\n", 1634 irq->vector); 1635 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1636 irq->vector, irq->res); 1637 irq->res = NULL; 1638 if (unlikely(rc != 0)) 1639 device_printf(adapter->pdev, "dev has no parent while " 1640 "releasing res for irq: %d\n", irq->vector); 1641 } 1642} 1643 1644static void 1645ena_free_io_irq(struct ena_adapter *adapter) 1646{ 1647 struct ena_irq *irq; 1648 int rc; 1649 1650 for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { 1651 irq = &adapter->irq_tbl[i]; 1652 if (irq->requested) { 1653 ena_trace(ENA_INFO | ENA_IOQ, "tear down irq: %d\n", 1654 irq->vector); 1655 rc = bus_teardown_intr(adapter->pdev, irq->res, 1656 irq->cookie); 1657 if (unlikely(rc != 0)) { 1658 device_printf(adapter->pdev, "failed to tear " 1659 "down irq: %d\n", irq->vector); 1660 } 1661 irq->requested = 0; 1662 } 1663 1664 if (irq->res != NULL) { 1665 ena_trace(ENA_INFO | ENA_IOQ, "release resource irq: %d\n", 1666 irq->vector); 1667 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1668 irq->vector, irq->res); 1669 irq->res = NULL; 1670 if (unlikely(rc != 0)) { 1671 device_printf(adapter->pdev, "dev has no parent" 1672 " while releasing res for irq: %d\n", 1673 irq->vector); 1674 } 1675 } 1676 } 1677} 1678 1679static void 1680ena_free_irqs(struct ena_adapter* adapter) 1681{ 1682 1683 ena_free_io_irq(adapter); 1684 ena_free_mgmnt_irq(adapter); 1685 ena_disable_msix(adapter); 1686} 1687 1688static void 1689ena_disable_msix(struct ena_adapter *adapter) 1690{ 1691 1692 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { 1693 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); 1694 pci_release_msi(adapter->pdev); 1695 } 1696 1697 adapter->msix_vecs = 0; 1698 if (adapter->msix_entries != NULL) 1699 free(adapter->msix_entries, M_DEVBUF); 1700 adapter->msix_entries = NULL; 1701} 1702 1703static void 1704ena_unmask_all_io_irqs(struct ena_adapter *adapter) 1705{ 1706 struct ena_com_io_cq* io_cq; 1707 struct ena_eth_io_intr_reg intr_reg; 1708 uint16_t ena_qid; 1709 int i; 1710 1711 /* Unmask interrupts for all queues */ 1712 for (i = 0; i < adapter->num_queues; i++) { 1713 ena_qid = ENA_IO_TXQ_IDX(i); 1714 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 1715 ena_com_update_intr_reg(&intr_reg, 0, 0, true); 1716 ena_com_unmask_intr(io_cq, &intr_reg); 1717 } 1718} 1719 1720/* Configure the Rx forwarding */ 1721static int 1722ena_rss_configure(struct ena_adapter *adapter) 1723{ 1724 struct ena_com_dev *ena_dev = adapter->ena_dev; 1725 int rc; 1726 1727 /* Set indirect table */ 1728 rc = ena_com_indirect_table_set(ena_dev); 1729 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 1730 return (rc); 1731 1732 /* Configure hash function (if supported) */ 1733 rc = ena_com_set_hash_function(ena_dev); 1734 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 1735 return (rc); 1736 1737 /* Configure hash inputs (if supported) */ 1738 rc = ena_com_set_hash_ctrl(ena_dev); 1739 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) 1740 return (rc); 1741 1742 return (0); 1743} 1744 1745static int 1746ena_up_complete(struct ena_adapter *adapter) 1747{ 1748 int rc; 1749 1750 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) { 1751 rc = ena_rss_configure(adapter); 1752 if (rc != 0) 1753 return (rc); 1754 } 1755 1756 rc = ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu); 1757 if (unlikely(rc != 0)) 1758 return (rc); 1759 1760 ena_refill_all_rx_bufs(adapter); 1761 ena_reset_counters((counter_u64_t *)&adapter->hw_stats, 1762 sizeof(adapter->hw_stats)); 1763 1764 return (0); 1765} 1766 1767int 1768ena_up(struct ena_adapter *adapter) 1769{ 1770 int rc = 0; 1771 1772 if (unlikely(device_is_attached(adapter->pdev) == 0)) { 1773 device_printf(adapter->pdev, "device is not attached!\n"); 1774 return (ENXIO); 1775 } 1776 1777 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { 1778 device_printf(adapter->pdev, "device is going UP\n"); 1779 1780 /* setup interrupts for IO queues */ 1781 rc = ena_setup_io_intr(adapter); 1782 if (unlikely(rc != 0)) { 1783 ena_trace(ENA_ALERT, "error setting up IO interrupt\n"); 1784 goto error; 1785 } 1786 rc = ena_request_io_irq(adapter); 1787 if (unlikely(rc != 0)) { 1788 ena_trace(ENA_ALERT, "err_req_irq\n"); 1789 goto error; 1790 } 1791 1792 /* allocate transmit descriptors */ 1793 rc = ena_setup_all_tx_resources(adapter); 1794 if (unlikely(rc != 0)) { 1795 ena_trace(ENA_ALERT, "err_setup_tx\n"); 1796 goto err_setup_tx; 1797 } 1798 1799 /* allocate receive descriptors */ 1800 rc = ena_setup_all_rx_resources(adapter); 1801 if (unlikely(rc != 0)) { 1802 ena_trace(ENA_ALERT, "err_setup_rx\n"); 1803 goto err_setup_rx; 1804 } 1805 1806 /* create IO queues for Rx & Tx */ 1807 rc = ena_create_io_queues(adapter); 1808 if (unlikely(rc != 0)) { 1809 ena_trace(ENA_ALERT, 1810 "create IO queues failed\n"); 1811 goto err_io_que; 1812 } 1813 1814 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) 1815 if_link_state_change(adapter->ifp, LINK_STATE_UP); 1816 1817 rc = ena_up_complete(adapter); 1818 if (unlikely(rc != 0)) 1819 goto err_up_complete; 1820 1821 counter_u64_add(adapter->dev_stats.interface_up, 1); 1822 1823 ena_update_hwassist(adapter); 1824 1825 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, 1826 IFF_DRV_OACTIVE); 1827 1828 /* Activate timer service only if the device is running. 1829 * If this flag is not set, it means that the driver is being 1830 * reset and timer service will be activated afterwards. 1831 */ 1832 if (ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) { 1833 callout_reset_sbt(&adapter->timer_service, SBT_1S, 1834 SBT_1S, ena_timer_service, (void *)adapter, 0); 1835 } 1836 1837 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter); 1838 1839 ena_unmask_all_io_irqs(adapter); 1840 } 1841 1842 return (0); 1843 1844err_up_complete: 1845 ena_destroy_all_io_queues(adapter); 1846err_io_que: 1847 ena_free_all_rx_resources(adapter); 1848err_setup_rx: 1849 ena_free_all_tx_resources(adapter); 1850err_setup_tx: 1851 ena_free_io_irq(adapter); 1852error: 1853 return (rc); 1854} 1855 1856static uint64_t 1857ena_get_counter(if_t ifp, ift_counter cnt) 1858{ 1859 struct ena_adapter *adapter; 1860 struct ena_hw_stats *stats; 1861 1862 adapter = if_getsoftc(ifp); 1863 stats = &adapter->hw_stats; 1864 1865 switch (cnt) { 1866 case IFCOUNTER_IPACKETS: 1867 return (counter_u64_fetch(stats->rx_packets)); 1868 case IFCOUNTER_OPACKETS: 1869 return (counter_u64_fetch(stats->tx_packets)); 1870 case IFCOUNTER_IBYTES: 1871 return (counter_u64_fetch(stats->rx_bytes)); 1872 case IFCOUNTER_OBYTES: 1873 return (counter_u64_fetch(stats->tx_bytes)); 1874 case IFCOUNTER_IQDROPS: 1875 return (counter_u64_fetch(stats->rx_drops)); 1876 default: 1877 return (if_get_counter_default(ifp, cnt)); 1878 } 1879} 1880 1881static int 1882ena_media_change(if_t ifp) 1883{ 1884 /* Media Change is not supported by firmware */ 1885 return (0); 1886} 1887 1888static void 1889ena_media_status(if_t ifp, struct ifmediareq *ifmr) 1890{ 1891 struct ena_adapter *adapter = if_getsoftc(ifp); 1892 ena_trace(ENA_DBG, "enter\n"); 1893 1894 mtx_lock(&adapter->global_mtx); 1895 1896 ifmr->ifm_status = IFM_AVALID; 1897 ifmr->ifm_active = IFM_ETHER; 1898 1899 if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) { 1900 mtx_unlock(&adapter->global_mtx); 1901 ena_trace(ENA_INFO, "Link is down\n"); 1902 return; 1903 } 1904 1905 ifmr->ifm_status |= IFM_ACTIVE; 1906 ifmr->ifm_active |= IFM_UNKNOWN | IFM_FDX; 1907 1908 mtx_unlock(&adapter->global_mtx); 1909} 1910 1911static void 1912ena_init(void *arg) 1913{ 1914 struct ena_adapter *adapter = (struct ena_adapter *)arg; 1915 1916 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { 1917 sx_xlock(&adapter->ioctl_sx); 1918 ena_up(adapter); 1919 sx_unlock(&adapter->ioctl_sx); 1920 } 1921} 1922 1923static int 1924ena_ioctl(if_t ifp, u_long command, caddr_t data) 1925{ 1926 struct ena_adapter *adapter; 1927 struct ifreq *ifr; 1928 int rc; 1929 1930 adapter = ifp->if_softc; 1931 ifr = (struct ifreq *)data; 1932 1933 /* 1934 * Acquiring lock to prevent from running up and down routines parallel. 1935 */ 1936 rc = 0; 1937 switch (command) { 1938 case SIOCSIFMTU: 1939 if (ifp->if_mtu == ifr->ifr_mtu) 1940 break; 1941 sx_xlock(&adapter->ioctl_sx); 1942 ena_down(adapter); 1943 1944 ena_change_mtu(ifp, ifr->ifr_mtu); 1945 1946 rc = ena_up(adapter); 1947 sx_unlock(&adapter->ioctl_sx); 1948 break; 1949 1950 case SIOCSIFFLAGS: 1951 if ((ifp->if_flags & IFF_UP) != 0) { 1952 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 1953 if ((ifp->if_flags & (IFF_PROMISC | 1954 IFF_ALLMULTI)) != 0) { 1955 device_printf(adapter->pdev, 1956 "ioctl promisc/allmulti\n"); 1957 } 1958 } else { 1959 sx_xlock(&adapter->ioctl_sx); 1960 rc = ena_up(adapter); 1961 sx_unlock(&adapter->ioctl_sx); 1962 } 1963 } else { 1964 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 1965 sx_xlock(&adapter->ioctl_sx); 1966 ena_down(adapter); 1967 sx_unlock(&adapter->ioctl_sx); 1968 } 1969 } 1970 break; 1971 1972 case SIOCADDMULTI: 1973 case SIOCDELMULTI: 1974 break; 1975 1976 case SIOCSIFMEDIA: 1977 case SIOCGIFMEDIA: 1978 rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 1979 break; 1980 1981 case SIOCSIFCAP: 1982 { 1983 int reinit = 0; 1984 1985 if (ifr->ifr_reqcap != ifp->if_capenable) { 1986 ifp->if_capenable = ifr->ifr_reqcap; 1987 reinit = 1; 1988 } 1989 1990 if ((reinit != 0) && 1991 ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)) { 1992 sx_xlock(&adapter->ioctl_sx); 1993 ena_down(adapter); 1994 rc = ena_up(adapter); 1995 sx_unlock(&adapter->ioctl_sx); 1996 } 1997 } 1998 1999 break; 2000 default: 2001 rc = ether_ioctl(ifp, command, data); 2002 break; 2003 } 2004 2005 return (rc); 2006} 2007 2008static int 2009ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *feat) 2010{ 2011 int caps = 0; 2012 2013 if ((feat->offload.tx & 2014 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK | 2015 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK | 2016 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)) != 0) 2017 caps |= IFCAP_TXCSUM; 2018 2019 if ((feat->offload.tx & 2020 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK | 2021 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)) != 0) 2022 caps |= IFCAP_TXCSUM_IPV6; 2023 2024 if ((feat->offload.tx & 2025 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0) 2026 caps |= IFCAP_TSO4; 2027 2028 if ((feat->offload.tx & 2029 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) != 0) 2030 caps |= IFCAP_TSO6; 2031 2032 if ((feat->offload.rx_supported & 2033 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK | 2034 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK)) != 0) 2035 caps |= IFCAP_RXCSUM; 2036 2037 if ((feat->offload.rx_supported & 2038 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) != 0) 2039 caps |= IFCAP_RXCSUM_IPV6; 2040 2041 caps |= IFCAP_LRO | IFCAP_JUMBO_MTU; 2042 2043 return (caps); 2044} 2045 2046static void 2047ena_update_host_info(struct ena_admin_host_info *host_info, if_t ifp) 2048{ 2049 2050 host_info->supported_network_features[0] = 2051 (uint32_t)if_getcapabilities(ifp); 2052} 2053 2054static void 2055ena_update_hwassist(struct ena_adapter *adapter) 2056{ 2057 if_t ifp = adapter->ifp; 2058 uint32_t feat = adapter->tx_offload_cap; 2059 int cap = if_getcapenable(ifp); 2060 int flags = 0; 2061 2062 if_clearhwassist(ifp); 2063 2064 if ((cap & IFCAP_TXCSUM) != 0) { 2065 if ((feat & 2066 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK) != 0) 2067 flags |= CSUM_IP; 2068 if ((feat & 2069 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK | 2070 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)) != 0) 2071 flags |= CSUM_IP_UDP | CSUM_IP_TCP; 2072 } 2073 2074 if ((cap & IFCAP_TXCSUM_IPV6) != 0) 2075 flags |= CSUM_IP6_UDP | CSUM_IP6_TCP; 2076 2077 if ((cap & IFCAP_TSO4) != 0) 2078 flags |= CSUM_IP_TSO; 2079 2080 if ((cap & IFCAP_TSO6) != 0) 2081 flags |= CSUM_IP6_TSO; 2082 2083 if_sethwassistbits(ifp, flags, 0); 2084} 2085 2086static int 2087ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter, 2088 struct ena_com_dev_get_features_ctx *feat) 2089{ 2090 if_t ifp; 2091 int caps = 0; 2092 2093 ifp = adapter->ifp = if_gethandle(IFT_ETHER); 2094 if (unlikely(ifp == NULL)) { 2095 ena_trace(ENA_ALERT, "can not allocate ifnet structure\n"); 2096 return (ENXIO); 2097 } 2098 if_initname(ifp, device_get_name(pdev), device_get_unit(pdev)); 2099 if_setdev(ifp, pdev); 2100 if_setsoftc(ifp, adapter); 2101 2102 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); 2103 if_setinitfn(ifp, ena_init); 2104 if_settransmitfn(ifp, ena_mq_start); 2105 if_setqflushfn(ifp, ena_qflush); 2106 if_setioctlfn(ifp, ena_ioctl); 2107 if_setgetcounterfn(ifp, ena_get_counter); 2108 2109 if_setsendqlen(ifp, adapter->tx_ring_size); 2110 if_setsendqready(ifp); 2111 if_setmtu(ifp, ETHERMTU); 2112 if_setbaudrate(ifp, 0); 2113 /* Zeroize capabilities... */ 2114 if_setcapabilities(ifp, 0); 2115 if_setcapenable(ifp, 0); 2116 /* check hardware support */ 2117 caps = ena_get_dev_offloads(feat); 2118 /* ... and set them */ 2119 if_setcapabilitiesbit(ifp, caps, 0); 2120 2121 /* TSO parameters */ 2122 ifp->if_hw_tsomax = ENA_TSO_MAXSIZE - 2123 (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2124 ifp->if_hw_tsomaxsegcount = adapter->max_tx_sgl_size - 1; 2125 ifp->if_hw_tsomaxsegsize = ENA_TSO_MAXSIZE; 2126 2127 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 2128 if_setcapenable(ifp, if_getcapabilities(ifp)); 2129 2130 /* 2131 * Specify the media types supported by this adapter and register 2132 * callbacks to update media and link information 2133 */ 2134 ifmedia_init(&adapter->media, IFM_IMASK, 2135 ena_media_change, ena_media_status); 2136 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2137 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2138 2139 ether_ifattach(ifp, adapter->mac_addr); 2140 2141 return (0); 2142} 2143 2144void 2145ena_down(struct ena_adapter *adapter) 2146{ 2147 int rc; 2148 2149 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { 2150 device_printf(adapter->pdev, "device is going DOWN\n"); 2151 2152 callout_drain(&adapter->timer_service); 2153 2154 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter); 2155 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, 2156 IFF_DRV_RUNNING); 2157 2158 ena_free_io_irq(adapter); 2159 2160 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) { 2161 rc = ena_com_dev_reset(adapter->ena_dev, 2162 adapter->reset_reason); 2163 if (unlikely(rc != 0)) 2164 device_printf(adapter->pdev, 2165 "Device reset failed\n"); 2166 } 2167 2168 ena_destroy_all_io_queues(adapter); 2169 2170 ena_free_all_tx_bufs(adapter); 2171 ena_free_all_rx_bufs(adapter); 2172 ena_free_all_tx_resources(adapter); 2173 ena_free_all_rx_resources(adapter); 2174 2175 counter_u64_add(adapter->dev_stats.interface_down, 1); 2176 } 2177} 2178 2179static int 2180ena_calc_io_queue_num(struct ena_adapter *adapter, 2181 struct ena_com_dev_get_features_ctx *get_feat_ctx) 2182{ 2183 struct ena_com_dev *ena_dev = adapter->ena_dev; 2184 int io_tx_sq_num, io_tx_cq_num, io_rx_num, io_queue_num; 2185 2186 /* Regular queues capabilities */ 2187 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 2188 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 2189 &get_feat_ctx->max_queue_ext.max_queue_ext; 2190 io_rx_num = min_t(int, max_queue_ext->max_rx_sq_num, 2191 max_queue_ext->max_rx_cq_num); 2192 2193 io_tx_sq_num = max_queue_ext->max_tx_sq_num; 2194 io_tx_cq_num = max_queue_ext->max_tx_cq_num; 2195 } else { 2196 struct ena_admin_queue_feature_desc *max_queues = 2197 &get_feat_ctx->max_queues; 2198 io_tx_sq_num = max_queues->max_sq_num; 2199 io_tx_cq_num = max_queues->max_cq_num; 2200 io_rx_num = min_t(int, io_tx_sq_num, io_tx_cq_num); 2201 } 2202 2203 /* In case of LLQ use the llq fields for the tx SQ/CQ */ 2204 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 2205 io_tx_sq_num = get_feat_ctx->llq.max_llq_num; 2206 2207 io_queue_num = min_t(int, mp_ncpus, ENA_MAX_NUM_IO_QUEUES); 2208 io_queue_num = min_t(int, io_queue_num, io_rx_num); 2209 io_queue_num = min_t(int, io_queue_num, io_tx_sq_num); 2210 io_queue_num = min_t(int, io_queue_num, io_tx_cq_num); 2211 /* 1 IRQ for for mgmnt and 1 IRQ for each TX/RX pair */ 2212 io_queue_num = min_t(int, io_queue_num, 2213 pci_msix_count(adapter->pdev) - 1); 2214 2215 return (io_queue_num); 2216} 2217 2218static int 2219ena_enable_wc(struct resource *res) 2220{ 2221#if defined(__i386) || defined(__amd64) 2222 vm_offset_t va; 2223 vm_size_t len; 2224 int rc; 2225 2226 va = (vm_offset_t)rman_get_virtual(res); 2227 len = rman_get_size(res); 2228 /* Enable write combining */ 2229 rc = pmap_change_attr(va, len, PAT_WRITE_COMBINING); 2230 if (unlikely(rc != 0)) { 2231 ena_trace(ENA_ALERT, "pmap_change_attr failed, %d\n", rc); 2232 return (rc); 2233 } 2234 2235 return (0); 2236#endif 2237 return (EOPNOTSUPP); 2238} 2239 2240static int 2241ena_set_queues_placement_policy(device_t pdev, struct ena_com_dev *ena_dev, 2242 struct ena_admin_feature_llq_desc *llq, 2243 struct ena_llq_configurations *llq_default_configurations) 2244{ 2245 struct ena_adapter *adapter = device_get_softc(pdev); 2246 int rc, rid; 2247 uint32_t llq_feature_mask; 2248 2249 llq_feature_mask = 1 << ENA_ADMIN_LLQ; 2250 if (!(ena_dev->supported_features & llq_feature_mask)) { 2251 device_printf(pdev, 2252 "LLQ is not supported. Fallback to host mode policy.\n"); 2253 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2254 return (0); 2255 } 2256 2257 rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); 2258 if (unlikely(rc != 0)) { 2259 device_printf(pdev, "Failed to configure the device mode. " 2260 "Fallback to host mode policy.\n"); 2261 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2262 return (0); 2263 } 2264 2265 /* Nothing to config, exit */ 2266 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 2267 return (0); 2268 2269 /* Try to allocate resources for LLQ bar */ 2270 rid = PCIR_BAR(ENA_MEM_BAR); 2271 adapter->memory = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, 2272 &rid, RF_ACTIVE); 2273 if (unlikely(adapter->memory == NULL)) { 2274 device_printf(pdev, "unable to allocate LLQ bar resource. " 2275 "Fallback to host mode policy.\n"); 2276 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2277 return (0); 2278 } 2279 2280 /* Enable write combining for better LLQ performance */ 2281 rc = ena_enable_wc(adapter->memory); 2282 if (unlikely(rc != 0)) { 2283 device_printf(pdev, "failed to enable write combining.\n"); 2284 return (rc); 2285 } 2286 2287 /* 2288 * Save virtual address of the device's memory region 2289 * for the ena_com layer. 2290 */ 2291 ena_dev->mem_bar = rman_get_virtual(adapter->memory); 2292 2293 return (0); 2294} 2295 2296static inline 2297void set_default_llq_configurations(struct ena_llq_configurations *llq_config) 2298{ 2299 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; 2300 llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B; 2301 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; 2302 llq_config->llq_num_decs_before_header = 2303 ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; 2304 llq_config->llq_ring_entry_size_value = 128; 2305} 2306 2307static int 2308ena_calc_queue_size(struct ena_adapter *adapter, 2309 struct ena_calc_queue_size_ctx *ctx) 2310{ 2311 struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq; 2312 struct ena_com_dev *ena_dev = ctx->ena_dev; 2313 uint32_t tx_queue_size = ENA_DEFAULT_RING_SIZE; 2314 uint32_t rx_queue_size = adapter->rx_ring_size; 2315 2316 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 2317 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 2318 &ctx->get_feat_ctx->max_queue_ext.max_queue_ext; 2319 rx_queue_size = min_t(uint32_t, rx_queue_size, 2320 max_queue_ext->max_rx_cq_depth); 2321 rx_queue_size = min_t(uint32_t, rx_queue_size, 2322 max_queue_ext->max_rx_sq_depth); 2323 tx_queue_size = min_t(uint32_t, tx_queue_size, 2324 max_queue_ext->max_tx_cq_depth); 2325 2326 if (ena_dev->tx_mem_queue_type == 2327 ENA_ADMIN_PLACEMENT_POLICY_DEV) 2328 tx_queue_size = min_t(uint32_t, tx_queue_size, 2329 llq->max_llq_depth); 2330 else 2331 tx_queue_size = min_t(uint32_t, tx_queue_size, 2332 max_queue_ext->max_tx_sq_depth); 2333 2334 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2335 max_queue_ext->max_per_packet_rx_descs); 2336 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2337 max_queue_ext->max_per_packet_tx_descs); 2338 } else { 2339 struct ena_admin_queue_feature_desc *max_queues = 2340 &ctx->get_feat_ctx->max_queues; 2341 rx_queue_size = min_t(uint32_t, rx_queue_size, 2342 max_queues->max_cq_depth); 2343 rx_queue_size = min_t(uint32_t, rx_queue_size, 2344 max_queues->max_sq_depth); 2345 tx_queue_size = min_t(uint32_t, tx_queue_size, 2346 max_queues->max_cq_depth); 2347 2348 if (ena_dev->tx_mem_queue_type == 2349 ENA_ADMIN_PLACEMENT_POLICY_DEV) 2350 tx_queue_size = min_t(uint32_t, tx_queue_size, 2351 llq->max_llq_depth); 2352 else 2353 tx_queue_size = min_t(uint32_t, tx_queue_size, 2354 max_queues->max_sq_depth); 2355 2356 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2357 max_queues->max_packet_tx_descs); 2358 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2359 max_queues->max_packet_rx_descs); 2360 } 2361 2362 /* round down to the nearest power of 2 */ 2363 rx_queue_size = 1 << (fls(rx_queue_size) - 1); 2364 tx_queue_size = 1 << (fls(tx_queue_size) - 1); 2365 2366 if (unlikely(rx_queue_size == 0 || tx_queue_size == 0)) { 2367 device_printf(ctx->pdev, "Invalid queue size\n"); 2368 return (EFAULT); 2369 } 2370 2371 ctx->rx_queue_size = rx_queue_size; 2372 ctx->tx_queue_size = tx_queue_size; 2373 2374 return (0); 2375} 2376 2377static int 2378ena_handle_updated_queues(struct ena_adapter *adapter, 2379 struct ena_com_dev_get_features_ctx *get_feat_ctx) 2380{ 2381 struct ena_com_dev *ena_dev = adapter->ena_dev; 2382 struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 }; 2383 device_t pdev = adapter->pdev; 2384 bool are_queues_changed = false; 2385 int io_queue_num, rc; 2386 2387 calc_queue_ctx.ena_dev = ena_dev; 2388 calc_queue_ctx.get_feat_ctx = get_feat_ctx; 2389 calc_queue_ctx.pdev = pdev; 2390 2391 io_queue_num = ena_calc_io_queue_num(adapter, get_feat_ctx); 2392 rc = ena_calc_queue_size(adapter, &calc_queue_ctx); 2393 if (unlikely(rc != 0 || io_queue_num <= 0)) 2394 return EFAULT; 2395 2396 if (adapter->tx_ring->buf_ring_size != adapter->buf_ring_size) 2397 are_queues_changed = true; 2398 2399 if (unlikely(adapter->tx_ring_size > calc_queue_ctx.tx_queue_size || 2400 adapter->rx_ring_size > calc_queue_ctx.rx_queue_size)) { 2401 device_printf(pdev, 2402 "Not enough resources to allocate requested queue sizes " 2403 "(TX,RX)=(%d,%d), falling back to queue sizes " 2404 "(TX,RX)=(%d,%d)\n", 2405 adapter->tx_ring_size, 2406 adapter->rx_ring_size, 2407 calc_queue_ctx.tx_queue_size, 2408 calc_queue_ctx.rx_queue_size); 2409 adapter->tx_ring_size = calc_queue_ctx.tx_queue_size; 2410 adapter->rx_ring_size = calc_queue_ctx.rx_queue_size; 2411 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; 2412 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; 2413 are_queues_changed = true; 2414 } 2415 2416 if (unlikely(adapter->num_queues > io_queue_num)) { 2417 device_printf(pdev, 2418 "Not enough resources to allocate %d queues, " 2419 "falling back to %d queues\n", 2420 adapter->num_queues, io_queue_num); 2421 adapter->num_queues = io_queue_num; 2422 if (ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter)) { 2423 ena_com_rss_destroy(ena_dev); 2424 rc = ena_rss_init_default(adapter); 2425 if (unlikely(rc != 0) && (rc != EOPNOTSUPP)) { 2426 device_printf(pdev, "Cannot init RSS rc: %d\n", 2427 rc); 2428 return (rc); 2429 } 2430 } 2431 are_queues_changed = true; 2432 } 2433 2434 if (unlikely(are_queues_changed)) { 2435 ena_free_all_io_rings_resources(adapter); 2436 ena_init_io_rings(adapter); 2437 } 2438 2439 return (0); 2440} 2441 2442static int 2443ena_rss_init_default(struct ena_adapter *adapter) 2444{ 2445 struct ena_com_dev *ena_dev = adapter->ena_dev; 2446 device_t dev = adapter->pdev; 2447 int qid, rc, i; 2448 2449 rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE); 2450 if (unlikely(rc != 0)) { 2451 device_printf(dev, "Cannot init indirect table\n"); 2452 return (rc); 2453 } 2454 2455 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 2456 qid = i % adapter->num_queues; 2457 rc = ena_com_indirect_table_fill_entry(ena_dev, i, 2458 ENA_IO_RXQ_IDX(qid)); 2459 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 2460 device_printf(dev, "Cannot fill indirect table\n"); 2461 goto err_rss_destroy; 2462 } 2463 } 2464 2465 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL, 2466 ENA_HASH_KEY_SIZE, 0xFFFFFFFF); 2467 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 2468 device_printf(dev, "Cannot fill hash function\n"); 2469 goto err_rss_destroy; 2470 } 2471 2472 rc = ena_com_set_default_hash_ctrl(ena_dev); 2473 if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) { 2474 device_printf(dev, "Cannot fill hash control\n"); 2475 goto err_rss_destroy; 2476 } 2477 2478 return (0); 2479 2480err_rss_destroy: 2481 ena_com_rss_destroy(ena_dev); 2482 return (rc); 2483} 2484 2485static void 2486ena_rss_init_default_deferred(void *arg) 2487{ 2488 struct ena_adapter *adapter; 2489 devclass_t dc; 2490 int max; 2491 int rc; 2492 2493 dc = devclass_find("ena"); 2494 if (unlikely(dc == NULL)) { 2495 ena_trace(ENA_ALERT, "No devclass ena\n"); 2496 return; 2497 } 2498 2499 max = devclass_get_maxunit(dc); 2500 while (max-- >= 0) { 2501 adapter = devclass_get_softc(dc, max); 2502 if (adapter != NULL) { 2503 rc = ena_rss_init_default(adapter); 2504 ENA_FLAG_SET_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 2505 if (unlikely(rc != 0)) { 2506 device_printf(adapter->pdev, 2507 "WARNING: RSS was not properly initialized," 2508 " it will affect bandwidth\n"); 2509 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_RSS_ACTIVE, adapter); 2510 } 2511 } 2512 } 2513} 2514SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, ena_rss_init_default_deferred, NULL); 2515 2516static void 2517ena_config_host_info(struct ena_com_dev *ena_dev, device_t dev) 2518{ 2519 struct ena_admin_host_info *host_info; 2520 uintptr_t rid; 2521 int rc; 2522 2523 /* Allocate only the host info */ 2524 rc = ena_com_allocate_host_info(ena_dev); 2525 if (unlikely(rc != 0)) { 2526 ena_trace(ENA_ALERT, "Cannot allocate host info\n"); 2527 return; 2528 } 2529 2530 host_info = ena_dev->host_attr.host_info; 2531 2532 if (pci_get_id(dev, PCI_ID_RID, &rid) == 0) 2533 host_info->bdf = rid; 2534 host_info->os_type = ENA_ADMIN_OS_FREEBSD; 2535 host_info->kernel_ver = osreldate; 2536 2537 sprintf(host_info->kernel_ver_str, "%d", osreldate); 2538 host_info->os_dist = 0; 2539 strncpy(host_info->os_dist_str, osrelease, 2540 sizeof(host_info->os_dist_str) - 1); 2541 2542 host_info->driver_version = 2543 (DRV_MODULE_VER_MAJOR) | 2544 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 2545 (DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 2546 host_info->num_cpus = mp_ncpus; 2547 2548 rc = ena_com_set_host_attributes(ena_dev); 2549 if (unlikely(rc != 0)) { 2550 if (rc == EOPNOTSUPP) 2551 ena_trace(ENA_WARNING, "Cannot set host attributes\n"); 2552 else 2553 ena_trace(ENA_ALERT, "Cannot set host attributes\n"); 2554 2555 goto err; 2556 } 2557 2558 return; 2559 2560err: 2561 ena_com_delete_host_info(ena_dev); 2562} 2563 2564static int 2565ena_device_init(struct ena_adapter *adapter, device_t pdev, 2566 struct ena_com_dev_get_features_ctx *get_feat_ctx, int *wd_active) 2567{ 2568 struct ena_com_dev* ena_dev = adapter->ena_dev; 2569 bool readless_supported; 2570 uint32_t aenq_groups; 2571 int dma_width; 2572 int rc; 2573 2574 rc = ena_com_mmio_reg_read_request_init(ena_dev); 2575 if (unlikely(rc != 0)) { 2576 device_printf(pdev, "failed to init mmio read less\n"); 2577 return (rc); 2578 } 2579 2580 /* 2581 * The PCIe configuration space revision id indicate if mmio reg 2582 * read is disabled 2583 */ 2584 readless_supported = !(pci_get_revid(pdev) & ENA_MMIO_DISABLE_REG_READ); 2585 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 2586 2587 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 2588 if (unlikely(rc != 0)) { 2589 device_printf(pdev, "Can not reset device\n"); 2590 goto err_mmio_read_less; 2591 } 2592 2593 rc = ena_com_validate_version(ena_dev); 2594 if (unlikely(rc != 0)) { 2595 device_printf(pdev, "device version is too low\n"); 2596 goto err_mmio_read_less; 2597 } 2598 2599 dma_width = ena_com_get_dma_width(ena_dev); 2600 if (unlikely(dma_width < 0)) { 2601 device_printf(pdev, "Invalid dma width value %d", dma_width); 2602 rc = dma_width; 2603 goto err_mmio_read_less; 2604 } 2605 adapter->dma_width = dma_width; 2606 2607 /* ENA admin level init */ 2608 rc = ena_com_admin_init(ena_dev, &aenq_handlers); 2609 if (unlikely(rc != 0)) { 2610 device_printf(pdev, 2611 "Can not initialize ena admin queue with device\n"); 2612 goto err_mmio_read_less; 2613 } 2614 2615 /* 2616 * To enable the msix interrupts the driver needs to know the number 2617 * of queues. So the driver uses polling mode to retrieve this 2618 * information 2619 */ 2620 ena_com_set_admin_polling_mode(ena_dev, true); 2621 2622 ena_config_host_info(ena_dev, pdev); 2623 2624 /* Get Device Attributes */ 2625 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 2626 if (unlikely(rc != 0)) { 2627 device_printf(pdev, 2628 "Cannot get attribute for ena device rc: %d\n", rc); 2629 goto err_admin_init; 2630 } 2631 2632 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | 2633 BIT(ENA_ADMIN_FATAL_ERROR) | 2634 BIT(ENA_ADMIN_WARNING) | 2635 BIT(ENA_ADMIN_NOTIFICATION) | 2636 BIT(ENA_ADMIN_KEEP_ALIVE); 2637 2638 aenq_groups &= get_feat_ctx->aenq.supported_groups; 2639 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 2640 if (unlikely(rc != 0)) { 2641 device_printf(pdev, "Cannot configure aenq groups rc: %d\n", rc); 2642 goto err_admin_init; 2643 } 2644 2645 *wd_active = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 2646 2647 return (0); 2648 2649err_admin_init: 2650 ena_com_delete_host_info(ena_dev); 2651 ena_com_admin_destroy(ena_dev); 2652err_mmio_read_less: 2653 ena_com_mmio_reg_read_request_destroy(ena_dev); 2654 2655 return (rc); 2656} 2657 2658static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter, 2659 int io_vectors) 2660{ 2661 struct ena_com_dev *ena_dev = adapter->ena_dev; 2662 int rc; 2663 2664 rc = ena_enable_msix(adapter); 2665 if (unlikely(rc != 0)) { 2666 device_printf(adapter->pdev, "Error with MSI-X enablement\n"); 2667 return (rc); 2668 } 2669 2670 ena_setup_mgmnt_intr(adapter); 2671 2672 rc = ena_request_mgmnt_irq(adapter); 2673 if (unlikely(rc != 0)) { 2674 device_printf(adapter->pdev, "Cannot setup mgmnt queue intr\n"); 2675 goto err_disable_msix; 2676 } 2677 2678 ena_com_set_admin_polling_mode(ena_dev, false); 2679 2680 ena_com_admin_aenq_enable(ena_dev); 2681 2682 return (0); 2683 2684err_disable_msix: 2685 ena_disable_msix(adapter); 2686 2687 return (rc); 2688} 2689 2690/* Function called on ENA_ADMIN_KEEP_ALIVE event */ 2691static void ena_keep_alive_wd(void *adapter_data, 2692 struct ena_admin_aenq_entry *aenq_e) 2693{ 2694 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 2695 struct ena_admin_aenq_keep_alive_desc *desc; 2696 sbintime_t stime; 2697 uint64_t rx_drops; 2698 2699 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; 2700 2701 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; 2702 counter_u64_zero(adapter->hw_stats.rx_drops); 2703 counter_u64_add(adapter->hw_stats.rx_drops, rx_drops); 2704 2705 stime = getsbinuptime(); 2706 atomic_store_rel_64(&adapter->keep_alive_timestamp, stime); 2707} 2708 2709/* Check for keep alive expiration */ 2710static void check_for_missing_keep_alive(struct ena_adapter *adapter) 2711{ 2712 sbintime_t timestamp, time; 2713 2714 if (adapter->wd_active == 0) 2715 return; 2716 2717 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2718 return; 2719 2720 timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp); 2721 time = getsbinuptime() - timestamp; 2722 if (unlikely(time > adapter->keep_alive_timeout)) { 2723 device_printf(adapter->pdev, 2724 "Keep alive watchdog timeout.\n"); 2725 counter_u64_add(adapter->dev_stats.wd_expired, 1); 2726 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 2727 adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO; 2728 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 2729 } 2730 } 2731} 2732 2733/* Check if admin queue is enabled */ 2734static void check_for_admin_com_state(struct ena_adapter *adapter) 2735{ 2736 if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) == 2737 false)) { 2738 device_printf(adapter->pdev, 2739 "ENA admin queue is not in running state!\n"); 2740 counter_u64_add(adapter->dev_stats.admin_q_pause, 1); 2741 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 2742 adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO; 2743 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 2744 } 2745 } 2746} 2747 2748static int 2749check_for_rx_interrupt_queue(struct ena_adapter *adapter, 2750 struct ena_ring *rx_ring) 2751{ 2752 if (likely(rx_ring->first_interrupt)) 2753 return (0); 2754 2755 if (ena_com_cq_empty(rx_ring->ena_com_io_cq)) 2756 return (0); 2757 2758 rx_ring->no_interrupt_event_cnt++; 2759 2760 if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) { 2761 device_printf(adapter->pdev, "Potential MSIX issue on Rx side " 2762 "Queue = %d. Reset the device\n", rx_ring->qid); 2763 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 2764 adapter->reset_reason = ENA_REGS_RESET_MISS_INTERRUPT; 2765 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 2766 } 2767 return (EIO); 2768 } 2769 2770 return (0); 2771} 2772 2773static int 2774check_missing_comp_in_tx_queue(struct ena_adapter *adapter, 2775 struct ena_ring *tx_ring) 2776{ 2777 struct bintime curtime, time; 2778 struct ena_tx_buffer *tx_buf; 2779 sbintime_t time_offset; 2780 uint32_t missed_tx = 0; 2781 int i, rc = 0; 2782 2783 getbinuptime(&curtime); 2784 2785 for (i = 0; i < tx_ring->ring_size; i++) { 2786 tx_buf = &tx_ring->tx_buffer_info[i]; 2787 2788 if (bintime_isset(&tx_buf->timestamp) == 0) 2789 continue; 2790 2791 time = curtime; 2792 bintime_sub(&time, &tx_buf->timestamp); 2793 time_offset = bttosbt(time); 2794 2795 if (unlikely(!tx_ring->first_interrupt && 2796 time_offset > 2 * adapter->missing_tx_timeout)) { 2797 /* 2798 * If after graceful period interrupt is still not 2799 * received, we schedule a reset. 2800 */ 2801 device_printf(adapter->pdev, 2802 "Potential MSIX issue on Tx side Queue = %d. " 2803 "Reset the device\n", tx_ring->qid); 2804 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, 2805 adapter))) { 2806 adapter->reset_reason = 2807 ENA_REGS_RESET_MISS_INTERRUPT; 2808 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, 2809 adapter); 2810 } 2811 return (EIO); 2812 } 2813 2814 /* Check again if packet is still waiting */ 2815 if (unlikely(time_offset > adapter->missing_tx_timeout)) { 2816 2817 if (!tx_buf->print_once) 2818 ena_trace(ENA_WARNING, "Found a Tx that wasn't " 2819 "completed on time, qid %d, index %d.\n", 2820 tx_ring->qid, i); 2821 2822 tx_buf->print_once = true; 2823 missed_tx++; 2824 } 2825 } 2826 2827 if (unlikely(missed_tx > adapter->missing_tx_threshold)) { 2828 device_printf(adapter->pdev, 2829 "The number of lost tx completion is above the threshold " 2830 "(%d > %d). Reset the device\n", 2831 missed_tx, adapter->missing_tx_threshold); 2832 if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 2833 adapter->reset_reason = ENA_REGS_RESET_MISS_TX_CMPL; 2834 ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 2835 } 2836 rc = EIO; 2837 } 2838 2839 counter_u64_add(tx_ring->tx_stats.missing_tx_comp, missed_tx); 2840 2841 return (rc); 2842} 2843 2844/* 2845 * Check for TX which were not completed on time. 2846 * Timeout is defined by "missing_tx_timeout". 2847 * Reset will be performed if number of incompleted 2848 * transactions exceeds "missing_tx_threshold". 2849 */ 2850static void 2851check_for_missing_completions(struct ena_adapter *adapter) 2852{ 2853 struct ena_ring *tx_ring; 2854 struct ena_ring *rx_ring; 2855 int i, budget, rc; 2856 2857 /* Make sure the driver doesn't turn the device in other process */ 2858 rmb(); 2859 2860 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 2861 return; 2862 2863 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) 2864 return; 2865 2866 if (adapter->missing_tx_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2867 return; 2868 2869 budget = adapter->missing_tx_max_queues; 2870 2871 for (i = adapter->next_monitored_tx_qid; i < adapter->num_queues; i++) { 2872 tx_ring = &adapter->tx_ring[i]; 2873 rx_ring = &adapter->rx_ring[i]; 2874 2875 rc = check_missing_comp_in_tx_queue(adapter, tx_ring); 2876 if (unlikely(rc != 0)) 2877 return; 2878 2879 rc = check_for_rx_interrupt_queue(adapter, rx_ring); 2880 if (unlikely(rc != 0)) 2881 return; 2882 2883 budget--; 2884 if (budget == 0) { 2885 i++; 2886 break; 2887 } 2888 } 2889 2890 adapter->next_monitored_tx_qid = i % adapter->num_queues; 2891} 2892 2893/* trigger rx cleanup after 2 consecutive detections */ 2894#define EMPTY_RX_REFILL 2 2895/* For the rare case where the device runs out of Rx descriptors and the 2896 * msix handler failed to refill new Rx descriptors (due to a lack of memory 2897 * for example). 2898 * This case will lead to a deadlock: 2899 * The device won't send interrupts since all the new Rx packets will be dropped 2900 * The msix handler won't allocate new Rx descriptors so the device won't be 2901 * able to send new packets. 2902 * 2903 * When such a situation is detected - execute rx cleanup task in another thread 2904 */ 2905static void 2906check_for_empty_rx_ring(struct ena_adapter *adapter) 2907{ 2908 struct ena_ring *rx_ring; 2909 int i, refill_required; 2910 2911 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 2912 return; 2913 2914 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) 2915 return; 2916 2917 for (i = 0; i < adapter->num_queues; i++) { 2918 rx_ring = &adapter->rx_ring[i]; 2919 2920 refill_required = ena_com_free_desc(rx_ring->ena_com_io_sq); 2921 if (unlikely(refill_required == (rx_ring->ring_size - 1))) { 2922 rx_ring->empty_rx_queue++; 2923 2924 if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) { 2925 counter_u64_add(rx_ring->rx_stats.empty_rx_ring, 2926 1); 2927 2928 device_printf(adapter->pdev, 2929 "trigger refill for ring %d\n", i); 2930 2931 taskqueue_enqueue(rx_ring->que->cleanup_tq, 2932 &rx_ring->que->cleanup_task); 2933 rx_ring->empty_rx_queue = 0; 2934 } 2935 } else { 2936 rx_ring->empty_rx_queue = 0; 2937 } 2938 } 2939} 2940 2941static void ena_update_hints(struct ena_adapter *adapter, 2942 struct ena_admin_ena_hw_hints *hints) 2943{ 2944 struct ena_com_dev *ena_dev = adapter->ena_dev; 2945 2946 if (hints->admin_completion_tx_timeout) 2947 ena_dev->admin_queue.completion_timeout = 2948 hints->admin_completion_tx_timeout * 1000; 2949 2950 if (hints->mmio_read_timeout) 2951 /* convert to usec */ 2952 ena_dev->mmio_read.reg_read_to = 2953 hints->mmio_read_timeout * 1000; 2954 2955 if (hints->missed_tx_completion_count_threshold_to_reset) 2956 adapter->missing_tx_threshold = 2957 hints->missed_tx_completion_count_threshold_to_reset; 2958 2959 if (hints->missing_tx_completion_timeout) { 2960 if (hints->missing_tx_completion_timeout == 2961 ENA_HW_HINTS_NO_TIMEOUT) 2962 adapter->missing_tx_timeout = ENA_HW_HINTS_NO_TIMEOUT; 2963 else 2964 adapter->missing_tx_timeout = 2965 SBT_1MS * hints->missing_tx_completion_timeout; 2966 } 2967 2968 if (hints->driver_watchdog_timeout) { 2969 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2970 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; 2971 else 2972 adapter->keep_alive_timeout = 2973 SBT_1MS * hints->driver_watchdog_timeout; 2974 } 2975} 2976 2977static void 2978ena_timer_service(void *data) 2979{ 2980 struct ena_adapter *adapter = (struct ena_adapter *)data; 2981 struct ena_admin_host_info *host_info = 2982 adapter->ena_dev->host_attr.host_info; 2983 2984 check_for_missing_keep_alive(adapter); 2985 2986 check_for_admin_com_state(adapter); 2987 2988 check_for_missing_completions(adapter); 2989 2990 check_for_empty_rx_ring(adapter); 2991 2992 if (host_info != NULL) 2993 ena_update_host_info(host_info, adapter->ifp); 2994 2995 if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 2996 device_printf(adapter->pdev, "Trigger reset is on\n"); 2997 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task); 2998 return; 2999 } 3000 3001 /* 3002 * Schedule another timeout one second from now. 3003 */ 3004 callout_schedule_sbt(&adapter->timer_service, SBT_1S, SBT_1S, 0); 3005} 3006 3007void 3008ena_destroy_device(struct ena_adapter *adapter, bool graceful) 3009{ 3010 if_t ifp = adapter->ifp; 3011 struct ena_com_dev *ena_dev = adapter->ena_dev; 3012 bool dev_up; 3013 3014 if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) 3015 return; 3016 3017 if_link_state_change(ifp, LINK_STATE_DOWN); 3018 3019 callout_drain(&adapter->timer_service); 3020 3021 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 3022 if (dev_up) 3023 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 3024 else 3025 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 3026 3027 if (!graceful) 3028 ena_com_set_admin_running_state(ena_dev, false); 3029 3030 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3031 ena_down(adapter); 3032 3033 /* 3034 * Stop the device from sending AENQ events (if the device was up, and 3035 * the trigger reset was on, ena_down already performs device reset) 3036 */ 3037 if (!(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter) && dev_up)) 3038 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); 3039 3040 ena_free_mgmnt_irq(adapter); 3041 3042 ena_disable_msix(adapter); 3043 3044 ena_com_abort_admin_commands(ena_dev); 3045 3046 ena_com_wait_for_abort_completion(ena_dev); 3047 3048 ena_com_admin_destroy(ena_dev); 3049 3050 ena_com_mmio_reg_read_request_destroy(ena_dev); 3051 3052 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3053 3054 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 3055 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3056} 3057 3058static int 3059ena_device_validate_params(struct ena_adapter *adapter, 3060 struct ena_com_dev_get_features_ctx *get_feat_ctx) 3061{ 3062 3063 if (memcmp(get_feat_ctx->dev_attr.mac_addr, adapter->mac_addr, 3064 ETHER_ADDR_LEN) != 0) { 3065 device_printf(adapter->pdev, 3066 "Error, mac address are different\n"); 3067 return (EINVAL); 3068 } 3069 3070 if (get_feat_ctx->dev_attr.max_mtu < if_getmtu(adapter->ifp)) { 3071 device_printf(adapter->pdev, 3072 "Error, device max mtu is smaller than ifp MTU\n"); 3073 return (EINVAL); 3074 } 3075 3076 return 0; 3077} 3078 3079int 3080ena_restore_device(struct ena_adapter *adapter) 3081{ 3082 struct ena_com_dev_get_features_ctx get_feat_ctx; 3083 struct ena_com_dev *ena_dev = adapter->ena_dev; 3084 if_t ifp = adapter->ifp; 3085 device_t dev = adapter->pdev; 3086 int wd_active; 3087 int rc; 3088 3089 ENA_FLAG_SET_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3090 3091 rc = ena_device_init(adapter, dev, &get_feat_ctx, &wd_active); 3092 if (rc != 0) { 3093 device_printf(dev, "Cannot initialize device\n"); 3094 goto err; 3095 } 3096 /* 3097 * Only enable WD if it was enabled before reset, so it won't override 3098 * value set by the user by the sysctl. 3099 */ 3100 if (adapter->wd_active != 0) 3101 adapter->wd_active = wd_active; 3102 3103 rc = ena_device_validate_params(adapter, &get_feat_ctx); 3104 if (rc != 0) { 3105 device_printf(dev, "Validation of device parameters failed\n"); 3106 goto err_device_destroy; 3107 } 3108 3109 rc = ena_handle_updated_queues(adapter, &get_feat_ctx); 3110 if (rc != 0) 3111 goto err_device_destroy; 3112 3113 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3114 /* Make sure we don't have a race with AENQ Links state handler */ 3115 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) 3116 if_link_state_change(ifp, LINK_STATE_UP); 3117 3118 rc = ena_enable_msix_and_set_admin_interrupts(adapter, 3119 adapter->num_queues); 3120 if (rc != 0) { 3121 device_printf(dev, "Enable MSI-X failed\n"); 3122 goto err_device_destroy; 3123 } 3124 3125 /* If the interface was up before the reset bring it up */ 3126 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) { 3127 rc = ena_up(adapter); 3128 if (rc != 0) { 3129 device_printf(dev, "Failed to create I/O queues\n"); 3130 goto err_disable_msix; 3131 } 3132 } 3133 3134 /* Indicate that device is running again and ready to work */ 3135 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3136 3137 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) { 3138 /* 3139 * As the AENQ handlers weren't executed during reset because 3140 * the flag ENA_FLAG_DEVICE_RUNNING was turned off, the 3141 * timestamp must be updated again That will prevent next reset 3142 * caused by missing keep alive. 3143 */ 3144 adapter->keep_alive_timestamp = getsbinuptime(); 3145 callout_reset_sbt(&adapter->timer_service, SBT_1S, SBT_1S, 3146 ena_timer_service, (void *)adapter, 0); 3147 } 3148 3149 device_printf(dev, 3150 "Device reset completed successfully, Driver info: %s\n", ena_version); 3151 3152 return (rc); 3153 3154err_disable_msix: 3155 ena_free_mgmnt_irq(adapter); 3156 ena_disable_msix(adapter); 3157err_device_destroy: 3158 ena_com_abort_admin_commands(ena_dev); 3159 ena_com_wait_for_abort_completion(ena_dev); 3160 ena_com_admin_destroy(ena_dev); 3161 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE); 3162 ena_com_mmio_reg_read_request_destroy(ena_dev); 3163err: 3164 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3165 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3166 device_printf(dev, "Reset attempt failed. Can not reset the device\n"); 3167 3168 return (rc); 3169} 3170 3171static void 3172ena_reset_task(void *arg, int pending) 3173{ 3174 struct ena_adapter *adapter = (struct ena_adapter *)arg; 3175 3176 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 3177 device_printf(adapter->pdev, 3178 "device reset scheduled but trigger_reset is off\n"); 3179 return; 3180 } 3181 3182 sx_xlock(&adapter->ioctl_sx); 3183 ena_destroy_device(adapter, false); 3184 ena_restore_device(adapter); 3185 sx_unlock(&adapter->ioctl_sx); 3186} 3187 3188/** 3189 * ena_attach - Device Initialization Routine 3190 * @pdev: device information struct 3191 * 3192 * Returns 0 on success, otherwise on failure. 3193 * 3194 * ena_attach initializes an adapter identified by a device structure. 3195 * The OS initialization, configuring of the adapter private structure, 3196 * and a hardware reset occur. 3197 **/ 3198static int 3199ena_attach(device_t pdev) 3200{ 3201 struct ena_com_dev_get_features_ctx get_feat_ctx; 3202 struct ena_llq_configurations llq_config; 3203 struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 }; 3204 static int version_printed; 3205 struct ena_adapter *adapter; 3206 struct ena_com_dev *ena_dev = NULL; 3207 const char *queue_type_str; 3208 int io_queue_num; 3209 int rid, rc; 3210 3211 adapter = device_get_softc(pdev); 3212 adapter->pdev = pdev; 3213 3214 mtx_init(&adapter->global_mtx, "ENA global mtx", NULL, MTX_DEF); 3215 sx_init(&adapter->ioctl_sx, "ENA ioctl sx"); 3216 3217 /* Set up the timer service */ 3218 callout_init_mtx(&adapter->timer_service, &adapter->global_mtx, 0); 3219 adapter->keep_alive_timeout = DEFAULT_KEEP_ALIVE_TO; 3220 adapter->missing_tx_timeout = DEFAULT_TX_CMP_TO; 3221 adapter->missing_tx_max_queues = DEFAULT_TX_MONITORED_QUEUES; 3222 adapter->missing_tx_threshold = DEFAULT_TX_CMP_THRESHOLD; 3223 3224 if (version_printed++ == 0) 3225 device_printf(pdev, "%s\n", ena_version); 3226 3227 /* Allocate memory for ena_dev structure */ 3228 ena_dev = malloc(sizeof(struct ena_com_dev), M_DEVBUF, 3229 M_WAITOK | M_ZERO); 3230 3231 adapter->ena_dev = ena_dev; 3232 ena_dev->dmadev = pdev; 3233 3234 rid = PCIR_BAR(ENA_REG_BAR); 3235 adapter->memory = NULL; 3236 adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, 3237 &rid, RF_ACTIVE); 3238 if (unlikely(adapter->registers == NULL)) { 3239 device_printf(pdev, 3240 "unable to allocate bus resource: registers!\n"); 3241 rc = ENOMEM; 3242 goto err_dev_free; 3243 } 3244 3245 ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF, 3246 M_WAITOK | M_ZERO); 3247 3248 /* Store register resources */ 3249 ((struct ena_bus*)(ena_dev->bus))->reg_bar_t = 3250 rman_get_bustag(adapter->registers); 3251 ((struct ena_bus*)(ena_dev->bus))->reg_bar_h = 3252 rman_get_bushandle(adapter->registers); 3253 3254 if (unlikely(((struct ena_bus*)(ena_dev->bus))->reg_bar_h == 0)) { 3255 device_printf(pdev, "failed to pmap registers bar\n"); 3256 rc = ENXIO; 3257 goto err_bus_free; 3258 } 3259 3260 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 3261 3262 /* Initially clear all the flags */ 3263 ENA_FLAG_ZERO(adapter); 3264 3265 /* Device initialization */ 3266 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active); 3267 if (unlikely(rc != 0)) { 3268 device_printf(pdev, "ENA device init failed! (err: %d)\n", rc); 3269 rc = ENXIO; 3270 goto err_bus_free; 3271 } 3272 3273 set_default_llq_configurations(&llq_config); 3274 3275#if defined(__arm__) || defined(__aarch64__) 3276 /* 3277 * Force LLQ disable, as the driver is not supporting WC enablement 3278 * on the ARM architecture. Using LLQ without WC would affect 3279 * performance in a negative way. 3280 */ 3281 ena_dev->supported_features &= ~(1 << ENA_ADMIN_LLQ); 3282#endif 3283 rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx.llq, 3284 &llq_config); 3285 if (unlikely(rc != 0)) { 3286 device_printf(pdev, "failed to set placement policy\n"); 3287 goto err_com_free; 3288 } 3289 3290 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 3291 queue_type_str = "Regular"; 3292 else 3293 queue_type_str = "Low Latency"; 3294 device_printf(pdev, "Placement policy: %s\n", queue_type_str); 3295 3296 adapter->keep_alive_timestamp = getsbinuptime(); 3297 3298 adapter->tx_offload_cap = get_feat_ctx.offload.tx; 3299 3300 memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr, 3301 ETHER_ADDR_LEN); 3302 3303 calc_queue_ctx.ena_dev = ena_dev; 3304 calc_queue_ctx.get_feat_ctx = &get_feat_ctx; 3305 calc_queue_ctx.pdev = pdev; 3306 3307 /* calculate IO queue number to create */ 3308 io_queue_num = ena_calc_io_queue_num(adapter, &get_feat_ctx); 3309 3310 ENA_ASSERT(io_queue_num > 0, "Invalid queue number: %d\n", 3311 io_queue_num); 3312 adapter->num_queues = io_queue_num; 3313 3314 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 3315 // Set the requested Rx ring size 3316 adapter->rx_ring_size = ENA_DEFAULT_RING_SIZE; 3317 /* calculatre ring sizes */ 3318 rc = ena_calc_queue_size(adapter, &calc_queue_ctx); 3319 if (unlikely((rc != 0) || (io_queue_num <= 0))) { 3320 rc = EFAULT; 3321 goto err_com_free; 3322 } 3323 3324 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3325 3326 adapter->tx_ring_size = calc_queue_ctx.tx_queue_size; 3327 adapter->rx_ring_size = calc_queue_ctx.rx_queue_size; 3328 3329 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; 3330 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; 3331 3332 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE; 3333 3334 /* set up dma tags for rx and tx buffers */ 3335 rc = ena_setup_tx_dma_tag(adapter); 3336 if (unlikely(rc != 0)) { 3337 device_printf(pdev, "Failed to create TX DMA tag\n"); 3338 goto err_com_free; 3339 } 3340 3341 rc = ena_setup_rx_dma_tag(adapter); 3342 if (unlikely(rc != 0)) { 3343 device_printf(pdev, "Failed to create RX DMA tag\n"); 3344 goto err_tx_tag_free; 3345 } 3346 3347 /* initialize rings basic information */ 3348 device_printf(pdev, 3349 "Creating %d io queues. Rx queue size: %d, Tx queue size: %d\n", 3350 io_queue_num, 3351 calc_queue_ctx.rx_queue_size, 3352 calc_queue_ctx.tx_queue_size); 3353 ena_init_io_rings(adapter); 3354 3355 rc = ena_enable_msix_and_set_admin_interrupts(adapter, io_queue_num); 3356 if (unlikely(rc != 0)) { 3357 device_printf(pdev, 3358 "Failed to enable and set the admin interrupts\n"); 3359 goto err_io_free; 3360 } 3361 3362 /* setup network interface */ 3363 rc = ena_setup_ifnet(pdev, adapter, &get_feat_ctx); 3364 if (unlikely(rc != 0)) { 3365 device_printf(pdev, "Error with network interface setup\n"); 3366 goto err_msix_free; 3367 } 3368 3369 /* Initialize reset task queue */ 3370 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter); 3371 adapter->reset_tq = taskqueue_create("ena_reset_enqueue", 3372 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq); 3373 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, 3374 "%s rstq", device_get_nameunit(adapter->pdev)); 3375 3376 /* Initialize statistics */ 3377 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats, 3378 sizeof(struct ena_stats_dev)); 3379 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats, 3380 sizeof(struct ena_hw_stats)); 3381 ena_sysctl_add_nodes(adapter); 3382 3383#ifdef DEV_NETMAP 3384 rc = ena_netmap_attach(adapter); 3385 if (rc != 0) { 3386 device_printf(pdev, "netmap attach failed: %d\n", rc); 3387 goto err_detach; 3388 } 3389#endif /* DEV_NETMAP */ 3390 3391 /* Tell the stack that the interface is not active */ 3392 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 3393 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3394 3395 return (0); 3396 3397#ifdef DEV_NETMAP 3398err_detach: 3399 ether_ifdetach(adapter->ifp); 3400#endif /* DEV_NETMAP */ 3401err_msix_free: 3402 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR); 3403 ena_free_mgmnt_irq(adapter); 3404 ena_disable_msix(adapter); 3405err_io_free: 3406 ena_free_all_io_rings_resources(adapter); 3407 ena_free_rx_dma_tag(adapter); 3408err_tx_tag_free: 3409 ena_free_tx_dma_tag(adapter); 3410err_com_free: 3411 ena_com_admin_destroy(ena_dev); 3412 ena_com_delete_host_info(ena_dev); 3413 ena_com_mmio_reg_read_request_destroy(ena_dev); 3414err_bus_free: 3415 free(ena_dev->bus, M_DEVBUF); 3416 ena_free_pci_resources(adapter); 3417err_dev_free: 3418 free(ena_dev, M_DEVBUF); 3419 3420 return (rc); 3421} 3422 3423/** 3424 * ena_detach - Device Removal Routine 3425 * @pdev: device information struct 3426 * 3427 * ena_detach is called by the device subsystem to alert the driver 3428 * that it should release a PCI device. 3429 **/ 3430static int 3431ena_detach(device_t pdev) 3432{ 3433 struct ena_adapter *adapter = device_get_softc(pdev); 3434 struct ena_com_dev *ena_dev = adapter->ena_dev; 3435 int rc; 3436 3437 /* Make sure VLANS are not using driver */ 3438 if (adapter->ifp->if_vlantrunk != NULL) { 3439 device_printf(adapter->pdev ,"VLAN is in use, detach first\n"); 3440 return (EBUSY); 3441 } 3442 3443 ether_ifdetach(adapter->ifp); 3444 3445 /* Free reset task and callout */ 3446 callout_drain(&adapter->timer_service); 3447 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL)) 3448 taskqueue_drain(adapter->reset_tq, &adapter->reset_task); 3449 taskqueue_free(adapter->reset_tq); 3450 3451 sx_xlock(&adapter->ioctl_sx); 3452 ena_down(adapter); 3453 ena_destroy_device(adapter, true); 3454 sx_unlock(&adapter->ioctl_sx); 3455 3456#ifdef DEV_NETMAP 3457 netmap_detach(adapter->ifp); 3458#endif /* DEV_NETMAP */ 3459 3460 ena_free_all_io_rings_resources(adapter); 3461 3462 ena_free_counters((counter_u64_t *)&adapter->hw_stats, 3463 sizeof(struct ena_hw_stats)); 3464 ena_free_counters((counter_u64_t *)&adapter->dev_stats, 3465 sizeof(struct ena_stats_dev)); 3466 3467 rc = ena_free_rx_dma_tag(adapter); 3468 if (unlikely(rc != 0)) 3469 device_printf(adapter->pdev, 3470 "Unmapped RX DMA tag associations\n"); 3471 3472 rc = ena_free_tx_dma_tag(adapter); 3473 if (unlikely(rc != 0)) 3474 device_printf(adapter->pdev, 3475 "Unmapped TX DMA tag associations\n"); 3476 3477 ena_free_irqs(adapter); 3478 3479 ena_free_pci_resources(adapter); 3480 3481 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) 3482 ena_com_rss_destroy(ena_dev); 3483 3484 ena_com_delete_host_info(ena_dev); 3485 3486 mtx_destroy(&adapter->global_mtx); 3487 sx_destroy(&adapter->ioctl_sx); 3488 3489 if_free(adapter->ifp); 3490 3491 if (ena_dev->bus != NULL) 3492 free(ena_dev->bus, M_DEVBUF); 3493 3494 if (ena_dev != NULL) 3495 free(ena_dev, M_DEVBUF); 3496 3497 return (bus_generic_detach(pdev)); 3498} 3499 3500/****************************************************************************** 3501 ******************************** AENQ Handlers ******************************* 3502 *****************************************************************************/ 3503/** 3504 * ena_update_on_link_change: 3505 * Notify the network interface about the change in link status 3506 **/ 3507static void 3508ena_update_on_link_change(void *adapter_data, 3509 struct ena_admin_aenq_entry *aenq_e) 3510{ 3511 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3512 struct ena_admin_aenq_link_change_desc *aenq_desc; 3513 int status; 3514 if_t ifp; 3515 3516 aenq_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; 3517 ifp = adapter->ifp; 3518 status = aenq_desc->flags & 3519 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK; 3520 3521 if (status != 0) { 3522 device_printf(adapter->pdev, "link is UP\n"); 3523 ENA_FLAG_SET_ATOMIC(ENA_FLAG_LINK_UP, adapter); 3524 if (!ENA_FLAG_ISSET(ENA_FLAG_ONGOING_RESET, adapter)) 3525 if_link_state_change(ifp, LINK_STATE_UP); 3526 } else { 3527 device_printf(adapter->pdev, "link is DOWN\n"); 3528 if_link_state_change(ifp, LINK_STATE_DOWN); 3529 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_LINK_UP, adapter); 3530 } 3531} 3532 3533static void ena_notification(void *adapter_data, 3534 struct ena_admin_aenq_entry *aenq_e) 3535{ 3536 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3537 struct ena_admin_ena_hw_hints *hints; 3538 3539 ENA_WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION, 3540 "Invalid group(%x) expected %x\n", aenq_e->aenq_common_desc.group, 3541 ENA_ADMIN_NOTIFICATION); 3542 3543 switch (aenq_e->aenq_common_desc.syndrom) { 3544 case ENA_ADMIN_UPDATE_HINTS: 3545 hints = 3546 (struct ena_admin_ena_hw_hints *)(&aenq_e->inline_data_w4); 3547 ena_update_hints(adapter, hints); 3548 break; 3549 default: 3550 device_printf(adapter->pdev, 3551 "Invalid aenq notification link state %d\n", 3552 aenq_e->aenq_common_desc.syndrom); 3553 } 3554} 3555 3556/** 3557 * This handler will called for unknown event group or unimplemented handlers 3558 **/ 3559static void 3560unimplemented_aenq_handler(void *adapter_data, 3561 struct ena_admin_aenq_entry *aenq_e) 3562{ 3563 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3564 3565 device_printf(adapter->pdev, 3566 "Unknown event was received or event with unimplemented handler\n"); 3567} 3568 3569static struct ena_aenq_handlers aenq_handlers = { 3570 .handlers = { 3571 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 3572 [ENA_ADMIN_NOTIFICATION] = ena_notification, 3573 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd, 3574 }, 3575 .unimplemented_handler = unimplemented_aenq_handler 3576}; 3577 3578/********************************************************************* 3579 * FreeBSD Device Interface Entry Points 3580 *********************************************************************/ 3581 3582static device_method_t ena_methods[] = { 3583 /* Device interface */ 3584 DEVMETHOD(device_probe, ena_probe), 3585 DEVMETHOD(device_attach, ena_attach), 3586 DEVMETHOD(device_detach, ena_detach), 3587 DEVMETHOD_END 3588}; 3589 3590static driver_t ena_driver = { 3591 "ena", ena_methods, sizeof(struct ena_adapter), 3592}; 3593 3594devclass_t ena_devclass; 3595DRIVER_MODULE(ena, pci, ena_driver, ena_devclass, 0, 0); 3596MODULE_PNP_INFO("U16:vendor;U16:device", pci, ena, ena_vendor_info_array, 3597 sizeof(ena_vendor_info_array[0]), nitems(ena_vendor_info_array) - 1); 3598MODULE_DEPEND(ena, pci, 1, 1, 1); 3599MODULE_DEPEND(ena, ether, 1, 1, 1); 3600#ifdef DEV_NETMAP 3601MODULE_DEPEND(ena, netmap, 1, 1, 1); 3602#endif /* DEV_NETMAP */ 3603 3604/*********************************************************************/ 3605