nicvf_queues.c revision 296602
1249259Sdim/* 2249259Sdim * Copyright (C) 2015 Cavium Inc. 3249259Sdim * All rights reserved. 4249259Sdim * 5249259Sdim * Redistribution and use in source and binary forms, with or without 6249259Sdim * modification, are permitted provided that the following conditions 7249259Sdim * are met: 8249259Sdim * 1. Redistributions of source code must retain the above copyright 9249259Sdim * notice, this list of conditions and the following disclaimer. 10276479Sdim * 2. Redistributions in binary form must reproduce the above copyright 11249259Sdim * notice, this list of conditions and the following disclaimer in the 12249259Sdim * documentation and/or other materials provided with the distribution. 13249259Sdim * 14249259Sdim * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15249259Sdim * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16249259Sdim * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17249259Sdim * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18276479Sdim * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19249259Sdim * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20249259Sdim * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21249259Sdim * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22249259Sdim * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23249259Sdim * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24249259Sdim * SUCH DAMAGE. 25249259Sdim * 26249259Sdim * $FreeBSD: head/sys/dev/vnic/nicvf_queues.c 296602 2016-03-10 05:45:24Z zbb $ 27249259Sdim * 28249259Sdim */ 29249259Sdim#include <sys/cdefs.h> 30249259Sdim__FBSDID("$FreeBSD: head/sys/dev/vnic/nicvf_queues.c 296602 2016-03-10 05:45:24Z zbb $"); 31249259Sdim 32249259Sdim#include "opt_inet.h" 33249259Sdim#include "opt_inet6.h" 34249259Sdim 35249259Sdim#include <sys/param.h> 36249259Sdim#include <sys/systm.h> 37249259Sdim#include <sys/bitset.h> 38249259Sdim#include <sys/bitstring.h> 39249259Sdim#include <sys/buf_ring.h> 40249259Sdim#include <sys/bus.h> 41249259Sdim#include <sys/endian.h> 42249259Sdim#include <sys/kernel.h> 43249259Sdim#include <sys/malloc.h> 44249259Sdim#include <sys/module.h> 45261991Sdim#include <sys/rman.h> 46261991Sdim#include <sys/pciio.h> 47261991Sdim#include <sys/pcpu.h> 48261991Sdim#include <sys/proc.h> 49261991Sdim#include <sys/sockio.h> 50249259Sdim#include <sys/socket.h> 51276479Sdim#include <sys/stdatomic.h> 52261991Sdim#include <sys/cpuset.h> 53261991Sdim#include <sys/lock.h> 54249259Sdim#include <sys/mutex.h> 55249259Sdim#include <sys/smp.h> 56249259Sdim#include <sys/taskqueue.h> 57249259Sdim 58249259Sdim#include <vm/vm.h> 59249259Sdim#include <vm/pmap.h> 60276479Sdim 61249259Sdim#include <machine/bus.h> 62261991Sdim#include <machine/vmparam.h> 63261991Sdim 64261991Sdim#include <net/ethernet.h> 65261991Sdim#include <net/if.h> 66249259Sdim#include <net/if_var.h> 67276479Sdim#include <net/if_media.h> 68249259Sdim#include <net/ifq.h> 69249259Sdim 70249259Sdim#include <netinet/in_systm.h> 71249259Sdim#include <netinet/in.h> 72249259Sdim#include <netinet/if_ether.h> 73249259Sdim#include <netinet/ip.h> 74261991Sdim#include <netinet/ip6.h> 75261991Sdim#include <netinet/sctp.h> 76261991Sdim#include <netinet/tcp.h> 77261991Sdim#include <netinet/tcp_lro.h> 78261991Sdim#include <netinet/udp.h> 79261991Sdim 80249259Sdim#include <dev/pci/pcireg.h> 81249259Sdim#include <dev/pci/pcivar.h> 82249259Sdim 83249259Sdim#include "thunder_bgx.h" 84249259Sdim#include "nic_reg.h" 85249259Sdim#include "nic.h" 86249259Sdim#include "q_struct.h" 87249259Sdim#include "nicvf_queues.h" 88249259Sdim 89249259Sdim#define DEBUG 90249259Sdim#undef DEBUG 91249259Sdim 92276479Sdim#ifdef DEBUG 93276479Sdim#define dprintf(dev, fmt, ...) device_printf(dev, fmt, ##__VA_ARGS__) 94249259Sdim#else 95249259Sdim#define dprintf(dev, fmt, ...) 96261991Sdim#endif 97261991Sdim 98261991SdimMALLOC_DECLARE(M_NICVF); 99261991Sdim 100261991Sdimstatic void nicvf_free_snd_queue(struct nicvf *, struct snd_queue *); 101261991Sdimstatic struct mbuf * nicvf_get_rcv_mbuf(struct nicvf *, struct cqe_rx_t *); 102261991Sdimstatic void nicvf_sq_disable(struct nicvf *, int); 103261991Sdimstatic void nicvf_sq_enable(struct nicvf *, struct snd_queue *, int); 104261991Sdimstatic void nicvf_put_sq_desc(struct snd_queue *, int); 105261991Sdimstatic void nicvf_cmp_queue_config(struct nicvf *, struct queue_set *, int, 106249259Sdim boolean_t); 107249259Sdimstatic void nicvf_sq_free_used_descs(struct nicvf *, struct snd_queue *, int); 108249259Sdim 109261991Sdimstatic void nicvf_rbdr_task(void *, int); 110261991Sdimstatic void nicvf_rbdr_task_nowait(void *, int); 111249259Sdim 112249259Sdimstruct rbuf_info { 113249259Sdim bus_dma_tag_t dmat; 114249259Sdim bus_dmamap_t dmap; 115249259Sdim struct mbuf * mbuf; 116249259Sdim}; 117249259Sdim 118249259Sdim#define GET_RBUF_INFO(x) ((struct rbuf_info *)((x) - NICVF_RCV_BUF_ALIGN_BYTES)) 119249259Sdim 120249259Sdim/* Poll a register for a specific value */ 121261991Sdimstatic int nicvf_poll_reg(struct nicvf *nic, int qidx, 122261991Sdim uint64_t reg, int bit_pos, int bits, int val) 123261991Sdim{ 124261991Sdim uint64_t bit_mask; 125261991Sdim uint64_t reg_val; 126261991Sdim int timeout = 10; 127261991Sdim 128261991Sdim bit_mask = (1UL << bits) - 1; 129261991Sdim bit_mask = (bit_mask << bit_pos); 130249259Sdim 131249259Sdim while (timeout) { 132249259Sdim reg_val = nicvf_queue_reg_read(nic, reg, qidx); 133249259Sdim if (((reg_val & bit_mask) >> bit_pos) == val) 134249259Sdim return (0); 135249259Sdim 136249259Sdim DELAY(1000); 137249259Sdim timeout--; 138249259Sdim } 139249259Sdim device_printf(nic->dev, "Poll on reg 0x%lx failed\n", reg); 140249259Sdim return (ETIMEDOUT); 141249259Sdim} 142249259Sdim 143249259Sdim/* Callback for bus_dmamap_load() */ 144249259Sdimstatic void 145249259Sdimnicvf_dmamap_q_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 146249259Sdim{ 147249259Sdim bus_addr_t *paddr; 148249259Sdim 149249259Sdim KASSERT(nseg == 1, ("wrong number of segments, should be 1")); 150249259Sdim paddr = arg; 151249259Sdim *paddr = segs->ds_addr; 152249259Sdim} 153249259Sdim 154249259Sdim/* Allocate memory for a queue's descriptors */ 155249259Sdimstatic int 156261991Sdimnicvf_alloc_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem, 157261991Sdim int q_len, int desc_size, int align_bytes) 158249259Sdim{ 159249259Sdim int err, err_dmat; 160276479Sdim 161276479Sdim /* Create DMA tag first */ 162276479Sdim err = bus_dma_tag_create( 163249259Sdim bus_get_dma_tag(nic->dev), /* parent tag */ 164249259Sdim align_bytes, /* alignment */ 165249259Sdim 0, /* boundary */ 166249259Sdim BUS_SPACE_MAXADDR, /* lowaddr */ 167249259Sdim BUS_SPACE_MAXADDR, /* highaddr */ 168249259Sdim NULL, NULL, /* filtfunc, filtfuncarg */ 169249259Sdim (q_len * desc_size), /* maxsize */ 170249259Sdim 1, /* nsegments */ 171249259Sdim (q_len * desc_size), /* maxsegsize */ 172249259Sdim 0, /* flags */ 173249259Sdim NULL, NULL, /* lockfunc, lockfuncarg */ 174249259Sdim &dmem->dmat); /* dmat */ 175249259Sdim 176249259Sdim if (err != 0) { 177249259Sdim device_printf(nic->dev, 178249259Sdim "Failed to create busdma tag for descriptors ring\n"); 179249259Sdim return (err); 180249259Sdim } 181249259Sdim 182249259Sdim /* Allocate segment of continuous DMA safe memory */ 183249259Sdim err = bus_dmamem_alloc( 184249259Sdim dmem->dmat, /* DMA tag */ 185249259Sdim &dmem->base, /* virtual address */ 186249259Sdim (BUS_DMA_NOWAIT | BUS_DMA_ZERO), /* flags */ 187249259Sdim &dmem->dmap); /* DMA map */ 188249259Sdim if (err != 0) { 189249259Sdim device_printf(nic->dev, "Failed to allocate DMA safe memory for" 190249259Sdim "descriptors ring\n"); 191249259Sdim goto dmamem_fail; 192249259Sdim } 193249259Sdim 194249259Sdim err = bus_dmamap_load( 195249259Sdim dmem->dmat, 196249259Sdim dmem->dmap, 197249259Sdim dmem->base, 198249259Sdim (q_len * desc_size), /* allocation size */ 199249259Sdim nicvf_dmamap_q_cb, /* map to DMA address cb. */ 200249259Sdim &dmem->phys_base, /* physical address */ 201249259Sdim BUS_DMA_NOWAIT); 202249259Sdim if (err != 0) { 203249259Sdim device_printf(nic->dev, 204249259Sdim "Cannot load DMA map of descriptors ring\n"); 205249259Sdim goto dmamap_fail; 206249259Sdim } 207249259Sdim 208249259Sdim dmem->q_len = q_len; 209249259Sdim dmem->size = (desc_size * q_len); 210249259Sdim 211249259Sdim return (0); 212249259Sdim 213249259Sdimdmamap_fail: 214249259Sdim bus_dmamem_free(dmem->dmat, dmem->base, dmem->dmap); 215249259Sdim dmem->phys_base = 0; 216249259Sdimdmamem_fail: 217249259Sdim err_dmat = bus_dma_tag_destroy(dmem->dmat); 218249259Sdim dmem->base = NULL; 219249259Sdim KASSERT(err_dmat == 0, 220249259Sdim ("%s: Trying to destroy BUSY DMA tag", __func__)); 221249259Sdim 222249259Sdim return (err); 223249259Sdim} 224249259Sdim 225249259Sdim/* Free queue's descriptor memory */ 226249259Sdimstatic void 227249259Sdimnicvf_free_q_desc_mem(struct nicvf *nic, struct q_desc_mem *dmem) 228249259Sdim{ 229249259Sdim int err; 230249259Sdim 231249259Sdim if ((dmem == NULL) || (dmem->base == NULL)) 232249259Sdim return; 233249259Sdim 234249259Sdim /* Unload a map */ 235249259Sdim bus_dmamap_sync(dmem->dmat, dmem->dmap, BUS_DMASYNC_POSTREAD); 236249259Sdim bus_dmamap_unload(dmem->dmat, dmem->dmap); 237249259Sdim /* Free DMA memory */ 238249259Sdim bus_dmamem_free(dmem->dmat, dmem->base, dmem->dmap); 239249259Sdim /* Destroy DMA tag */ 240249259Sdim err = bus_dma_tag_destroy(dmem->dmat); 241249259Sdim 242249259Sdim KASSERT(err == 0, 243249259Sdim ("%s: Trying to destroy BUSY DMA tag", __func__)); 244249259Sdim 245249259Sdim dmem->phys_base = 0; 246249259Sdim dmem->base = NULL; 247249259Sdim} 248249259Sdim 249249259Sdim/* 250249259Sdim * Allocate buffer for packet reception 251249259Sdim * HW returns memory address where packet is DMA'ed but not a pointer 252249259Sdim * into RBDR ring, so save buffer address at the start of fragment and 253249259Sdim * align the start address to a cache aligned address 254249259Sdim */ 255249259Sdimstatic __inline int 256276479Sdimnicvf_alloc_rcv_buffer(struct nicvf *nic, struct rbdr *rbdr, 257276479Sdim bus_dmamap_t dmap, int mflags, uint32_t buf_len, bus_addr_t *rbuf) 258249259Sdim{ 259249259Sdim struct mbuf *mbuf; 260249259Sdim struct rbuf_info *rinfo; 261249259Sdim bus_dma_segment_t segs[1]; 262249259Sdim int nsegs; 263249259Sdim int err; 264249259Sdim 265249259Sdim mbuf = m_getjcl(mflags, MT_DATA, M_PKTHDR, MCLBYTES); 266249259Sdim if (mbuf == NULL) 267249259Sdim return (ENOMEM); 268249259Sdim 269249259Sdim /* 270249259Sdim * The length is equal to the actual length + one 128b line 271249259Sdim * used as a room for rbuf_info structure. 272249259Sdim */ 273249259Sdim mbuf->m_len = mbuf->m_pkthdr.len = buf_len; 274249259Sdim 275249259Sdim err = bus_dmamap_load_mbuf_sg(rbdr->rbdr_buff_dmat, dmap, mbuf, segs, 276249259Sdim &nsegs, BUS_DMA_NOWAIT); 277249259Sdim if (err != 0) { 278249259Sdim device_printf(nic->dev, 279249259Sdim "Failed to map mbuf into DMA visible memory, err: %d\n", 280249259Sdim err); 281249259Sdim m_freem(mbuf); 282249259Sdim bus_dmamap_destroy(rbdr->rbdr_buff_dmat, dmap); 283249259Sdim return (err); 284249259Sdim } 285249259Sdim if (nsegs != 1) 286249259Sdim panic("Unexpected number of DMA segments for RB: %d", nsegs); 287276479Sdim /* 288249259Sdim * Now use the room for rbuf_info structure 289249259Sdim * and adjust mbuf data and length. 290249259Sdim */ 291249259Sdim rinfo = (struct rbuf_info *)mbuf->m_data; 292249259Sdim m_adj(mbuf, NICVF_RCV_BUF_ALIGN_BYTES); 293249259Sdim 294249259Sdim rinfo->dmat = rbdr->rbdr_buff_dmat; 295249259Sdim rinfo->dmap = dmap; 296261991Sdim rinfo->mbuf = mbuf; 297249259Sdim 298249259Sdim *rbuf = segs[0].ds_addr + NICVF_RCV_BUF_ALIGN_BYTES; 299249259Sdim 300249259Sdim return (0); 301249259Sdim} 302249259Sdim 303249259Sdim/* Retrieve mbuf for received packet */ 304249259Sdimstatic struct mbuf * 305249259Sdimnicvf_rb_ptr_to_mbuf(struct nicvf *nic, bus_addr_t rb_ptr) 306249259Sdim{ 307249259Sdim struct mbuf *mbuf; 308249259Sdim struct rbuf_info *rinfo; 309249259Sdim 310249259Sdim /* Get buffer start address and alignment offset */ 311249259Sdim rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(rb_ptr)); 312249259Sdim 313249259Sdim /* Now retrieve mbuf to give to stack */ 314249259Sdim mbuf = rinfo->mbuf; 315249259Sdim if (__predict_false(mbuf == NULL)) { 316249259Sdim panic("%s: Received packet fragment with NULL mbuf", 317249259Sdim device_get_nameunit(nic->dev)); 318249259Sdim } 319276479Sdim /* 320276479Sdim * Clear the mbuf in the descriptor to indicate 321249259Sdim * that this slot is processed and free to use. 322249259Sdim */ 323249259Sdim rinfo->mbuf = NULL; 324249259Sdim 325249259Sdim bus_dmamap_sync(rinfo->dmat, rinfo->dmap, BUS_DMASYNC_POSTREAD); 326249259Sdim bus_dmamap_unload(rinfo->dmat, rinfo->dmap); 327249259Sdim 328276479Sdim return (mbuf); 329276479Sdim} 330249259Sdim 331249259Sdim/* Allocate RBDR ring and populate receive buffers */ 332249259Sdimstatic int 333249259Sdimnicvf_init_rbdr(struct nicvf *nic, struct rbdr *rbdr, int ring_len, 334249259Sdim int buf_size, int qidx) 335249259Sdim{ 336249259Sdim bus_dmamap_t dmap; 337249259Sdim bus_addr_t rbuf; 338249259Sdim struct rbdr_entry_t *desc; 339276479Sdim int idx; 340249259Sdim int err; 341249259Sdim 342249259Sdim /* Allocate rbdr descriptors ring */ 343249259Sdim err = nicvf_alloc_q_desc_mem(nic, &rbdr->dmem, ring_len, 344249259Sdim sizeof(struct rbdr_entry_t), NICVF_RCV_BUF_ALIGN_BYTES); 345249259Sdim if (err != 0) { 346249259Sdim device_printf(nic->dev, 347249259Sdim "Failed to create RBDR descriptors ring\n"); 348249259Sdim return (err); 349276479Sdim } 350249259Sdim 351249259Sdim rbdr->desc = rbdr->dmem.base; 352249259Sdim /* 353249259Sdim * Buffer size has to be in multiples of 128 bytes. 354276479Sdim * Make room for metadata of size of one line (128 bytes). 355276479Sdim */ 356249259Sdim rbdr->dma_size = buf_size - NICVF_RCV_BUF_ALIGN_BYTES; 357249259Sdim rbdr->enable = TRUE; 358249259Sdim rbdr->thresh = RBDR_THRESH; 359249259Sdim rbdr->nic = nic; 360249259Sdim rbdr->idx = qidx; 361249259Sdim 362249259Sdim /* 363249259Sdim * Create DMA tag for Rx buffers. 364249259Sdim * Each map created using this tag is intended to store Rx payload for 365261991Sdim * one fragment and one header structure containing rbuf_info (thus 366261991Sdim * additional 128 byte line since RB must be a multiple of 128 byte 367261991Sdim * cache line). 368261991Sdim */ 369249259Sdim if (buf_size > MCLBYTES) { 370276479Sdim device_printf(nic->dev, 371276479Sdim "Buffer size to large for mbuf cluster\n"); 372249259Sdim return (EINVAL); 373249259Sdim } 374249259Sdim err = bus_dma_tag_create( 375276479Sdim bus_get_dma_tag(nic->dev), /* parent tag */ 376276479Sdim NICVF_RCV_BUF_ALIGN_BYTES, /* alignment */ 377249259Sdim 0, /* boundary */ 378249259Sdim DMAP_MAX_PHYSADDR, /* lowaddr */ 379249259Sdim DMAP_MIN_PHYSADDR, /* highaddr */ 380249259Sdim NULL, NULL, /* filtfunc, filtfuncarg */ 381249259Sdim roundup2(buf_size, MCLBYTES), /* maxsize */ 382249259Sdim 1, /* nsegments */ 383249259Sdim roundup2(buf_size, MCLBYTES), /* maxsegsize */ 384249259Sdim 0, /* flags */ 385249259Sdim NULL, NULL, /* lockfunc, lockfuncarg */ 386249259Sdim &rbdr->rbdr_buff_dmat); /* dmat */ 387249259Sdim 388249259Sdim if (err != 0) { 389249259Sdim device_printf(nic->dev, 390249259Sdim "Failed to create busdma tag for RBDR buffers\n"); 391249259Sdim return (err); 392249259Sdim } 393249259Sdim 394249259Sdim rbdr->rbdr_buff_dmaps = malloc(sizeof(*rbdr->rbdr_buff_dmaps) * 395249259Sdim ring_len, M_NICVF, (M_WAITOK | M_ZERO)); 396249259Sdim 397249259Sdim for (idx = 0; idx < ring_len; idx++) { 398249259Sdim err = bus_dmamap_create(rbdr->rbdr_buff_dmat, 0, &dmap); 399276479Sdim if (err != 0) { 400249259Sdim device_printf(nic->dev, 401249259Sdim "Failed to create DMA map for RB\n"); 402249259Sdim return (err); 403249259Sdim } 404249259Sdim rbdr->rbdr_buff_dmaps[idx] = dmap; 405249259Sdim 406249259Sdim err = nicvf_alloc_rcv_buffer(nic, rbdr, dmap, M_WAITOK, 407249259Sdim DMA_BUFFER_LEN, &rbuf); 408261991Sdim if (err != 0) 409261991Sdim return (err); 410261991Sdim 411261991Sdim desc = GET_RBDR_DESC(rbdr, idx); 412261991Sdim desc->buf_addr = (rbuf >> NICVF_RCV_BUF_ALIGN); 413261991Sdim } 414261991Sdim 415261991Sdim /* Allocate taskqueue */ 416249259Sdim TASK_INIT(&rbdr->rbdr_task, 0, nicvf_rbdr_task, rbdr); 417249259Sdim TASK_INIT(&rbdr->rbdr_task_nowait, 0, nicvf_rbdr_task_nowait, rbdr); 418249259Sdim rbdr->rbdr_taskq = taskqueue_create_fast("nicvf_rbdr_taskq", M_WAITOK, 419249259Sdim taskqueue_thread_enqueue, &rbdr->rbdr_taskq); 420249259Sdim taskqueue_start_threads(&rbdr->rbdr_taskq, 1, PI_NET, "%s: rbdr_taskq", 421249259Sdim device_get_nameunit(nic->dev)); 422249259Sdim 423249259Sdim return (0); 424249259Sdim} 425249259Sdim 426249259Sdim/* Free RBDR ring and its receive buffers */ 427249259Sdimstatic void 428249259Sdimnicvf_free_rbdr(struct nicvf *nic, struct rbdr *rbdr) 429249259Sdim{ 430249259Sdim struct mbuf *mbuf; 431249259Sdim struct queue_set *qs; 432249259Sdim struct rbdr_entry_t *desc; 433249259Sdim struct rbuf_info *rinfo; 434249259Sdim bus_addr_t buf_addr; 435249259Sdim int head, tail, idx; 436249259Sdim int err; 437249259Sdim 438249259Sdim qs = nic->qs; 439249259Sdim 440249259Sdim if ((qs == NULL) || (rbdr == NULL)) 441249259Sdim return; 442249259Sdim 443249259Sdim rbdr->enable = FALSE; 444249259Sdim if (rbdr->rbdr_taskq != NULL) { 445249259Sdim /* Remove tasks */ 446249259Sdim while (taskqueue_cancel(rbdr->rbdr_taskq, 447249259Sdim &rbdr->rbdr_task_nowait, NULL) != 0) { 448249259Sdim /* Finish the nowait task first */ 449249259Sdim taskqueue_drain(rbdr->rbdr_taskq, 450249259Sdim &rbdr->rbdr_task_nowait); 451249259Sdim } 452249259Sdim taskqueue_free(rbdr->rbdr_taskq); 453249259Sdim rbdr->rbdr_taskq = NULL; 454249259Sdim 455249259Sdim while (taskqueue_cancel(taskqueue_thread, 456249259Sdim &rbdr->rbdr_task, NULL) != 0) { 457249259Sdim /* Now finish the sleepable task */ 458249259Sdim taskqueue_drain(taskqueue_thread, &rbdr->rbdr_task); 459249259Sdim } 460249259Sdim } 461249259Sdim 462249259Sdim /* 463249259Sdim * Free all of the memory under the RB descriptors. 464249259Sdim * There are assumptions here: 465249259Sdim * 1. Corresponding RBDR is disabled 466249259Sdim * - it is safe to operate using head and tail indexes 467249259Sdim * 2. All bffers that were received are properly freed by 468249259Sdim * the receive handler 469249259Sdim * - there is no need to unload DMA map and free MBUF for other 470249259Sdim * descriptors than unused ones 471249259Sdim */ 472249259Sdim if (rbdr->rbdr_buff_dmat != NULL) { 473249259Sdim head = rbdr->head; 474249259Sdim tail = rbdr->tail; 475249259Sdim while (head != tail) { 476249259Sdim desc = GET_RBDR_DESC(rbdr, head); 477249259Sdim buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN; 478249259Sdim rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(buf_addr)); 479249259Sdim bus_dmamap_unload(rbdr->rbdr_buff_dmat, rinfo->dmap); 480249259Sdim mbuf = rinfo->mbuf; 481249259Sdim /* This will destroy everything including rinfo! */ 482249259Sdim m_freem(mbuf); 483249259Sdim head++; 484249259Sdim head &= (rbdr->dmem.q_len - 1); 485249259Sdim } 486249259Sdim /* Free tail descriptor */ 487249259Sdim desc = GET_RBDR_DESC(rbdr, tail); 488249259Sdim buf_addr = desc->buf_addr << NICVF_RCV_BUF_ALIGN; 489249259Sdim rinfo = GET_RBUF_INFO(PHYS_TO_DMAP(buf_addr)); 490249259Sdim bus_dmamap_unload(rbdr->rbdr_buff_dmat, rinfo->dmap); 491249259Sdim mbuf = rinfo->mbuf; 492249259Sdim /* This will destroy everything including rinfo! */ 493249259Sdim m_freem(mbuf); 494249259Sdim 495249259Sdim /* Destroy DMA maps */ 496249259Sdim for (idx = 0; idx < qs->rbdr_len; idx++) { 497249259Sdim if (rbdr->rbdr_buff_dmaps[idx] == NULL) 498249259Sdim continue; 499249259Sdim err = bus_dmamap_destroy(rbdr->rbdr_buff_dmat, 500249259Sdim rbdr->rbdr_buff_dmaps[idx]); 501249259Sdim KASSERT(err == 0, 502249259Sdim ("%s: Could not destroy DMA map for RB, desc: %d", 503249259Sdim __func__, idx)); 504249259Sdim rbdr->rbdr_buff_dmaps[idx] = NULL; 505249259Sdim } 506249259Sdim 507249259Sdim /* Now destroy the tag */ 508249259Sdim err = bus_dma_tag_destroy(rbdr->rbdr_buff_dmat); 509249259Sdim KASSERT(err == 0, 510249259Sdim ("%s: Trying to destroy BUSY DMA tag", __func__)); 511249259Sdim 512249259Sdim rbdr->head = 0; 513249259Sdim rbdr->tail = 0; 514249259Sdim } 515249259Sdim 516249259Sdim /* Free RBDR ring */ 517249259Sdim nicvf_free_q_desc_mem(nic, &rbdr->dmem); 518249259Sdim} 519249259Sdim 520249259Sdim/* 521249259Sdim * Refill receive buffer descriptors with new buffers. 522249259Sdim */ 523249259Sdimstatic int 524249259Sdimnicvf_refill_rbdr(struct rbdr *rbdr, int mflags) 525249259Sdim{ 526249259Sdim struct nicvf *nic; 527249259Sdim struct queue_set *qs; 528249259Sdim int rbdr_idx; 529249259Sdim int tail, qcount; 530249259Sdim int refill_rb_cnt; 531249259Sdim struct rbdr_entry_t *desc; 532249259Sdim bus_dmamap_t dmap; 533249259Sdim bus_addr_t rbuf; 534249259Sdim boolean_t rb_alloc_fail; 535249259Sdim int new_rb; 536249259Sdim 537249259Sdim rb_alloc_fail = TRUE; 538249259Sdim new_rb = 0; 539249259Sdim nic = rbdr->nic; 540249259Sdim qs = nic->qs; 541249259Sdim rbdr_idx = rbdr->idx; 542249259Sdim 543276479Sdim /* Check if it's enabled */ 544249259Sdim if (!rbdr->enable) 545261991Sdim return (0); 546261991Sdim 547261991Sdim /* Get no of desc's to be refilled */ 548261991Sdim qcount = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, rbdr_idx); 549261991Sdim qcount &= 0x7FFFF; 550261991Sdim /* Doorbell can be ringed with a max of ring size minus 1 */ 551276479Sdim if (qcount >= (qs->rbdr_len - 1)) { 552276479Sdim rb_alloc_fail = FALSE; 553249259Sdim goto out; 554249259Sdim } else 555249259Sdim refill_rb_cnt = qs->rbdr_len - qcount - 1; 556249259Sdim 557249259Sdim /* Start filling descs from tail */ 558249259Sdim tail = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, rbdr_idx) >> 3; 559249259Sdim while (refill_rb_cnt) { 560249259Sdim tail++; 561249259Sdim tail &= (rbdr->dmem.q_len - 1); 562249259Sdim 563249259Sdim dmap = rbdr->rbdr_buff_dmaps[tail]; 564249259Sdim if (nicvf_alloc_rcv_buffer(nic, rbdr, dmap, mflags, 565249259Sdim DMA_BUFFER_LEN, &rbuf)) { 566249259Sdim /* Something went wrong. Resign */ 567249259Sdim break; 568249259Sdim } 569249259Sdim desc = GET_RBDR_DESC(rbdr, tail); 570249259Sdim desc->buf_addr = (rbuf >> NICVF_RCV_BUF_ALIGN); 571249259Sdim refill_rb_cnt--; 572249259Sdim new_rb++; 573249259Sdim } 574249259Sdim 575249259Sdim /* make sure all memory stores are done before ringing doorbell */ 576249259Sdim wmb(); 577261991Sdim 578261991Sdim /* Check if buffer allocation failed */ 579261991Sdim if (refill_rb_cnt == 0) 580261991Sdim rb_alloc_fail = FALSE; 581261991Sdim 582261991Sdim /* Notify HW */ 583261991Sdim nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, 584261991Sdim rbdr_idx, new_rb); 585261991Sdimout: 586261991Sdim if (!rb_alloc_fail) { 587261991Sdim /* 588261991Sdim * Re-enable RBDR interrupts only 589261991Sdim * if buffer allocation is success. 590249259Sdim */ 591249259Sdim nicvf_enable_intr(nic, NICVF_INTR_RBDR, rbdr_idx); 592249259Sdim 593249259Sdim return (0); 594249259Sdim } 595249259Sdim 596249259Sdim return (ENOMEM); 597249259Sdim} 598249259Sdim 599249259Sdim/* Refill RBs even if sleep is needed to reclaim memory */ 600249259Sdimstatic void 601249259Sdimnicvf_rbdr_task(void *arg, int pending) 602249259Sdim{ 603249259Sdim struct rbdr *rbdr; 604249259Sdim int err; 605249259Sdim 606249259Sdim rbdr = (struct rbdr *)arg; 607249259Sdim 608249259Sdim err = nicvf_refill_rbdr(rbdr, M_WAITOK); 609249259Sdim if (__predict_false(err != 0)) { 610249259Sdim panic("%s: Failed to refill RBs even when sleep enabled", 611249259Sdim __func__); 612249259Sdim } 613249259Sdim} 614249259Sdim 615249259Sdim/* Refill RBs as soon as possible without waiting */ 616249259Sdimstatic void 617249259Sdimnicvf_rbdr_task_nowait(void *arg, int pending) 618249259Sdim{ 619249259Sdim struct rbdr *rbdr; 620249259Sdim int err; 621249259Sdim 622249259Sdim rbdr = (struct rbdr *)arg; 623249259Sdim 624249259Sdim err = nicvf_refill_rbdr(rbdr, M_NOWAIT); 625249259Sdim if (err != 0) { 626249259Sdim /* 627249259Sdim * Schedule another, sleepable kernel thread 628249259Sdim * that will for sure refill the buffers. 629249259Sdim */ 630249259Sdim taskqueue_enqueue(taskqueue_thread, &rbdr->rbdr_task); 631249259Sdim } 632249259Sdim} 633249259Sdim 634249259Sdimstatic int 635249259Sdimnicvf_rcv_pkt_handler(struct nicvf *nic, struct cmp_queue *cq, 636249259Sdim struct cqe_rx_t *cqe_rx, int cqe_type) 637249259Sdim{ 638249259Sdim struct mbuf *mbuf; 639249259Sdim struct rcv_queue *rq; 640249259Sdim int rq_idx; 641249259Sdim int err = 0; 642249259Sdim 643249259Sdim rq_idx = cqe_rx->rq_idx; 644249259Sdim rq = &nic->qs->rq[rq_idx]; 645249259Sdim 646249259Sdim /* Check for errors */ 647249259Sdim err = nicvf_check_cqe_rx_errs(nic, cq, cqe_rx); 648249259Sdim if (err && !cqe_rx->rb_cnt) 649249259Sdim return (0); 650249259Sdim 651249259Sdim mbuf = nicvf_get_rcv_mbuf(nic, cqe_rx); 652249259Sdim if (mbuf == NULL) { 653249259Sdim dprintf(nic->dev, "Packet not received\n"); 654249259Sdim return (0); 655249259Sdim } 656249259Sdim 657249259Sdim /* If error packet */ 658249259Sdim if (err != 0) { 659249259Sdim m_freem(mbuf); 660249259Sdim return (0); 661249259Sdim } 662249259Sdim 663249259Sdim if (rq->lro_enabled && 664249259Sdim ((cqe_rx->l3_type == L3TYPE_IPV4) && (cqe_rx->l4_type == L4TYPE_TCP)) && 665249259Sdim (mbuf->m_pkthdr.csum_flags & (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) == 666249259Sdim (CSUM_DATA_VALID | CSUM_PSEUDO_HDR)) { 667249259Sdim /* 668249259Sdim * At this point it is known that there are no errors in the 669249259Sdim * packet. Attempt to LRO enqueue. Send to stack if no resources 670249259Sdim * or enqueue error. 671249259Sdim */ 672249259Sdim if ((rq->lro.lro_cnt != 0) && 673249259Sdim (tcp_lro_rx(&rq->lro, mbuf, 0) == 0)) 674249259Sdim return (0); 675249259Sdim } 676249259Sdim /* 677276479Sdim * Push this packet to the stack later to avoid 678276479Sdim * unlocking completion task in the middle of work. 679276479Sdim */ 680276479Sdim err = buf_ring_enqueue(cq->rx_br, mbuf); 681276479Sdim if (err != 0) { 682249259Sdim /* 683276479Sdim * Failed to enqueue this mbuf. 684276479Sdim * We don't drop it, just schedule another task. 685276479Sdim */ 686276479Sdim return (err); 687276479Sdim } 688276479Sdim 689249259Sdim return (0); 690249259Sdim} 691249259Sdim 692249259Sdimstatic int 693249259Sdimnicvf_snd_pkt_handler(struct nicvf *nic, struct cmp_queue *cq, 694249259Sdim struct cqe_send_t *cqe_tx, int cqe_type) 695249259Sdim{ 696249259Sdim bus_dmamap_t dmap; 697249259Sdim struct mbuf *mbuf; 698249259Sdim struct snd_queue *sq; 699249259Sdim struct sq_hdr_subdesc *hdr; 700249259Sdim 701249259Sdim mbuf = NULL; 702249259Sdim sq = &nic->qs->sq[cqe_tx->sq_idx]; 703249259Sdim /* Avoid blocking here since we hold a non-sleepable NICVF_CMP_LOCK */ 704249259Sdim if (NICVF_TX_TRYLOCK(sq) == 0) 705249259Sdim return (EAGAIN); 706249259Sdim 707249259Sdim hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, cqe_tx->sqe_ptr); 708249259Sdim if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) { 709249259Sdim NICVF_TX_UNLOCK(sq); 710249259Sdim return (0); 711249259Sdim } 712249259Sdim 713249259Sdim dprintf(nic->dev, 714249259Sdim "%s Qset #%d SQ #%d SQ ptr #%d subdesc count %d\n", 715249259Sdim __func__, cqe_tx->sq_qs, cqe_tx->sq_idx, 716249259Sdim cqe_tx->sqe_ptr, hdr->subdesc_cnt); 717249259Sdim 718249259Sdim dmap = (bus_dmamap_t)sq->snd_buff[cqe_tx->sqe_ptr].dmap; 719249259Sdim bus_dmamap_unload(sq->snd_buff_dmat, dmap); 720249259Sdim 721249259Sdim mbuf = (struct mbuf *)sq->snd_buff[cqe_tx->sqe_ptr].mbuf; 722249259Sdim if (mbuf != NULL) { 723249259Sdim m_freem(mbuf); 724249259Sdim sq->snd_buff[cqe_tx->sqe_ptr].mbuf = NULL; 725249259Sdim nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1); 726249259Sdim } 727249259Sdim 728249259Sdim nicvf_check_cqe_tx_errs(nic, cq, cqe_tx); 729249259Sdim 730249259Sdim NICVF_TX_UNLOCK(sq); 731249259Sdim return (0); 732249259Sdim} 733249259Sdim 734249259Sdimstatic int 735249259Sdimnicvf_cq_intr_handler(struct nicvf *nic, uint8_t cq_idx) 736249259Sdim{ 737249259Sdim struct mbuf *mbuf; 738249259Sdim struct ifnet *ifp; 739249259Sdim int processed_cqe, work_done = 0, tx_done = 0; 740249259Sdim int cqe_count, cqe_head; 741249259Sdim struct queue_set *qs = nic->qs; 742249259Sdim struct cmp_queue *cq = &qs->cq[cq_idx]; 743249259Sdim struct rcv_queue *rq; 744249259Sdim struct cqe_rx_t *cq_desc; 745249259Sdim struct lro_ctrl *lro; 746249259Sdim struct lro_entry *queued; 747249259Sdim int rq_idx; 748249259Sdim int cmp_err; 749249259Sdim 750249259Sdim NICVF_CMP_LOCK(cq); 751249259Sdim cmp_err = 0; 752249259Sdim processed_cqe = 0; 753249259Sdim /* Get no of valid CQ entries to process */ 754249259Sdim cqe_count = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, cq_idx); 755249259Sdim cqe_count &= CQ_CQE_COUNT; 756249259Sdim if (cqe_count == 0) 757249259Sdim goto out; 758249259Sdim 759249259Sdim /* Get head of the valid CQ entries */ 760249259Sdim cqe_head = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_HEAD, cq_idx) >> 9; 761249259Sdim cqe_head &= 0xFFFF; 762249259Sdim 763249259Sdim dprintf(nic->dev, "%s CQ%d cqe_count %d cqe_head %d\n", 764249259Sdim __func__, cq_idx, cqe_count, cqe_head); 765249259Sdim while (processed_cqe < cqe_count) { 766249259Sdim /* Get the CQ descriptor */ 767249259Sdim cq_desc = (struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head); 768249259Sdim cqe_head++; 769249259Sdim cqe_head &= (cq->dmem.q_len - 1); 770249259Sdim /* Prefetch next CQ descriptor */ 771249259Sdim __builtin_prefetch((struct cqe_rx_t *)GET_CQ_DESC(cq, cqe_head)); 772249259Sdim 773249259Sdim dprintf(nic->dev, "CQ%d cq_desc->cqe_type %d\n", cq_idx, 774249259Sdim cq_desc->cqe_type); 775249259Sdim switch (cq_desc->cqe_type) { 776249259Sdim case CQE_TYPE_RX: 777249259Sdim cmp_err = nicvf_rcv_pkt_handler(nic, cq, cq_desc, 778249259Sdim CQE_TYPE_RX); 779249259Sdim if (__predict_false(cmp_err != 0)) { 780249259Sdim /* 781249259Sdim * Ups. Cannot finish now. 782249259Sdim * Let's try again later. 783249259Sdim */ 784249259Sdim goto done; 785249259Sdim } 786249259Sdim work_done++; 787249259Sdim break; 788249259Sdim case CQE_TYPE_SEND: 789249259Sdim cmp_err = nicvf_snd_pkt_handler(nic, cq, 790249259Sdim (void *)cq_desc, CQE_TYPE_SEND); 791249259Sdim if (__predict_false(cmp_err != 0)) { 792249259Sdim /* 793249259Sdim * Ups. Cannot finish now. 794249259Sdim * Let's try again later. 795249259Sdim */ 796249259Sdim goto done; 797249259Sdim } 798249259Sdim 799249259Sdim tx_done++; 800249259Sdim break; 801249259Sdim case CQE_TYPE_INVALID: 802249259Sdim case CQE_TYPE_RX_SPLIT: 803249259Sdim case CQE_TYPE_RX_TCP: 804249259Sdim case CQE_TYPE_SEND_PTP: 805249259Sdim /* Ignore for now */ 806249259Sdim break; 807249259Sdim } 808249259Sdim processed_cqe++; 809249259Sdim } 810249259Sdimdone: 811249259Sdim dprintf(nic->dev, 812249259Sdim "%s CQ%d processed_cqe %d work_done %d\n", 813249259Sdim __func__, cq_idx, processed_cqe, work_done); 814249259Sdim 815249259Sdim /* Ring doorbell to inform H/W to reuse processed CQEs */ 816249259Sdim nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_DOOR, cq_idx, processed_cqe); 817249259Sdim 818249259Sdim if ((tx_done > 0) && 819249259Sdim ((if_getdrvflags(nic->ifp) & IFF_DRV_RUNNING) != 0)) { 820249259Sdim /* Reenable TXQ if its stopped earlier due to SQ full */ 821249259Sdim if_setdrvflagbits(nic->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 822249259Sdim } 823249259Sdimout: 824249259Sdim /* 825249259Sdim * Flush any outstanding LRO work 826249259Sdim */ 827249259Sdim rq_idx = cq_idx; 828249259Sdim rq = &nic->qs->rq[rq_idx]; 829249259Sdim lro = &rq->lro; 830249259Sdim while ((queued = SLIST_FIRST(&lro->lro_active)) != NULL) { 831249259Sdim SLIST_REMOVE_HEAD(&lro->lro_active, next); 832249259Sdim tcp_lro_flush(lro, queued); 833249259Sdim } 834249259Sdim 835249259Sdim NICVF_CMP_UNLOCK(cq); 836249259Sdim 837249259Sdim ifp = nic->ifp; 838249259Sdim /* Push received MBUFs to the stack */ 839249259Sdim while (!buf_ring_empty(cq->rx_br)) { 840249259Sdim mbuf = buf_ring_dequeue_mc(cq->rx_br); 841249259Sdim if (__predict_true(mbuf != NULL)) 842249259Sdim (*ifp->if_input)(ifp, mbuf); 843249259Sdim } 844249259Sdim 845249259Sdim return (cmp_err); 846249259Sdim} 847249259Sdim 848249259Sdim/* 849249259Sdim * Qset error interrupt handler 850249259Sdim * 851249259Sdim * As of now only CQ errors are handled 852249259Sdim */ 853249259Sdimstatic void 854249259Sdimnicvf_qs_err_task(void *arg, int pending) 855249259Sdim{ 856249259Sdim struct nicvf *nic; 857249259Sdim struct queue_set *qs; 858249259Sdim int qidx; 859249259Sdim uint64_t status; 860249259Sdim boolean_t enable = TRUE; 861249259Sdim 862249259Sdim nic = (struct nicvf *)arg; 863249259Sdim qs = nic->qs; 864249259Sdim 865249259Sdim /* Deactivate network interface */ 866249259Sdim if_setdrvflagbits(nic->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 867249259Sdim 868249259Sdim /* Check if it is CQ err */ 869249259Sdim for (qidx = 0; qidx < qs->cq_cnt; qidx++) { 870249259Sdim status = nicvf_queue_reg_read(nic, NIC_QSET_CQ_0_7_STATUS, 871249259Sdim qidx); 872249259Sdim if ((status & CQ_ERR_MASK) == 0) 873249259Sdim continue; 874249259Sdim /* Process already queued CQEs and reconfig CQ */ 875249259Sdim nicvf_disable_intr(nic, NICVF_INTR_CQ, qidx); 876249259Sdim nicvf_sq_disable(nic, qidx); 877249259Sdim (void)nicvf_cq_intr_handler(nic, qidx); 878249259Sdim nicvf_cmp_queue_config(nic, qs, qidx, enable); 879249259Sdim nicvf_sq_free_used_descs(nic, &qs->sq[qidx], qidx); 880249259Sdim nicvf_sq_enable(nic, &qs->sq[qidx], qidx); 881249259Sdim nicvf_enable_intr(nic, NICVF_INTR_CQ, qidx); 882249259Sdim } 883249259Sdim 884249259Sdim if_setdrvflagbits(nic->ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); 885249259Sdim /* Re-enable Qset error interrupt */ 886249259Sdim nicvf_enable_intr(nic, NICVF_INTR_QS_ERR, 0); 887249259Sdim} 888249259Sdim 889249259Sdimstatic void 890249259Sdimnicvf_cmp_task(void *arg, int pending) 891249259Sdim{ 892249259Sdim struct cmp_queue *cq; 893249259Sdim struct nicvf *nic; 894 int cmp_err; 895 896 cq = (struct cmp_queue *)arg; 897 nic = cq->nic; 898 899 /* Handle CQ descriptors */ 900 cmp_err = nicvf_cq_intr_handler(nic, cq->idx); 901 if (__predict_false(cmp_err != 0)) { 902 /* 903 * Schedule another thread here since we did not 904 * process the entire CQ due to Tx or Rx CQ parse error. 905 */ 906 taskqueue_enqueue(cq->cmp_taskq, &cq->cmp_task); 907 908 } 909 910 nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->idx); 911 /* Reenable interrupt (previously disabled in nicvf_intr_handler() */ 912 nicvf_enable_intr(nic, NICVF_INTR_CQ, cq->idx); 913 914} 915 916/* Initialize completion queue */ 917static int 918nicvf_init_cmp_queue(struct nicvf *nic, struct cmp_queue *cq, int q_len, 919 int qidx) 920{ 921 int err; 922 923 /* Initizalize lock */ 924 snprintf(cq->mtx_name, sizeof(cq->mtx_name), "%s: CQ(%d) lock", 925 device_get_nameunit(nic->dev), qidx); 926 mtx_init(&cq->mtx, cq->mtx_name, NULL, MTX_DEF); 927 928 err = nicvf_alloc_q_desc_mem(nic, &cq->dmem, q_len, CMP_QUEUE_DESC_SIZE, 929 NICVF_CQ_BASE_ALIGN_BYTES); 930 931 if (err != 0) { 932 device_printf(nic->dev, 933 "Could not allocate DMA memory for CQ\n"); 934 return (err); 935 } 936 937 cq->desc = cq->dmem.base; 938 cq->thresh = pass1_silicon(nic->dev) ? 0 : CMP_QUEUE_CQE_THRESH; 939 cq->nic = nic; 940 cq->idx = qidx; 941 nic->cq_coalesce_usecs = (CMP_QUEUE_TIMER_THRESH * 0.05) - 1; 942 943 cq->rx_br = buf_ring_alloc(CMP_QUEUE_LEN * 8, M_DEVBUF, M_WAITOK, 944 &cq->mtx); 945 946 /* Allocate taskqueue */ 947 TASK_INIT(&cq->cmp_task, 0, nicvf_cmp_task, cq); 948 cq->cmp_taskq = taskqueue_create_fast("nicvf_cmp_taskq", M_WAITOK, 949 taskqueue_thread_enqueue, &cq->cmp_taskq); 950 taskqueue_start_threads(&cq->cmp_taskq, 1, PI_NET, "%s: cmp_taskq(%d)", 951 device_get_nameunit(nic->dev), qidx); 952 953 return (0); 954} 955 956static void 957nicvf_free_cmp_queue(struct nicvf *nic, struct cmp_queue *cq) 958{ 959 960 if (cq == NULL) 961 return; 962 /* 963 * The completion queue itself should be disabled by now 964 * (ref. nicvf_snd_queue_config()). 965 * Ensure that it is safe to disable it or panic. 966 */ 967 if (cq->enable) 968 panic("%s: Trying to free working CQ(%d)", __func__, cq->idx); 969 970 if (cq->cmp_taskq != NULL) { 971 /* Remove task */ 972 while (taskqueue_cancel(cq->cmp_taskq, &cq->cmp_task, NULL) != 0) 973 taskqueue_drain(cq->cmp_taskq, &cq->cmp_task); 974 975 taskqueue_free(cq->cmp_taskq); 976 cq->cmp_taskq = NULL; 977 } 978 /* 979 * Completion interrupt will possibly enable interrupts again 980 * so disable interrupting now after we finished processing 981 * completion task. It is safe to do so since the corresponding CQ 982 * was already disabled. 983 */ 984 nicvf_disable_intr(nic, NICVF_INTR_CQ, cq->idx); 985 nicvf_clear_intr(nic, NICVF_INTR_CQ, cq->idx); 986 987 NICVF_CMP_LOCK(cq); 988 nicvf_free_q_desc_mem(nic, &cq->dmem); 989 drbr_free(cq->rx_br, M_DEVBUF); 990 NICVF_CMP_UNLOCK(cq); 991 mtx_destroy(&cq->mtx); 992 memset(cq->mtx_name, 0, sizeof(cq->mtx_name)); 993} 994 995static void 996nicvf_snd_task(void *arg, int pending) 997{ 998 struct snd_queue *sq = (struct snd_queue *)arg; 999 struct mbuf *mbuf; 1000 1001 NICVF_TX_LOCK(sq); 1002 while (1) { 1003 mbuf = drbr_dequeue(NULL, sq->br); 1004 if (mbuf == NULL) 1005 break; 1006 1007 if (nicvf_tx_mbuf_locked(sq, mbuf) != 0) { 1008 /* XXX ARM64TODO: Increase Tx drop counter */ 1009 m_freem(mbuf); 1010 break; 1011 } 1012 } 1013 NICVF_TX_UNLOCK(sq); 1014} 1015 1016/* Initialize transmit queue */ 1017static int 1018nicvf_init_snd_queue(struct nicvf *nic, struct snd_queue *sq, int q_len, 1019 int qidx) 1020{ 1021 size_t i; 1022 int err; 1023 1024 /* Initizalize TX lock for this queue */ 1025 snprintf(sq->mtx_name, sizeof(sq->mtx_name), "%s: SQ(%d) lock", 1026 device_get_nameunit(nic->dev), qidx); 1027 mtx_init(&sq->mtx, sq->mtx_name, NULL, MTX_DEF); 1028 1029 NICVF_TX_LOCK(sq); 1030 /* Allocate buffer ring */ 1031 sq->br = buf_ring_alloc(q_len / MIN_SQ_DESC_PER_PKT_XMIT, M_DEVBUF, 1032 M_NOWAIT, &sq->mtx); 1033 if (sq->br == NULL) { 1034 device_printf(nic->dev, 1035 "ERROR: Could not set up buf ring for SQ(%d)\n", qidx); 1036 err = ENOMEM; 1037 goto error; 1038 } 1039 1040 /* Allocate DMA memory for Tx descriptors */ 1041 err = nicvf_alloc_q_desc_mem(nic, &sq->dmem, q_len, SND_QUEUE_DESC_SIZE, 1042 NICVF_SQ_BASE_ALIGN_BYTES); 1043 if (err != 0) { 1044 device_printf(nic->dev, 1045 "Could not allocate DMA memory for SQ\n"); 1046 goto error; 1047 } 1048 1049 sq->desc = sq->dmem.base; 1050 sq->head = sq->tail = 0; 1051 atomic_store_rel_int(&sq->free_cnt, q_len - 1); 1052 sq->thresh = SND_QUEUE_THRESH; 1053 sq->idx = qidx; 1054 sq->nic = nic; 1055 1056 /* 1057 * Allocate DMA maps for Tx buffers 1058 */ 1059 1060 /* Create DMA tag first */ 1061 err = bus_dma_tag_create( 1062 bus_get_dma_tag(nic->dev), /* parent tag */ 1063 1, /* alignment */ 1064 0, /* boundary */ 1065 BUS_SPACE_MAXADDR, /* lowaddr */ 1066 BUS_SPACE_MAXADDR, /* highaddr */ 1067 NULL, NULL, /* filtfunc, filtfuncarg */ 1068 NICVF_TSO_MAXSIZE, /* maxsize */ 1069 NICVF_TSO_NSEGS, /* nsegments */ 1070 MCLBYTES, /* maxsegsize */ 1071 0, /* flags */ 1072 NULL, NULL, /* lockfunc, lockfuncarg */ 1073 &sq->snd_buff_dmat); /* dmat */ 1074 1075 if (err != 0) { 1076 device_printf(nic->dev, 1077 "Failed to create busdma tag for Tx buffers\n"); 1078 goto error; 1079 } 1080 1081 /* Allocate send buffers array */ 1082 sq->snd_buff = malloc(sizeof(*sq->snd_buff) * q_len, M_NICVF, 1083 (M_NOWAIT | M_ZERO)); 1084 if (sq->snd_buff == NULL) { 1085 device_printf(nic->dev, 1086 "Could not allocate memory for Tx buffers array\n"); 1087 err = ENOMEM; 1088 goto error; 1089 } 1090 1091 /* Now populate maps */ 1092 for (i = 0; i < q_len; i++) { 1093 err = bus_dmamap_create(sq->snd_buff_dmat, 0, 1094 &sq->snd_buff[i].dmap); 1095 if (err != 0) { 1096 device_printf(nic->dev, 1097 "Failed to create DMA maps for Tx buffers\n"); 1098 goto error; 1099 } 1100 } 1101 NICVF_TX_UNLOCK(sq); 1102 1103 /* Allocate taskqueue */ 1104 TASK_INIT(&sq->snd_task, 0, nicvf_snd_task, sq); 1105 sq->snd_taskq = taskqueue_create_fast("nicvf_snd_taskq", M_WAITOK, 1106 taskqueue_thread_enqueue, &sq->snd_taskq); 1107 taskqueue_start_threads(&sq->snd_taskq, 1, PI_NET, "%s: snd_taskq(%d)", 1108 device_get_nameunit(nic->dev), qidx); 1109 1110 return (0); 1111error: 1112 NICVF_TX_UNLOCK(sq); 1113 return (err); 1114} 1115 1116static void 1117nicvf_free_snd_queue(struct nicvf *nic, struct snd_queue *sq) 1118{ 1119 struct queue_set *qs = nic->qs; 1120 size_t i; 1121 int err; 1122 1123 if (sq == NULL) 1124 return; 1125 1126 if (sq->snd_taskq != NULL) { 1127 /* Remove task */ 1128 while (taskqueue_cancel(sq->snd_taskq, &sq->snd_task, NULL) != 0) 1129 taskqueue_drain(sq->snd_taskq, &sq->snd_task); 1130 1131 taskqueue_free(sq->snd_taskq); 1132 sq->snd_taskq = NULL; 1133 } 1134 1135 NICVF_TX_LOCK(sq); 1136 if (sq->snd_buff_dmat != NULL) { 1137 if (sq->snd_buff != NULL) { 1138 for (i = 0; i < qs->sq_len; i++) { 1139 m_freem(sq->snd_buff[i].mbuf); 1140 sq->snd_buff[i].mbuf = NULL; 1141 1142 bus_dmamap_unload(sq->snd_buff_dmat, 1143 sq->snd_buff[i].dmap); 1144 err = bus_dmamap_destroy(sq->snd_buff_dmat, 1145 sq->snd_buff[i].dmap); 1146 /* 1147 * If bus_dmamap_destroy fails it can cause 1148 * random panic later if the tag is also 1149 * destroyed in the process. 1150 */ 1151 KASSERT(err == 0, 1152 ("%s: Could not destroy DMA map for SQ", 1153 __func__)); 1154 } 1155 } 1156 1157 free(sq->snd_buff, M_NICVF); 1158 1159 err = bus_dma_tag_destroy(sq->snd_buff_dmat); 1160 KASSERT(err == 0, 1161 ("%s: Trying to destroy BUSY DMA tag", __func__)); 1162 } 1163 1164 /* Free private driver ring for this send queue */ 1165 if (sq->br != NULL) 1166 drbr_free(sq->br, M_DEVBUF); 1167 1168 if (sq->dmem.base != NULL) 1169 nicvf_free_q_desc_mem(nic, &sq->dmem); 1170 1171 NICVF_TX_UNLOCK(sq); 1172 /* Destroy Tx lock */ 1173 mtx_destroy(&sq->mtx); 1174 memset(sq->mtx_name, 0, sizeof(sq->mtx_name)); 1175} 1176 1177static void 1178nicvf_reclaim_snd_queue(struct nicvf *nic, struct queue_set *qs, int qidx) 1179{ 1180 1181 /* Disable send queue */ 1182 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, 0); 1183 /* Check if SQ is stopped */ 1184 if (nicvf_poll_reg(nic, qidx, NIC_QSET_SQ_0_7_STATUS, 21, 1, 0x01)) 1185 return; 1186 /* Reset send queue */ 1187 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET); 1188} 1189 1190static void 1191nicvf_reclaim_rcv_queue(struct nicvf *nic, struct queue_set *qs, int qidx) 1192{ 1193 union nic_mbx mbx = {}; 1194 1195 /* Make sure all packets in the pipeline are written back into mem */ 1196 mbx.msg.msg = NIC_MBOX_MSG_RQ_SW_SYNC; 1197 nicvf_send_msg_to_pf(nic, &mbx); 1198} 1199 1200static void 1201nicvf_reclaim_cmp_queue(struct nicvf *nic, struct queue_set *qs, int qidx) 1202{ 1203 1204 /* Disable timer threshold (doesn't get reset upon CQ reset */ 1205 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 0); 1206 /* Disable completion queue */ 1207 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, 0); 1208 /* Reset completion queue */ 1209 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET); 1210} 1211 1212static void 1213nicvf_reclaim_rbdr(struct nicvf *nic, struct rbdr *rbdr, int qidx) 1214{ 1215 uint64_t tmp, fifo_state; 1216 int timeout = 10; 1217 1218 /* Save head and tail pointers for feeing up buffers */ 1219 rbdr->head = 1220 nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_HEAD, qidx) >> 3; 1221 rbdr->tail = 1222 nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_TAIL, qidx) >> 3; 1223 1224 /* 1225 * If RBDR FIFO is in 'FAIL' state then do a reset first 1226 * before relaiming. 1227 */ 1228 fifo_state = nicvf_queue_reg_read(nic, NIC_QSET_RBDR_0_1_STATUS0, qidx); 1229 if (((fifo_state >> 62) & 0x03) == 0x3) { 1230 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, 1231 qidx, NICVF_RBDR_RESET); 1232 } 1233 1234 /* Disable RBDR */ 1235 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0); 1236 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00)) 1237 return; 1238 while (1) { 1239 tmp = nicvf_queue_reg_read(nic, 1240 NIC_QSET_RBDR_0_1_PREFETCH_STATUS, qidx); 1241 if ((tmp & 0xFFFFFFFF) == ((tmp >> 32) & 0xFFFFFFFF)) 1242 break; 1243 1244 DELAY(1000); 1245 timeout--; 1246 if (!timeout) { 1247 device_printf(nic->dev, 1248 "Failed polling on prefetch status\n"); 1249 return; 1250 } 1251 } 1252 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 1253 NICVF_RBDR_RESET); 1254 1255 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x02)) 1256 return; 1257 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 0x00); 1258 if (nicvf_poll_reg(nic, qidx, NIC_QSET_RBDR_0_1_STATUS0, 62, 2, 0x00)) 1259 return; 1260} 1261 1262/* Configures receive queue */ 1263static void 1264nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, 1265 int qidx, bool enable) 1266{ 1267 union nic_mbx mbx = {}; 1268 struct rcv_queue *rq; 1269 struct rq_cfg rq_cfg; 1270 struct ifnet *ifp; 1271 struct lro_ctrl *lro; 1272 1273 ifp = nic->ifp; 1274 1275 rq = &qs->rq[qidx]; 1276 rq->enable = enable; 1277 1278 lro = &rq->lro; 1279 1280 /* Disable receive queue */ 1281 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 0); 1282 1283 if (!rq->enable) { 1284 nicvf_reclaim_rcv_queue(nic, qs, qidx); 1285 /* Free LRO memory */ 1286 tcp_lro_free(lro); 1287 rq->lro_enabled = FALSE; 1288 return; 1289 } 1290 1291 /* Configure LRO if enabled */ 1292 rq->lro_enabled = FALSE; 1293 if ((if_getcapenable(ifp) & IFCAP_LRO) != 0) { 1294 if (tcp_lro_init(lro) != 0) { 1295 device_printf(nic->dev, 1296 "Failed to initialize LRO for RXQ%d\n", qidx); 1297 } else { 1298 rq->lro_enabled = TRUE; 1299 lro->ifp = nic->ifp; 1300 } 1301 } 1302 1303 rq->cq_qs = qs->vnic_id; 1304 rq->cq_idx = qidx; 1305 rq->start_rbdr_qs = qs->vnic_id; 1306 rq->start_qs_rbdr_idx = qs->rbdr_cnt - 1; 1307 rq->cont_rbdr_qs = qs->vnic_id; 1308 rq->cont_qs_rbdr_idx = qs->rbdr_cnt - 1; 1309 /* all writes of RBDR data to be loaded into L2 Cache as well*/ 1310 rq->caching = 1; 1311 1312 /* Send a mailbox msg to PF to config RQ */ 1313 mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG; 1314 mbx.rq.qs_num = qs->vnic_id; 1315 mbx.rq.rq_num = qidx; 1316 mbx.rq.cfg = (rq->caching << 26) | (rq->cq_qs << 19) | 1317 (rq->cq_idx << 16) | (rq->cont_rbdr_qs << 9) | 1318 (rq->cont_qs_rbdr_idx << 8) | (rq->start_rbdr_qs << 1) | 1319 (rq->start_qs_rbdr_idx); 1320 nicvf_send_msg_to_pf(nic, &mbx); 1321 1322 mbx.rq.msg = NIC_MBOX_MSG_RQ_BP_CFG; 1323 mbx.rq.cfg = (1UL << 63) | (1UL << 62) | (qs->vnic_id << 0); 1324 nicvf_send_msg_to_pf(nic, &mbx); 1325 1326 /* 1327 * RQ drop config 1328 * Enable CQ drop to reserve sufficient CQEs for all tx packets 1329 */ 1330 mbx.rq.msg = NIC_MBOX_MSG_RQ_DROP_CFG; 1331 mbx.rq.cfg = (1UL << 62) | (RQ_CQ_DROP << 8); 1332 nicvf_send_msg_to_pf(nic, &mbx); 1333 1334 nicvf_queue_reg_write(nic, NIC_QSET_RQ_GEN_CFG, 0, 0x00); 1335 1336 /* Enable Receive queue */ 1337 rq_cfg.ena = 1; 1338 rq_cfg.tcp_ena = 0; 1339 nicvf_queue_reg_write(nic, NIC_QSET_RQ_0_7_CFG, qidx, 1340 *(uint64_t *)&rq_cfg); 1341} 1342 1343/* Configures completion queue */ 1344static void 1345nicvf_cmp_queue_config(struct nicvf *nic, struct queue_set *qs, 1346 int qidx, boolean_t enable) 1347{ 1348 struct cmp_queue *cq; 1349 struct cq_cfg cq_cfg; 1350 1351 cq = &qs->cq[qidx]; 1352 cq->enable = enable; 1353 1354 if (!cq->enable) { 1355 nicvf_reclaim_cmp_queue(nic, qs, qidx); 1356 return; 1357 } 1358 1359 /* Reset completion queue */ 1360 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, NICVF_CQ_RESET); 1361 1362 /* Set completion queue base address */ 1363 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_BASE, qidx, 1364 (uint64_t)(cq->dmem.phys_base)); 1365 1366 /* Enable Completion queue */ 1367 cq_cfg.ena = 1; 1368 cq_cfg.reset = 0; 1369 cq_cfg.caching = 0; 1370 cq_cfg.qsize = CMP_QSIZE; 1371 cq_cfg.avg_con = 0; 1372 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG, qidx, *(uint64_t *)&cq_cfg); 1373 1374 /* Set threshold value for interrupt generation */ 1375 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_THRESH, qidx, cq->thresh); 1376 nicvf_queue_reg_write(nic, NIC_QSET_CQ_0_7_CFG2, qidx, 1377 nic->cq_coalesce_usecs); 1378} 1379 1380/* Configures transmit queue */ 1381static void 1382nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs, int qidx, 1383 boolean_t enable) 1384{ 1385 union nic_mbx mbx = {}; 1386 struct snd_queue *sq; 1387 struct sq_cfg sq_cfg; 1388 1389 sq = &qs->sq[qidx]; 1390 sq->enable = enable; 1391 1392 if (!sq->enable) { 1393 nicvf_reclaim_snd_queue(nic, qs, qidx); 1394 return; 1395 } 1396 1397 /* Reset send queue */ 1398 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, NICVF_SQ_RESET); 1399 1400 sq->cq_qs = qs->vnic_id; 1401 sq->cq_idx = qidx; 1402 1403 /* Send a mailbox msg to PF to config SQ */ 1404 mbx.sq.msg = NIC_MBOX_MSG_SQ_CFG; 1405 mbx.sq.qs_num = qs->vnic_id; 1406 mbx.sq.sq_num = qidx; 1407 mbx.sq.sqs_mode = nic->sqs_mode; 1408 mbx.sq.cfg = (sq->cq_qs << 3) | sq->cq_idx; 1409 nicvf_send_msg_to_pf(nic, &mbx); 1410 1411 /* Set queue base address */ 1412 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_BASE, qidx, 1413 (uint64_t)(sq->dmem.phys_base)); 1414 1415 /* Enable send queue & set queue size */ 1416 sq_cfg.ena = 1; 1417 sq_cfg.reset = 0; 1418 sq_cfg.ldwb = 0; 1419 sq_cfg.qsize = SND_QSIZE; 1420 sq_cfg.tstmp_bgx_intf = 0; 1421 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, *(uint64_t *)&sq_cfg); 1422 1423 /* Set threshold value for interrupt generation */ 1424 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_THRESH, qidx, sq->thresh); 1425} 1426 1427/* Configures receive buffer descriptor ring */ 1428static void 1429nicvf_rbdr_config(struct nicvf *nic, struct queue_set *qs, int qidx, 1430 boolean_t enable) 1431{ 1432 struct rbdr *rbdr; 1433 struct rbdr_cfg rbdr_cfg; 1434 1435 rbdr = &qs->rbdr[qidx]; 1436 nicvf_reclaim_rbdr(nic, rbdr, qidx); 1437 if (!enable) 1438 return; 1439 1440 /* Set descriptor base address */ 1441 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_BASE, qidx, 1442 (uint64_t)(rbdr->dmem.phys_base)); 1443 1444 /* Enable RBDR & set queue size */ 1445 /* Buffer size should be in multiples of 128 bytes */ 1446 rbdr_cfg.ena = 1; 1447 rbdr_cfg.reset = 0; 1448 rbdr_cfg.ldwb = 0; 1449 rbdr_cfg.qsize = RBDR_SIZE; 1450 rbdr_cfg.avg_con = 0; 1451 rbdr_cfg.lines = rbdr->dma_size / 128; 1452 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_CFG, qidx, 1453 *(uint64_t *)&rbdr_cfg); 1454 1455 /* Notify HW */ 1456 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_DOOR, qidx, 1457 qs->rbdr_len - 1); 1458 1459 /* Set threshold value for interrupt generation */ 1460 nicvf_queue_reg_write(nic, NIC_QSET_RBDR_0_1_THRESH, qidx, 1461 rbdr->thresh - 1); 1462} 1463 1464/* Requests PF to assign and enable Qset */ 1465void 1466nicvf_qset_config(struct nicvf *nic, boolean_t enable) 1467{ 1468 union nic_mbx mbx = {}; 1469 struct queue_set *qs; 1470 struct qs_cfg *qs_cfg; 1471 1472 qs = nic->qs; 1473 if (qs == NULL) { 1474 device_printf(nic->dev, 1475 "Qset is still not allocated, don't init queues\n"); 1476 return; 1477 } 1478 1479 qs->enable = enable; 1480 qs->vnic_id = nic->vf_id; 1481 1482 /* Send a mailbox msg to PF to config Qset */ 1483 mbx.qs.msg = NIC_MBOX_MSG_QS_CFG; 1484 mbx.qs.num = qs->vnic_id; 1485 1486 mbx.qs.cfg = 0; 1487 qs_cfg = (struct qs_cfg *)&mbx.qs.cfg; 1488 if (qs->enable) { 1489 qs_cfg->ena = 1; 1490 qs_cfg->vnic = qs->vnic_id; 1491 } 1492 nicvf_send_msg_to_pf(nic, &mbx); 1493} 1494 1495static void 1496nicvf_free_resources(struct nicvf *nic) 1497{ 1498 int qidx; 1499 struct queue_set *qs; 1500 1501 qs = nic->qs; 1502 /* 1503 * Remove QS error task first since it has to be dead 1504 * to safely free completion queue tasks. 1505 */ 1506 if (qs->qs_err_taskq != NULL) { 1507 /* Shut down QS error tasks */ 1508 while (taskqueue_cancel(qs->qs_err_taskq, 1509 &qs->qs_err_task, NULL) != 0) { 1510 taskqueue_drain(qs->qs_err_taskq, &qs->qs_err_task); 1511 1512 } 1513 taskqueue_free(qs->qs_err_taskq); 1514 qs->qs_err_taskq = NULL; 1515 } 1516 /* Free receive buffer descriptor ring */ 1517 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 1518 nicvf_free_rbdr(nic, &qs->rbdr[qidx]); 1519 1520 /* Free completion queue */ 1521 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 1522 nicvf_free_cmp_queue(nic, &qs->cq[qidx]); 1523 1524 /* Free send queue */ 1525 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1526 nicvf_free_snd_queue(nic, &qs->sq[qidx]); 1527} 1528 1529static int 1530nicvf_alloc_resources(struct nicvf *nic) 1531{ 1532 struct queue_set *qs = nic->qs; 1533 int qidx; 1534 1535 /* Alloc receive buffer descriptor ring */ 1536 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { 1537 if (nicvf_init_rbdr(nic, &qs->rbdr[qidx], qs->rbdr_len, 1538 DMA_BUFFER_LEN, qidx)) 1539 goto alloc_fail; 1540 } 1541 1542 /* Alloc send queue */ 1543 for (qidx = 0; qidx < qs->sq_cnt; qidx++) { 1544 if (nicvf_init_snd_queue(nic, &qs->sq[qidx], qs->sq_len, qidx)) 1545 goto alloc_fail; 1546 } 1547 1548 /* Alloc completion queue */ 1549 for (qidx = 0; qidx < qs->cq_cnt; qidx++) { 1550 if (nicvf_init_cmp_queue(nic, &qs->cq[qidx], qs->cq_len, qidx)) 1551 goto alloc_fail; 1552 } 1553 1554 /* Allocate QS error taskqueue */ 1555 TASK_INIT(&qs->qs_err_task, 0, nicvf_qs_err_task, nic); 1556 qs->qs_err_taskq = taskqueue_create_fast("nicvf_qs_err_taskq", M_WAITOK, 1557 taskqueue_thread_enqueue, &qs->qs_err_taskq); 1558 taskqueue_start_threads(&qs->qs_err_taskq, 1, PI_NET, "%s: qs_taskq", 1559 device_get_nameunit(nic->dev)); 1560 1561 return (0); 1562alloc_fail: 1563 nicvf_free_resources(nic); 1564 return (ENOMEM); 1565} 1566 1567int 1568nicvf_set_qset_resources(struct nicvf *nic) 1569{ 1570 struct queue_set *qs; 1571 1572 qs = malloc(sizeof(*qs), M_NICVF, (M_ZERO | M_WAITOK)); 1573 nic->qs = qs; 1574 1575 /* Set count of each queue */ 1576 qs->rbdr_cnt = RBDR_CNT; 1577 /* With no RSS we stay with single RQ */ 1578 qs->rq_cnt = 1; 1579 1580 qs->sq_cnt = SND_QUEUE_CNT; 1581 qs->cq_cnt = CMP_QUEUE_CNT; 1582 1583 /* Set queue lengths */ 1584 qs->rbdr_len = RCV_BUF_COUNT; 1585 qs->sq_len = SND_QUEUE_LEN; 1586 qs->cq_len = CMP_QUEUE_LEN; 1587 1588 nic->rx_queues = qs->rq_cnt; 1589 nic->tx_queues = qs->sq_cnt; 1590 1591 return (0); 1592} 1593 1594int 1595nicvf_config_data_transfer(struct nicvf *nic, boolean_t enable) 1596{ 1597 boolean_t disable = FALSE; 1598 struct queue_set *qs; 1599 int qidx; 1600 1601 qs = nic->qs; 1602 if (qs == NULL) 1603 return (0); 1604 1605 if (enable) { 1606 if (nicvf_alloc_resources(nic) != 0) 1607 return (ENOMEM); 1608 1609 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1610 nicvf_snd_queue_config(nic, qs, qidx, enable); 1611 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 1612 nicvf_cmp_queue_config(nic, qs, qidx, enable); 1613 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 1614 nicvf_rbdr_config(nic, qs, qidx, enable); 1615 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 1616 nicvf_rcv_queue_config(nic, qs, qidx, enable); 1617 } else { 1618 for (qidx = 0; qidx < qs->rq_cnt; qidx++) 1619 nicvf_rcv_queue_config(nic, qs, qidx, disable); 1620 for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) 1621 nicvf_rbdr_config(nic, qs, qidx, disable); 1622 for (qidx = 0; qidx < qs->sq_cnt; qidx++) 1623 nicvf_snd_queue_config(nic, qs, qidx, disable); 1624 for (qidx = 0; qidx < qs->cq_cnt; qidx++) 1625 nicvf_cmp_queue_config(nic, qs, qidx, disable); 1626 1627 nicvf_free_resources(nic); 1628 } 1629 1630 return (0); 1631} 1632 1633/* 1634 * Get a free desc from SQ 1635 * returns descriptor ponter & descriptor number 1636 */ 1637static __inline int 1638nicvf_get_sq_desc(struct snd_queue *sq, int desc_cnt) 1639{ 1640 int qentry; 1641 1642 qentry = sq->tail; 1643 atomic_subtract_int(&sq->free_cnt, desc_cnt); 1644 sq->tail += desc_cnt; 1645 sq->tail &= (sq->dmem.q_len - 1); 1646 1647 return (qentry); 1648} 1649 1650/* Free descriptor back to SQ for future use */ 1651static void 1652nicvf_put_sq_desc(struct snd_queue *sq, int desc_cnt) 1653{ 1654 1655 atomic_add_int(&sq->free_cnt, desc_cnt); 1656 sq->head += desc_cnt; 1657 sq->head &= (sq->dmem.q_len - 1); 1658} 1659 1660static __inline int 1661nicvf_get_nxt_sqentry(struct snd_queue *sq, int qentry) 1662{ 1663 qentry++; 1664 qentry &= (sq->dmem.q_len - 1); 1665 return (qentry); 1666} 1667 1668static void 1669nicvf_sq_enable(struct nicvf *nic, struct snd_queue *sq, int qidx) 1670{ 1671 uint64_t sq_cfg; 1672 1673 sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx); 1674 sq_cfg |= NICVF_SQ_EN; 1675 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg); 1676 /* Ring doorbell so that H/W restarts processing SQEs */ 1677 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, qidx, 0); 1678} 1679 1680static void 1681nicvf_sq_disable(struct nicvf *nic, int qidx) 1682{ 1683 uint64_t sq_cfg; 1684 1685 sq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_CFG, qidx); 1686 sq_cfg &= ~NICVF_SQ_EN; 1687 nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_CFG, qidx, sq_cfg); 1688} 1689 1690static void 1691nicvf_sq_free_used_descs(struct nicvf *nic, struct snd_queue *sq, int qidx) 1692{ 1693 uint64_t head, tail; 1694 struct snd_buff *snd_buff; 1695 struct sq_hdr_subdesc *hdr; 1696 1697 NICVF_TX_LOCK(sq); 1698 head = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_HEAD, qidx) >> 4; 1699 tail = nicvf_queue_reg_read(nic, NIC_QSET_SQ_0_7_TAIL, qidx) >> 4; 1700 while (sq->head != head) { 1701 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, sq->head); 1702 if (hdr->subdesc_type != SQ_DESC_TYPE_HEADER) { 1703 nicvf_put_sq_desc(sq, 1); 1704 continue; 1705 } 1706 snd_buff = &sq->snd_buff[sq->head]; 1707 if (snd_buff->mbuf != NULL) { 1708 bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap); 1709 m_freem(snd_buff->mbuf); 1710 sq->snd_buff[sq->head].mbuf = NULL; 1711 } 1712 nicvf_put_sq_desc(sq, hdr->subdesc_cnt + 1); 1713 } 1714 NICVF_TX_UNLOCK(sq); 1715} 1716 1717/* 1718 * Add SQ HEADER subdescriptor. 1719 * First subdescriptor for every send descriptor. 1720 */ 1721static __inline int 1722nicvf_sq_add_hdr_subdesc(struct snd_queue *sq, int qentry, 1723 int subdesc_cnt, struct mbuf *mbuf, int len) 1724{ 1725 struct nicvf *nic; 1726 struct sq_hdr_subdesc *hdr; 1727 struct ether_vlan_header *eh; 1728#ifdef INET 1729 struct ip *ip; 1730 struct tcphdr *th; 1731#endif 1732 uint16_t etype; 1733 int ehdrlen, iphlen, poff; 1734 1735 nic = sq->nic; 1736 1737 hdr = (struct sq_hdr_subdesc *)GET_SQ_DESC(sq, qentry); 1738 sq->snd_buff[qentry].mbuf = mbuf; 1739 1740 memset(hdr, 0, SND_QUEUE_DESC_SIZE); 1741 hdr->subdesc_type = SQ_DESC_TYPE_HEADER; 1742 /* Enable notification via CQE after processing SQE */ 1743 hdr->post_cqe = 1; 1744 /* No of subdescriptors following this */ 1745 hdr->subdesc_cnt = subdesc_cnt; 1746 hdr->tot_len = len; 1747 1748 eh = mtod(mbuf, struct ether_vlan_header *); 1749 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 1750 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 1751 etype = ntohs(eh->evl_proto); 1752 } else { 1753 ehdrlen = ETHER_HDR_LEN; 1754 etype = ntohs(eh->evl_encap_proto); 1755 } 1756 1757 switch (etype) { 1758#ifdef INET6 1759 case ETHERTYPE_IPV6: 1760 /* ARM64TODO: Add support for IPv6 */ 1761 hdr->csum_l3 = 0; 1762 sq->snd_buff[qentry].mbuf = NULL; 1763 return (ENXIO); 1764#endif 1765#ifdef INET 1766 case ETHERTYPE_IP: 1767 if (mbuf->m_len < ehdrlen + sizeof(struct ip)) { 1768 mbuf = m_pullup(mbuf, ehdrlen + sizeof(struct ip)); 1769 sq->snd_buff[qentry].mbuf = mbuf; 1770 if (mbuf == NULL) 1771 return (ENOBUFS); 1772 } 1773 1774 ip = (struct ip *)(mbuf->m_data + ehdrlen); 1775 ip->ip_sum = 0; 1776 iphlen = ip->ip_hl << 2; 1777 poff = ehdrlen + iphlen; 1778 1779 if (mbuf->m_pkthdr.csum_flags != 0) { 1780 hdr->csum_l3 = 1; /* Enable IP csum calculation */ 1781 switch (ip->ip_p) { 1782 case IPPROTO_TCP: 1783 if ((mbuf->m_pkthdr.csum_flags & CSUM_TCP) == 0) 1784 break; 1785 1786 if (mbuf->m_len < (poff + sizeof(struct tcphdr))) { 1787 mbuf = m_pullup(mbuf, poff + sizeof(struct tcphdr)); 1788 sq->snd_buff[qentry].mbuf = mbuf; 1789 if (mbuf == NULL) 1790 return (ENOBUFS); 1791 } 1792 hdr->csum_l4 = SEND_L4_CSUM_TCP; 1793 break; 1794 case IPPROTO_UDP: 1795 if ((mbuf->m_pkthdr.csum_flags & CSUM_UDP) == 0) 1796 break; 1797 1798 if (mbuf->m_len < (poff + sizeof(struct udphdr))) { 1799 mbuf = m_pullup(mbuf, poff + sizeof(struct udphdr)); 1800 sq->snd_buff[qentry].mbuf = mbuf; 1801 if (mbuf == NULL) 1802 return (ENOBUFS); 1803 } 1804 hdr->csum_l4 = SEND_L4_CSUM_UDP; 1805 break; 1806 case IPPROTO_SCTP: 1807 if ((mbuf->m_pkthdr.csum_flags & CSUM_SCTP) == 0) 1808 break; 1809 1810 if (mbuf->m_len < (poff + sizeof(struct sctphdr))) { 1811 mbuf = m_pullup(mbuf, poff + sizeof(struct sctphdr)); 1812 sq->snd_buff[qentry].mbuf = mbuf; 1813 if (mbuf == NULL) 1814 return (ENOBUFS); 1815 } 1816 hdr->csum_l4 = SEND_L4_CSUM_SCTP; 1817 break; 1818 default: 1819 break; 1820 } 1821 hdr->l3_offset = ehdrlen; 1822 hdr->l4_offset = ehdrlen + iphlen; 1823 } 1824 1825 if ((mbuf->m_pkthdr.tso_segsz != 0) && nic->hw_tso) { 1826 /* 1827 * Extract ip again as m_data could have been modified. 1828 */ 1829 ip = (struct ip *)(mbuf->m_data + ehdrlen); 1830 th = (struct tcphdr *)((caddr_t)ip + iphlen); 1831 1832 hdr->tso = 1; 1833 hdr->tso_start = ehdrlen + iphlen + (th->th_off * 4); 1834 hdr->tso_max_paysize = mbuf->m_pkthdr.tso_segsz; 1835 hdr->inner_l3_offset = ehdrlen - 2; 1836 nic->drv_stats.tx_tso++; 1837 } 1838 break; 1839#endif 1840 default: 1841 hdr->csum_l3 = 0; 1842 } 1843 1844 return (0); 1845} 1846 1847/* 1848 * SQ GATHER subdescriptor 1849 * Must follow HDR descriptor 1850 */ 1851static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry, 1852 int size, uint64_t data) 1853{ 1854 struct sq_gather_subdesc *gather; 1855 1856 qentry &= (sq->dmem.q_len - 1); 1857 gather = (struct sq_gather_subdesc *)GET_SQ_DESC(sq, qentry); 1858 1859 memset(gather, 0, SND_QUEUE_DESC_SIZE); 1860 gather->subdesc_type = SQ_DESC_TYPE_GATHER; 1861 gather->ld_type = NIC_SEND_LD_TYPE_E_LDD; 1862 gather->size = size; 1863 gather->addr = data; 1864} 1865 1866/* Put an mbuf to a SQ for packet transfer. */ 1867int 1868nicvf_tx_mbuf_locked(struct snd_queue *sq, struct mbuf *mbuf) 1869{ 1870 bus_dma_segment_t segs[256]; 1871 struct nicvf *nic; 1872 struct snd_buff *snd_buff; 1873 size_t seg; 1874 int nsegs, qentry; 1875 int subdesc_cnt; 1876 int err; 1877 1878 NICVF_TX_LOCK_ASSERT(sq); 1879 1880 if (sq->free_cnt == 0) 1881 return (ENOBUFS); 1882 1883 snd_buff = &sq->snd_buff[sq->tail]; 1884 1885 err = bus_dmamap_load_mbuf_sg(sq->snd_buff_dmat, snd_buff->dmap, 1886 mbuf, segs, &nsegs, BUS_DMA_NOWAIT); 1887 if (err != 0) { 1888 /* ARM64TODO: Add mbuf defragmenting if we lack maps */ 1889 return (err); 1890 } 1891 1892 /* Set how many subdescriptors is required */ 1893 nic = sq->nic; 1894 if (mbuf->m_pkthdr.tso_segsz != 0 && nic->hw_tso) 1895 subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT; 1896 else 1897 subdesc_cnt = MIN_SQ_DESC_PER_PKT_XMIT + nsegs - 1; 1898 1899 if (subdesc_cnt > sq->free_cnt) { 1900 /* ARM64TODO: Add mbuf defragmentation if we lack descriptors */ 1901 bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap); 1902 return (ENOBUFS); 1903 } 1904 1905 qentry = nicvf_get_sq_desc(sq, subdesc_cnt); 1906 1907 /* Add SQ header subdesc */ 1908 err = nicvf_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, mbuf, 1909 mbuf->m_pkthdr.len); 1910 if (err != 0) { 1911 bus_dmamap_unload(sq->snd_buff_dmat, snd_buff->dmap); 1912 return (err); 1913 } 1914 1915 /* Add SQ gather subdescs */ 1916 for (seg = 0; seg < nsegs; seg++) { 1917 qentry = nicvf_get_nxt_sqentry(sq, qentry); 1918 nicvf_sq_add_gather_subdesc(sq, qentry, segs[seg].ds_len, 1919 segs[seg].ds_addr); 1920 } 1921 1922 /* make sure all memory stores are done before ringing doorbell */ 1923 bus_dmamap_sync(sq->dmem.dmat, sq->dmem.dmap, BUS_DMASYNC_PREWRITE); 1924 1925 dprintf(sq->nic->dev, "%s: sq->idx: %d, subdesc_cnt: %d\n", 1926 __func__, sq->idx, subdesc_cnt); 1927 /* Inform HW to xmit new packet */ 1928 nicvf_queue_reg_write(sq->nic, NIC_QSET_SQ_0_7_DOOR, 1929 sq->idx, subdesc_cnt); 1930 return (0); 1931} 1932 1933static __inline u_int 1934frag_num(u_int i) 1935{ 1936#if BYTE_ORDER == BIG_ENDIAN 1937 return ((i & ~3) + 3 - (i & 3)); 1938#else 1939 return (i); 1940#endif 1941} 1942 1943/* Returns MBUF for a received packet */ 1944struct mbuf * 1945nicvf_get_rcv_mbuf(struct nicvf *nic, struct cqe_rx_t *cqe_rx) 1946{ 1947 int frag; 1948 int payload_len = 0; 1949 struct mbuf *mbuf; 1950 struct mbuf *mbuf_frag; 1951 uint16_t *rb_lens = NULL; 1952 uint64_t *rb_ptrs = NULL; 1953 1954 mbuf = NULL; 1955 rb_lens = (uint16_t *)((uint8_t *)cqe_rx + (3 * sizeof(uint64_t))); 1956 rb_ptrs = (uint64_t *)((uint8_t *)cqe_rx + (6 * sizeof(uint64_t))); 1957 1958 dprintf(nic->dev, "%s rb_cnt %d rb0_ptr %lx rb0_sz %d\n", 1959 __func__, cqe_rx->rb_cnt, cqe_rx->rb0_ptr, cqe_rx->rb0_sz); 1960 1961 for (frag = 0; frag < cqe_rx->rb_cnt; frag++) { 1962 payload_len = rb_lens[frag_num(frag)]; 1963 if (frag == 0) { 1964 /* First fragment */ 1965 mbuf = nicvf_rb_ptr_to_mbuf(nic, 1966 (*rb_ptrs - cqe_rx->align_pad)); 1967 mbuf->m_len = payload_len; 1968 mbuf->m_data += cqe_rx->align_pad; 1969 if_setrcvif(mbuf, nic->ifp); 1970 } else { 1971 /* Add fragments */ 1972 mbuf_frag = nicvf_rb_ptr_to_mbuf(nic, *rb_ptrs); 1973 m_append(mbuf, payload_len, mbuf_frag->m_data); 1974 m_freem(mbuf_frag); 1975 } 1976 /* Next buffer pointer */ 1977 rb_ptrs++; 1978 } 1979 1980 if (__predict_true(mbuf != NULL)) { 1981 m_fixhdr(mbuf); 1982 mbuf->m_pkthdr.flowid = cqe_rx->rq_idx; 1983 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE); 1984 if (__predict_true((if_getcapenable(nic->ifp) & IFCAP_RXCSUM) != 0)) { 1985 /* 1986 * HW by default verifies IP & TCP/UDP/SCTP checksums 1987 */ 1988 1989 /* XXX: Do we need to include IP with options too? */ 1990 if (__predict_true(cqe_rx->l3_type == L3TYPE_IPV4 || 1991 cqe_rx->l3_type == L3TYPE_IPV6)) { 1992 mbuf->m_pkthdr.csum_flags = 1993 (CSUM_IP_CHECKED | CSUM_IP_VALID); 1994 } 1995 if (cqe_rx->l4_type == L4TYPE_TCP || 1996 cqe_rx->l4_type == L4TYPE_UDP || 1997 cqe_rx->l4_type == L4TYPE_SCTP) { 1998 mbuf->m_pkthdr.csum_flags |= 1999 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 2000 mbuf->m_pkthdr.csum_data = htons(0xffff); 2001 } 2002 } 2003 } 2004 2005 return (mbuf); 2006} 2007 2008/* Enable interrupt */ 2009void 2010nicvf_enable_intr(struct nicvf *nic, int int_type, int q_idx) 2011{ 2012 uint64_t reg_val; 2013 2014 reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S); 2015 2016 switch (int_type) { 2017 case NICVF_INTR_CQ: 2018 reg_val |= ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT); 2019 break; 2020 case NICVF_INTR_SQ: 2021 reg_val |= ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT); 2022 break; 2023 case NICVF_INTR_RBDR: 2024 reg_val |= ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT); 2025 break; 2026 case NICVF_INTR_PKT_DROP: 2027 reg_val |= (1UL << NICVF_INTR_PKT_DROP_SHIFT); 2028 break; 2029 case NICVF_INTR_TCP_TIMER: 2030 reg_val |= (1UL << NICVF_INTR_TCP_TIMER_SHIFT); 2031 break; 2032 case NICVF_INTR_MBOX: 2033 reg_val |= (1UL << NICVF_INTR_MBOX_SHIFT); 2034 break; 2035 case NICVF_INTR_QS_ERR: 2036 reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT); 2037 break; 2038 default: 2039 device_printf(nic->dev, 2040 "Failed to enable interrupt: unknown type\n"); 2041 break; 2042 } 2043 2044 nicvf_reg_write(nic, NIC_VF_ENA_W1S, reg_val); 2045} 2046 2047/* Disable interrupt */ 2048void 2049nicvf_disable_intr(struct nicvf *nic, int int_type, int q_idx) 2050{ 2051 uint64_t reg_val = 0; 2052 2053 switch (int_type) { 2054 case NICVF_INTR_CQ: 2055 reg_val |= ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT); 2056 break; 2057 case NICVF_INTR_SQ: 2058 reg_val |= ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT); 2059 break; 2060 case NICVF_INTR_RBDR: 2061 reg_val |= ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT); 2062 break; 2063 case NICVF_INTR_PKT_DROP: 2064 reg_val |= (1UL << NICVF_INTR_PKT_DROP_SHIFT); 2065 break; 2066 case NICVF_INTR_TCP_TIMER: 2067 reg_val |= (1UL << NICVF_INTR_TCP_TIMER_SHIFT); 2068 break; 2069 case NICVF_INTR_MBOX: 2070 reg_val |= (1UL << NICVF_INTR_MBOX_SHIFT); 2071 break; 2072 case NICVF_INTR_QS_ERR: 2073 reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT); 2074 break; 2075 default: 2076 device_printf(nic->dev, 2077 "Failed to disable interrupt: unknown type\n"); 2078 break; 2079 } 2080 2081 nicvf_reg_write(nic, NIC_VF_ENA_W1C, reg_val); 2082} 2083 2084/* Clear interrupt */ 2085void 2086nicvf_clear_intr(struct nicvf *nic, int int_type, int q_idx) 2087{ 2088 uint64_t reg_val = 0; 2089 2090 switch (int_type) { 2091 case NICVF_INTR_CQ: 2092 reg_val = ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT); 2093 break; 2094 case NICVF_INTR_SQ: 2095 reg_val = ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT); 2096 break; 2097 case NICVF_INTR_RBDR: 2098 reg_val = ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT); 2099 break; 2100 case NICVF_INTR_PKT_DROP: 2101 reg_val = (1UL << NICVF_INTR_PKT_DROP_SHIFT); 2102 break; 2103 case NICVF_INTR_TCP_TIMER: 2104 reg_val = (1UL << NICVF_INTR_TCP_TIMER_SHIFT); 2105 break; 2106 case NICVF_INTR_MBOX: 2107 reg_val = (1UL << NICVF_INTR_MBOX_SHIFT); 2108 break; 2109 case NICVF_INTR_QS_ERR: 2110 reg_val |= (1UL << NICVF_INTR_QS_ERR_SHIFT); 2111 break; 2112 default: 2113 device_printf(nic->dev, 2114 "Failed to clear interrupt: unknown type\n"); 2115 break; 2116 } 2117 2118 nicvf_reg_write(nic, NIC_VF_INT, reg_val); 2119} 2120 2121/* Check if interrupt is enabled */ 2122int 2123nicvf_is_intr_enabled(struct nicvf *nic, int int_type, int q_idx) 2124{ 2125 uint64_t reg_val; 2126 uint64_t mask = 0xff; 2127 2128 reg_val = nicvf_reg_read(nic, NIC_VF_ENA_W1S); 2129 2130 switch (int_type) { 2131 case NICVF_INTR_CQ: 2132 mask = ((1UL << q_idx) << NICVF_INTR_CQ_SHIFT); 2133 break; 2134 case NICVF_INTR_SQ: 2135 mask = ((1UL << q_idx) << NICVF_INTR_SQ_SHIFT); 2136 break; 2137 case NICVF_INTR_RBDR: 2138 mask = ((1UL << q_idx) << NICVF_INTR_RBDR_SHIFT); 2139 break; 2140 case NICVF_INTR_PKT_DROP: 2141 mask = NICVF_INTR_PKT_DROP_MASK; 2142 break; 2143 case NICVF_INTR_TCP_TIMER: 2144 mask = NICVF_INTR_TCP_TIMER_MASK; 2145 break; 2146 case NICVF_INTR_MBOX: 2147 mask = NICVF_INTR_MBOX_MASK; 2148 break; 2149 case NICVF_INTR_QS_ERR: 2150 mask = NICVF_INTR_QS_ERR_MASK; 2151 break; 2152 default: 2153 device_printf(nic->dev, 2154 "Failed to check interrupt enable: unknown type\n"); 2155 break; 2156 } 2157 2158 return (reg_val & mask); 2159} 2160 2161void 2162nicvf_update_rq_stats(struct nicvf *nic, int rq_idx) 2163{ 2164 struct rcv_queue *rq; 2165 2166#define GET_RQ_STATS(reg) \ 2167 nicvf_reg_read(nic, NIC_QSET_RQ_0_7_STAT_0_1 |\ 2168 (rq_idx << NIC_Q_NUM_SHIFT) | (reg << 3)) 2169 2170 rq = &nic->qs->rq[rq_idx]; 2171 rq->stats.bytes = GET_RQ_STATS(RQ_SQ_STATS_OCTS); 2172 rq->stats.pkts = GET_RQ_STATS(RQ_SQ_STATS_PKTS); 2173} 2174 2175void 2176nicvf_update_sq_stats(struct nicvf *nic, int sq_idx) 2177{ 2178 struct snd_queue *sq; 2179 2180#define GET_SQ_STATS(reg) \ 2181 nicvf_reg_read(nic, NIC_QSET_SQ_0_7_STAT_0_1 |\ 2182 (sq_idx << NIC_Q_NUM_SHIFT) | (reg << 3)) 2183 2184 sq = &nic->qs->sq[sq_idx]; 2185 sq->stats.bytes = GET_SQ_STATS(RQ_SQ_STATS_OCTS); 2186 sq->stats.pkts = GET_SQ_STATS(RQ_SQ_STATS_PKTS); 2187} 2188 2189/* Check for errors in the receive cmp.queue entry */ 2190int 2191nicvf_check_cqe_rx_errs(struct nicvf *nic, struct cmp_queue *cq, 2192 struct cqe_rx_t *cqe_rx) 2193{ 2194 struct nicvf_hw_stats *stats = &nic->hw_stats; 2195 struct nicvf_drv_stats *drv_stats = &nic->drv_stats; 2196 2197 if (!cqe_rx->err_level && !cqe_rx->err_opcode) { 2198 drv_stats->rx_frames_ok++; 2199 return (0); 2200 } 2201 2202 switch (cqe_rx->err_opcode) { 2203 case CQ_RX_ERROP_RE_PARTIAL: 2204 stats->rx_bgx_truncated_pkts++; 2205 break; 2206 case CQ_RX_ERROP_RE_JABBER: 2207 stats->rx_jabber_errs++; 2208 break; 2209 case CQ_RX_ERROP_RE_FCS: 2210 stats->rx_fcs_errs++; 2211 break; 2212 case CQ_RX_ERROP_RE_RX_CTL: 2213 stats->rx_bgx_errs++; 2214 break; 2215 case CQ_RX_ERROP_PREL2_ERR: 2216 stats->rx_prel2_errs++; 2217 break; 2218 case CQ_RX_ERROP_L2_MAL: 2219 stats->rx_l2_hdr_malformed++; 2220 break; 2221 case CQ_RX_ERROP_L2_OVERSIZE: 2222 stats->rx_oversize++; 2223 break; 2224 case CQ_RX_ERROP_L2_UNDERSIZE: 2225 stats->rx_undersize++; 2226 break; 2227 case CQ_RX_ERROP_L2_LENMISM: 2228 stats->rx_l2_len_mismatch++; 2229 break; 2230 case CQ_RX_ERROP_L2_PCLP: 2231 stats->rx_l2_pclp++; 2232 break; 2233 case CQ_RX_ERROP_IP_NOT: 2234 stats->rx_ip_ver_errs++; 2235 break; 2236 case CQ_RX_ERROP_IP_CSUM_ERR: 2237 stats->rx_ip_csum_errs++; 2238 break; 2239 case CQ_RX_ERROP_IP_MAL: 2240 stats->rx_ip_hdr_malformed++; 2241 break; 2242 case CQ_RX_ERROP_IP_MALD: 2243 stats->rx_ip_payload_malformed++; 2244 break; 2245 case CQ_RX_ERROP_IP_HOP: 2246 stats->rx_ip_ttl_errs++; 2247 break; 2248 case CQ_RX_ERROP_L3_PCLP: 2249 stats->rx_l3_pclp++; 2250 break; 2251 case CQ_RX_ERROP_L4_MAL: 2252 stats->rx_l4_malformed++; 2253 break; 2254 case CQ_RX_ERROP_L4_CHK: 2255 stats->rx_l4_csum_errs++; 2256 break; 2257 case CQ_RX_ERROP_UDP_LEN: 2258 stats->rx_udp_len_errs++; 2259 break; 2260 case CQ_RX_ERROP_L4_PORT: 2261 stats->rx_l4_port_errs++; 2262 break; 2263 case CQ_RX_ERROP_TCP_FLAG: 2264 stats->rx_tcp_flag_errs++; 2265 break; 2266 case CQ_RX_ERROP_TCP_OFFSET: 2267 stats->rx_tcp_offset_errs++; 2268 break; 2269 case CQ_RX_ERROP_L4_PCLP: 2270 stats->rx_l4_pclp++; 2271 break; 2272 case CQ_RX_ERROP_RBDR_TRUNC: 2273 stats->rx_truncated_pkts++; 2274 break; 2275 } 2276 2277 return (1); 2278} 2279 2280/* Check for errors in the send cmp.queue entry */ 2281int 2282nicvf_check_cqe_tx_errs(struct nicvf *nic, struct cmp_queue *cq, 2283 struct cqe_send_t *cqe_tx) 2284{ 2285 struct cmp_queue_stats *stats = &cq->stats; 2286 2287 switch (cqe_tx->send_status) { 2288 case CQ_TX_ERROP_GOOD: 2289 stats->tx.good++; 2290 return (0); 2291 case CQ_TX_ERROP_DESC_FAULT: 2292 stats->tx.desc_fault++; 2293 break; 2294 case CQ_TX_ERROP_HDR_CONS_ERR: 2295 stats->tx.hdr_cons_err++; 2296 break; 2297 case CQ_TX_ERROP_SUBDC_ERR: 2298 stats->tx.subdesc_err++; 2299 break; 2300 case CQ_TX_ERROP_IMM_SIZE_OFLOW: 2301 stats->tx.imm_size_oflow++; 2302 break; 2303 case CQ_TX_ERROP_DATA_SEQUENCE_ERR: 2304 stats->tx.data_seq_err++; 2305 break; 2306 case CQ_TX_ERROP_MEM_SEQUENCE_ERR: 2307 stats->tx.mem_seq_err++; 2308 break; 2309 case CQ_TX_ERROP_LOCK_VIOL: 2310 stats->tx.lock_viol++; 2311 break; 2312 case CQ_TX_ERROP_DATA_FAULT: 2313 stats->tx.data_fault++; 2314 break; 2315 case CQ_TX_ERROP_TSTMP_CONFLICT: 2316 stats->tx.tstmp_conflict++; 2317 break; 2318 case CQ_TX_ERROP_TSTMP_TIMEOUT: 2319 stats->tx.tstmp_timeout++; 2320 break; 2321 case CQ_TX_ERROP_MEM_FAULT: 2322 stats->tx.mem_fault++; 2323 break; 2324 case CQ_TX_ERROP_CK_OVERLAP: 2325 stats->tx.csum_overlap++; 2326 break; 2327 case CQ_TX_ERROP_CK_OFLOW: 2328 stats->tx.csum_overflow++; 2329 break; 2330 } 2331 2332 return (1); 2333} 2334