1279232Sluigi/* 2279232Sluigi * Copyright (C) 2015, Luigi Rizzo. All rights reserved. 3279232Sluigi * 4279232Sluigi * Redistribution and use in source and binary forms, with or without 5279232Sluigi * modification, are permitted provided that the following conditions 6279232Sluigi * are met: 7279232Sluigi * 1. Redistributions of source code must retain the above copyright 8279232Sluigi * notice, this list of conditions and the following disclaimer. 9279232Sluigi * 2. Redistributions in binary form must reproduce the above copyright 10279232Sluigi * notice, this list of conditions and the following disclaimer in the 11279232Sluigi * documentation and/or other materials provided with the distribution. 12279232Sluigi * 13279232Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14279232Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15279232Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16279232Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17279232Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18279232Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19279232Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20279232Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21279232Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22279232Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23279232Sluigi * SUCH DAMAGE. 24279232Sluigi */ 25279232Sluigi 26279232Sluigi/* 27279232Sluigi * $FreeBSD: stable/11/sys/dev/netmap/if_ixl_netmap.h 359310 2020-03-25 23:06:04Z vmaffione $ 28279232Sluigi * 29279232Sluigi * netmap support for: ixl 30279232Sluigi * 31279232Sluigi * derived from ixgbe 32279232Sluigi * netmap support for a network driver. 33279232Sluigi * This file contains code but only static or inline functions used 34279232Sluigi * by a single driver. To avoid replication of code we just #include 35279232Sluigi * it near the beginning of the standard driver. 36279232Sluigi * For ixl the file is imported in two places, hence the conditional at the 37279232Sluigi * beginning. 38279232Sluigi */ 39279232Sluigi 40279232Sluigi#include <net/netmap.h> 41279232Sluigi#include <sys/selinfo.h> 42279232Sluigi 43279232Sluigi/* 44279232Sluigi * Some drivers may need the following headers. Others 45279232Sluigi * already include them by default 46279232Sluigi 47279232Sluigi#include <vm/vm.h> 48279232Sluigi#include <vm/pmap.h> 49279232Sluigi 50279232Sluigi */ 51279232Sluigi#include <dev/netmap/netmap_kern.h> 52279232Sluigi 53279232Sluigiint ixl_netmap_txsync(struct netmap_kring *kring, int flags); 54279232Sluigiint ixl_netmap_rxsync(struct netmap_kring *kring, int flags); 55279232Sluigi 56279232Sluigiextern int ixl_rx_miss, ixl_rx_miss_bufs, ixl_crcstrip; 57279232Sluigi 58279232Sluigi#ifdef NETMAP_IXL_MAIN 59279232Sluigi/* 60279232Sluigi * device-specific sysctl variables: 61279232Sluigi * 62341477Svmaffione * ixl_crcstrip: 0: NIC keeps CRC in rx frames, 1: NIC strips it (default). 63279232Sluigi * During regular operations the CRC is stripped, but on some 64279232Sluigi * hardware reception of frames not multiple of 64 is slower, 65279232Sluigi * so using crcstrip=0 helps in benchmarks. 66279232Sluigi * 67279232Sluigi * ixl_rx_miss, ixl_rx_miss_bufs: 68279232Sluigi * count packets that might be missed due to lost interrupts. 69279232Sluigi */ 70341477Svmaffioneint ixl_rx_miss, ixl_rx_miss_bufs, ixl_crcstrip = 1; 71279232SluigiSYSCTL_DECL(_dev_netmap); 72285349Sluigi/* 73285349Sluigi * The xl driver by default strips CRCs and we do not override it. 74285349Sluigi */ 75285349Sluigi#if 0 76279232SluigiSYSCTL_INT(_dev_netmap, OID_AUTO, ixl_crcstrip, 77341477Svmaffione CTLFLAG_RW, &ixl_crcstrip, 1, "NIC strips CRC on rx frames"); 78285349Sluigi#endif 79279232SluigiSYSCTL_INT(_dev_netmap, OID_AUTO, ixl_rx_miss, 80279232Sluigi CTLFLAG_RW, &ixl_rx_miss, 0, "potentially missed rx intr"); 81279232SluigiSYSCTL_INT(_dev_netmap, OID_AUTO, ixl_rx_miss_bufs, 82279232Sluigi CTLFLAG_RW, &ixl_rx_miss_bufs, 0, "potentially missed rx intr bufs"); 83279232Sluigi 84279232Sluigi 85279232Sluigi/* 86279232Sluigi * Register/unregister. We are already under netmap lock. 87279232Sluigi * Only called on the first register or the last unregister. 88279232Sluigi */ 89279232Sluigistatic int 90279232Sluigiixl_netmap_reg(struct netmap_adapter *na, int onoff) 91279232Sluigi{ 92279232Sluigi struct ifnet *ifp = na->ifp; 93279232Sluigi struct ixl_vsi *vsi = ifp->if_softc; 94279232Sluigi struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 95279232Sluigi 96279232Sluigi IXL_PF_LOCK(pf); 97279232Sluigi ixl_disable_intr(vsi); 98279232Sluigi 99279232Sluigi /* Tell the stack that the interface is no longer active */ 100279232Sluigi ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 101279232Sluigi 102279232Sluigi //set_crcstrip(&adapter->hw, onoff); 103279232Sluigi /* enable or disable flags and callbacks in na and ifp */ 104279232Sluigi if (onoff) { 105279232Sluigi nm_set_native_flags(na); 106279232Sluigi } else { 107279232Sluigi nm_clear_native_flags(na); 108279232Sluigi } 109279232Sluigi ixl_init_locked(pf); /* also enables intr */ 110279232Sluigi //set_crcstrip(&adapter->hw, onoff); // XXX why twice ? 111279232Sluigi IXL_PF_UNLOCK(pf); 112279232Sluigi return (ifp->if_drv_flags & IFF_DRV_RUNNING ? 0 : 1); 113279232Sluigi} 114279232Sluigi 115279232Sluigi 116279232Sluigi/* 117279232Sluigi * The attach routine, called near the end of ixl_attach(), 118279232Sluigi * fills the parameters for netmap_attach() and calls it. 119279232Sluigi * It cannot fail, in the worst case (such as no memory) 120279232Sluigi * netmap mode will be disabled and the driver will only 121279232Sluigi * operate in standard mode. 122279232Sluigi */ 123279232Sluigistatic void 124279232Sluigiixl_netmap_attach(struct ixl_vsi *vsi) 125279232Sluigi{ 126279232Sluigi struct netmap_adapter na; 127279232Sluigi 128279232Sluigi bzero(&na, sizeof(na)); 129279232Sluigi 130279232Sluigi na.ifp = vsi->ifp; 131279232Sluigi na.na_flags = NAF_BDG_MAYSLEEP; 132343559Svmaffione na.num_tx_desc = vsi->num_tx_desc; 133343559Svmaffione na.num_rx_desc = vsi->num_rx_desc; 134279232Sluigi na.nm_txsync = ixl_netmap_txsync; 135279232Sluigi na.nm_rxsync = ixl_netmap_rxsync; 136279232Sluigi na.nm_register = ixl_netmap_reg; 137279232Sluigi na.num_tx_rings = na.num_rx_rings = vsi->num_queues; 138279232Sluigi netmap_attach(&na); 139279232Sluigi} 140279232Sluigi 141279232Sluigi 142279232Sluigi#else /* !NETMAP_IXL_MAIN, code for ixl_txrx.c */ 143279232Sluigi 144279232Sluigi/* 145279232Sluigi * Reconcile kernel and user view of the transmit ring. 146279232Sluigi * 147279232Sluigi * All information is in the kring. 148279232Sluigi * Userspace wants to send packets up to the one before kring->rhead, 149279232Sluigi * kernel knows kring->nr_hwcur is the first unsent packet. 150279232Sluigi * 151279232Sluigi * Here we push packets out (as many as possible), and possibly 152279232Sluigi * reclaim buffers from previously completed transmission. 153279232Sluigi * 154279232Sluigi * The caller (netmap) guarantees that there is only one instance 155279232Sluigi * running at any time. Any interference with other driver 156279232Sluigi * methods should be handled by the individual drivers. 157279232Sluigi */ 158279232Sluigiint 159279232Sluigiixl_netmap_txsync(struct netmap_kring *kring, int flags) 160279232Sluigi{ 161279232Sluigi struct netmap_adapter *na = kring->na; 162279232Sluigi struct ifnet *ifp = na->ifp; 163279232Sluigi struct netmap_ring *ring = kring->ring; 164279232Sluigi u_int nm_i; /* index into the netmap ring */ 165279232Sluigi u_int nic_i; /* index into the NIC ring */ 166279232Sluigi u_int n; 167279232Sluigi u_int const lim = kring->nkr_num_slots - 1; 168279232Sluigi u_int const head = kring->rhead; 169279232Sluigi /* 170279232Sluigi * interrupts on every tx packet are expensive so request 171279232Sluigi * them every half ring, or where NS_REPORT is set 172279232Sluigi */ 173279232Sluigi u_int report_frequency = kring->nkr_num_slots >> 1; 174279232Sluigi 175279232Sluigi /* device-specific */ 176279232Sluigi struct ixl_vsi *vsi = ifp->if_softc; 177279232Sluigi struct ixl_queue *que = &vsi->queues[kring->ring_id]; 178279232Sluigi struct tx_ring *txr = &que->txr; 179279232Sluigi 180279232Sluigi bus_dmamap_sync(txr->dma.tag, txr->dma.map, 181279232Sluigi BUS_DMASYNC_POSTREAD); 182279232Sluigi 183279232Sluigi /* 184279232Sluigi * First part: process new packets to send. 185279232Sluigi * nm_i is the current index in the netmap ring, 186279232Sluigi * nic_i is the corresponding index in the NIC ring. 187279232Sluigi * 188279232Sluigi * If we have packets to send (nm_i != head) 189279232Sluigi * iterate over the netmap ring, fetch length and update 190279232Sluigi * the corresponding slot in the NIC ring. Some drivers also 191279232Sluigi * need to update the buffer's physical address in the NIC slot 192279232Sluigi * even NS_BUF_CHANGED is not set (PNMB computes the addresses). 193279232Sluigi * 194279232Sluigi * The netmap_reload_map() calls is especially expensive, 195279232Sluigi * even when (as in this case) the tag is 0, so do only 196279232Sluigi * when the buffer has actually changed. 197279232Sluigi * 198279232Sluigi * If possible do not set the report/intr bit on all slots, 199279232Sluigi * but only a few times per ring or when NS_REPORT is set. 200279232Sluigi * 201279232Sluigi * Finally, on 10G and faster drivers, it might be useful 202279232Sluigi * to prefetch the next slot and txr entry. 203279232Sluigi */ 204279232Sluigi 205279232Sluigi nm_i = kring->nr_hwcur; 206279232Sluigi if (nm_i != head) { /* we have new packets to send */ 207279232Sluigi nic_i = netmap_idx_k2n(kring, nm_i); 208279232Sluigi 209279232Sluigi __builtin_prefetch(&ring->slot[nm_i]); 210279232Sluigi __builtin_prefetch(&txr->buffers[nic_i]); 211279232Sluigi 212279232Sluigi for (n = 0; nm_i != head; n++) { 213279232Sluigi struct netmap_slot *slot = &ring->slot[nm_i]; 214279232Sluigi u_int len = slot->len; 215279232Sluigi uint64_t paddr; 216279232Sluigi void *addr = PNMB(na, slot, &paddr); 217279232Sluigi 218279232Sluigi /* device-specific */ 219279232Sluigi struct i40e_tx_desc *curr = &txr->base[nic_i]; 220279232Sluigi struct ixl_tx_buf *txbuf = &txr->buffers[nic_i]; 221279232Sluigi u64 flags = (slot->flags & NS_REPORT || 222279232Sluigi nic_i == 0 || nic_i == report_frequency) ? 223279232Sluigi ((u64)I40E_TX_DESC_CMD_RS << I40E_TXD_QW1_CMD_SHIFT) : 0; 224279232Sluigi 225279232Sluigi /* prefetch for next round */ 226279232Sluigi __builtin_prefetch(&ring->slot[nm_i + 1]); 227279232Sluigi __builtin_prefetch(&txr->buffers[nic_i + 1]); 228279232Sluigi 229279232Sluigi NM_CHECK_ADDR_LEN(na, addr, len); 230279232Sluigi 231279232Sluigi if (slot->flags & NS_BUF_CHANGED) { 232279232Sluigi /* buffer has changed, reload map */ 233279232Sluigi netmap_reload_map(na, txr->dma.tag, txbuf->map, addr); 234279232Sluigi } 235279232Sluigi slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); 236279232Sluigi 237279232Sluigi /* Fill the slot in the NIC ring. */ 238279232Sluigi curr->buffer_addr = htole64(paddr); 239279232Sluigi curr->cmd_type_offset_bsz = htole64( 240279232Sluigi ((u64)len << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | 241279232Sluigi flags | 242359310Svmaffione ((u64)I40E_TX_DESC_CMD_EOP << I40E_TXD_QW1_CMD_SHIFT) | 243359310Svmaffione ((u64)I40E_TX_DESC_CMD_ICRC << I40E_TXD_QW1_CMD_SHIFT) 244279232Sluigi ); // XXX more ? 245279232Sluigi 246279232Sluigi /* make sure changes to the buffer are synced */ 247279232Sluigi bus_dmamap_sync(txr->dma.tag, txbuf->map, 248279232Sluigi BUS_DMASYNC_PREWRITE); 249279232Sluigi 250279232Sluigi nm_i = nm_next(nm_i, lim); 251279232Sluigi nic_i = nm_next(nic_i, lim); 252279232Sluigi } 253279232Sluigi kring->nr_hwcur = head; 254279232Sluigi 255279232Sluigi /* synchronize the NIC ring */ 256279232Sluigi bus_dmamap_sync(txr->dma.tag, txr->dma.map, 257279232Sluigi BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 258279232Sluigi 259279232Sluigi /* (re)start the tx unit up to slot nic_i (excluded) */ 260279232Sluigi wr32(vsi->hw, txr->tail, nic_i); 261279232Sluigi } 262279232Sluigi 263279232Sluigi /* 264279232Sluigi * Second part: reclaim buffers for completed transmissions. 265279232Sluigi */ 266343559Svmaffione nic_i = LE32_TO_CPU(*(volatile __le32 *)&txr->base[que->num_tx_desc]); 267343559Svmaffione if (unlikely(nic_i >= que->num_tx_desc)) { 268343559Svmaffione nm_prerr("error: invalid value of hw head index %u", nic_i); 269343559Svmaffione } else if (nic_i != txr->next_to_clean) { 270279232Sluigi /* some tx completed, increment avail */ 271279232Sluigi txr->next_to_clean = nic_i; 272279232Sluigi kring->nr_hwtail = nm_prev(netmap_idx_n2k(kring, nic_i), lim); 273279232Sluigi } 274279232Sluigi 275279232Sluigi return 0; 276279232Sluigi} 277279232Sluigi 278279232Sluigi 279279232Sluigi/* 280279232Sluigi * Reconcile kernel and user view of the receive ring. 281279232Sluigi * Same as for the txsync, this routine must be efficient. 282279232Sluigi * The caller guarantees a single invocations, but races against 283279232Sluigi * the rest of the driver should be handled here. 284279232Sluigi * 285279232Sluigi * On call, kring->rhead is the first packet that userspace wants 286279232Sluigi * to keep, and kring->rcur is the wakeup point. 287279232Sluigi * The kernel has previously reported packets up to kring->rtail. 288279232Sluigi * 289279232Sluigi * If (flags & NAF_FORCE_READ) also check for incoming packets irrespective 290279232Sluigi * of whether or not we received an interrupt. 291279232Sluigi */ 292279232Sluigiint 293279232Sluigiixl_netmap_rxsync(struct netmap_kring *kring, int flags) 294279232Sluigi{ 295279232Sluigi struct netmap_adapter *na = kring->na; 296279232Sluigi struct ifnet *ifp = na->ifp; 297279232Sluigi struct netmap_ring *ring = kring->ring; 298279232Sluigi u_int nm_i; /* index into the netmap ring */ 299279232Sluigi u_int nic_i; /* index into the NIC ring */ 300279232Sluigi u_int n; 301279232Sluigi u_int const lim = kring->nkr_num_slots - 1; 302285349Sluigi u_int const head = kring->rhead; 303279232Sluigi int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; 304279232Sluigi 305279232Sluigi /* device-specific */ 306279232Sluigi struct ixl_vsi *vsi = ifp->if_softc; 307279232Sluigi struct ixl_queue *que = &vsi->queues[kring->ring_id]; 308279232Sluigi struct rx_ring *rxr = &que->rxr; 309279232Sluigi 310279232Sluigi if (head > lim) 311279232Sluigi return netmap_ring_reinit(kring); 312279232Sluigi 313279232Sluigi /* XXX check sync modes */ 314279232Sluigi bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, 315279232Sluigi BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 316279232Sluigi 317279232Sluigi /* 318279232Sluigi * First part: import newly received packets. 319279232Sluigi * 320279232Sluigi * nm_i is the index of the next free slot in the netmap ring, 321279232Sluigi * nic_i is the index of the next received packet in the NIC ring, 322279232Sluigi * and they may differ in case if_init() has been called while 323279232Sluigi * in netmap mode. For the receive ring we have 324279232Sluigi * 325279232Sluigi * nic_i = rxr->next_check; 326279232Sluigi * nm_i = kring->nr_hwtail (previous) 327279232Sluigi * and 328279232Sluigi * nm_i == (nic_i + kring->nkr_hwofs) % ring_size 329279232Sluigi * 330279232Sluigi * rxr->next_check is set to 0 on a ring reinit 331279232Sluigi */ 332279232Sluigi if (netmap_no_pendintr || force_update) { 333279232Sluigi int crclen = ixl_crcstrip ? 0 : 4; 334279232Sluigi 335279232Sluigi nic_i = rxr->next_check; // or also k2n(kring->nr_hwtail) 336279232Sluigi nm_i = netmap_idx_n2k(kring, nic_i); 337279232Sluigi 338279232Sluigi for (n = 0; ; n++) { 339279232Sluigi union i40e_32byte_rx_desc *curr = &rxr->base[nic_i]; 340279232Sluigi uint64_t qword = le64toh(curr->wb.qword1.status_error_len); 341279232Sluigi uint32_t staterr = (qword & I40E_RXD_QW1_STATUS_MASK) 342279232Sluigi >> I40E_RXD_QW1_STATUS_SHIFT; 343279232Sluigi 344279232Sluigi if ((staterr & (1<<I40E_RX_DESC_STATUS_DD_SHIFT)) == 0) 345279232Sluigi break; 346279232Sluigi ring->slot[nm_i].len = ((qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) 347279232Sluigi >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - crclen; 348341477Svmaffione ring->slot[nm_i].flags = 0; 349279232Sluigi bus_dmamap_sync(rxr->ptag, 350279232Sluigi rxr->buffers[nic_i].pmap, BUS_DMASYNC_POSTREAD); 351279232Sluigi nm_i = nm_next(nm_i, lim); 352279232Sluigi nic_i = nm_next(nic_i, lim); 353279232Sluigi } 354279232Sluigi if (n) { /* update the state variables */ 355279232Sluigi if (netmap_no_pendintr && !force_update) { 356279232Sluigi /* diagnostics */ 357279232Sluigi ixl_rx_miss ++; 358279232Sluigi ixl_rx_miss_bufs += n; 359279232Sluigi } 360279232Sluigi rxr->next_check = nic_i; 361279232Sluigi kring->nr_hwtail = nm_i; 362279232Sluigi } 363279232Sluigi kring->nr_kflags &= ~NKR_PENDINTR; 364279232Sluigi } 365279232Sluigi 366279232Sluigi /* 367279232Sluigi * Second part: skip past packets that userspace has released. 368279232Sluigi * (kring->nr_hwcur to head excluded), 369279232Sluigi * and make the buffers available for reception. 370279232Sluigi * As usual nm_i is the index in the netmap ring, 371279232Sluigi * nic_i is the index in the NIC ring, and 372279232Sluigi * nm_i == (nic_i + kring->nkr_hwofs) % ring_size 373279232Sluigi */ 374279232Sluigi nm_i = kring->nr_hwcur; 375279232Sluigi if (nm_i != head) { 376279232Sluigi nic_i = netmap_idx_k2n(kring, nm_i); 377279232Sluigi for (n = 0; nm_i != head; n++) { 378279232Sluigi struct netmap_slot *slot = &ring->slot[nm_i]; 379279232Sluigi uint64_t paddr; 380279232Sluigi void *addr = PNMB(na, slot, &paddr); 381279232Sluigi 382279232Sluigi union i40e_32byte_rx_desc *curr = &rxr->base[nic_i]; 383279232Sluigi struct ixl_rx_buf *rxbuf = &rxr->buffers[nic_i]; 384279232Sluigi 385279232Sluigi if (addr == NETMAP_BUF_BASE(na)) /* bad buf */ 386279232Sluigi goto ring_reset; 387279232Sluigi 388279232Sluigi if (slot->flags & NS_BUF_CHANGED) { 389279232Sluigi /* buffer has changed, reload map */ 390279232Sluigi netmap_reload_map(na, rxr->ptag, rxbuf->pmap, addr); 391279232Sluigi slot->flags &= ~NS_BUF_CHANGED; 392279232Sluigi } 393279232Sluigi curr->read.pkt_addr = htole64(paddr); 394279232Sluigi curr->read.hdr_addr = 0; // XXX needed 395279232Sluigi bus_dmamap_sync(rxr->ptag, rxbuf->pmap, 396279232Sluigi BUS_DMASYNC_PREREAD); 397279232Sluigi nm_i = nm_next(nm_i, lim); 398279232Sluigi nic_i = nm_next(nic_i, lim); 399279232Sluigi } 400279232Sluigi kring->nr_hwcur = head; 401279232Sluigi 402279232Sluigi bus_dmamap_sync(rxr->dma.tag, rxr->dma.map, 403279232Sluigi BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 404279232Sluigi /* 405279232Sluigi * IMPORTANT: we must leave one free slot in the ring, 406279232Sluigi * so move nic_i back by one unit 407279232Sluigi */ 408279232Sluigi nic_i = nm_prev(nic_i, lim); 409279232Sluigi wr32(vsi->hw, rxr->tail, nic_i); 410279232Sluigi } 411279232Sluigi 412279232Sluigi return 0; 413279232Sluigi 414279232Sluigiring_reset: 415279232Sluigi return netmap_ring_reinit(kring); 416279232Sluigi} 417279232Sluigi 418279232Sluigi#endif /* !NETMAP_IXL_MAIN */ 419279232Sluigi 420279232Sluigi/* end of file */ 421