if_re_netmap.h revision 231594
1227614Sluigi/* 2227614Sluigi * Copyright (C) 2011 Luigi Rizzo. All rights reserved. 3227614Sluigi * 4227614Sluigi * Redistribution and use in source and binary forms, with or without 5227614Sluigi * modification, are permitted provided that the following conditions 6227614Sluigi * are met: 7227614Sluigi * 1. Redistributions of source code must retain the above copyright 8227614Sluigi * notice, this list of conditions and the following disclaimer. 9227614Sluigi * 2. Redistributions in binary form must reproduce the above copyright 10227614Sluigi * notice, this list of conditions and the following disclaimer in the 11227614Sluigi * documentation and/or other materials provided with the distribution. 12227614Sluigi * 13227614Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14227614Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15227614Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16227614Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17227614Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18227614Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19227614Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20227614Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21227614Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22227614Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23227614Sluigi * SUCH DAMAGE. 24227614Sluigi */ 25227614Sluigi 26227614Sluigi/* 27227614Sluigi * $FreeBSD: head/sys/dev/netmap/if_re_netmap.h 231594 2012-02-13 18:56:34Z luigi $ 28230055Sluigi * $Id: if_re_netmap.h 10075 2011-12-25 22:55:48Z luigi $ 29227614Sluigi * 30227614Sluigi * netmap support for if_re 31227614Sluigi */ 32227614Sluigi 33227614Sluigi#include <net/netmap.h> 34227614Sluigi#include <sys/selinfo.h> 35227614Sluigi#include <vm/vm.h> 36227614Sluigi#include <vm/pmap.h> /* vtophys ? */ 37227614Sluigi#include <dev/netmap/netmap_kern.h> 38227614Sluigi 39227614Sluigistatic int re_netmap_reg(struct ifnet *, int onoff); 40231594Sluigistatic int re_netmap_txsync(struct ifnet *, u_int, int); 41231594Sluigistatic int re_netmap_rxsync(struct ifnet *, u_int, int); 42231594Sluigistatic void re_netmap_lock_wrapper(struct ifnet *, int, u_int); 43227614Sluigi 44227614Sluigistatic void 45227614Sluigire_netmap_attach(struct rl_softc *sc) 46227614Sluigi{ 47227614Sluigi struct netmap_adapter na; 48227614Sluigi 49227614Sluigi bzero(&na, sizeof(na)); 50227614Sluigi 51227614Sluigi na.ifp = sc->rl_ifp; 52227614Sluigi na.separate_locks = 0; 53227614Sluigi na.num_tx_desc = sc->rl_ldata.rl_tx_desc_cnt; 54227614Sluigi na.num_rx_desc = sc->rl_ldata.rl_rx_desc_cnt; 55227614Sluigi na.nm_txsync = re_netmap_txsync; 56227614Sluigi na.nm_rxsync = re_netmap_rxsync; 57227614Sluigi na.nm_lock = re_netmap_lock_wrapper; 58227614Sluigi na.nm_register = re_netmap_reg; 59227614Sluigi netmap_attach(&na, 1); 60227614Sluigi} 61227614Sluigi 62227614Sluigi 63227614Sluigi/* 64227614Sluigi * wrapper to export locks to the generic code 65227614Sluigi * We should not use the tx/rx locks 66227614Sluigi */ 67227614Sluigistatic void 68231594Sluigire_netmap_lock_wrapper(struct ifnet *ifp, int what, u_int queueid) 69227614Sluigi{ 70231594Sluigi struct rl_softc *adapter = ifp->if_softc; 71227614Sluigi 72227614Sluigi switch (what) { 73227614Sluigi case NETMAP_CORE_LOCK: 74227614Sluigi RL_LOCK(adapter); 75227614Sluigi break; 76227614Sluigi case NETMAP_CORE_UNLOCK: 77227614Sluigi RL_UNLOCK(adapter); 78227614Sluigi break; 79227614Sluigi 80227614Sluigi case NETMAP_TX_LOCK: 81227614Sluigi case NETMAP_RX_LOCK: 82227614Sluigi case NETMAP_TX_UNLOCK: 83227614Sluigi case NETMAP_RX_UNLOCK: 84227614Sluigi D("invalid lock call %d, no tx/rx locks here", what); 85227614Sluigi break; 86227614Sluigi } 87227614Sluigi} 88227614Sluigi 89227614Sluigi 90227614Sluigi/* 91227614Sluigi * support for netmap register/unregisted. We are already under core lock. 92227614Sluigi * only called on the first register or the last unregister. 93227614Sluigi */ 94227614Sluigistatic int 95227614Sluigire_netmap_reg(struct ifnet *ifp, int onoff) 96227614Sluigi{ 97227614Sluigi struct rl_softc *adapter = ifp->if_softc; 98227614Sluigi struct netmap_adapter *na = NA(ifp); 99227614Sluigi int error = 0; 100227614Sluigi 101228276Sluigi if (na == NULL) 102227614Sluigi return EINVAL; 103227614Sluigi /* Tell the stack that the interface is no longer active */ 104227614Sluigi ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 105227614Sluigi 106227614Sluigi re_stop(adapter); 107227614Sluigi 108227614Sluigi if (onoff) { 109227614Sluigi ifp->if_capenable |= IFCAP_NETMAP; 110227614Sluigi 111228276Sluigi /* save if_transmit to restore it later */ 112227614Sluigi na->if_transmit = ifp->if_transmit; 113227614Sluigi ifp->if_transmit = netmap_start; 114227614Sluigi 115227614Sluigi re_init_locked(adapter); 116227614Sluigi 117227614Sluigi if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == 0) { 118227614Sluigi error = ENOMEM; 119227614Sluigi goto fail; 120227614Sluigi } 121227614Sluigi } else { 122227614Sluigifail: 123227614Sluigi /* restore if_transmit */ 124227614Sluigi ifp->if_transmit = na->if_transmit; 125227614Sluigi ifp->if_capenable &= ~IFCAP_NETMAP; 126227614Sluigi re_init_locked(adapter); /* also enables intr */ 127227614Sluigi } 128228276Sluigi return (error); 129227614Sluigi} 130227614Sluigi 131227614Sluigi 132227614Sluigi/* 133227614Sluigi * Reconcile kernel and user view of the transmit ring. 134227614Sluigi */ 135227614Sluigistatic int 136231594Sluigire_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 137227614Sluigi{ 138231594Sluigi struct rl_softc *sc = ifp->if_softc; 139227614Sluigi struct rl_txdesc *txd = sc->rl_ldata.rl_tx_desc; 140227614Sluigi struct netmap_adapter *na = NA(sc->rl_ifp); 141227614Sluigi struct netmap_kring *kring = &na->tx_rings[ring_nr]; 142227614Sluigi struct netmap_ring *ring = kring->ring; 143228276Sluigi int j, k, l, n, lim = kring->nkr_num_slots - 1; 144227614Sluigi 145227614Sluigi k = ring->cur; 146228276Sluigi if (k > lim) 147227614Sluigi return netmap_ring_reinit(kring); 148227614Sluigi 149227614Sluigi if (do_lock) 150227614Sluigi RL_LOCK(sc); 151227614Sluigi 152227614Sluigi /* Sync the TX descriptor list */ 153227614Sluigi bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag, 154227614Sluigi sc->rl_ldata.rl_tx_list_map, 155227614Sluigi BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 156227614Sluigi 157228276Sluigi /* XXX move after the transmissions */ 158227614Sluigi /* record completed transmissions */ 159228276Sluigi for (n = 0, l = sc->rl_ldata.rl_tx_considx; 160228276Sluigi l != sc->rl_ldata.rl_tx_prodidx; 161228276Sluigi n++, l = RL_TX_DESC_NXT(sc, l)) { 162227614Sluigi uint32_t cmdstat = 163228276Sluigi le32toh(sc->rl_ldata.rl_tx_list[l].rl_cmdstat); 164227614Sluigi if (cmdstat & RL_TDESC_STAT_OWN) 165227614Sluigi break; 166227614Sluigi } 167227614Sluigi if (n > 0) { 168228276Sluigi sc->rl_ldata.rl_tx_considx = l; 169227614Sluigi sc->rl_ldata.rl_tx_free += n; 170227614Sluigi kring->nr_hwavail += n; 171227614Sluigi } 172227614Sluigi 173227614Sluigi /* update avail to what the hardware knows */ 174227614Sluigi ring->avail = kring->nr_hwavail; 175227614Sluigi 176228276Sluigi j = kring->nr_hwcur; 177227614Sluigi if (j != k) { /* we have new packets to send */ 178227614Sluigi n = 0; 179228276Sluigi l = sc->rl_ldata.rl_tx_prodidx; 180227614Sluigi while (j != k) { 181227614Sluigi struct netmap_slot *slot = &ring->slot[j]; 182228276Sluigi struct rl_desc *desc = &sc->rl_ldata.rl_tx_list[l]; 183227614Sluigi int cmd = slot->len | RL_TDESC_CMD_EOF | 184227614Sluigi RL_TDESC_CMD_OWN | RL_TDESC_CMD_SOF ; 185229939Sluigi uint64_t paddr; 186229939Sluigi void *addr = PNMB(slot, &paddr); 187227614Sluigi int len = slot->len; 188227614Sluigi 189227614Sluigi if (addr == netmap_buffer_base || len > NETMAP_BUF_SIZE) { 190227614Sluigi if (do_lock) 191227614Sluigi RL_UNLOCK(sc); 192228276Sluigi // XXX what about prodidx ? 193227614Sluigi return netmap_ring_reinit(kring); 194227614Sluigi } 195227614Sluigi 196228276Sluigi if (l == lim) /* mark end of ring */ 197227614Sluigi cmd |= RL_TDESC_CMD_EOR; 198227614Sluigi 199227614Sluigi if (slot->flags & NS_BUF_CHANGED) { 200227614Sluigi desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 201227614Sluigi desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 202227614Sluigi /* buffer has changed, unload and reload map */ 203227614Sluigi netmap_reload_map(sc->rl_ldata.rl_tx_mtag, 204229939Sluigi txd[l].tx_dmamap, addr); 205227614Sluigi slot->flags &= ~NS_BUF_CHANGED; 206227614Sluigi } 207227614Sluigi slot->flags &= ~NS_REPORT; 208227614Sluigi desc->rl_cmdstat = htole32(cmd); 209227614Sluigi bus_dmamap_sync(sc->rl_ldata.rl_tx_mtag, 210228276Sluigi txd[l].tx_dmamap, BUS_DMASYNC_PREWRITE); 211227614Sluigi j = (j == lim) ? 0 : j + 1; 212228276Sluigi l = (l == lim) ? 0 : l + 1; 213227614Sluigi n++; 214227614Sluigi } 215228276Sluigi sc->rl_ldata.rl_tx_prodidx = l; 216228276Sluigi kring->nr_hwcur = k; 217227614Sluigi 218227614Sluigi /* decrease avail by number of sent packets */ 219227614Sluigi ring->avail -= n; 220227614Sluigi kring->nr_hwavail = ring->avail; 221227614Sluigi 222227614Sluigi bus_dmamap_sync(sc->rl_ldata.rl_tx_list_tag, 223227614Sluigi sc->rl_ldata.rl_tx_list_map, 224227614Sluigi BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 225227614Sluigi 226227614Sluigi /* start ? */ 227227614Sluigi CSR_WRITE_1(sc, sc->rl_txstart, RL_TXSTART_START); 228227614Sluigi } 229227614Sluigi if (do_lock) 230227614Sluigi RL_UNLOCK(sc); 231227614Sluigi return 0; 232227614Sluigi} 233227614Sluigi 234227614Sluigi 235227614Sluigi/* 236227614Sluigi * Reconcile kernel and user view of the receive ring. 237227614Sluigi */ 238227614Sluigistatic int 239231594Sluigire_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 240227614Sluigi{ 241231594Sluigi struct rl_softc *sc = ifp->if_softc; 242227614Sluigi struct rl_rxdesc *rxd = sc->rl_ldata.rl_rx_desc; 243227614Sluigi struct netmap_adapter *na = NA(sc->rl_ifp); 244227614Sluigi struct netmap_kring *kring = &na->rx_rings[ring_nr]; 245227614Sluigi struct netmap_ring *ring = kring->ring; 246228276Sluigi int j, k, l, n, lim = kring->nkr_num_slots - 1; 247227614Sluigi 248227614Sluigi k = ring->cur; 249228276Sluigi if (k > lim) 250227614Sluigi return netmap_ring_reinit(kring); 251227614Sluigi 252227614Sluigi if (do_lock) 253227614Sluigi RL_LOCK(sc); 254227614Sluigi /* XXX check sync modes */ 255227614Sluigi bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag, 256227614Sluigi sc->rl_ldata.rl_rx_list_map, 257227614Sluigi BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 258227614Sluigi 259227614Sluigi /* 260227614Sluigi * The device uses all the buffers in the ring, so we need 261227614Sluigi * another termination condition in addition to RL_RDESC_STAT_OWN 262227614Sluigi * cleared (all buffers could have it cleared. The easiest one 263227614Sluigi * is to limit the amount of data reported up to 'lim' 264227614Sluigi */ 265228276Sluigi l = sc->rl_ldata.rl_rx_prodidx; /* next pkt to check */ 266228276Sluigi j = l + kring->nkr_hwofs; 267227614Sluigi for (n = kring->nr_hwavail; n < lim ; n++) { 268228276Sluigi struct rl_desc *cur_rx = &sc->rl_ldata.rl_rx_list[l]; 269227614Sluigi uint32_t rxstat = le32toh(cur_rx->rl_cmdstat); 270227614Sluigi uint32_t total_len; 271227614Sluigi 272227614Sluigi if ((rxstat & RL_RDESC_STAT_OWN) != 0) 273227614Sluigi break; 274227614Sluigi total_len = rxstat & sc->rl_rxlenmask; 275227614Sluigi /* XXX subtract crc */ 276227614Sluigi total_len = (total_len < 4) ? 0 : total_len - 4; 277227614Sluigi kring->ring->slot[j].len = total_len; 278227614Sluigi /* sync was in re_newbuf() */ 279227614Sluigi bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, 280228276Sluigi rxd[l].rx_dmamap, BUS_DMASYNC_POSTREAD); 281228276Sluigi j = (j == lim) ? 0 : j + 1; 282228276Sluigi l = (l == lim) ? 0 : l + 1; 283227614Sluigi } 284227614Sluigi if (n != kring->nr_hwavail) { 285228276Sluigi sc->rl_ldata.rl_rx_prodidx = l; 286227614Sluigi sc->rl_ifp->if_ipackets += n - kring->nr_hwavail; 287227614Sluigi kring->nr_hwavail = n; 288227614Sluigi } 289227614Sluigi 290227614Sluigi /* skip past packets that userspace has already processed, 291227614Sluigi * making them available for reception. 292227614Sluigi * advance nr_hwcur and issue a bus_dmamap_sync on the 293227614Sluigi * buffers so it is safe to write to them. 294227614Sluigi * Also increase nr_hwavail 295227614Sluigi */ 296227614Sluigi j = kring->nr_hwcur; 297227614Sluigi if (j != k) { /* userspace has read some packets. */ 298227614Sluigi n = 0; 299228276Sluigi l = kring->nr_hwcur - kring->nkr_hwofs; 300228276Sluigi if (l < 0) 301228276Sluigi l += lim + 1; 302227614Sluigi while (j != k) { 303227614Sluigi struct netmap_slot *slot = ring->slot + j; 304228276Sluigi struct rl_desc *desc = &sc->rl_ldata.rl_rx_list[l]; 305227614Sluigi int cmd = na->buff_size | RL_RDESC_CMD_OWN; 306229939Sluigi uint64_t paddr; 307229939Sluigi void *addr = PNMB(slot, &paddr); 308227614Sluigi 309227614Sluigi if (addr == netmap_buffer_base) { /* bad buf */ 310227614Sluigi if (do_lock) 311227614Sluigi RL_UNLOCK(sc); 312227614Sluigi return netmap_ring_reinit(kring); 313227614Sluigi } 314227614Sluigi 315228276Sluigi if (l == lim) /* mark end of ring */ 316227614Sluigi cmd |= RL_RDESC_CMD_EOR; 317227614Sluigi 318227614Sluigi desc->rl_cmdstat = htole32(cmd); 319227614Sluigi slot->flags &= ~NS_REPORT; 320227614Sluigi if (slot->flags & NS_BUF_CHANGED) { 321227614Sluigi desc->rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 322227614Sluigi desc->rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 323227614Sluigi netmap_reload_map(sc->rl_ldata.rl_rx_mtag, 324229939Sluigi rxd[l].rx_dmamap, addr); 325227614Sluigi slot->flags &= ~NS_BUF_CHANGED; 326227614Sluigi } 327227614Sluigi bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, 328228276Sluigi rxd[l].rx_dmamap, BUS_DMASYNC_PREREAD); 329227614Sluigi j = (j == lim) ? 0 : j + 1; 330228276Sluigi l = (l == lim) ? 0 : l + 1; 331227614Sluigi n++; 332227614Sluigi } 333227614Sluigi kring->nr_hwavail -= n; 334227614Sluigi kring->nr_hwcur = k; 335227614Sluigi /* Flush the RX DMA ring */ 336227614Sluigi 337227614Sluigi bus_dmamap_sync(sc->rl_ldata.rl_rx_list_tag, 338227614Sluigi sc->rl_ldata.rl_rx_list_map, 339227614Sluigi BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD); 340227614Sluigi } 341227614Sluigi /* tell userspace that there are new packets */ 342228276Sluigi ring->avail = kring->nr_hwavail; 343227614Sluigi if (do_lock) 344227614Sluigi RL_UNLOCK(sc); 345227614Sluigi return 0; 346227614Sluigi} 347227614Sluigi 348228276Sluigi/* 349228276Sluigi * Additional routines to init the tx and rx rings. 350228276Sluigi * In other drivers we do that inline in the main code. 351228276Sluigi */ 352227614Sluigistatic void 353227614Sluigire_netmap_tx_init(struct rl_softc *sc) 354227614Sluigi{ 355227614Sluigi struct rl_txdesc *txd; 356227614Sluigi struct rl_desc *desc; 357228276Sluigi int i, n; 358227614Sluigi struct netmap_adapter *na = NA(sc->rl_ifp); 359227614Sluigi struct netmap_slot *slot = netmap_reset(na, NR_TX, 0, 0); 360227614Sluigi 361227614Sluigi /* slot is NULL if we are not in netmap mode */ 362227614Sluigi if (!slot) 363227614Sluigi return; 364227614Sluigi /* in netmap mode, overwrite addresses and maps */ 365227614Sluigi txd = sc->rl_ldata.rl_tx_desc; 366227614Sluigi desc = sc->rl_ldata.rl_tx_list; 367228276Sluigi n = sc->rl_ldata.rl_tx_desc_cnt; 368227614Sluigi 369228276Sluigi /* l points in the netmap ring, i points in the NIC ring */ 370228276Sluigi for (i = 0; i < n; i++) { 371228276Sluigi void *addr; 372228276Sluigi uint64_t paddr; 373228276Sluigi struct netmap_kring *kring = &na->tx_rings[0]; 374228276Sluigi int l = i + kring->nkr_hwofs; 375227614Sluigi 376228276Sluigi if (l >= n) 377228276Sluigi l -= n; 378228276Sluigi 379229939Sluigi addr = PNMB(slot + l, &paddr); 380227614Sluigi desc[i].rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 381227614Sluigi desc[i].rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 382227614Sluigi netmap_load_map(sc->rl_ldata.rl_tx_mtag, 383229939Sluigi txd[i].tx_dmamap, addr); 384227614Sluigi } 385227614Sluigi} 386227614Sluigi 387227614Sluigistatic void 388227614Sluigire_netmap_rx_init(struct rl_softc *sc) 389227614Sluigi{ 390227614Sluigi struct netmap_adapter *na = NA(sc->rl_ifp); 391227614Sluigi struct netmap_slot *slot = netmap_reset(na, NR_RX, 0, 0); 392227614Sluigi struct rl_desc *desc = sc->rl_ldata.rl_rx_list; 393227614Sluigi uint32_t cmdstat; 394228276Sluigi int i, n; 395227614Sluigi 396227614Sluigi if (!slot) 397227614Sluigi return; 398228276Sluigi n = sc->rl_ldata.rl_rx_desc_cnt; 399228276Sluigi for (i = 0; i < n; i++) { 400228276Sluigi void *addr; 401228276Sluigi uint64_t paddr; 402228276Sluigi struct netmap_kring *kring = &na->rx_rings[0]; 403228276Sluigi int l = i + kring->nkr_hwofs; 404227614Sluigi 405228276Sluigi if (l >= n) 406228276Sluigi l -= n; 407227614Sluigi 408229939Sluigi addr = PNMB(slot + l, &paddr); 409229939Sluigi 410229939Sluigi netmap_reload_map(sc->rl_ldata.rl_rx_mtag, 411230055Sluigi sc->rl_ldata.rl_rx_desc[i].rx_dmamap, addr); 412229939Sluigi bus_dmamap_sync(sc->rl_ldata.rl_rx_mtag, 413229939Sluigi sc->rl_ldata.rl_rx_desc[i].rx_dmamap, BUS_DMASYNC_PREREAD); 414227614Sluigi desc[i].rl_bufaddr_lo = htole32(RL_ADDR_LO(paddr)); 415227614Sluigi desc[i].rl_bufaddr_hi = htole32(RL_ADDR_HI(paddr)); 416228276Sluigi cmdstat = na->buff_size; 417228276Sluigi if (i == n - 1) 418227614Sluigi cmdstat |= RL_RDESC_CMD_EOR; 419228276Sluigi /* 420228276Sluigi * userspace knows that hwavail packets were ready before the 421228276Sluigi * reset, so we need to tell the NIC that last hwavail 422228276Sluigi * descriptors of the ring are still owned by the driver. 423228276Sluigi */ 424228276Sluigi if (i < n - 1 - kring->nr_hwavail) // XXX + 1 ? 425228276Sluigi cmdstat |= RL_RDESC_CMD_OWN; 426228276Sluigi desc[i].rl_cmdstat = htole32(cmdstat); 427227614Sluigi } 428227614Sluigi} 429