if_em_netmap.h revision 231594
1/* 2 * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26/* 27 * $FreeBSD: head/sys/dev/netmap/if_em_netmap.h 231594 2012-02-13 18:56:34Z luigi $ 28 * $Id: if_em_netmap.h 9802 2011-12-02 18:42:37Z luigi $ 29 * 30 * netmap changes for if_em. 31 * 32 * For structure and details on the individual functions please see 33 * ixgbe_netmap.h 34 */ 35 36#include <net/netmap.h> 37#include <sys/selinfo.h> 38#include <vm/vm.h> 39#include <vm/pmap.h> /* vtophys ? */ 40#include <dev/netmap/netmap_kern.h> 41 42static void em_netmap_block_tasks(struct adapter *); 43static void em_netmap_unblock_tasks(struct adapter *); 44static int em_netmap_reg(struct ifnet *, int onoff); 45static int em_netmap_txsync(struct ifnet *, u_int, int); 46static int em_netmap_rxsync(struct ifnet *, u_int, int); 47static void em_netmap_lock_wrapper(struct ifnet *, int, u_int); 48 49static void 50em_netmap_attach(struct adapter *adapter) 51{ 52 struct netmap_adapter na; 53 54 bzero(&na, sizeof(na)); 55 56 na.ifp = adapter->ifp; 57 na.separate_locks = 1; 58 na.num_tx_desc = adapter->num_tx_desc; 59 na.num_rx_desc = adapter->num_rx_desc; 60 na.nm_txsync = em_netmap_txsync; 61 na.nm_rxsync = em_netmap_rxsync; 62 na.nm_lock = em_netmap_lock_wrapper; 63 na.nm_register = em_netmap_reg; 64 netmap_attach(&na, adapter->num_queues); 65} 66 67 68/* 69 * wrapper to export locks to the generic code 70 */ 71static void 72em_netmap_lock_wrapper(struct ifnet *ifp, int what, u_int queueid) 73{ 74 struct adapter *adapter = ifp->if_softc; 75 76 ASSERT(queueid < adapter->num_queues); 77 switch (what) { 78 case NETMAP_CORE_LOCK: 79 EM_CORE_LOCK(adapter); 80 break; 81 case NETMAP_CORE_UNLOCK: 82 EM_CORE_UNLOCK(adapter); 83 break; 84 case NETMAP_TX_LOCK: 85 EM_TX_LOCK(&adapter->tx_rings[queueid]); 86 break; 87 case NETMAP_TX_UNLOCK: 88 EM_TX_UNLOCK(&adapter->tx_rings[queueid]); 89 break; 90 case NETMAP_RX_LOCK: 91 EM_RX_LOCK(&adapter->rx_rings[queueid]); 92 break; 93 case NETMAP_RX_UNLOCK: 94 EM_RX_UNLOCK(&adapter->rx_rings[queueid]); 95 break; 96 } 97} 98 99 100// XXX do we need to block/unblock the tasks ? 101static void 102em_netmap_block_tasks(struct adapter *adapter) 103{ 104 if (adapter->msix > 1) { /* MSIX */ 105 int i; 106 struct tx_ring *txr = adapter->tx_rings; 107 struct rx_ring *rxr = adapter->rx_rings; 108 109 for (i = 0; i < adapter->num_queues; i++, txr++, rxr++) { 110 taskqueue_block(txr->tq); 111 taskqueue_drain(txr->tq, &txr->tx_task); 112 taskqueue_block(rxr->tq); 113 taskqueue_drain(rxr->tq, &rxr->rx_task); 114 } 115 } else { /* legacy */ 116 taskqueue_block(adapter->tq); 117 taskqueue_drain(adapter->tq, &adapter->link_task); 118 taskqueue_drain(adapter->tq, &adapter->que_task); 119 } 120} 121 122 123static void 124em_netmap_unblock_tasks(struct adapter *adapter) 125{ 126 if (adapter->msix > 1) { 127 struct tx_ring *txr = adapter->tx_rings; 128 struct rx_ring *rxr = adapter->rx_rings; 129 int i; 130 131 for (i = 0; i < adapter->num_queues; i++) { 132 taskqueue_unblock(txr->tq); 133 taskqueue_unblock(rxr->tq); 134 } 135 } else { /* legacy */ 136 taskqueue_unblock(adapter->tq); 137 } 138} 139 140/* 141 * register-unregister routine 142 */ 143static int 144em_netmap_reg(struct ifnet *ifp, int onoff) 145{ 146 struct adapter *adapter = ifp->if_softc; 147 struct netmap_adapter *na = NA(ifp); 148 int error = 0; 149 150 if (na == NULL) 151 return EINVAL; /* no netmap support here */ 152 153 em_disable_intr(adapter); 154 155 /* Tell the stack that the interface is no longer active */ 156 ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); 157 158 em_netmap_block_tasks(adapter); 159 160 if (onoff) { 161 ifp->if_capenable |= IFCAP_NETMAP; 162 163 na->if_transmit = ifp->if_transmit; 164 ifp->if_transmit = netmap_start; 165 166 em_init_locked(adapter); 167 if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) == 0) { 168 error = ENOMEM; 169 goto fail; 170 } 171 } else { 172fail: 173 /* restore if_transmit */ 174 ifp->if_transmit = na->if_transmit; 175 ifp->if_capenable &= ~IFCAP_NETMAP; 176 em_init_locked(adapter); /* also enable intr */ 177 } 178 em_netmap_unblock_tasks(adapter); 179 return (error); 180} 181 182/* 183 * Reconcile hardware and user view of the transmit ring. 184 */ 185static int 186em_netmap_txsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 187{ 188 struct adapter *adapter = ifp->if_softc; 189 struct tx_ring *txr = &adapter->tx_rings[ring_nr]; 190 struct netmap_adapter *na = NA(adapter->ifp); 191 struct netmap_kring *kring = &na->tx_rings[ring_nr]; 192 struct netmap_ring *ring = kring->ring; 193 int j, k, l, n = 0, lim = kring->nkr_num_slots - 1; 194 195 /* generate an interrupt approximately every half ring */ 196 int report_frequency = kring->nkr_num_slots >> 1; 197 198 k = ring->cur; 199 if (k > lim) 200 return netmap_ring_reinit(kring); 201 202 if (do_lock) 203 EM_TX_LOCK(txr); 204 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 205 BUS_DMASYNC_POSTREAD); 206 207 /* check for new packets to send. 208 * j indexes the netmap ring, l indexes the nic ring, and 209 * j = kring->nr_hwcur, l = E1000_TDT (not tracked), 210 * j == (l + kring->nkr_hwofs) % ring_size 211 */ 212 j = kring->nr_hwcur; 213 if (j != k) { /* we have packets to send */ 214 l = j - kring->nkr_hwofs; 215 if (l < 0) 216 l += lim + 1; 217 while (j != k) { 218 struct netmap_slot *slot = &ring->slot[j]; 219 struct e1000_tx_desc *curr = &txr->tx_base[l]; 220 struct em_buffer *txbuf = &txr->tx_buffers[l]; 221 int flags = ((slot->flags & NS_REPORT) || 222 j == 0 || j == report_frequency) ? 223 E1000_TXD_CMD_RS : 0; 224 uint64_t paddr; 225 void *addr = PNMB(slot, &paddr); 226 int len = slot->len; 227 if (addr == netmap_buffer_base || len > NETMAP_BUF_SIZE) { 228 if (do_lock) 229 EM_TX_UNLOCK(txr); 230 return netmap_ring_reinit(kring); 231 } 232 233 slot->flags &= ~NS_REPORT; 234 curr->upper.data = 0; 235 curr->lower.data = 236 htole32(adapter->txd_cmd | len | 237 (E1000_TXD_CMD_EOP | flags) ); 238 if (slot->flags & NS_BUF_CHANGED) { 239 curr->buffer_addr = htole64(paddr); 240 /* buffer has changed, reload map */ 241 netmap_reload_map(txr->txtag, txbuf->map, addr); 242 slot->flags &= ~NS_BUF_CHANGED; 243 } 244 245 bus_dmamap_sync(txr->txtag, txbuf->map, 246 BUS_DMASYNC_PREWRITE); 247 j = (j == lim) ? 0 : j + 1; 248 l = (l == lim) ? 0 : l + 1; 249 n++; 250 } 251 kring->nr_hwcur = k; 252 253 /* decrease avail by number of sent packets */ 254 kring->nr_hwavail -= n; 255 256 bus_dmamap_sync(txr->txdma.dma_tag, txr->txdma.dma_map, 257 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 258 259 E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), l); 260 } 261 262 if (n == 0 || kring->nr_hwavail < 1) { 263 int delta; 264 265 /* record completed transmissions using THD. */ 266 l = E1000_READ_REG(&adapter->hw, E1000_TDH(ring_nr)); 267 if (l >= kring->nkr_num_slots) { /* XXX can happen */ 268 D("TDH wrap %d", l); 269 l -= kring->nkr_num_slots; 270 } 271 delta = l - txr->next_to_clean; 272 if (delta) { 273 /* some completed, increment hwavail. */ 274 if (delta < 0) 275 delta += kring->nkr_num_slots; 276 txr->next_to_clean = l; 277 kring->nr_hwavail += delta; 278 } 279 } 280 /* update avail to what the hardware knows */ 281 ring->avail = kring->nr_hwavail; 282 283 if (do_lock) 284 EM_TX_UNLOCK(txr); 285 return 0; 286} 287 288/* 289 * Reconcile kernel and user view of the receive ring. 290 */ 291static int 292em_netmap_rxsync(struct ifnet *ifp, u_int ring_nr, int do_lock) 293{ 294 struct adapter *adapter = ifp->if_softc; 295 struct rx_ring *rxr = &adapter->rx_rings[ring_nr]; 296 struct netmap_adapter *na = NA(adapter->ifp); 297 struct netmap_kring *kring = &na->rx_rings[ring_nr]; 298 struct netmap_ring *ring = kring->ring; 299 int j, k, l, n, lim = kring->nkr_num_slots - 1; 300 301 k = ring->cur; 302 if (k > lim) 303 return netmap_ring_reinit(kring); 304 305 if (do_lock) 306 EM_RX_LOCK(rxr); 307 /* XXX check sync modes */ 308 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 309 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 310 311 /* import newly received packets into the netmap ring. 312 * j is an index in the netmap ring, l in the NIC ring, and 313 * j = (kring->nr_hwcur + kring->nr_hwavail) % ring_size 314 * l = rxr->next_to_check; 315 * and 316 * j == (l + kring->nkr_hwofs) % ring_size 317 */ 318 l = rxr->next_to_check; 319 j = l + kring->nkr_hwofs; 320 /* here nkr_hwofs can be negative so must check for j < 0 */ 321 if (j < 0) 322 j += lim + 1; 323 else if (j > lim) 324 j -= lim + 1; 325 for (n = 0; ; n++) { 326 struct e1000_rx_desc *curr = &rxr->rx_base[l]; 327 328 if ((curr->status & E1000_RXD_STAT_DD) == 0) 329 break; 330 ring->slot[j].len = le16toh(curr->length); 331 bus_dmamap_sync(rxr->rxtag, rxr->rx_buffers[l].map, 332 BUS_DMASYNC_POSTREAD); 333 j = (j == lim) ? 0 : j + 1; 334 /* make sure next_to_refresh follows next_to_check */ 335 rxr->next_to_refresh = l; // XXX 336 l = (l == lim) ? 0 : l + 1; 337 } 338 if (n) { 339 rxr->next_to_check = l; 340 kring->nr_hwavail += n; 341 } 342 343 /* skip past packets that userspace has already processed */ 344 j = kring->nr_hwcur; 345 if (j != k) { /* userspace has read some packets. */ 346 n = 0; 347 l = j - kring->nkr_hwofs; /* NIC ring index */ 348 /* here nkr_hwofs can be negative so check for l > lim */ 349 if (l < 0) 350 l += lim + 1; 351 else if (l > lim) 352 l -= lim + 1; 353 while (j != k) { 354 struct netmap_slot *slot = &ring->slot[j]; 355 struct e1000_rx_desc *curr = &rxr->rx_base[l]; 356 struct em_buffer *rxbuf = &rxr->rx_buffers[l]; 357 uint64_t paddr; 358 void *addr = PNMB(slot, &paddr); 359 360 if (addr == netmap_buffer_base) { /* bad buf */ 361 if (do_lock) 362 EM_RX_UNLOCK(rxr); 363 return netmap_ring_reinit(kring); 364 } 365 366 curr->status = 0; 367 if (slot->flags & NS_BUF_CHANGED) { 368 curr->buffer_addr = htole64(paddr); 369 /* buffer has changed, reload map */ 370 netmap_reload_map(rxr->rxtag, rxbuf->map, addr); 371 slot->flags &= ~NS_BUF_CHANGED; 372 } 373 374 bus_dmamap_sync(rxr->rxtag, rxbuf->map, 375 BUS_DMASYNC_PREREAD); 376 377 j = (j == lim) ? 0 : j + 1; 378 l = (l == lim) ? 0 : l + 1; 379 n++; 380 } 381 kring->nr_hwavail -= n; 382 kring->nr_hwcur = k; 383 bus_dmamap_sync(rxr->rxdma.dma_tag, rxr->rxdma.dma_map, 384 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 385 /* 386 * IMPORTANT: we must leave one free slot in the ring, 387 * so move l back by one unit 388 */ 389 l = (l == 0) ? lim : l - 1; 390 E1000_WRITE_REG(&adapter->hw, E1000_RDT(rxr->me), l); 391 } 392 /* tell userspace that there are new packets */ 393 ring->avail = kring->nr_hwavail ; 394 if (do_lock) 395 EM_RX_UNLOCK(rxr); 396 return 0; 397} 398