1/*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (C) 2013-2016 Vincenzo Maffione 5 * Copyright (C) 2013-2016 Luigi Rizzo 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30/* 31 * This module implements netmap support on top of standard, 32 * unmodified device drivers. 33 * 34 * A NIOCREGIF request is handled here if the device does not 35 * have native support. TX and RX rings are emulated as follows: 36 * 37 * NIOCREGIF 38 * We preallocate a block of TX mbufs (roughly as many as 39 * tx descriptors; the number is not critical) to speed up 40 * operation during transmissions. The refcount on most of 41 * these buffers is artificially bumped up so we can recycle 42 * them more easily. Also, the destructor is intercepted 43 * so we use it as an interrupt notification to wake up 44 * processes blocked on a poll(). 45 * 46 * For each receive ring we allocate one "struct mbq" 47 * (an mbuf tailq plus a spinlock). We intercept packets 48 * (through if_input) 49 * on the receive path and put them in the mbq from which 50 * netmap receive routines can grab them. 51 * 52 * TX: 53 * in the generic_txsync() routine, netmap buffers are copied 54 * (or linked, in a future) to the preallocated mbufs 55 * and pushed to the transmit queue. Some of these mbufs 56 * (those with NS_REPORT, or otherwise every half ring) 57 * have the refcount=1, others have refcount=2. 58 * When the destructor is invoked, we take that as 59 * a notification that all mbufs up to that one in 60 * the specific ring have been completed, and generate 61 * the equivalent of a transmit interrupt. 62 * 63 * RX: 64 * 65 */ 66 67#ifdef __FreeBSD__ 68 69#include <sys/cdefs.h> /* prerequisite */ 70#include <sys/types.h> 71#include <sys/errno.h> 72#include <sys/malloc.h> 73#include <sys/lock.h> /* PROT_EXEC */ 74#include <sys/rwlock.h> 75#include <sys/socket.h> /* sockaddrs */ 76#include <sys/selinfo.h> 77#include <net/if.h> 78#include <net/if_types.h> 79#include <net/if_var.h> 80#include <machine/bus.h> /* bus_dmamap_* in netmap_kern.h */ 81 82#include <net/netmap.h> 83#include <dev/netmap/netmap_kern.h> 84#include <dev/netmap/netmap_mem2.h> 85 86#define MBUF_RXQ(m) ((m)->m_pkthdr.flowid) 87#define smp_mb() 88 89#elif defined _WIN32 90 91#include "win_glue.h" 92 93#define MBUF_TXQ(m) 0//((m)->m_pkthdr.flowid) 94#define MBUF_RXQ(m) 0//((m)->m_pkthdr.flowid) 95#define smp_mb() //XXX: to be correctly defined 96 97#else /* linux */ 98 99#include "bsd_glue.h" 100 101#include <linux/ethtool.h> /* struct ethtool_ops, get_ringparam */ 102#include <linux/hrtimer.h> 103 104static inline struct mbuf * 105nm_os_get_mbuf(struct ifnet *ifp, int len) 106{ 107 return alloc_skb(LL_RESERVED_SPACE(ifp) + len + 108 ifp->needed_tailroom, GFP_ATOMIC); 109} 110 111#endif /* linux */ 112 113 114/* Common headers. */ 115#include <net/netmap.h> 116#include <dev/netmap/netmap_kern.h> 117#include <dev/netmap/netmap_mem2.h> 118 119 120#define for_each_kring_n(_i, _k, _karr, _n) \ 121 for ((_k)=*(_karr), (_i) = 0; (_i) < (_n); (_i)++, (_k) = (_karr)[(_i)]) 122 123#define for_each_tx_kring(_i, _k, _na) \ 124 for_each_kring_n(_i, _k, (_na)->tx_rings, (_na)->num_tx_rings) 125#define for_each_tx_kring_h(_i, _k, _na) \ 126 for_each_kring_n(_i, _k, (_na)->tx_rings, (_na)->num_tx_rings + 1) 127 128#define for_each_rx_kring(_i, _k, _na) \ 129 for_each_kring_n(_i, _k, (_na)->rx_rings, (_na)->num_rx_rings) 130#define for_each_rx_kring_h(_i, _k, _na) \ 131 for_each_kring_n(_i, _k, (_na)->rx_rings, (_na)->num_rx_rings + 1) 132 133 134/* ======================== PERFORMANCE STATISTICS =========================== */ 135 136#ifdef RATE_GENERIC 137#define IFRATE(x) x 138struct rate_stats { 139 unsigned long txpkt; 140 unsigned long txsync; 141 unsigned long txirq; 142 unsigned long txrepl; 143 unsigned long txdrop; 144 unsigned long rxpkt; 145 unsigned long rxirq; 146 unsigned long rxsync; 147}; 148 149struct rate_context { 150 unsigned refcount; 151 struct timer_list timer; 152 struct rate_stats new; 153 struct rate_stats old; 154}; 155 156#define RATE_PRINTK(_NAME_) \ 157 printk( #_NAME_ " = %lu Hz\n", (cur._NAME_ - ctx->old._NAME_)/RATE_PERIOD); 158#define RATE_PERIOD 2 159static void rate_callback(unsigned long arg) 160{ 161 struct rate_context * ctx = (struct rate_context *)arg; 162 struct rate_stats cur = ctx->new; 163 int r; 164 165 RATE_PRINTK(txpkt); 166 RATE_PRINTK(txsync); 167 RATE_PRINTK(txirq); 168 RATE_PRINTK(txrepl); 169 RATE_PRINTK(txdrop); 170 RATE_PRINTK(rxpkt); 171 RATE_PRINTK(rxsync); 172 RATE_PRINTK(rxirq); 173 printk("\n"); 174 175 ctx->old = cur; 176 r = mod_timer(&ctx->timer, jiffies + 177 msecs_to_jiffies(RATE_PERIOD * 1000)); 178 if (unlikely(r)) 179 nm_prerr("mod_timer() failed"); 180} 181 182static struct rate_context rate_ctx; 183 184void generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi) 185{ 186 if (txp) rate_ctx.new.txpkt++; 187 if (txs) rate_ctx.new.txsync++; 188 if (txi) rate_ctx.new.txirq++; 189 if (rxp) rate_ctx.new.rxpkt++; 190 if (rxs) rate_ctx.new.rxsync++; 191 if (rxi) rate_ctx.new.rxirq++; 192} 193 194#else /* !RATE */ 195#define IFRATE(x) 196#endif /* !RATE */ 197 198 199/* ========== GENERIC (EMULATED) NETMAP ADAPTER SUPPORT ============= */ 200 201/* 202 * Wrapper used by the generic adapter layer to notify 203 * the poller threads. Differently from netmap_rx_irq(), we check 204 * only NAF_NETMAP_ON instead of NAF_NATIVE_ON to enable the irq. 205 */ 206void 207netmap_generic_irq(struct netmap_adapter *na, u_int q, u_int *work_done) 208{ 209 if (unlikely(!nm_netmap_on(na))) 210 return; 211 212 netmap_common_irq(na, q, work_done); 213#ifdef RATE_GENERIC 214 if (work_done) 215 rate_ctx.new.rxirq++; 216 else 217 rate_ctx.new.txirq++; 218#endif /* RATE_GENERIC */ 219} 220 221static int 222generic_netmap_unregister(struct netmap_adapter *na) 223{ 224 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 225 struct netmap_kring *kring = NULL; 226 int i, r; 227 228 if (na->active_fds == 0) { 229 na->na_flags &= ~NAF_NETMAP_ON; 230 231 /* Stop intercepting packets on the RX path. */ 232 nm_os_catch_rx(gna, 0); 233 234 /* Release packet steering control. */ 235 nm_os_catch_tx(gna, 0); 236 } 237 238 netmap_krings_mode_commit(na, /*onoff=*/0); 239 240 for_each_rx_kring(r, kring, na) { 241 /* Free the mbufs still pending in the RX queues, 242 * that did not end up into the corresponding netmap 243 * RX rings. */ 244 mbq_safe_purge(&kring->rx_queue); 245 nm_os_mitigation_cleanup(&gna->mit[r]); 246 } 247 248 /* Decrement reference counter for the mbufs in the 249 * TX pools. These mbufs can be still pending in drivers, 250 * (e.g. this happens with virtio-net driver, which 251 * does lazy reclaiming of transmitted mbufs). */ 252 for_each_tx_kring(r, kring, na) { 253 /* We must remove the destructor on the TX event, 254 * because the destructor invokes netmap code, and 255 * the netmap module may disappear before the 256 * TX event is consumed. */ 257 mtx_lock_spin(&kring->tx_event_lock); 258 if (kring->tx_event) { 259 SET_MBUF_DESTRUCTOR(kring->tx_event, NULL, NULL); 260 } 261 kring->tx_event = NULL; 262 mtx_unlock_spin(&kring->tx_event_lock); 263 } 264 265 if (na->active_fds == 0) { 266 nm_os_free(gna->mit); 267 268 for_each_rx_kring(r, kring, na) { 269 mbq_safe_fini(&kring->rx_queue); 270 } 271 272 for_each_tx_kring(r, kring, na) { 273 callout_drain(&kring->tx_event_callout); 274 275 if (kring->tx_pool == NULL) { 276 continue; 277 } 278 279 for (i=0; i<na->num_tx_desc; i++) { 280 if (kring->tx_pool[i]) { 281 m_free(kring->tx_pool[i]); 282 kring->tx_pool[i] = NULL; 283 } 284 } 285 mtx_destroy(&kring->tx_event_lock); 286 nm_os_free(kring->tx_pool); 287 kring->tx_pool = NULL; 288 } 289 290#ifdef RATE_GENERIC 291 if (--rate_ctx.refcount == 0) { 292 nm_prinf("del_timer()"); 293 del_timer(&rate_ctx.timer); 294 } 295#endif 296 nm_prinf("Emulated adapter for %s deactivated", na->name); 297 } 298 299 return 0; 300} 301 302/* Enable/disable netmap mode for a generic network interface. */ 303static int 304generic_netmap_register(struct netmap_adapter *na, int enable) 305{ 306 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 307 struct netmap_kring *kring = NULL; 308 int error; 309 int i, r; 310 311 if (!na) { 312 return EINVAL; 313 } 314 315 if (!enable) { 316 /* This is actually an unregif. */ 317 return generic_netmap_unregister(na); 318 } 319 320 if (na->active_fds == 0) { 321 nm_prinf("Emulated adapter for %s activated", na->name); 322 /* Do all memory allocations when (na->active_fds == 0), to 323 * simplify error management. */ 324 325 /* Allocate memory for mitigation support on all the rx queues. */ 326 gna->mit = nm_os_malloc(na->num_rx_rings * sizeof(struct nm_generic_mit)); 327 if (!gna->mit) { 328 nm_prerr("mitigation allocation failed"); 329 error = ENOMEM; 330 goto out; 331 } 332 333 for_each_rx_kring(r, kring, na) { 334 /* Init mitigation support. */ 335 nm_os_mitigation_init(&gna->mit[r], r, na); 336 337 /* Initialize the rx queue, as generic_rx_handler() can 338 * be called as soon as nm_os_catch_rx() returns. 339 */ 340 mbq_safe_init(&kring->rx_queue); 341 } 342 343 /* 344 * Prepare mbuf pools (parallel to the tx rings), for packet 345 * transmission. Don't preallocate the mbufs here, it's simpler 346 * to leave this task to txsync. 347 */ 348 for_each_tx_kring(r, kring, na) { 349 kring->tx_pool = NULL; 350 } 351 for_each_tx_kring(r, kring, na) { 352 kring->tx_pool = 353 nm_os_malloc(na->num_tx_desc * sizeof(struct mbuf *)); 354 if (!kring->tx_pool) { 355 nm_prerr("tx_pool allocation failed"); 356 error = ENOMEM; 357 goto free_tx_pools; 358 } 359 mtx_init(&kring->tx_event_lock, "tx_event_lock", 360 NULL, MTX_SPIN); 361 callout_init_mtx(&kring->tx_event_callout, 362 &kring->tx_event_lock, 363 CALLOUT_RETURNUNLOCKED); 364 } 365 } 366 367 netmap_krings_mode_commit(na, /*onoff=*/1); 368 369 for_each_tx_kring(r, kring, na) { 370 /* Initialize tx_pool and tx_event. */ 371 for (i=0; i<na->num_tx_desc; i++) { 372 kring->tx_pool[i] = NULL; 373 } 374 375 kring->tx_event = NULL; 376 } 377 378 if (na->active_fds == 0) { 379 /* Prepare to intercept incoming traffic. */ 380 error = nm_os_catch_rx(gna, 1); 381 if (error) { 382 nm_prerr("nm_os_catch_rx(1) failed (%d)", error); 383 goto free_tx_pools; 384 } 385 386 /* Let netmap control the packet steering. */ 387 error = nm_os_catch_tx(gna, 1); 388 if (error) { 389 nm_prerr("nm_os_catch_tx(1) failed (%d)", error); 390 goto catch_rx; 391 } 392 393 na->na_flags |= NAF_NETMAP_ON; 394 395#ifdef RATE_GENERIC 396 if (rate_ctx.refcount == 0) { 397 nm_prinf("setup_timer()"); 398 memset(&rate_ctx, 0, sizeof(rate_ctx)); 399 setup_timer(&rate_ctx.timer, &rate_callback, (unsigned long)&rate_ctx); 400 if (mod_timer(&rate_ctx.timer, jiffies + msecs_to_jiffies(1500))) { 401 nm_prerr("Error: mod_timer()"); 402 } 403 } 404 rate_ctx.refcount++; 405#endif /* RATE */ 406 } 407 408 return 0; 409 410 /* Here (na->active_fds == 0) holds. */ 411catch_rx: 412 nm_os_catch_rx(gna, 0); 413free_tx_pools: 414 for_each_tx_kring(r, kring, na) { 415 mtx_destroy(&kring->tx_event_lock); 416 if (kring->tx_pool == NULL) { 417 continue; 418 } 419 nm_os_free(kring->tx_pool); 420 kring->tx_pool = NULL; 421 } 422 for_each_rx_kring(r, kring, na) { 423 mbq_safe_fini(&kring->rx_queue); 424 } 425 nm_os_free(gna->mit); 426out: 427 428 return error; 429} 430 431/* 432 * Callback invoked when the device driver frees an mbuf used 433 * by netmap to transmit a packet. This usually happens when 434 * the NIC notifies the driver that transmission is completed. 435 */ 436static void 437generic_mbuf_dtor(struct mbuf *m) 438{ 439 struct netmap_adapter *na = GEN_TX_MBUF_NA(m); 440 struct netmap_kring *kring; 441 unsigned int r = MBUF_TXQ(m); 442 unsigned int r_orig = r; 443 444 if (unlikely(!nm_netmap_on(na) || r >= na->num_tx_rings)) { 445 nm_prerr("Error: no netmap adapter on device %p", 446 GEN_TX_MBUF_IFP(m)); 447 return; 448 } 449 450 /* 451 * First, clear the event mbuf. 452 * In principle, the event 'm' should match the one stored 453 * on ring 'r'. However we check it explicitly to stay 454 * safe against lower layers (qdisc, driver, etc.) changing 455 * MBUF_TXQ(m) under our feet. If the match is not found 456 * on 'r', we try to see if it belongs to some other ring. 457 */ 458 for (;;) { 459 bool match = false; 460 461 kring = na->tx_rings[r]; 462 mtx_lock_spin(&kring->tx_event_lock); 463 464 /* 465 * The netmap destructor can be called between us getting the 466 * reference and taking the lock, in that case the ring 467 * reference won't be valid. The destructor will free this mbuf 468 * so we can stop here. 469 */ 470 if (GEN_TX_MBUF_NA(m) == NULL) { 471 mtx_unlock_spin(&kring->tx_event_lock); 472 return; 473 } 474 475 if (kring->tx_event == m) { 476 kring->tx_event = NULL; 477 match = true; 478 } 479 mtx_unlock_spin(&kring->tx_event_lock); 480 481 if (match) { 482 if (r != r_orig) { 483 nm_prlim(1, "event %p migrated: ring %u --> %u", 484 m, r_orig, r); 485 } 486 break; 487 } 488 489 if (++r == na->num_tx_rings) r = 0; 490 491 if (r == r_orig) { 492#ifndef __FreeBSD__ 493 /* 494 * On FreeBSD this situation can arise if the tx_event 495 * callout handler cleared a stuck packet. 496 */ 497 nm_prlim(1, "Cannot match event %p", m); 498#endif 499 nm_generic_mbuf_dtor(m); 500 return; 501 } 502 } 503 504 /* Second, wake up clients. They will reclaim the event through 505 * txsync. */ 506 netmap_generic_irq(na, r, NULL); 507 nm_generic_mbuf_dtor(m); 508} 509 510/* Record completed transmissions and update hwtail. 511 * 512 * The oldest tx buffer not yet completed is at nr_hwtail + 1, 513 * nr_hwcur is the first unsent buffer. 514 */ 515static u_int 516generic_netmap_tx_clean(struct netmap_kring *kring, int txqdisc) 517{ 518 u_int const lim = kring->nkr_num_slots - 1; 519 u_int nm_i = nm_next(kring->nr_hwtail, lim); 520 u_int hwcur = kring->nr_hwcur; 521 u_int n = 0; 522 struct mbuf **tx_pool = kring->tx_pool; 523 524 nm_prdis("hwcur = %d, hwtail = %d", kring->nr_hwcur, kring->nr_hwtail); 525 526 while (nm_i != hwcur) { /* buffers not completed */ 527 struct mbuf *m = tx_pool[nm_i]; 528 529 if (txqdisc) { 530 if (m == NULL) { 531 /* Nothing to do, this is going 532 * to be replenished. */ 533 nm_prlim(3, "Is this happening?"); 534 535 } else if (MBUF_QUEUED(m)) { 536 break; /* Not dequeued yet. */ 537 538 } else if (MBUF_REFCNT(m) != 1) { 539 /* This mbuf has been dequeued but is still busy 540 * (refcount is 2). 541 * Leave it to the driver and replenish. */ 542 m_free(m); 543 tx_pool[nm_i] = NULL; 544 } 545 546 } else { 547 if (unlikely(m == NULL)) { 548 int event_consumed; 549 550 /* This slot was used to place an event. */ 551 mtx_lock_spin(&kring->tx_event_lock); 552 event_consumed = (kring->tx_event == NULL); 553 mtx_unlock_spin(&kring->tx_event_lock); 554 if (!event_consumed) { 555 /* The event has not been consumed yet, 556 * still busy in the driver. */ 557 break; 558 } 559 /* The event has been consumed, we can go 560 * ahead. */ 561 } else if (MBUF_REFCNT(m) != 1) { 562 /* This mbuf is still busy: its refcnt is 2. */ 563 break; 564 } 565 } 566 567 n++; 568 nm_i = nm_next(nm_i, lim); 569 } 570 kring->nr_hwtail = nm_prev(nm_i, lim); 571 nm_prdis("tx completed [%d] -> hwtail %d", n, kring->nr_hwtail); 572 573 return n; 574} 575 576/* Compute a slot index in the middle between inf and sup. */ 577static inline u_int 578ring_middle(u_int inf, u_int sup, u_int lim) 579{ 580 u_int n = lim + 1; 581 u_int e; 582 583 if (sup >= inf) { 584 e = (sup + inf) / 2; 585 } else { /* wrap around */ 586 e = (sup + n + inf) / 2; 587 if (e >= n) { 588 e -= n; 589 } 590 } 591 592 if (unlikely(e >= n)) { 593 nm_prerr("This cannot happen"); 594 e = 0; 595 } 596 597 return e; 598} 599 600#ifdef __FreeBSD__ 601static void 602generic_tx_callout(void *arg) 603{ 604 struct netmap_kring *kring = arg; 605 606 kring->tx_event = NULL; 607 mtx_unlock_spin(&kring->tx_event_lock); 608 netmap_generic_irq(kring->na, kring->ring_id, NULL); 609} 610#endif 611 612static void 613generic_set_tx_event(struct netmap_kring *kring, u_int hwcur) 614{ 615 u_int lim = kring->nkr_num_slots - 1; 616 struct mbuf *m; 617 u_int e; 618 u_int ntc = nm_next(kring->nr_hwtail, lim); /* next to clean */ 619 620 if (ntc == hwcur) { 621 return; /* all buffers are free */ 622 } 623 624 /* 625 * We have pending packets in the driver between hwtail+1 626 * and hwcur, and we have to chose one of these slot to 627 * generate a notification. 628 * There is a race but this is only called within txsync which 629 * does a double check. 630 */ 631#if 0 632 /* Choose a slot in the middle, so that we don't risk ending 633 * up in a situation where the client continuously wake up, 634 * fills one or a few TX slots and go to sleep again. */ 635 e = ring_middle(ntc, hwcur, lim); 636#else 637 /* Choose the first pending slot, to be safe against driver 638 * reordering mbuf transmissions. */ 639 e = ntc; 640#endif 641 642 m = kring->tx_pool[e]; 643 if (m == NULL) { 644 /* An event is already in place. */ 645 return; 646 } 647 648 mtx_lock_spin(&kring->tx_event_lock); 649 if (kring->tx_event) { 650 /* An event is already in place. */ 651 mtx_unlock_spin(&kring->tx_event_lock); 652 return; 653 } 654 655 SET_MBUF_DESTRUCTOR(m, generic_mbuf_dtor, kring->na); 656 657 kring->tx_event = m; 658#ifdef __FreeBSD__ 659 /* 660 * Handle the possibility that the transmitted buffer isn't reclaimed 661 * within a bounded period of time. This can arise when transmitting 662 * out of multiple ports via a lagg or bridge interface, since the 663 * member ports may legitimately only free transmitted buffers in 664 * batches. 665 * 666 * The callout handler clears the stuck packet from the ring, allowing 667 * transmission to proceed. In the common case we let 668 * generic_mbuf_dtor() unstick the ring, allowing mbufs to be 669 * reused most of the time. 670 */ 671 callout_reset_sbt_curcpu(&kring->tx_event_callout, SBT_1MS, 0, 672 generic_tx_callout, kring, 0); 673#endif 674 mtx_unlock_spin(&kring->tx_event_lock); 675 676 kring->tx_pool[e] = NULL; 677 678 nm_prdis("Request Event at %d mbuf %p refcnt %d", e, m, m ? MBUF_REFCNT(m) : -2 ); 679 680 /* Decrement the refcount. This will free it if we lose the race 681 * with the driver. */ 682 m_free(m); 683} 684 685/* 686 * generic_netmap_txsync() transforms netmap buffers into mbufs 687 * and passes them to the standard device driver 688 * (ndo_start_xmit() or ifp->if_transmit() ). 689 * On linux this is not done directly, but using dev_queue_xmit(), 690 * since it implements the TX flow control (and takes some locks). 691 */ 692static int 693generic_netmap_txsync(struct netmap_kring *kring, int flags) 694{ 695 struct netmap_adapter *na = kring->na; 696 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 697 if_t ifp = na->ifp; 698 struct netmap_ring *ring = kring->ring; 699 u_int nm_i; /* index into the netmap ring */ // j 700 u_int const lim = kring->nkr_num_slots - 1; 701 u_int const head = kring->rhead; 702 u_int ring_nr = kring->ring_id; 703 704 IFRATE(rate_ctx.new.txsync++); 705 706 rmb(); 707 708 /* 709 * First part: process new packets to send. 710 */ 711 nm_i = kring->nr_hwcur; 712 if (nm_i != head) { /* we have new packets to send */ 713 struct nm_os_gen_arg a; 714 u_int event = -1; 715#ifdef __FreeBSD__ 716 struct epoch_tracker et; 717 718 NET_EPOCH_ENTER(et); 719#endif 720 721 if (gna->txqdisc && nm_kr_txempty(kring)) { 722 /* In txqdisc mode, we ask for a delayed notification, 723 * but only when cur == hwtail, which means that the 724 * client is going to block. */ 725 event = ring_middle(nm_i, head, lim); 726 nm_prdis("Place txqdisc event (hwcur=%u,event=%u," 727 "head=%u,hwtail=%u)", nm_i, event, head, 728 kring->nr_hwtail); 729 } 730 731 a.ifp = ifp; 732 a.ring_nr = ring_nr; 733 a.head = a.tail = NULL; 734 735 while (nm_i != head) { 736 struct netmap_slot *slot = &ring->slot[nm_i]; 737 u_int len = slot->len; 738 void *addr = NMB(na, slot); 739 /* device-specific */ 740 struct mbuf *m; 741 int tx_ret; 742 743 NM_CHECK_ADDR_LEN(na, addr, len); 744 745 /* Tale a mbuf from the tx pool (replenishing the pool 746 * entry if necessary) and copy in the user packet. */ 747 m = kring->tx_pool[nm_i]; 748 if (unlikely(m == NULL)) { 749 kring->tx_pool[nm_i] = m = 750 nm_os_get_mbuf(ifp, NETMAP_BUF_SIZE(na)); 751 if (m == NULL) { 752 nm_prlim(2, "Failed to replenish mbuf"); 753 /* Here we could schedule a timer which 754 * retries to replenish after a while, 755 * and notifies the client when it 756 * manages to replenish some slots. In 757 * any case we break early to avoid 758 * crashes. */ 759 break; 760 } 761 IFRATE(rate_ctx.new.txrepl++); 762 } else { 763 nm_os_mbuf_reinit(m); 764 } 765 766 a.m = m; 767 a.addr = addr; 768 a.len = len; 769 a.qevent = (nm_i == event); 770 /* When not in txqdisc mode, we should ask 771 * notifications when NS_REPORT is set, or roughly 772 * every half ring. To optimize this, we set a 773 * notification event when the client runs out of 774 * TX ring space, or when transmission fails. In 775 * the latter case we also break early. 776 */ 777 tx_ret = nm_os_generic_xmit_frame(&a); 778 if (unlikely(tx_ret)) { 779 if (!gna->txqdisc) { 780 /* 781 * No room for this mbuf in the device driver. 782 * Request a notification FOR A PREVIOUS MBUF, 783 * then call generic_netmap_tx_clean(kring) to do the 784 * double check and see if we can free more buffers. 785 * If there is space continue, else break; 786 * NOTE: the double check is necessary if the problem 787 * occurs in the txsync call after selrecord(). 788 * Also, we need some way to tell the caller that not 789 * all buffers were queued onto the device (this was 790 * not a problem with native netmap driver where space 791 * is preallocated). The bridge has a similar problem 792 * and we solve it there by dropping the excess packets. 793 */ 794 generic_set_tx_event(kring, nm_i); 795 if (generic_netmap_tx_clean(kring, gna->txqdisc)) { 796 /* space now available */ 797 continue; 798 } else { 799 break; 800 } 801 } 802 803 /* In txqdisc mode, the netmap-aware qdisc 804 * queue has the same length as the number of 805 * netmap slots (N). Since tail is advanced 806 * only when packets are dequeued, qdisc 807 * queue overrun cannot happen, so 808 * nm_os_generic_xmit_frame() did not fail 809 * because of that. 810 * However, packets can be dropped because 811 * carrier is off, or because our qdisc is 812 * being deactivated, or possibly for other 813 * reasons. In these cases, we just let the 814 * packet to be dropped. */ 815 IFRATE(rate_ctx.new.txdrop++); 816 } 817 818 slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED); 819 nm_i = nm_next(nm_i, lim); 820 IFRATE(rate_ctx.new.txpkt++); 821 } 822 if (a.head != NULL) { 823 a.addr = NULL; 824 nm_os_generic_xmit_frame(&a); 825 } 826 /* Update hwcur to the next slot to transmit. Here nm_i 827 * is not necessarily head, we could break early. */ 828 kring->nr_hwcur = nm_i; 829 830#ifdef __FreeBSD__ 831 NET_EPOCH_EXIT(et); 832#endif 833 } 834 835 if (!gna->txqdisc && (flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring))) { 836 /* No more available slots? Set a notification event 837 * on a netmap slot that will be cleaned in the future. 838 * No doublecheck is performed, since txsync() will be 839 * called twice by netmap_poll(). 840 */ 841 generic_set_tx_event(kring, nm_i); 842 } 843 844 /* 845 * Second, reclaim completed buffers 846 */ 847 generic_netmap_tx_clean(kring, gna->txqdisc); 848 849 return 0; 850} 851 852 853/* 854 * This handler is registered (through nm_os_catch_rx()) 855 * within the attached network interface 856 * in the RX subsystem, so that every mbuf passed up by 857 * the driver can be stolen to the network stack. 858 * Stolen packets are put in a queue where the 859 * generic_netmap_rxsync() callback can extract them. 860 * Returns 1 if the packet was stolen, 0 otherwise. 861 */ 862int 863generic_rx_handler(if_t ifp, struct mbuf *m) 864{ 865 struct netmap_adapter *na = NA(ifp); 866 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na; 867 struct netmap_kring *kring; 868 u_int work_done; 869 u_int r = MBUF_RXQ(m); /* receive ring number */ 870 871 if (r >= na->num_rx_rings) { 872 r = r % na->num_rx_rings; 873 } 874 875 kring = na->rx_rings[r]; 876 877 if (kring->nr_mode == NKR_NETMAP_OFF) { 878 /* We must not intercept this mbuf. */ 879 return 0; 880 } 881 882 /* limit the size of the queue */ 883 if (unlikely(!gna->rxsg && MBUF_LEN(m) > NETMAP_BUF_SIZE(na))) { 884 /* This may happen when GRO/LRO features are enabled for 885 * the NIC driver when the generic adapter does not 886 * support RX scatter-gather. */ 887 nm_prlim(2, "Warning: driver pushed up big packet " 888 "(size=%d)", (int)MBUF_LEN(m)); 889 if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); 890 m_freem(m); 891 } else if (unlikely(mbq_len(&kring->rx_queue) > na->num_rx_desc)) { 892 if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); 893 m_freem(m); 894 } else { 895 mbq_safe_enqueue(&kring->rx_queue, m); 896 } 897 898 if (netmap_generic_mit < 32768) { 899 /* no rx mitigation, pass notification up */ 900 netmap_generic_irq(na, r, &work_done); 901 } else { 902 /* same as send combining, filter notification if there is a 903 * pending timer, otherwise pass it up and start a timer. 904 */ 905 if (likely(nm_os_mitigation_active(&gna->mit[r]))) { 906 /* Record that there is some pending work. */ 907 gna->mit[r].mit_pending = 1; 908 } else { 909 netmap_generic_irq(na, r, &work_done); 910 nm_os_mitigation_start(&gna->mit[r]); 911 } 912 } 913 914 /* We have intercepted the mbuf. */ 915 return 1; 916} 917 918/* 919 * generic_netmap_rxsync() extracts mbufs from the queue filled by 920 * generic_netmap_rx_handler() and puts their content in the netmap 921 * receive ring. 922 * Access must be protected because the rx handler is asynchronous, 923 */ 924static int 925generic_netmap_rxsync(struct netmap_kring *kring, int flags) 926{ 927 struct netmap_ring *ring = kring->ring; 928 struct netmap_adapter *na = kring->na; 929 u_int nm_i; /* index into the netmap ring */ //j, 930 u_int n; 931 u_int const lim = kring->nkr_num_slots - 1; 932 u_int const head = kring->rhead; 933 int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR; 934 935 /* Adapter-specific variables. */ 936 u_int nm_buf_len = NETMAP_BUF_SIZE(na); 937 struct mbq tmpq; 938 struct mbuf *m; 939 int avail; /* in bytes */ 940 int mlen; 941 int copy; 942 943 if (head > lim) 944 return netmap_ring_reinit(kring); 945 946 IFRATE(rate_ctx.new.rxsync++); 947 948 /* 949 * First part: skip past packets that userspace has released. 950 * This can possibly make room for the second part. 951 */ 952 nm_i = kring->nr_hwcur; 953 if (nm_i != head) { 954 /* Userspace has released some packets. */ 955 for (n = 0; nm_i != head; n++) { 956 struct netmap_slot *slot = &ring->slot[nm_i]; 957 958 slot->flags &= ~NS_BUF_CHANGED; 959 nm_i = nm_next(nm_i, lim); 960 } 961 kring->nr_hwcur = head; 962 } 963 964 /* 965 * Second part: import newly received packets. 966 */ 967 if (!netmap_no_pendintr && !force_update) { 968 return 0; 969 } 970 971 nm_i = kring->nr_hwtail; /* First empty slot in the receive ring. */ 972 973 /* Compute the available space (in bytes) in this netmap ring. 974 * The first slot that is not considered in is the one before 975 * nr_hwcur. */ 976 977 avail = nm_prev(kring->nr_hwcur, lim) - nm_i; 978 if (avail < 0) 979 avail += lim + 1; 980 avail *= nm_buf_len; 981 982 /* First pass: While holding the lock on the RX mbuf queue, 983 * extract as many mbufs as they fit the available space, 984 * and put them in a temporary queue. 985 * To avoid performing a per-mbuf division (mlen / nm_buf_len) to 986 * to update avail, we do the update in a while loop that we 987 * also use to set the RX slots, but without performing the copy. */ 988 mbq_init(&tmpq); 989 mbq_lock(&kring->rx_queue); 990 for (n = 0;; n++) { 991 m = mbq_peek(&kring->rx_queue); 992 if (!m) { 993 /* No more packets from the driver. */ 994 break; 995 } 996 997 mlen = MBUF_LEN(m); 998 if (mlen > avail) { 999 /* No more space in the ring. */ 1000 break; 1001 } 1002 1003 mbq_dequeue(&kring->rx_queue); 1004 1005 while (mlen) { 1006 copy = nm_buf_len; 1007 if (mlen < copy) { 1008 copy = mlen; 1009 } 1010 mlen -= copy; 1011 avail -= nm_buf_len; 1012 1013 ring->slot[nm_i].len = copy; 1014 ring->slot[nm_i].flags = (mlen ? NS_MOREFRAG : 0); 1015 nm_i = nm_next(nm_i, lim); 1016 } 1017 1018 mbq_enqueue(&tmpq, m); 1019 } 1020 mbq_unlock(&kring->rx_queue); 1021 1022 /* Second pass: Drain the temporary queue, going over the used RX slots, 1023 * and perform the copy out of the RX queue lock. */ 1024 nm_i = kring->nr_hwtail; 1025 1026 for (;;) { 1027 void *nmaddr; 1028 int ofs = 0; 1029 int morefrag; 1030 1031 m = mbq_dequeue(&tmpq); 1032 if (!m) { 1033 break; 1034 } 1035 1036 do { 1037 nmaddr = NMB(na, &ring->slot[nm_i]); 1038 /* We only check the address here on generic rx rings. */ 1039 if (nmaddr == NETMAP_BUF_BASE(na)) { /* Bad buffer */ 1040 m_freem(m); 1041 mbq_purge(&tmpq); 1042 mbq_fini(&tmpq); 1043 return netmap_ring_reinit(kring); 1044 } 1045 1046 copy = ring->slot[nm_i].len; 1047 m_copydata(m, ofs, copy, nmaddr); 1048 ofs += copy; 1049 morefrag = ring->slot[nm_i].flags & NS_MOREFRAG; 1050 nm_i = nm_next(nm_i, lim); 1051 } while (morefrag); 1052 1053 m_freem(m); 1054 } 1055 1056 mbq_fini(&tmpq); 1057 1058 if (n) { 1059 kring->nr_hwtail = nm_i; 1060 IFRATE(rate_ctx.new.rxpkt += n); 1061 } 1062 kring->nr_kflags &= ~NKR_PENDINTR; 1063 1064 return 0; 1065} 1066 1067static void 1068generic_netmap_dtor(struct netmap_adapter *na) 1069{ 1070 struct netmap_generic_adapter *gna = (struct netmap_generic_adapter*)na; 1071 if_t ifp = netmap_generic_getifp(gna); 1072 struct netmap_adapter *prev_na = gna->prev; 1073 1074 if (prev_na != NULL) { 1075 netmap_adapter_put(prev_na); 1076 if (nm_iszombie(na)) { 1077 /* 1078 * The driver has been removed without releasing 1079 * the reference so we need to do it here. 1080 */ 1081 netmap_adapter_put(prev_na); 1082 } 1083 nm_prinf("Native netmap adapter for %s restored", prev_na->name); 1084 } 1085 NM_RESTORE_NA(ifp, prev_na); 1086 na->ifp = NULL; 1087 nm_prinf("Emulated netmap adapter for %s destroyed", na->name); 1088} 1089 1090int 1091na_is_generic(struct netmap_adapter *na) 1092{ 1093 return na->nm_register == generic_netmap_register; 1094} 1095 1096/* 1097 * generic_netmap_attach() makes it possible to use netmap on 1098 * a device without native netmap support. 1099 * This is less performant than native support but potentially 1100 * faster than raw sockets or similar schemes. 1101 * 1102 * In this "emulated" mode, netmap rings do not necessarily 1103 * have the same size as those in the NIC. We use a default 1104 * value and possibly override it if the OS has ways to fetch the 1105 * actual configuration. 1106 */ 1107int 1108generic_netmap_attach(if_t ifp) 1109{ 1110 struct netmap_adapter *na; 1111 struct netmap_generic_adapter *gna; 1112 int retval; 1113 u_int num_tx_desc, num_rx_desc; 1114 1115#ifdef __FreeBSD__ 1116 if (if_gettype(ifp) == IFT_LOOP) { 1117 nm_prerr("if_loop is not supported by %s", __func__); 1118 return EINVAL; 1119 } 1120#endif 1121 1122 if (NM_NA_CLASH(ifp)) { 1123 /* If NA(ifp) is not null but there is no valid netmap 1124 * adapter it means that someone else is using the same 1125 * pointer (e.g. ax25_ptr on linux). This happens for 1126 * instance when also PF_RING is in use. */ 1127 nm_prerr("Error: netmap adapter hook is busy"); 1128 return EBUSY; 1129 } 1130 1131 num_tx_desc = num_rx_desc = netmap_generic_ringsize; /* starting point */ 1132 1133 nm_os_generic_find_num_desc(ifp, &num_tx_desc, &num_rx_desc); /* ignore errors */ 1134 if (num_tx_desc == 0 || num_rx_desc == 0) { 1135 nm_prerr("Device has no hw slots (tx %u, rx %u)", num_tx_desc, num_rx_desc); 1136 return EINVAL; 1137 } 1138 1139 gna = nm_os_malloc(sizeof(*gna)); 1140 if (gna == NULL) { 1141 nm_prerr("no memory on attach, give up"); 1142 return ENOMEM; 1143 } 1144 na = (struct netmap_adapter *)gna; 1145 strlcpy(na->name, if_name(ifp), sizeof(na->name)); 1146 na->ifp = ifp; 1147 na->num_tx_desc = num_tx_desc; 1148 na->num_rx_desc = num_rx_desc; 1149 na->rx_buf_maxsize = 32768; 1150 na->nm_register = &generic_netmap_register; 1151 na->nm_txsync = &generic_netmap_txsync; 1152 na->nm_rxsync = &generic_netmap_rxsync; 1153 na->nm_dtor = &generic_netmap_dtor; 1154 /* when using generic, NAF_NETMAP_ON is set so we force 1155 * NAF_SKIP_INTR to use the regular interrupt handler 1156 */ 1157 na->na_flags = NAF_SKIP_INTR | NAF_HOST_RINGS; 1158 1159 nm_prdis("[GNA] num_tx_queues(%d), real_num_tx_queues(%d), len(%lu)", 1160 ifp->num_tx_queues, ifp->real_num_tx_queues, 1161 ifp->tx_queue_len); 1162 nm_prdis("[GNA] num_rx_queues(%d), real_num_rx_queues(%d)", 1163 ifp->num_rx_queues, ifp->real_num_rx_queues); 1164 1165 nm_os_generic_find_num_queues(ifp, &na->num_tx_rings, &na->num_rx_rings); 1166 1167 retval = netmap_attach_common(na); 1168 if (retval) { 1169 nm_os_free(gna); 1170 return retval; 1171 } 1172 1173 if (NM_NA_VALID(ifp)) { 1174 gna->prev = NA(ifp); /* save old na */ 1175 netmap_adapter_get(gna->prev); 1176 } 1177 NM_ATTACH_NA(ifp, na); 1178 1179 nm_os_generic_set_features(gna); 1180 1181 nm_prinf("Emulated adapter for %s created (prev was %s)", na->name, 1182 gna->prev ? gna->prev->name : "NULL"); 1183 1184 return retval; 1185} 1186