1/* 2 * Copyright (C) 2014-2016 Giuseppe Lettieri 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27/* 28 * $FreeBSD: stable/11/sys/dev/netmap/netmap_monitor.c 344047 2019-02-12 09:26:05Z vmaffione $ 29 * 30 * Monitors 31 * 32 * netmap monitors can be used to do monitoring of network traffic 33 * on another adapter, when the latter adapter is working in netmap mode. 34 * 35 * Monitors offer to userspace the same interface as any other netmap port, 36 * with as many pairs of netmap rings as the monitored adapter. 37 * However, only the rx rings are actually used. Each monitor rx ring receives 38 * the traffic transiting on both the tx and rx corresponding rings in the 39 * monitored adapter. During registration, the user can choose if she wants 40 * to intercept tx only, rx only, or both tx and rx traffic. 41 * 42 * If the monitor is not able to cope with the stream of frames, excess traffic 43 * will be dropped. 44 * 45 * If the monitored adapter leaves netmap mode, the monitor has to be restarted. 46 * 47 * Monitors can be either zero-copy or copy-based. 48 * 49 * Copy monitors see the frames before they are consumed: 50 * 51 * - For tx traffic, this is when the application sends them, before they are 52 * passed down to the adapter. 53 * 54 * - For rx traffic, this is when they are received by the adapter, before 55 * they are sent up to the application, if any (note that, if no 56 * application is reading from a monitored ring, the ring will eventually 57 * fill up and traffic will stop). 58 * 59 * Zero-copy monitors only see the frames after they have been consumed: 60 * 61 * - For tx traffic, this is after the slots containing the frames have been 62 * marked as free. Note that this may happen at a considerably delay after 63 * frame transmission, since freeing of slots is often done lazily. 64 * 65 * - For rx traffic, this is after the consumer on the monitored adapter 66 * has released them. In most cases, the consumer is a userspace 67 * application which may have modified the frame contents. 68 * 69 * Several copy or zero-copy monitors may be active on any ring. 70 * 71 */ 72 73 74#if defined(__FreeBSD__) 75#include <sys/cdefs.h> /* prerequisite */ 76 77#include <sys/types.h> 78#include <sys/errno.h> 79#include <sys/param.h> /* defines used in kernel.h */ 80#include <sys/kernel.h> /* types used in module initialization */ 81#include <sys/malloc.h> 82#include <sys/poll.h> 83#include <sys/lock.h> 84#include <sys/rwlock.h> 85#include <sys/selinfo.h> 86#include <sys/sysctl.h> 87#include <sys/socket.h> /* sockaddrs */ 88#include <net/if.h> 89#include <net/if_var.h> 90#include <machine/bus.h> /* bus_dmamap_* */ 91#include <sys/refcount.h> 92 93 94#elif defined(linux) 95 96#include "bsd_glue.h" 97 98#elif defined(__APPLE__) 99 100#warning OSX support is only partial 101#include "osx_glue.h" 102 103#elif defined(_WIN32) 104#include "win_glue.h" 105#else 106 107#error Unsupported platform 108 109#endif /* unsupported */ 110 111/* 112 * common headers 113 */ 114 115#include <net/netmap.h> 116#include <dev/netmap/netmap_kern.h> 117#include <dev/netmap/netmap_mem2.h> 118 119#ifdef WITH_MONITOR 120 121#define NM_MONITOR_MAXSLOTS 4096 122 123/* 124 ******************************************************************** 125 * functions common to both kind of monitors 126 ******************************************************************** 127 */ 128 129static int netmap_zmon_reg(struct netmap_adapter *, int); 130static int 131nm_is_zmon(struct netmap_adapter *na) 132{ 133 return na->nm_register == netmap_zmon_reg; 134} 135 136/* nm_sync callback for the monitor's own tx rings. 137 * This makes no sense and always returns error 138 */ 139static int 140netmap_monitor_txsync(struct netmap_kring *kring, int flags) 141{ 142 nm_prlim(1, "%s %x", kring->name, flags); 143 return EIO; 144} 145 146/* nm_sync callback for the monitor's own rx rings. 147 * Note that the lock in netmap_zmon_parent_sync only protects 148 * writers among themselves. Synchronization between writers 149 * (i.e., netmap_zmon_parent_txsync and netmap_zmon_parent_rxsync) 150 * and readers (i.e., netmap_zmon_rxsync) relies on memory barriers. 151 */ 152static int 153netmap_monitor_rxsync(struct netmap_kring *kring, int flags) 154{ 155 struct netmap_monitor_adapter *mna = 156 (struct netmap_monitor_adapter *)kring->na; 157 if (unlikely(mna->priv.np_na == NULL)) { 158 /* parent left netmap mode */ 159 return EIO; 160 } 161 nm_prdis("%s %x", kring->name, flags); 162 kring->nr_hwcur = kring->rhead; 163 mb(); 164 return 0; 165} 166 167/* nm_krings_create callbacks for monitors. 168 */ 169static int 170netmap_monitor_krings_create(struct netmap_adapter *na) 171{ 172 int error = netmap_krings_create(na, 0); 173 enum txrx t; 174 175 if (error) 176 return error; 177 /* override the host rings callbacks */ 178 for_rx_tx(t) { 179 int i; 180 u_int first = nma_get_nrings(na, t); 181 for (i = 0; i < nma_get_host_nrings(na, t); i++) { 182 struct netmap_kring *kring = NMR(na, t)[first + i]; 183 kring->nm_sync = t == NR_TX ? netmap_monitor_txsync : 184 netmap_monitor_rxsync; 185 } 186 } 187 return 0; 188} 189 190/* nm_krings_delete callback for monitors */ 191static void 192netmap_monitor_krings_delete(struct netmap_adapter *na) 193{ 194 netmap_krings_delete(na); 195} 196 197 198static u_int 199nm_txrx2flag(enum txrx t) 200{ 201 return (t == NR_RX ? NR_MONITOR_RX : NR_MONITOR_TX); 202} 203 204/* allocate the monitors array in the monitored kring */ 205static int 206nm_monitor_alloc(struct netmap_kring *kring, u_int n) 207{ 208 size_t old_len, len; 209 struct netmap_kring **nm; 210 211 if (n <= kring->max_monitors) 212 /* we already have more entries that requested */ 213 return 0; 214 215 old_len = sizeof(struct netmap_kring *)*kring->max_monitors; 216 len = sizeof(struct netmap_kring *) * n; 217 nm = nm_os_realloc(kring->monitors, len, old_len); 218 if (nm == NULL) 219 return ENOMEM; 220 221 kring->monitors = nm; 222 kring->max_monitors = n; 223 224 return 0; 225} 226 227/* deallocate the parent array in the parent adapter */ 228static void 229nm_monitor_dealloc(struct netmap_kring *kring) 230{ 231 if (kring->monitors) { 232 if (kring->n_monitors > 0) { 233 nm_prerr("freeing not empty monitor array for %s (%d dangling monitors)!", 234 kring->name, kring->n_monitors); 235 } 236 nm_os_free(kring->monitors); 237 kring->monitors = NULL; 238 kring->max_monitors = 0; 239 kring->n_monitors = 0; 240 } 241} 242 243/* returns 1 iff kring has no monitors */ 244static inline int 245nm_monitor_none(struct netmap_kring *kring) 246{ 247 return kring->n_monitors == 0 && 248 kring->zmon_list[NR_TX].next == NULL && 249 kring->zmon_list[NR_RX].next == NULL; 250} 251 252/* 253 * monitors work by replacing the nm_sync() and possibly the 254 * nm_notify() callbacks in the monitored rings. 255 */ 256static int netmap_zmon_parent_txsync(struct netmap_kring *, int); 257static int netmap_zmon_parent_rxsync(struct netmap_kring *, int); 258static int netmap_monitor_parent_txsync(struct netmap_kring *, int); 259static int netmap_monitor_parent_rxsync(struct netmap_kring *, int); 260static int netmap_monitor_parent_notify(struct netmap_kring *, int); 261 262static int 263nm_monitor_dummycb(struct netmap_kring *kring, int flags) 264{ 265 (void)kring; 266 (void)flags; 267 return 0; 268} 269 270static void 271nm_monitor_intercept_callbacks(struct netmap_kring *kring) 272{ 273 nm_prdis("intercept callbacks on %s", kring->name); 274 kring->mon_sync = kring->nm_sync != NULL ? 275 kring->nm_sync : nm_monitor_dummycb; 276 kring->mon_notify = kring->nm_notify; 277 if (kring->tx == NR_TX) { 278 kring->nm_sync = netmap_monitor_parent_txsync; 279 } else { 280 kring->nm_sync = netmap_monitor_parent_rxsync; 281 kring->nm_notify = netmap_monitor_parent_notify; 282 kring->mon_tail = kring->nr_hwtail; 283 } 284} 285 286static void 287nm_monitor_restore_callbacks(struct netmap_kring *kring) 288{ 289 nm_prdis("restoring callbacks on %s", kring->name); 290 kring->nm_sync = kring->mon_sync; 291 kring->mon_sync = NULL; 292 if (kring->tx == NR_RX) { 293 kring->nm_notify = kring->mon_notify; 294 } 295 kring->mon_notify = NULL; 296} 297 298static struct netmap_kring * 299nm_zmon_list_head(struct netmap_kring *mkring, enum txrx t) 300{ 301 struct netmap_adapter *na = mkring->na; 302 struct netmap_kring *kring = mkring; 303 struct netmap_zmon_list *z = &kring->zmon_list[t]; 304 /* reach the head of the list */ 305 while (nm_is_zmon(na) && z->prev != NULL) { 306 kring = z->prev; 307 na = kring->na; 308 z = &kring->zmon_list[t]; 309 } 310 return nm_is_zmon(na) ? NULL : kring; 311} 312 313/* add the monitor mkring to the list of monitors of kring. 314 * If this is the first monitor, intercept the callbacks 315 */ 316static int 317netmap_monitor_add(struct netmap_kring *mkring, struct netmap_kring *kring, int zmon) 318{ 319 int error = NM_IRQ_COMPLETED; 320 enum txrx t = kring->tx; 321 struct netmap_zmon_list *z = &kring->zmon_list[t]; 322 struct netmap_zmon_list *mz = &mkring->zmon_list[t]; 323 struct netmap_kring *ikring = kring; 324 325 /* a zero-copy monitor which is not the first in the list 326 * must monitor the previous monitor 327 */ 328 if (zmon && z->prev != NULL) 329 ikring = z->prev; /* tail of the list */ 330 331 /* synchronize with concurrently running nm_sync()s */ 332 nm_kr_stop(kring, NM_KR_LOCKED); 333 334 if (nm_monitor_none(ikring)) { 335 /* this is the first monitor, intercept the callbacks */ 336 nm_prdis("%s: intercept callbacks on %s", mkring->name, ikring->name); 337 nm_monitor_intercept_callbacks(ikring); 338 } 339 340 if (zmon) { 341 /* append the zmon to the list */ 342 ikring->zmon_list[t].next = mkring; 343 z->prev = mkring; /* new tail */ 344 mz->prev = ikring; 345 mz->next = NULL; 346 /* grab a reference to the previous netmap adapter 347 * in the chain (this may be the monitored port 348 * or another zero-copy monitor) 349 */ 350 netmap_adapter_get(ikring->na); 351 } else { 352 /* make sure the monitor array exists and is big enough */ 353 error = nm_monitor_alloc(kring, kring->n_monitors + 1); 354 if (error) 355 goto out; 356 kring->monitors[kring->n_monitors] = mkring; 357 mkring->mon_pos[kring->tx] = kring->n_monitors; 358 kring->n_monitors++; 359 } 360 361out: 362 nm_kr_start(kring); 363 return error; 364} 365 366/* remove the monitor mkring from the list of monitors of kring. 367 * If this is the last monitor, restore the original callbacks 368 */ 369static void 370netmap_monitor_del(struct netmap_kring *mkring, struct netmap_kring *kring, enum txrx t) 371{ 372 int zmon = nm_is_zmon(mkring->na); 373 struct netmap_zmon_list *mz = &mkring->zmon_list[t]; 374 struct netmap_kring *ikring = kring; 375 376 377 if (zmon) { 378 /* get to the head of the list */ 379 kring = nm_zmon_list_head(mkring, t); 380 ikring = mz->prev; 381 } 382 383 /* synchronize with concurrently running nm_sync()s 384 * if kring is NULL (orphaned list) the monitored port 385 * has exited netmap mode, so there is nothing to stop 386 */ 387 if (kring != NULL) 388 nm_kr_stop(kring, NM_KR_LOCKED); 389 390 if (zmon) { 391 /* remove the monitor from the list */ 392 if (mz->next != NULL) { 393 mz->next->zmon_list[t].prev = mz->prev; 394 /* we also need to let the next monitor drop the 395 * reference to us and grab the reference to the 396 * previous ring owner, instead 397 */ 398 if (mz->prev != NULL) 399 netmap_adapter_get(mz->prev->na); 400 netmap_adapter_put(mkring->na); 401 } else if (kring != NULL) { 402 /* in the monitored kring, prev is actually the 403 * pointer to the tail of the list 404 */ 405 kring->zmon_list[t].prev = 406 (mz->prev != kring ? mz->prev : NULL); 407 } 408 if (mz->prev != NULL) { 409 netmap_adapter_put(mz->prev->na); 410 mz->prev->zmon_list[t].next = mz->next; 411 } 412 mz->prev = NULL; 413 mz->next = NULL; 414 } else { 415 /* this is a copy monitor */ 416 uint32_t mon_pos = mkring->mon_pos[kring->tx]; 417 kring->n_monitors--; 418 if (mon_pos != kring->n_monitors) { 419 kring->monitors[mon_pos] = 420 kring->monitors[kring->n_monitors]; 421 kring->monitors[mon_pos]->mon_pos[kring->tx] = mon_pos; 422 } 423 kring->monitors[kring->n_monitors] = NULL; 424 if (kring->n_monitors == 0) { 425 nm_monitor_dealloc(kring); 426 } 427 } 428 429 if (ikring != NULL && nm_monitor_none(ikring)) { 430 /* this was the last monitor, restore the callbacks */ 431 nm_monitor_restore_callbacks(ikring); 432 } 433 434 if (kring != NULL) 435 nm_kr_start(kring); 436} 437 438 439/* This is called when the monitored adapter leaves netmap mode 440 * (see netmap_do_unregif). 441 * We need to notify the monitors that the monitored rings are gone. 442 * We do this by setting their mna->priv.np_na to NULL. 443 * Note that the rings are already stopped when this happens, so 444 * no monitor ring callback can be active. 445 */ 446void 447netmap_monitor_stop(struct netmap_adapter *na) 448{ 449 enum txrx t; 450 451 for_rx_tx(t) { 452 u_int i; 453 454 for (i = 0; i < netmap_all_rings(na, t); i++) { 455 struct netmap_kring *kring = NMR(na, t)[i]; 456 struct netmap_zmon_list *z = &kring->zmon_list[t]; 457 u_int j; 458 459 for (j = 0; j < kring->n_monitors; j++) { 460 struct netmap_kring *mkring = 461 kring->monitors[j]; 462 struct netmap_monitor_adapter *mna = 463 (struct netmap_monitor_adapter *)mkring->na; 464 /* forget about this adapter */ 465 if (mna->priv.np_na != NULL) { 466 netmap_adapter_put(mna->priv.np_na); 467 mna->priv.np_na = NULL; 468 } 469 kring->monitors[j] = NULL; 470 } 471 472 if (!nm_is_zmon(na)) { 473 /* we are the head of at most one list */ 474 struct netmap_kring *zkring; 475 for (zkring = z->next; zkring != NULL; 476 zkring = zkring->zmon_list[t].next) 477 { 478 struct netmap_monitor_adapter *next = 479 (struct netmap_monitor_adapter *)zkring->na; 480 /* let the monitor forget about us */ 481 netmap_adapter_put(next->priv.np_na); /* nop if null */ 482 next->priv.np_na = NULL; 483 } 484 /* orhpan the zmon list */ 485 if (z->next != NULL) 486 z->next->zmon_list[t].prev = NULL; 487 z->next = NULL; 488 z->prev = NULL; 489 } 490 491 if (!nm_monitor_none(kring)) { 492 493 kring->n_monitors = 0; 494 nm_monitor_dealloc(kring); 495 nm_monitor_restore_callbacks(kring); 496 } 497 } 498 } 499} 500 501 502/* common functions for the nm_register() callbacks of both kind of 503 * monitors. 504 */ 505static int 506netmap_monitor_reg_common(struct netmap_adapter *na, int onoff, int zmon) 507{ 508 struct netmap_monitor_adapter *mna = 509 (struct netmap_monitor_adapter *)na; 510 struct netmap_priv_d *priv = &mna->priv; 511 struct netmap_adapter *pna = priv->np_na; 512 struct netmap_kring *kring, *mkring; 513 int i; 514 enum txrx t, s; 515 516 nm_prdis("%p: onoff %d", na, onoff); 517 if (onoff) { 518 if (pna == NULL) { 519 /* parent left netmap mode, fatal */ 520 nm_prerr("%s: parent left netmap mode", na->name); 521 return ENXIO; 522 } 523 for_rx_tx(t) { 524 for (i = 0; i < netmap_all_rings(na, t); i++) { 525 mkring = NMR(na, t)[i]; 526 if (!nm_kring_pending_on(mkring)) 527 continue; 528 mkring->nr_mode = NKR_NETMAP_ON; 529 if (t == NR_TX) 530 continue; 531 for_rx_tx(s) { 532 if (i > nma_get_nrings(pna, s)) 533 continue; 534 if (mna->flags & nm_txrx2flag(s)) { 535 kring = NMR(pna, s)[i]; 536 netmap_monitor_add(mkring, kring, zmon); 537 } 538 } 539 } 540 } 541 na->na_flags |= NAF_NETMAP_ON; 542 } else { 543 if (na->active_fds == 0) 544 na->na_flags &= ~NAF_NETMAP_ON; 545 for_rx_tx(t) { 546 for (i = 0; i < netmap_all_rings(na, t); i++) { 547 mkring = NMR(na, t)[i]; 548 if (!nm_kring_pending_off(mkring)) 549 continue; 550 mkring->nr_mode = NKR_NETMAP_OFF; 551 if (t == NR_TX) 552 continue; 553 /* we cannot access the parent krings if the parent 554 * has left netmap mode. This is signaled by a NULL 555 * pna pointer 556 */ 557 if (pna == NULL) 558 continue; 559 for_rx_tx(s) { 560 if (i > nma_get_nrings(pna, s)) 561 continue; 562 if (mna->flags & nm_txrx2flag(s)) { 563 kring = NMR(pna, s)[i]; 564 netmap_monitor_del(mkring, kring, s); 565 } 566 } 567 } 568 } 569 } 570 return 0; 571} 572 573/* 574 **************************************************************** 575 * functions specific for zero-copy monitors 576 **************************************************************** 577 */ 578 579/* 580 * Common function for both zero-copy tx and rx nm_sync() 581 * callbacks 582 */ 583static int 584netmap_zmon_parent_sync(struct netmap_kring *kring, int flags, enum txrx tx) 585{ 586 struct netmap_kring *mkring = kring->zmon_list[tx].next; 587 struct netmap_ring *ring = kring->ring, *mring; 588 int error = 0; 589 int rel_slots, free_slots, busy, sent = 0; 590 u_int beg, end, i; 591 u_int lim = kring->nkr_num_slots - 1, 592 mlim; // = mkring->nkr_num_slots - 1; 593 594 if (mkring == NULL) { 595 nm_prlim(5, "NULL monitor on %s", kring->name); 596 return 0; 597 } 598 mring = mkring->ring; 599 mlim = mkring->nkr_num_slots - 1; 600 601 /* get the relased slots (rel_slots) */ 602 if (tx == NR_TX) { 603 beg = kring->nr_hwtail + 1; 604 error = kring->mon_sync(kring, flags); 605 if (error) 606 return error; 607 end = kring->nr_hwtail + 1; 608 } else { /* NR_RX */ 609 beg = kring->nr_hwcur; 610 end = kring->rhead; 611 } 612 613 rel_slots = end - beg; 614 if (rel_slots < 0) 615 rel_slots += kring->nkr_num_slots; 616 617 if (!rel_slots) { 618 /* no released slots, but we still need 619 * to call rxsync if this is a rx ring 620 */ 621 goto out_rxsync; 622 } 623 624 /* we need to lock the monitor receive ring, since it 625 * is the target of bot tx and rx traffic from the monitored 626 * adapter 627 */ 628 mtx_lock(&mkring->q_lock); 629 /* get the free slots available on the monitor ring */ 630 i = mkring->nr_hwtail; 631 busy = i - mkring->nr_hwcur; 632 if (busy < 0) 633 busy += mkring->nkr_num_slots; 634 free_slots = mlim - busy; 635 636 if (!free_slots) 637 goto out; 638 639 /* swap min(free_slots, rel_slots) slots */ 640 if (free_slots < rel_slots) { 641 beg += (rel_slots - free_slots); 642 rel_slots = free_slots; 643 } 644 if (unlikely(beg >= kring->nkr_num_slots)) 645 beg -= kring->nkr_num_slots; 646 647 sent = rel_slots; 648 for ( ; rel_slots; rel_slots--) { 649 struct netmap_slot *s = &ring->slot[beg]; 650 struct netmap_slot *ms = &mring->slot[i]; 651 uint32_t tmp; 652 653 tmp = ms->buf_idx; 654 ms->buf_idx = s->buf_idx; 655 s->buf_idx = tmp; 656 nm_prdis(5, "beg %d buf_idx %d", beg, tmp); 657 658 tmp = ms->len; 659 ms->len = s->len; 660 s->len = tmp; 661 662 ms->flags = s->flags; 663 s->flags |= NS_BUF_CHANGED; 664 665 beg = nm_next(beg, lim); 666 i = nm_next(i, mlim); 667 668 } 669 mb(); 670 mkring->nr_hwtail = i; 671 672out: 673 mtx_unlock(&mkring->q_lock); 674 675 if (sent) { 676 /* notify the new frames to the monitor */ 677 mkring->nm_notify(mkring, 0); 678 } 679 680out_rxsync: 681 if (tx == NR_RX) 682 error = kring->mon_sync(kring, flags); 683 684 return error; 685} 686 687/* callback used to replace the nm_sync callback in the monitored tx rings */ 688static int 689netmap_zmon_parent_txsync(struct netmap_kring *kring, int flags) 690{ 691 return netmap_zmon_parent_sync(kring, flags, NR_TX); 692} 693 694/* callback used to replace the nm_sync callback in the monitored rx rings */ 695static int 696netmap_zmon_parent_rxsync(struct netmap_kring *kring, int flags) 697{ 698 return netmap_zmon_parent_sync(kring, flags, NR_RX); 699} 700 701static int 702netmap_zmon_reg(struct netmap_adapter *na, int onoff) 703{ 704 return netmap_monitor_reg_common(na, onoff, 1 /* zcopy */); 705} 706 707/* nm_dtor callback for monitors */ 708static void 709netmap_zmon_dtor(struct netmap_adapter *na) 710{ 711 struct netmap_monitor_adapter *mna = 712 (struct netmap_monitor_adapter *)na; 713 struct netmap_priv_d *priv = &mna->priv; 714 struct netmap_adapter *pna = priv->np_na; 715 716 netmap_adapter_put(pna); 717} 718 719/* 720 **************************************************************** 721 * functions specific for copy monitors 722 **************************************************************** 723 */ 724 725static void 726netmap_monitor_parent_sync(struct netmap_kring *kring, u_int first_new, int new_slots) 727{ 728 u_int j; 729 730 for (j = 0; j < kring->n_monitors; j++) { 731 struct netmap_kring *mkring = kring->monitors[j]; 732 u_int i, mlim, beg; 733 int free_slots, busy, sent = 0, m; 734 u_int lim = kring->nkr_num_slots - 1; 735 struct netmap_ring *ring = kring->ring, *mring = mkring->ring; 736 u_int max_len = NETMAP_BUF_SIZE(mkring->na); 737 738 mlim = mkring->nkr_num_slots - 1; 739 740 /* we need to lock the monitor receive ring, since it 741 * is the target of bot tx and rx traffic from the monitored 742 * adapter 743 */ 744 mtx_lock(&mkring->q_lock); 745 /* get the free slots available on the monitor ring */ 746 i = mkring->nr_hwtail; 747 busy = i - mkring->nr_hwcur; 748 if (busy < 0) 749 busy += mkring->nkr_num_slots; 750 free_slots = mlim - busy; 751 752 if (!free_slots) 753 goto out; 754 755 /* copy min(free_slots, new_slots) slots */ 756 m = new_slots; 757 beg = first_new; 758 if (free_slots < m) { 759 beg += (m - free_slots); 760 if (beg >= kring->nkr_num_slots) 761 beg -= kring->nkr_num_slots; 762 m = free_slots; 763 } 764 765 for ( ; m; m--) { 766 struct netmap_slot *s = &ring->slot[beg]; 767 struct netmap_slot *ms = &mring->slot[i]; 768 u_int copy_len = s->len; 769 char *src = NMB(kring->na, s), 770 *dst = NMB(mkring->na, ms); 771 772 if (unlikely(copy_len > max_len)) { 773 nm_prlim(5, "%s->%s: truncating %d to %d", kring->name, 774 mkring->name, copy_len, max_len); 775 copy_len = max_len; 776 } 777 778 memcpy(dst, src, copy_len); 779 ms->len = copy_len; 780 ms->flags = s->flags; 781 sent++; 782 783 beg = nm_next(beg, lim); 784 i = nm_next(i, mlim); 785 } 786 mb(); 787 mkring->nr_hwtail = i; 788 out: 789 mtx_unlock(&mkring->q_lock); 790 791 if (sent) { 792 /* notify the new frames to the monitor */ 793 mkring->nm_notify(mkring, 0); 794 } 795 } 796} 797 798/* callback used to replace the nm_sync callback in the monitored tx rings */ 799static int 800netmap_monitor_parent_txsync(struct netmap_kring *kring, int flags) 801{ 802 u_int first_new; 803 int new_slots; 804 805 /* get the new slots */ 806 if (kring->n_monitors > 0) { 807 first_new = kring->nr_hwcur; 808 new_slots = kring->rhead - first_new; 809 if (new_slots < 0) 810 new_slots += kring->nkr_num_slots; 811 if (new_slots) 812 netmap_monitor_parent_sync(kring, first_new, new_slots); 813 } 814 if (kring->zmon_list[NR_TX].next != NULL) { 815 return netmap_zmon_parent_txsync(kring, flags); 816 } 817 return kring->mon_sync(kring, flags); 818} 819 820/* callback used to replace the nm_sync callback in the monitored rx rings */ 821static int 822netmap_monitor_parent_rxsync(struct netmap_kring *kring, int flags) 823{ 824 u_int first_new; 825 int new_slots, error; 826 827 /* get the new slots */ 828 if (kring->zmon_list[NR_RX].next != NULL) { 829 error = netmap_zmon_parent_rxsync(kring, flags); 830 } else { 831 error = kring->mon_sync(kring, flags); 832 } 833 if (error) 834 return error; 835 if (kring->n_monitors > 0) { 836 first_new = kring->mon_tail; 837 new_slots = kring->nr_hwtail - first_new; 838 if (new_slots < 0) 839 new_slots += kring->nkr_num_slots; 840 if (new_slots) 841 netmap_monitor_parent_sync(kring, first_new, new_slots); 842 kring->mon_tail = kring->nr_hwtail; 843 } 844 return 0; 845} 846 847/* callback used to replace the nm_notify() callback in the monitored rx rings */ 848static int 849netmap_monitor_parent_notify(struct netmap_kring *kring, int flags) 850{ 851 int (*notify)(struct netmap_kring*, int); 852 nm_prdis(5, "%s %x", kring->name, flags); 853 /* ?xsync callbacks have tryget called by their callers 854 * (NIOCREGIF and poll()), but here we have to call it 855 * by ourself 856 */ 857 if (nm_kr_tryget(kring, 0, NULL)) { 858 /* in all cases, just skip the sync */ 859 return NM_IRQ_COMPLETED; 860 } 861 if (kring->n_monitors > 0) { 862 netmap_monitor_parent_rxsync(kring, NAF_FORCE_READ); 863 } 864 if (nm_monitor_none(kring)) { 865 /* we are no longer monitoring this ring, so both 866 * mon_sync and mon_notify are NULL 867 */ 868 notify = kring->nm_notify; 869 } else { 870 notify = kring->mon_notify; 871 } 872 nm_kr_put(kring); 873 return notify(kring, flags); 874} 875 876 877static int 878netmap_monitor_reg(struct netmap_adapter *na, int onoff) 879{ 880 return netmap_monitor_reg_common(na, onoff, 0 /* no zcopy */); 881} 882 883static void 884netmap_monitor_dtor(struct netmap_adapter *na) 885{ 886 struct netmap_monitor_adapter *mna = 887 (struct netmap_monitor_adapter *)na; 888 struct netmap_priv_d *priv = &mna->priv; 889 struct netmap_adapter *pna = priv->np_na; 890 891 netmap_adapter_put(pna); 892} 893 894 895/* check if req is a request for a monitor adapter that we can satisfy */ 896int 897netmap_get_monitor_na(struct nmreq_header *hdr, struct netmap_adapter **na, 898 struct netmap_mem_d *nmd, int create) 899{ 900 struct nmreq_register *req = (struct nmreq_register *)(uintptr_t)hdr->nr_body; 901 struct nmreq_register preq; 902 struct netmap_adapter *pna; /* parent adapter */ 903 struct netmap_monitor_adapter *mna; 904 struct ifnet *ifp = NULL; 905 int error; 906 int zcopy = (req->nr_flags & NR_ZCOPY_MON); 907 908 if (zcopy) { 909 req->nr_flags |= (NR_MONITOR_TX | NR_MONITOR_RX); 910 } 911 if ((req->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX)) == 0) { 912 nm_prdis("not a monitor"); 913 return 0; 914 } 915 /* this is a request for a monitor adapter */ 916 917 nm_prdis("flags %lx", req->nr_flags); 918 919 /* First, try to find the adapter that we want to monitor. 920 * We use the same req, after we have turned off the monitor flags. 921 * In this way we can potentially monitor everything netmap understands, 922 * except other monitors. 923 */ 924 memcpy(&preq, req, sizeof(preq)); 925 preq.nr_flags &= ~(NR_MONITOR_TX | NR_MONITOR_RX | NR_ZCOPY_MON); 926 hdr->nr_body = (uintptr_t)&preq; 927 error = netmap_get_na(hdr, &pna, &ifp, nmd, create); 928 hdr->nr_body = (uintptr_t)req; 929 if (error) { 930 nm_prerr("parent lookup failed: %d", error); 931 return error; 932 } 933 nm_prdis("found parent: %s", pna->name); 934 935 if (!nm_netmap_on(pna)) { 936 /* parent not in netmap mode */ 937 /* XXX we can wait for the parent to enter netmap mode, 938 * by intercepting its nm_register callback (2014-03-16) 939 */ 940 nm_prerr("%s not in netmap mode", pna->name); 941 error = EINVAL; 942 goto put_out; 943 } 944 945 mna = nm_os_malloc(sizeof(*mna)); 946 if (mna == NULL) { 947 error = ENOMEM; 948 goto put_out; 949 } 950 mna->priv.np_na = pna; 951 952 /* grab all the rings we need in the parent */ 953 error = netmap_interp_ringid(&mna->priv, req->nr_mode, req->nr_ringid, 954 req->nr_flags); 955 if (error) { 956 nm_prerr("ringid error"); 957 goto free_out; 958 } 959 snprintf(mna->up.name, sizeof(mna->up.name), "%s/%s%s%s#%lu", pna->name, 960 zcopy ? "z" : "", 961 (req->nr_flags & NR_MONITOR_RX) ? "r" : "", 962 (req->nr_flags & NR_MONITOR_TX) ? "t" : "", 963 pna->monitor_id++); 964 965 /* the monitor supports the host rings iff the parent does */ 966 mna->up.na_flags |= (pna->na_flags & NAF_HOST_RINGS); 967 /* a do-nothing txsync: monitors cannot be used to inject packets */ 968 mna->up.nm_txsync = netmap_monitor_txsync; 969 mna->up.nm_rxsync = netmap_monitor_rxsync; 970 mna->up.nm_krings_create = netmap_monitor_krings_create; 971 mna->up.nm_krings_delete = netmap_monitor_krings_delete; 972 mna->up.num_tx_rings = 1; // XXX what should we do here with chained zmons? 973 /* we set the number of our rx_rings to be max(num_rx_rings, num_rx_rings) 974 * in the parent 975 */ 976 mna->up.num_rx_rings = pna->num_rx_rings; 977 if (pna->num_tx_rings > pna->num_rx_rings) 978 mna->up.num_rx_rings = pna->num_tx_rings; 979 /* by default, the number of slots is the same as in 980 * the parent rings, but the user may ask for a different 981 * number 982 */ 983 mna->up.num_tx_desc = req->nr_tx_slots; 984 nm_bound_var(&mna->up.num_tx_desc, pna->num_tx_desc, 985 1, NM_MONITOR_MAXSLOTS, NULL); 986 mna->up.num_rx_desc = req->nr_rx_slots; 987 nm_bound_var(&mna->up.num_rx_desc, pna->num_rx_desc, 988 1, NM_MONITOR_MAXSLOTS, NULL); 989 if (zcopy) { 990 mna->up.nm_register = netmap_zmon_reg; 991 mna->up.nm_dtor = netmap_zmon_dtor; 992 /* to have zero copy, we need to use the same memory allocator 993 * as the monitored port 994 */ 995 mna->up.nm_mem = netmap_mem_get(pna->nm_mem); 996 /* and the allocator cannot be changed */ 997 mna->up.na_flags |= NAF_MEM_OWNER; 998 } else { 999 mna->up.nm_register = netmap_monitor_reg; 1000 mna->up.nm_dtor = netmap_monitor_dtor; 1001 mna->up.nm_mem = netmap_mem_private_new( 1002 mna->up.num_tx_rings, 1003 mna->up.num_tx_desc, 1004 mna->up.num_rx_rings, 1005 mna->up.num_rx_desc, 1006 0, /* extra bufs */ 1007 0, /* pipes */ 1008 &error); 1009 if (mna->up.nm_mem == NULL) 1010 goto put_out; 1011 } 1012 1013 error = netmap_attach_common(&mna->up); 1014 if (error) { 1015 nm_prerr("netmap_attach_common failed"); 1016 goto mem_put_out; 1017 } 1018 1019 /* remember the traffic directions we have to monitor */ 1020 mna->flags = (req->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX | NR_ZCOPY_MON)); 1021 1022 *na = &mna->up; 1023 netmap_adapter_get(*na); 1024 1025 /* keep the reference to the parent */ 1026 nm_prdis("monitor ok"); 1027 1028 /* drop the reference to the ifp, if any */ 1029 if (ifp) 1030 if_rele(ifp); 1031 1032 return 0; 1033 1034mem_put_out: 1035 netmap_mem_put(mna->up.nm_mem); 1036free_out: 1037 nm_os_free(mna); 1038put_out: 1039 netmap_unget_na(pna, ifp); 1040 return error; 1041} 1042 1043 1044#endif /* WITH_MONITOR */ 1045