Deleted Added
sdiff udiff text old ( 267164 ) new ( 270063 )
full compact
netmap_vale.c (267164) netmap_vale.c (270063)
1/*
2 * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 43 unchanged lines hidden (view full) ---

52/*
53 * OS-specific code that is used only within this file.
54 * Other OS-specific code that must be accessed by drivers
55 * is present in netmap_kern.h
56 */
57
58#if defined(__FreeBSD__)
59#include <sys/cdefs.h> /* prerequisite */
1/*
2 * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 43 unchanged lines hidden (view full) ---

52/*
53 * OS-specific code that is used only within this file.
54 * Other OS-specific code that must be accessed by drivers
55 * is present in netmap_kern.h
56 */
57
58#if defined(__FreeBSD__)
59#include <sys/cdefs.h> /* prerequisite */
60__FBSDID("$FreeBSD: head/sys/dev/netmap/netmap_vale.c 267164 2014-06-06 14:57:40Z luigi $");
60__FBSDID("$FreeBSD: head/sys/dev/netmap/netmap_vale.c 270063 2014-08-16 15:00:01Z luigi $");
61
62#include <sys/types.h>
63#include <sys/errno.h>
64#include <sys/param.h> /* defines used in kernel.h */
65#include <sys/kernel.h> /* types used in module initialization */
66#include <sys/conf.h> /* cdevsw struct, UID, GID */
67#include <sys/sockio.h>
68#include <sys/socketvar.h> /* struct socket */

--- 83 unchanged lines hidden (view full) ---

152 * used in the bridge. The actual value may be larger as the
153 * last packet in the block may overflow the size.
154 */
155int bridge_batch = NM_BDG_BATCH; /* bridge batch size */
156SYSCTL_DECL(_dev_netmap);
157SYSCTL_INT(_dev_netmap, OID_AUTO, bridge_batch, CTLFLAG_RW, &bridge_batch, 0 , "");
158
159
61
62#include <sys/types.h>
63#include <sys/errno.h>
64#include <sys/param.h> /* defines used in kernel.h */
65#include <sys/kernel.h> /* types used in module initialization */
66#include <sys/conf.h> /* cdevsw struct, UID, GID */
67#include <sys/sockio.h>
68#include <sys/socketvar.h> /* struct socket */

--- 83 unchanged lines hidden (view full) ---

152 * used in the bridge. The actual value may be larger as the
153 * last packet in the block may overflow the size.
154 */
155int bridge_batch = NM_BDG_BATCH; /* bridge batch size */
156SYSCTL_DECL(_dev_netmap);
157SYSCTL_INT(_dev_netmap, OID_AUTO, bridge_batch, CTLFLAG_RW, &bridge_batch, 0 , "");
158
159
160static int bdg_netmap_attach(struct nmreq *nmr, struct ifnet *ifp);
161static int bdg_netmap_reg(struct netmap_adapter *na, int onoff);
162static int netmap_bwrap_attach(struct ifnet *, struct ifnet *);
160static int netmap_vp_create(struct nmreq *, struct ifnet *, struct netmap_vp_adapter **);
161static int netmap_vp_reg(struct netmap_adapter *na, int onoff);
163static int netmap_bwrap_register(struct netmap_adapter *, int onoff);
162static int netmap_bwrap_register(struct netmap_adapter *, int onoff);
164int kern_netmap_regif(struct nmreq *nmr);
165
166/*
167 * For each output interface, nm_bdg_q is used to construct a list.
168 * bq_len is the number of output buffers (we can have coalescing
169 * during the copy).
170 */
171struct nm_bdg_q {
172 uint16_t bq_head;

--- 39 unchanged lines hidden (view full) ---

212 * The function to decide the destination port.
213 * It returns either of an index of the destination port,
214 * NM_BDG_BROADCAST to broadcast this packet, or NM_BDG_NOPORT not to
215 * forward this packet. ring_nr is the source ring index, and the
216 * function may overwrite this value to forward this packet to a
217 * different ring index.
218 * This function must be set by netmap_bdgctl().
219 */
163
164/*
165 * For each output interface, nm_bdg_q is used to construct a list.
166 * bq_len is the number of output buffers (we can have coalescing
167 * during the copy).
168 */
169struct nm_bdg_q {
170 uint16_t bq_head;

--- 39 unchanged lines hidden (view full) ---

210 * The function to decide the destination port.
211 * It returns either of an index of the destination port,
212 * NM_BDG_BROADCAST to broadcast this packet, or NM_BDG_NOPORT not to
213 * forward this packet. ring_nr is the source ring index, and the
214 * function may overwrite this value to forward this packet to a
215 * different ring index.
216 * This function must be set by netmap_bdgctl().
217 */
220 bdg_lookup_fn_t nm_bdg_lookup;
218 struct netmap_bdg_ops bdg_ops;
221
222 /* the forwarding table, MAC+ports.
223 * XXX should be changed to an argument to be passed to
224 * the lookup function, and allocated on attach
225 */
226 struct nm_hash_ent ht[NM_BDG_HASH];
227};
228
219
220 /* the forwarding table, MAC+ports.
221 * XXX should be changed to an argument to be passed to
222 * the lookup function, and allocated on attach
223 */
224 struct nm_hash_ent ht[NM_BDG_HASH];
225};
226
227const char*
228netmap_bdg_name(struct netmap_vp_adapter *vp)
229{
230 struct nm_bridge *b = vp->na_bdg;
231 if (b == NULL)
232 return NULL;
233 return b->bdg_basename;
234}
229
235
236
230/*
231 * XXX in principle nm_bridges could be created dynamically
232 * Right now we have a static array and deletions are protected
233 * by an exclusive lock.
234 */
235struct nm_bridge nm_bridges[NM_BRIDGES];
236
237

--- 78 unchanged lines hidden (view full) ---

316 strncpy(b->bdg_basename, name, namelen);
317 ND("create new bridge %s with ports %d", b->bdg_basename,
318 b->bdg_active_ports);
319 b->bdg_namelen = namelen;
320 b->bdg_active_ports = 0;
321 for (i = 0; i < NM_BDG_MAXPORTS; i++)
322 b->bdg_port_index[i] = i;
323 /* set the default function */
237/*
238 * XXX in principle nm_bridges could be created dynamically
239 * Right now we have a static array and deletions are protected
240 * by an exclusive lock.
241 */
242struct nm_bridge nm_bridges[NM_BRIDGES];
243
244

--- 78 unchanged lines hidden (view full) ---

323 strncpy(b->bdg_basename, name, namelen);
324 ND("create new bridge %s with ports %d", b->bdg_basename,
325 b->bdg_active_ports);
326 b->bdg_namelen = namelen;
327 b->bdg_active_ports = 0;
328 for (i = 0; i < NM_BDG_MAXPORTS; i++)
329 b->bdg_port_index[i] = i;
330 /* set the default function */
324 b->nm_bdg_lookup = netmap_bdg_learning;
331 b->bdg_ops.lookup = netmap_bdg_learning;
325 /* reset the MAC address table */
326 bzero(b->ht, sizeof(struct nm_hash_ent) * NM_BDG_HASH);
327 }
328 return b;
329}
330
331
332/*

--- 51 unchanged lines hidden (view full) ---

384 dstq[j].bq_len = 0;
385 }
386 kring[i].nkr_ft = ft;
387 }
388 return 0;
389}
390
391
332 /* reset the MAC address table */
333 bzero(b->ht, sizeof(struct nm_hash_ent) * NM_BDG_HASH);
334 }
335 return b;
336}
337
338
339/*

--- 51 unchanged lines hidden (view full) ---

391 dstq[j].bq_len = 0;
392 }
393 kring[i].nkr_ft = ft;
394 }
395 return 0;
396}
397
398
399/* remove from bridge b the ports in slots hw and sw
400 * (sw can be -1 if not needed)
401 */
392static void
393netmap_bdg_detach_common(struct nm_bridge *b, int hw, int sw)
394{
395 int s_hw = hw, s_sw = sw;
396 int i, lim =b->bdg_active_ports;
397 uint8_t tmp[NM_BDG_MAXPORTS];
398
399 /*

--- 29 unchanged lines hidden (view full) ---

429 i++;
430 }
431 }
432 if (hw >= 0 || sw >= 0) {
433 D("XXX delete failed hw %d sw %d, should panic...", hw, sw);
434 }
435
436 BDG_WLOCK(b);
402static void
403netmap_bdg_detach_common(struct nm_bridge *b, int hw, int sw)
404{
405 int s_hw = hw, s_sw = sw;
406 int i, lim =b->bdg_active_ports;
407 uint8_t tmp[NM_BDG_MAXPORTS];
408
409 /*

--- 29 unchanged lines hidden (view full) ---

439 i++;
440 }
441 }
442 if (hw >= 0 || sw >= 0) {
443 D("XXX delete failed hw %d sw %d, should panic...", hw, sw);
444 }
445
446 BDG_WLOCK(b);
447 if (b->bdg_ops.dtor)
448 b->bdg_ops.dtor(b->bdg_ports[s_hw]);
437 b->bdg_ports[s_hw] = NULL;
438 if (s_sw >= 0) {
439 b->bdg_ports[s_sw] = NULL;
440 }
441 memcpy(b->bdg_port_index, tmp, sizeof(tmp));
442 b->bdg_active_ports = lim;
443 BDG_WUNLOCK(b);
444
445 ND("now %d active ports", lim);
446 if (lim == 0) {
447 ND("marking bridge %s as free", b->bdg_basename);
449 b->bdg_ports[s_hw] = NULL;
450 if (s_sw >= 0) {
451 b->bdg_ports[s_sw] = NULL;
452 }
453 memcpy(b->bdg_port_index, tmp, sizeof(tmp));
454 b->bdg_active_ports = lim;
455 BDG_WUNLOCK(b);
456
457 ND("now %d active ports", lim);
458 if (lim == 0) {
459 ND("marking bridge %s as free", b->bdg_basename);
448 b->nm_bdg_lookup = NULL;
460 bzero(&b->bdg_ops, sizeof(b->bdg_ops));
449 }
450}
451
461 }
462}
463
464/* nm_bdg_ctl callback for VALE ports */
465static int
466netmap_vp_bdg_ctl(struct netmap_adapter *na, struct nmreq *nmr, int attach)
467{
468 struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter *)na;
469 struct nm_bridge *b = vpna->na_bdg;
452
470
471 if (attach)
472 return 0; /* nothing to do */
473 if (b) {
474 netmap_set_all_rings(na, 0 /* disable */);
475 netmap_bdg_detach_common(b, vpna->bdg_port, -1);
476 vpna->na_bdg = NULL;
477 netmap_set_all_rings(na, 1 /* enable */);
478 }
479 /* I have took reference just for attach */
480 netmap_adapter_put(na);
481 return 0;
482}
483
484/* nm_dtor callback for ephemeral VALE ports */
453static void
485static void
454netmap_adapter_vp_dtor(struct netmap_adapter *na)
486netmap_vp_dtor(struct netmap_adapter *na)
455{
456 struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter*)na;
457 struct nm_bridge *b = vpna->na_bdg;
487{
488 struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter*)na;
489 struct nm_bridge *b = vpna->na_bdg;
458 struct ifnet *ifp = na->ifp;
459
490
460 ND("%s has %d references", NM_IFPNAME(ifp), na->na_refcount);
491 ND("%s has %d references", na->name, na->na_refcount);
461
462 if (b) {
463 netmap_bdg_detach_common(b, vpna->bdg_port, -1);
464 }
492
493 if (b) {
494 netmap_bdg_detach_common(b, vpna->bdg_port, -1);
495 }
496}
465
497
466 bzero(ifp, sizeof(*ifp));
467 free(ifp, M_DEVBUF);
498/* nm_dtor callback for persistent VALE ports */
499static void
500netmap_persist_vp_dtor(struct netmap_adapter *na)
501{
502 struct ifnet *ifp = na->ifp;
503
504 netmap_vp_dtor(na);
468 na->ifp = NULL;
505 na->ifp = NULL;
506 nm_vi_detach(ifp);
469}
470
507}
508
509/* remove a persistent VALE port from the system */
510static int
511nm_vi_destroy(const char *name)
512{
513 struct ifnet *ifp;
514 int error;
471
515
516 ifp = ifunit_ref(name);
517 if (!ifp)
518 return ENXIO;
519 NMG_LOCK();
520 /* make sure this is actually a VALE port */
521 if (!NETMAP_CAPABLE(ifp) || NA(ifp)->nm_register != netmap_vp_reg) {
522 error = EINVAL;
523 goto err;
524 }
525
526 if (NA(ifp)->na_refcount > 1) {
527 error = EBUSY;
528 goto err;
529 }
530 NMG_UNLOCK();
531
532 D("destroying a persistent vale interface %s", ifp->if_xname);
533 /* Linux requires all the references are released
534 * before unregister
535 */
536 if_rele(ifp);
537 netmap_detach(ifp);
538 return 0;
539
540err:
541 NMG_UNLOCK();
542 if_rele(ifp);
543 return error;
544}
545
546/*
547 * Create a virtual interface registered to the system.
548 * The interface will be attached to a bridge later.
549 */
550static int
551nm_vi_create(struct nmreq *nmr)
552{
553 struct ifnet *ifp;
554 struct netmap_vp_adapter *vpna;
555 int error;
556
557 /* don't include VALE prefix */
558 if (!strncmp(nmr->nr_name, NM_NAME, strlen(NM_NAME)))
559 return EINVAL;
560 ifp = ifunit_ref(nmr->nr_name);
561 if (ifp) { /* already exist, cannot create new one */
562 if_rele(ifp);
563 return EEXIST;
564 }
565 error = nm_vi_persist(nmr->nr_name, &ifp);
566 if (error)
567 return error;
568
569 NMG_LOCK();
570 /* netmap_vp_create creates a struct netmap_vp_adapter */
571 error = netmap_vp_create(nmr, ifp, &vpna);
572 if (error) {
573 D("error %d", error);
574 nm_vi_detach(ifp);
575 return error;
576 }
577 /* persist-specific routines */
578 vpna->up.nm_bdg_ctl = netmap_vp_bdg_ctl;
579 vpna->up.nm_dtor = netmap_persist_vp_dtor;
580 netmap_adapter_get(&vpna->up);
581 NMG_UNLOCK();
582 D("created %s", ifp->if_xname);
583 return 0;
584}
585
472/* Try to get a reference to a netmap adapter attached to a VALE switch.
473 * If the adapter is found (or is created), this function returns 0, a
474 * non NULL pointer is returned into *na, and the caller holds a
475 * reference to the adapter.
476 * If an adapter is not found, then no reference is grabbed and the
477 * function returns an error code, or 0 if there is just a VALE prefix
478 * mismatch. Therefore the caller holds a reference when
479 * (*na != NULL && return == 0).
480 */
481int
482netmap_get_bdg_na(struct nmreq *nmr, struct netmap_adapter **na, int create)
483{
586/* Try to get a reference to a netmap adapter attached to a VALE switch.
587 * If the adapter is found (or is created), this function returns 0, a
588 * non NULL pointer is returned into *na, and the caller holds a
589 * reference to the adapter.
590 * If an adapter is not found, then no reference is grabbed and the
591 * function returns an error code, or 0 if there is just a VALE prefix
592 * mismatch. Therefore the caller holds a reference when
593 * (*na != NULL && return == 0).
594 */
595int
596netmap_get_bdg_na(struct nmreq *nmr, struct netmap_adapter **na, int create)
597{
484 const char *name = nmr->nr_name;
598 char *nr_name = nmr->nr_name;
599 const char *ifname;
485 struct ifnet *ifp;
486 int error = 0;
600 struct ifnet *ifp;
601 int error = 0;
487 struct netmap_adapter *ret;
488 struct netmap_vp_adapter *vpna;
602 struct netmap_vp_adapter *vpna, *hostna = NULL;
489 struct nm_bridge *b;
490 int i, j, cand = -1, cand2 = -1;
491 int needed;
492
493 *na = NULL; /* default return value */
494
495 /* first try to see if this is a bridge port. */
496 NMG_LOCK_ASSERT();
603 struct nm_bridge *b;
604 int i, j, cand = -1, cand2 = -1;
605 int needed;
606
607 *na = NULL; /* default return value */
608
609 /* first try to see if this is a bridge port. */
610 NMG_LOCK_ASSERT();
497 if (strncmp(name, NM_NAME, sizeof(NM_NAME) - 1)) {
611 if (strncmp(nr_name, NM_NAME, sizeof(NM_NAME) - 1)) {
498 return 0; /* no error, but no VALE prefix */
499 }
500
612 return 0; /* no error, but no VALE prefix */
613 }
614
501 b = nm_find_bridge(name, create);
615 b = nm_find_bridge(nr_name, create);
502 if (b == NULL) {
616 if (b == NULL) {
503 D("no bridges available for '%s'", name);
617 D("no bridges available for '%s'", nr_name);
504 return (create ? ENOMEM : ENXIO);
505 }
618 return (create ? ENOMEM : ENXIO);
619 }
620 if (strlen(nr_name) < b->bdg_namelen) /* impossible */
621 panic("x");
506
507 /* Now we are sure that name starts with the bridge's name,
508 * lookup the port in the bridge. We need to scan the entire
509 * list. It is not important to hold a WLOCK on the bridge
510 * during the search because NMG_LOCK already guarantees
511 * that there are no other possible writers.
512 */
513
514 /* lookup in the local list of ports */
515 for (j = 0; j < b->bdg_active_ports; j++) {
516 i = b->bdg_port_index[j];
517 vpna = b->bdg_ports[i];
518 // KASSERT(na != NULL);
622
623 /* Now we are sure that name starts with the bridge's name,
624 * lookup the port in the bridge. We need to scan the entire
625 * list. It is not important to hold a WLOCK on the bridge
626 * during the search because NMG_LOCK already guarantees
627 * that there are no other possible writers.
628 */
629
630 /* lookup in the local list of ports */
631 for (j = 0; j < b->bdg_active_ports; j++) {
632 i = b->bdg_port_index[j];
633 vpna = b->bdg_ports[i];
634 // KASSERT(na != NULL);
519 ifp = vpna->up.ifp;
520 /* XXX make sure the name only contains one : */
521 if (!strcmp(NM_IFPNAME(ifp), name)) {
635 D("checking %s", vpna->up.name);
636 if (!strcmp(vpna->up.name, nr_name)) {
522 netmap_adapter_get(&vpna->up);
637 netmap_adapter_get(&vpna->up);
523 ND("found existing if %s refs %d", name,
524 vpna->na_bdg_refcount);
525 *na = (struct netmap_adapter *)vpna;
638 ND("found existing if %s refs %d", nr_name)
639 *na = &vpna->up;
526 return 0;
527 }
528 }
529 /* not found, should we create it? */
530 if (!create)
531 return ENXIO;
532 /* yes we should, see if we have space to attach entries */
533 needed = 2; /* in some cases we only need 1 */
534 if (b->bdg_active_ports + needed >= NM_BDG_MAXPORTS) {
535 D("bridge full %d, cannot create new port", b->bdg_active_ports);
536 return ENOMEM;
537 }
538 /* record the next two ports available, but do not allocate yet */
539 cand = b->bdg_port_index[b->bdg_active_ports];
540 cand2 = b->bdg_port_index[b->bdg_active_ports + 1];
541 ND("+++ bridge %s port %s used %d avail %d %d",
640 return 0;
641 }
642 }
643 /* not found, should we create it? */
644 if (!create)
645 return ENXIO;
646 /* yes we should, see if we have space to attach entries */
647 needed = 2; /* in some cases we only need 1 */
648 if (b->bdg_active_ports + needed >= NM_BDG_MAXPORTS) {
649 D("bridge full %d, cannot create new port", b->bdg_active_ports);
650 return ENOMEM;
651 }
652 /* record the next two ports available, but do not allocate yet */
653 cand = b->bdg_port_index[b->bdg_active_ports];
654 cand2 = b->bdg_port_index[b->bdg_active_ports + 1];
655 ND("+++ bridge %s port %s used %d avail %d %d",
542 b->bdg_basename, name, b->bdg_active_ports, cand, cand2);
656 b->bdg_basename, ifname, b->bdg_active_ports, cand, cand2);
543
544 /*
545 * try see if there is a matching NIC with this name
546 * (after the bridge's name)
547 */
657
658 /*
659 * try see if there is a matching NIC with this name
660 * (after the bridge's name)
661 */
548 ifp = ifunit_ref(name + b->bdg_namelen + 1);
549 if (!ifp) { /* this is a virtual port */
662 ifname = nr_name + b->bdg_namelen + 1;
663 ifp = ifunit_ref(ifname);
664 if (!ifp) {
665 /* Create an ephemeral virtual port
666 * This block contains all the ephemeral-specific logics
667 */
550 if (nmr->nr_cmd) {
551 /* nr_cmd must be 0 for a virtual port */
552 return EINVAL;
553 }
554
668 if (nmr->nr_cmd) {
669 /* nr_cmd must be 0 for a virtual port */
670 return EINVAL;
671 }
672
555 /* create a struct ifnet for the new port.
556 * need M_NOWAIT as we are under nma_lock
557 */
558 ifp = malloc(sizeof(*ifp), M_DEVBUF, M_NOWAIT | M_ZERO);
559 if (!ifp)
560 return ENOMEM;
561
562 strcpy(ifp->if_xname, name);
563 /* bdg_netmap_attach creates a struct netmap_adapter */
673 /* bdg_netmap_attach creates a struct netmap_adapter */
564 error = bdg_netmap_attach(nmr, ifp);
674 error = netmap_vp_create(nmr, NULL, &vpna);
565 if (error) {
566 D("error %d", error);
567 free(ifp, M_DEVBUF);
568 return error;
569 }
675 if (error) {
676 D("error %d", error);
677 free(ifp, M_DEVBUF);
678 return error;
679 }
570 ret = NA(ifp);
571 cand2 = -1; /* only need one port */
572 } else { /* this is a NIC */
573 struct ifnet *fake_ifp;
680 /* shortcut - we can skip get_hw_na(),
681 * ownership check and nm_bdg_attach()
682 */
683 } else {
684 struct netmap_adapter *hw;
574
685
575 error = netmap_get_hw_na(ifp, &ret);
576 if (error || ret == NULL)
686 error = netmap_get_hw_na(ifp, &hw);
687 if (error || hw == NULL)
577 goto out;
578
688 goto out;
689
579 /* make sure the NIC is not already in use */
580 if (NETMAP_OWNED_BY_ANY(ret)) {
581 D("NIC %s busy, cannot attach to bridge",
582 NM_IFPNAME(ifp));
583 error = EBUSY;
690 /* host adapter might not be created */
691 error = hw->nm_bdg_attach(nr_name, hw);
692 if (error)
584 goto out;
693 goto out;
585 }
586 /* create a fake interface */
587 fake_ifp = malloc(sizeof(*ifp), M_DEVBUF, M_NOWAIT | M_ZERO);
588 if (!fake_ifp) {
589 error = ENOMEM;
590 goto out;
591 }
592 strcpy(fake_ifp->if_xname, name);
593 error = netmap_bwrap_attach(fake_ifp, ifp);
594 if (error) {
595 free(fake_ifp, M_DEVBUF);
596 goto out;
597 }
598 ret = NA(fake_ifp);
599 if (nmr->nr_arg1 != NETMAP_BDG_HOST)
600 cand2 = -1; /* only need one port */
694 vpna = hw->na_vp;
695 hostna = hw->na_hostvp;
601 if_rele(ifp);
696 if_rele(ifp);
697 if (nmr->nr_arg1 != NETMAP_BDG_HOST)
698 hostna = NULL;
602 }
699 }
603 vpna = (struct netmap_vp_adapter *)ret;
604
605 BDG_WLOCK(b);
606 vpna->bdg_port = cand;
607 ND("NIC %p to bridge port %d", vpna, cand);
608 /* bind the port to the bridge (virtual ports are not active) */
609 b->bdg_ports[cand] = vpna;
610 vpna->na_bdg = b;
611 b->bdg_active_ports++;
700
701 BDG_WLOCK(b);
702 vpna->bdg_port = cand;
703 ND("NIC %p to bridge port %d", vpna, cand);
704 /* bind the port to the bridge (virtual ports are not active) */
705 b->bdg_ports[cand] = vpna;
706 vpna->na_bdg = b;
707 b->bdg_active_ports++;
612 if (cand2 >= 0) {
613 struct netmap_vp_adapter *hostna = vpna + 1;
708 if (hostna != NULL) {
614 /* also bind the host stack to the bridge */
615 b->bdg_ports[cand2] = hostna;
616 hostna->bdg_port = cand2;
617 hostna->na_bdg = b;
618 b->bdg_active_ports++;
619 ND("host %p to bridge port %d", hostna, cand2);
620 }
709 /* also bind the host stack to the bridge */
710 b->bdg_ports[cand2] = hostna;
711 hostna->bdg_port = cand2;
712 hostna->na_bdg = b;
713 b->bdg_active_ports++;
714 ND("host %p to bridge port %d", hostna, cand2);
715 }
621 ND("if %s refs %d", name, vpna->up.na_refcount);
716 ND("if %s refs %d", ifname, vpna->up.na_refcount);
622 BDG_WUNLOCK(b);
717 BDG_WUNLOCK(b);
623 *na = ret;
624 netmap_adapter_get(ret);
718 *na = &vpna->up;
719 netmap_adapter_get(*na);
625 return 0;
626
627out:
628 if_rele(ifp);
629
630 return error;
631}
632
633
720 return 0;
721
722out:
723 if_rele(ifp);
724
725 return error;
726}
727
728
634/* Process NETMAP_BDG_ATTACH and NETMAP_BDG_DETACH */
729/* Process NETMAP_BDG_ATTACH */
635static int
730static int
636nm_bdg_attach(struct nmreq *nmr)
731nm_bdg_ctl_attach(struct nmreq *nmr)
637{
638 struct netmap_adapter *na;
732{
733 struct netmap_adapter *na;
639 struct netmap_if *nifp;
640 struct netmap_priv_d *npriv;
641 struct netmap_bwrap_adapter *bna;
642 int error;
643
734 int error;
735
644 npriv = malloc(sizeof(*npriv), M_DEVBUF, M_NOWAIT|M_ZERO);
645 if (npriv == NULL)
646 return ENOMEM;
647
648 NMG_LOCK();
649
650 error = netmap_get_bdg_na(nmr, &na, 1 /* create if not exists */);
736 NMG_LOCK();
737
738 error = netmap_get_bdg_na(nmr, &na, 1 /* create if not exists */);
651 if (error) /* no device, or another bridge or user owns the device */
739 if (error) /* no device */
652 goto unlock_exit;
653
654 if (na == NULL) { /* VALE prefix missing */
655 error = EINVAL;
656 goto unlock_exit;
657 }
658
740 goto unlock_exit;
741
742 if (na == NULL) { /* VALE prefix missing */
743 error = EINVAL;
744 goto unlock_exit;
745 }
746
659 if (na->active_fds > 0) { /* already registered */
747 if (NETMAP_OWNED_BY_ANY(na)) {
660 error = EBUSY;
661 goto unref_exit;
662 }
663
748 error = EBUSY;
749 goto unref_exit;
750 }
751
664 nifp = netmap_do_regif(npriv, na, nmr->nr_ringid, nmr->nr_flags, &error);
665 if (!nifp) {
666 goto unref_exit;
752 if (na->nm_bdg_ctl) {
753 /* nop for VALE ports. The bwrap needs to put the hwna
754 * in netmap mode (see netmap_bwrap_bdg_ctl)
755 */
756 error = na->nm_bdg_ctl(na, nmr, 1);
757 if (error)
758 goto unref_exit;
759 ND("registered %s to netmap-mode", na->name);
667 }
760 }
668
669 bna = (struct netmap_bwrap_adapter*)na;
670 bna->na_kpriv = npriv;
671 NMG_UNLOCK();
761 NMG_UNLOCK();
672 ND("registered %s to netmap-mode", NM_IFPNAME(na->ifp));
673 return 0;
674
675unref_exit:
676 netmap_adapter_put(na);
677unlock_exit:
678 NMG_UNLOCK();
762 return 0;
763
764unref_exit:
765 netmap_adapter_put(na);
766unlock_exit:
767 NMG_UNLOCK();
679 bzero(npriv, sizeof(*npriv));
680 free(npriv, M_DEVBUF);
681 return error;
682}
683
684
768 return error;
769}
770
771
772/* process NETMAP_BDG_DETACH */
685static int
773static int
686nm_bdg_detach(struct nmreq *nmr)
774nm_bdg_ctl_detach(struct nmreq *nmr)
687{
688 struct netmap_adapter *na;
689 int error;
775{
776 struct netmap_adapter *na;
777 int error;
690 struct netmap_bwrap_adapter *bna;
691 int last_instance;
692
693 NMG_LOCK();
694 error = netmap_get_bdg_na(nmr, &na, 0 /* don't create */);
695 if (error) { /* no device, or another bridge or user owns the device */
696 goto unlock_exit;
697 }
698
699 if (na == NULL) { /* VALE prefix missing */
700 error = EINVAL;
701 goto unlock_exit;
702 }
703
778
779 NMG_LOCK();
780 error = netmap_get_bdg_na(nmr, &na, 0 /* don't create */);
781 if (error) { /* no device, or another bridge or user owns the device */
782 goto unlock_exit;
783 }
784
785 if (na == NULL) { /* VALE prefix missing */
786 error = EINVAL;
787 goto unlock_exit;
788 }
789
704 bna = (struct netmap_bwrap_adapter *)na;
705
706 if (na->active_fds == 0) { /* not registered */
707 error = EINVAL;
708 goto unref_exit;
790 if (na->nm_bdg_ctl) {
791 /* remove the port from bridge. The bwrap
792 * also needs to put the hwna in normal mode
793 */
794 error = na->nm_bdg_ctl(na, nmr, 0);
709 }
710
795 }
796
711 last_instance = netmap_dtor_locked(bna->na_kpriv); /* unregister */
712 if (!last_instance) {
713 D("--- error, trying to detach an entry with active mmaps");
714 error = EINVAL;
715 } else {
716 struct netmap_priv_d *npriv = bna->na_kpriv;
717
718 bna->na_kpriv = NULL;
719 D("deleting priv");
720
721 bzero(npriv, sizeof(*npriv));
722 free(npriv, M_DEVBUF);
723 }
724
725unref_exit:
726 netmap_adapter_put(na);
727unlock_exit:
728 NMG_UNLOCK();
729 return error;
730
731}
732
733
797 netmap_adapter_put(na);
798unlock_exit:
799 NMG_UNLOCK();
800 return error;
801
802}
803
804
734/* exported to kernel callers, e.g. OVS ?
735 * Entry point.
805/* Called by either user's context (netmap_ioctl())
806 * or external kernel modules (e.g., Openvswitch).
807 * Operation is indicated in nmr->nr_cmd.
808 * NETMAP_BDG_OPS that sets configure/lookup/dtor functions to the bridge
809 * requires bdg_ops argument; the other commands ignore this argument.
810 *
736 * Called without NMG_LOCK.
737 */
738int
811 * Called without NMG_LOCK.
812 */
813int
739netmap_bdg_ctl(struct nmreq *nmr, bdg_lookup_fn_t func)
814netmap_bdg_ctl(struct nmreq *nmr, struct netmap_bdg_ops *bdg_ops)
740{
741 struct nm_bridge *b;
742 struct netmap_adapter *na;
743 struct netmap_vp_adapter *vpna;
815{
816 struct nm_bridge *b;
817 struct netmap_adapter *na;
818 struct netmap_vp_adapter *vpna;
744 struct ifnet *iter;
745 char *name = nmr->nr_name;
746 int cmd = nmr->nr_cmd, namelen = strlen(name);
747 int error = 0, i, j;
748
749 switch (cmd) {
819 char *name = nmr->nr_name;
820 int cmd = nmr->nr_cmd, namelen = strlen(name);
821 int error = 0, i, j;
822
823 switch (cmd) {
824 case NETMAP_BDG_NEWIF:
825 error = nm_vi_create(nmr);
826 break;
827
828 case NETMAP_BDG_DELIF:
829 error = nm_vi_destroy(nmr->nr_name);
830 break;
831
750 case NETMAP_BDG_ATTACH:
832 case NETMAP_BDG_ATTACH:
751 error = nm_bdg_attach(nmr);
833 error = nm_bdg_ctl_attach(nmr);
752 break;
753
754 case NETMAP_BDG_DETACH:
834 break;
835
836 case NETMAP_BDG_DETACH:
755 error = nm_bdg_detach(nmr);
837 error = nm_bdg_ctl_detach(nmr);
756 break;
757
758 case NETMAP_BDG_LIST:
759 /* this is used to enumerate bridges and ports */
760 if (namelen) { /* look up indexes of bridge and port */
761 if (strncmp(name, NM_NAME, strlen(NM_NAME))) {
762 error = EINVAL;
763 break;
764 }
765 NMG_LOCK();
766 b = nm_find_bridge(name, 0 /* don't create */);
767 if (!b) {
768 error = ENOENT;
769 NMG_UNLOCK();
770 break;
771 }
772
838 break;
839
840 case NETMAP_BDG_LIST:
841 /* this is used to enumerate bridges and ports */
842 if (namelen) { /* look up indexes of bridge and port */
843 if (strncmp(name, NM_NAME, strlen(NM_NAME))) {
844 error = EINVAL;
845 break;
846 }
847 NMG_LOCK();
848 b = nm_find_bridge(name, 0 /* don't create */);
849 if (!b) {
850 error = ENOENT;
851 NMG_UNLOCK();
852 break;
853 }
854
855 name = name + b->bdg_namelen + 1;
773 error = ENOENT;
774 for (j = 0; j < b->bdg_active_ports; j++) {
775 i = b->bdg_port_index[j];
776 vpna = b->bdg_ports[i];
777 if (vpna == NULL) {
778 D("---AAAAAAAAARGH-------");
779 continue;
780 }
856 error = ENOENT;
857 for (j = 0; j < b->bdg_active_ports; j++) {
858 i = b->bdg_port_index[j];
859 vpna = b->bdg_ports[i];
860 if (vpna == NULL) {
861 D("---AAAAAAAAARGH-------");
862 continue;
863 }
781 iter = vpna->up.ifp;
782 /* the former and the latter identify a
783 * virtual port and a NIC, respectively
784 */
864 /* the former and the latter identify a
865 * virtual port and a NIC, respectively
866 */
785 if (!strcmp(iter->if_xname, name)) {
867 if (!strcmp(vpna->up.name, name)) {
786 /* bridge index */
787 nmr->nr_arg1 = b - nm_bridges;
788 nmr->nr_arg2 = i; /* port index */
789 error = 0;
790 break;
791 }
792 }
793 NMG_UNLOCK();

--- 14 unchanged lines hidden (view full) ---

808 if (j >= b->bdg_active_ports) {
809 j = 0; /* following bridges scan from 0 */
810 continue;
811 }
812 nmr->nr_arg1 = i;
813 nmr->nr_arg2 = j;
814 j = b->bdg_port_index[j];
815 vpna = b->bdg_ports[j];
868 /* bridge index */
869 nmr->nr_arg1 = b - nm_bridges;
870 nmr->nr_arg2 = i; /* port index */
871 error = 0;
872 break;
873 }
874 }
875 NMG_UNLOCK();

--- 14 unchanged lines hidden (view full) ---

890 if (j >= b->bdg_active_ports) {
891 j = 0; /* following bridges scan from 0 */
892 continue;
893 }
894 nmr->nr_arg1 = i;
895 nmr->nr_arg2 = j;
896 j = b->bdg_port_index[j];
897 vpna = b->bdg_ports[j];
816 iter = vpna->up.ifp;
817 strncpy(name, iter->if_xname, (size_t)IFNAMSIZ);
898 strncpy(name, vpna->up.name, (size_t)IFNAMSIZ);
818 error = 0;
819 break;
820 }
821 NMG_UNLOCK();
822 }
823 break;
824
899 error = 0;
900 break;
901 }
902 NMG_UNLOCK();
903 }
904 break;
905
825 case NETMAP_BDG_LOOKUP_REG:
826 /* register a lookup function to the given bridge.
906 case NETMAP_BDG_REGOPS: /* XXX this should not be available from userspace */
907 /* register callbacks to the given bridge.
827 * nmr->nr_name may be just bridge's name (including ':'
828 * if it is not just NM_NAME).
829 */
908 * nmr->nr_name may be just bridge's name (including ':'
909 * if it is not just NM_NAME).
910 */
830 if (!func) {
911 if (!bdg_ops) {
831 error = EINVAL;
832 break;
833 }
834 NMG_LOCK();
835 b = nm_find_bridge(name, 0 /* don't create */);
836 if (!b) {
837 error = EINVAL;
838 } else {
912 error = EINVAL;
913 break;
914 }
915 NMG_LOCK();
916 b = nm_find_bridge(name, 0 /* don't create */);
917 if (!b) {
918 error = EINVAL;
919 } else {
839 b->nm_bdg_lookup = func;
920 b->bdg_ops = *bdg_ops;
840 }
841 NMG_UNLOCK();
842 break;
843
844 case NETMAP_BDG_VNET_HDR:
845 /* Valid lengths for the virtio-net header are 0 (no header),
846 10 and 12. */
847 if (nmr->nr_arg1 != 0 &&
848 nmr->nr_arg1 != sizeof(struct nm_vnet_hdr) &&
849 nmr->nr_arg1 != 12) {
850 error = EINVAL;
851 break;
852 }
853 NMG_LOCK();
854 error = netmap_get_bdg_na(nmr, &na, 0);
855 if (na && !error) {
856 vpna = (struct netmap_vp_adapter *)na;
857 vpna->virt_hdr_len = nmr->nr_arg1;
858 if (vpna->virt_hdr_len)
921 }
922 NMG_UNLOCK();
923 break;
924
925 case NETMAP_BDG_VNET_HDR:
926 /* Valid lengths for the virtio-net header are 0 (no header),
927 10 and 12. */
928 if (nmr->nr_arg1 != 0 &&
929 nmr->nr_arg1 != sizeof(struct nm_vnet_hdr) &&
930 nmr->nr_arg1 != 12) {
931 error = EINVAL;
932 break;
933 }
934 NMG_LOCK();
935 error = netmap_get_bdg_na(nmr, &na, 0);
936 if (na && !error) {
937 vpna = (struct netmap_vp_adapter *)na;
938 vpna->virt_hdr_len = nmr->nr_arg1;
939 if (vpna->virt_hdr_len)
859 vpna->mfs = NETMAP_BDG_BUF_SIZE(na->nm_mem);
940 vpna->mfs = NETMAP_BUF_SIZE(na);
860 D("Using vnet_hdr_len %d for %p", vpna->virt_hdr_len, vpna);
861 netmap_adapter_put(na);
862 }
863 NMG_UNLOCK();
864 break;
865
866 default:
867 D("invalid cmd (nmr->nr_cmd) (0x%x)", cmd);
868 error = EINVAL;
869 break;
870 }
871 return error;
872}
873
941 D("Using vnet_hdr_len %d for %p", vpna->virt_hdr_len, vpna);
942 netmap_adapter_put(na);
943 }
944 NMG_UNLOCK();
945 break;
946
947 default:
948 D("invalid cmd (nmr->nr_cmd) (0x%x)", cmd);
949 error = EINVAL;
950 break;
951 }
952 return error;
953}
954
955int
956netmap_bdg_config(struct nmreq *nmr)
957{
958 struct nm_bridge *b;
959 int error = EINVAL;
960
961 NMG_LOCK();
962 b = nm_find_bridge(nmr->nr_name, 0);
963 if (!b) {
964 NMG_UNLOCK();
965 return error;
966 }
967 NMG_UNLOCK();
968 /* Don't call config() with NMG_LOCK() held */
969 BDG_RLOCK(b);
970 if (b->bdg_ops.config != NULL)
971 error = b->bdg_ops.config((struct nm_ifreq *)nmr);
972 BDG_RUNLOCK(b);
973 return error;
974}
975
976
977/* nm_krings_create callback for VALE ports.
978 * Calls the standard netmap_krings_create, then adds leases on rx
979 * rings and bdgfwd on tx rings.
980 */
874static int
875netmap_vp_krings_create(struct netmap_adapter *na)
876{
877 u_int tailroom;
878 int error, i;
879 uint32_t *leases;
880 u_int nrx = netmap_real_rx_rings(na);
881

--- 18 unchanged lines hidden (view full) ---

900 netmap_krings_delete(na);
901 return error;
902 }
903
904 return 0;
905}
906
907
981static int
982netmap_vp_krings_create(struct netmap_adapter *na)
983{
984 u_int tailroom;
985 int error, i;
986 uint32_t *leases;
987 u_int nrx = netmap_real_rx_rings(na);
988

--- 18 unchanged lines hidden (view full) ---

1007 netmap_krings_delete(na);
1008 return error;
1009 }
1010
1011 return 0;
1012}
1013
1014
1015/* nm_krings_delete callback for VALE ports. */
908static void
909netmap_vp_krings_delete(struct netmap_adapter *na)
910{
911 nm_free_bdgfwd(na);
912 netmap_krings_delete(na);
913}
914
915
916static int
917nm_bdg_flush(struct nm_bdg_fwd *ft, u_int n,
918 struct netmap_vp_adapter *na, u_int ring_nr);
919
920
921/*
1016static void
1017netmap_vp_krings_delete(struct netmap_adapter *na)
1018{
1019 nm_free_bdgfwd(na);
1020 netmap_krings_delete(na);
1021}
1022
1023
1024static int
1025nm_bdg_flush(struct nm_bdg_fwd *ft, u_int n,
1026 struct netmap_vp_adapter *na, u_int ring_nr);
1027
1028
1029/*
1030 * main dispatch routine for the bridge.
922 * Grab packets from a kring, move them into the ft structure
923 * associated to the tx (input) port. Max one instance per port,
924 * filtered on input (ioctl, poll or XXX).
925 * Returns the next position in the ring.
926 */
927static int
1031 * Grab packets from a kring, move them into the ft structure
1032 * associated to the tx (input) port. Max one instance per port,
1033 * filtered on input (ioctl, poll or XXX).
1034 * Returns the next position in the ring.
1035 */
1036static int
928nm_bdg_preflush(struct netmap_vp_adapter *na, u_int ring_nr,
929 struct netmap_kring *kring, u_int end)
1037nm_bdg_preflush(struct netmap_kring *kring, u_int end)
930{
1038{
1039 struct netmap_vp_adapter *na =
1040 (struct netmap_vp_adapter*)kring->na;
931 struct netmap_ring *ring = kring->ring;
932 struct nm_bdg_fwd *ft;
1041 struct netmap_ring *ring = kring->ring;
1042 struct nm_bdg_fwd *ft;
1043 u_int ring_nr = kring->ring_id;
933 u_int j = kring->nr_hwcur, lim = kring->nkr_num_slots - 1;
934 u_int ft_i = 0; /* start from 0 */
935 u_int frags = 1; /* how many frags ? */
936 struct nm_bridge *b = na->na_bdg;
937
938 /* To protect against modifications to the bridge we acquire a
939 * shared lock, waiting if we can sleep (if the source port is
940 * attached to a user process) or with a trylock otherwise (NICs).

--- 12 unchanged lines hidden (view full) ---

953
954 ft[ft_i].ft_len = slot->len;
955 ft[ft_i].ft_flags = slot->flags;
956
957 ND("flags is 0x%x", slot->flags);
958 /* this slot goes into a list so initialize the link field */
959 ft[ft_i].ft_next = NM_FT_NULL;
960 buf = ft[ft_i].ft_buf = (slot->flags & NS_INDIRECT) ?
1044 u_int j = kring->nr_hwcur, lim = kring->nkr_num_slots - 1;
1045 u_int ft_i = 0; /* start from 0 */
1046 u_int frags = 1; /* how many frags ? */
1047 struct nm_bridge *b = na->na_bdg;
1048
1049 /* To protect against modifications to the bridge we acquire a
1050 * shared lock, waiting if we can sleep (if the source port is
1051 * attached to a user process) or with a trylock otherwise (NICs).

--- 12 unchanged lines hidden (view full) ---

1064
1065 ft[ft_i].ft_len = slot->len;
1066 ft[ft_i].ft_flags = slot->flags;
1067
1068 ND("flags is 0x%x", slot->flags);
1069 /* this slot goes into a list so initialize the link field */
1070 ft[ft_i].ft_next = NM_FT_NULL;
1071 buf = ft[ft_i].ft_buf = (slot->flags & NS_INDIRECT) ?
961 (void *)(uintptr_t)slot->ptr : BDG_NMB(&na->up, slot);
1072 (void *)(uintptr_t)slot->ptr : NMB(&na->up, slot);
962 if (unlikely(buf == NULL)) {
963 RD(5, "NULL %s buffer pointer from %s slot %d len %d",
964 (slot->flags & NS_INDIRECT) ? "INDIRECT" : "DIRECT",
965 kring->name, j, ft[ft_i].ft_len);
1073 if (unlikely(buf == NULL)) {
1074 RD(5, "NULL %s buffer pointer from %s slot %d len %d",
1075 (slot->flags & NS_INDIRECT) ? "INDIRECT" : "DIRECT",
1076 kring->name, j, ft[ft_i].ft_len);
966 buf = ft[ft_i].ft_buf = NMB_VA(0); /* the 'null' buffer */
1077 buf = ft[ft_i].ft_buf = NETMAP_BUF_BASE(&na->up);
967 ft[ft_i].ft_len = 0;
968 ft[ft_i].ft_flags = 0;
969 }
970 __builtin_prefetch(buf);
971 ++ft_i;
972 if (slot->flags & NS_MOREFRAG) {
973 frags++;
974 continue;

--- 55 unchanged lines hidden (view full) ---

1030 mix(a, b, c);
1031#define BRIDGE_RTHASH_MASK (NM_BDG_HASH-1)
1032 return (c & BRIDGE_RTHASH_MASK);
1033}
1034
1035#undef mix
1036
1037
1078 ft[ft_i].ft_len = 0;
1079 ft[ft_i].ft_flags = 0;
1080 }
1081 __builtin_prefetch(buf);
1082 ++ft_i;
1083 if (slot->flags & NS_MOREFRAG) {
1084 frags++;
1085 continue;

--- 55 unchanged lines hidden (view full) ---

1141 mix(a, b, c);
1142#define BRIDGE_RTHASH_MASK (NM_BDG_HASH-1)
1143 return (c & BRIDGE_RTHASH_MASK);
1144}
1145
1146#undef mix
1147
1148
1149/* nm_register callback for VALE ports */
1038static int
1150static int
1039bdg_netmap_reg(struct netmap_adapter *na, int onoff)
1151netmap_vp_reg(struct netmap_adapter *na, int onoff)
1040{
1041 struct netmap_vp_adapter *vpna =
1042 (struct netmap_vp_adapter*)na;
1152{
1153 struct netmap_vp_adapter *vpna =
1154 (struct netmap_vp_adapter*)na;
1043 struct ifnet *ifp = na->ifp;
1044
1155
1045 /* the interface is already attached to the bridge,
1046 * so we only need to toggle IFCAP_NETMAP.
1156 /* persistent ports may be put in netmap mode
1157 * before being attached to a bridge
1047 */
1158 */
1048 BDG_WLOCK(vpna->na_bdg);
1159 if (vpna->na_bdg)
1160 BDG_WLOCK(vpna->na_bdg);
1049 if (onoff) {
1161 if (onoff) {
1050 ifp->if_capenable |= IFCAP_NETMAP;
1162 na->na_flags |= NAF_NETMAP_ON;
1163 /* XXX on FreeBSD, persistent VALE ports should also
1164 * toggle IFCAP_NETMAP in na->ifp (2014-03-16)
1165 */
1051 } else {
1166 } else {
1052 ifp->if_capenable &= ~IFCAP_NETMAP;
1167 na->na_flags &= ~NAF_NETMAP_ON;
1053 }
1168 }
1054 BDG_WUNLOCK(vpna->na_bdg);
1169 if (vpna->na_bdg)
1170 BDG_WUNLOCK(vpna->na_bdg);
1055 return 0;
1056}
1057
1058
1059/*
1060 * Lookup function for a learning bridge.
1061 * Update the hash table with the source address,
1062 * and then returns the destination port index, and the
1063 * ring in *dst_ring (at the moment, always use ring 0)
1064 */
1065u_int
1171 return 0;
1172}
1173
1174
1175/*
1176 * Lookup function for a learning bridge.
1177 * Update the hash table with the source address,
1178 * and then returns the destination port index, and the
1179 * ring in *dst_ring (at the moment, always use ring 0)
1180 */
1181u_int
1066netmap_bdg_learning(char *buf, u_int buf_len, uint8_t *dst_ring,
1067 struct netmap_vp_adapter *na)
1182netmap_bdg_learning(struct nm_bdg_fwd *ft, uint8_t *dst_ring,
1183 const struct netmap_vp_adapter *na)
1068{
1184{
1185 uint8_t *buf = ft->ft_buf;
1186 u_int buf_len = ft->ft_len;
1069 struct nm_hash_ent *ht = na->na_bdg->ht;
1070 uint32_t sh, dh;
1071 u_int dst, mysrc = na->bdg_port;
1072 uint64_t smac, dmac;
1073
1187 struct nm_hash_ent *ht = na->na_bdg->ht;
1188 uint32_t sh, dh;
1189 u_int dst, mysrc = na->bdg_port;
1190 uint64_t smac, dmac;
1191
1074 if (buf_len < 14) {
1075 RD(5, "invalid buf length %d", buf_len);
1192 /* safety check, unfortunately we have many cases */
1193 if (buf_len >= 14 + na->virt_hdr_len) {
1194 /* virthdr + mac_hdr in the same slot */
1195 buf += na->virt_hdr_len;
1196 buf_len -= na->virt_hdr_len;
1197 } else if (buf_len == na->virt_hdr_len && ft->ft_flags & NS_MOREFRAG) {
1198 /* only header in first fragment */
1199 ft++;
1200 buf = ft->ft_buf;
1201 buf_len = ft->ft_len;
1202 } else {
1203 RD(5, "invalid buf format, length %d", buf_len);
1076 return NM_BDG_NOPORT;
1077 }
1078 dmac = le64toh(*(uint64_t *)(buf)) & 0xffffffffffff;
1079 smac = le64toh(*(uint64_t *)(buf + 4));
1080 smac >>= 16;
1081
1082 /*
1083 * The hash is somewhat expensive, there might be some

--- 81 unchanged lines hidden (view full) ---

1165 if (k->nkr_hwlease > lim)
1166 k->nkr_hwlease -= lim + 1;
1167
1168 if (k->nkr_hwlease >= k->nkr_num_slots ||
1169 k->nr_hwcur >= k->nkr_num_slots ||
1170 k->nr_hwtail >= k->nkr_num_slots ||
1171 k->nkr_lease_idx >= k->nkr_num_slots) {
1172 D("invalid kring %s, cur %d tail %d lease %d lease_idx %d lim %d",
1204 return NM_BDG_NOPORT;
1205 }
1206 dmac = le64toh(*(uint64_t *)(buf)) & 0xffffffffffff;
1207 smac = le64toh(*(uint64_t *)(buf + 4));
1208 smac >>= 16;
1209
1210 /*
1211 * The hash is somewhat expensive, there might be some

--- 81 unchanged lines hidden (view full) ---

1293 if (k->nkr_hwlease > lim)
1294 k->nkr_hwlease -= lim + 1;
1295
1296 if (k->nkr_hwlease >= k->nkr_num_slots ||
1297 k->nr_hwcur >= k->nkr_num_slots ||
1298 k->nr_hwtail >= k->nkr_num_slots ||
1299 k->nkr_lease_idx >= k->nkr_num_slots) {
1300 D("invalid kring %s, cur %d tail %d lease %d lease_idx %d lim %d",
1173 k->na->ifp->if_xname,
1301 k->na->name,
1174 k->nr_hwcur, k->nr_hwtail, k->nkr_hwlease,
1175 k->nkr_lease_idx, k->nkr_num_slots);
1176 }
1177 return lease_idx;
1178}
1179
1180/*
1302 k->nr_hwcur, k->nr_hwtail, k->nkr_hwlease,
1303 k->nkr_lease_idx, k->nkr_num_slots);
1304 }
1305 return lease_idx;
1306}
1307
1308/*
1309 *
1181 * This flush routine supports only unicast and broadcast but a large
1182 * number of ports, and lets us replace the learn and dispatch functions.
1183 */
1184int
1185nm_bdg_flush(struct nm_bdg_fwd *ft, u_int n, struct netmap_vp_adapter *na,
1186 u_int ring_nr)
1187{
1188 struct nm_bdg_q *dst_ents, *brddst;

--- 10 unchanged lines hidden (view full) ---

1199 dst_ents = (struct nm_bdg_q *)(ft + NM_BDG_BATCH_MAX);
1200 dsts = (uint16_t *)(dst_ents + NM_BDG_MAXPORTS * NM_BDG_MAXRINGS + 1);
1201
1202 /* first pass: find a destination for each packet in the batch */
1203 for (i = 0; likely(i < n); i += ft[i].ft_frags) {
1204 uint8_t dst_ring = ring_nr; /* default, same ring as origin */
1205 uint16_t dst_port, d_i;
1206 struct nm_bdg_q *d;
1310 * This flush routine supports only unicast and broadcast but a large
1311 * number of ports, and lets us replace the learn and dispatch functions.
1312 */
1313int
1314nm_bdg_flush(struct nm_bdg_fwd *ft, u_int n, struct netmap_vp_adapter *na,
1315 u_int ring_nr)
1316{
1317 struct nm_bdg_q *dst_ents, *brddst;

--- 10 unchanged lines hidden (view full) ---

1328 dst_ents = (struct nm_bdg_q *)(ft + NM_BDG_BATCH_MAX);
1329 dsts = (uint16_t *)(dst_ents + NM_BDG_MAXPORTS * NM_BDG_MAXRINGS + 1);
1330
1331 /* first pass: find a destination for each packet in the batch */
1332 for (i = 0; likely(i < n); i += ft[i].ft_frags) {
1333 uint8_t dst_ring = ring_nr; /* default, same ring as origin */
1334 uint16_t dst_port, d_i;
1335 struct nm_bdg_q *d;
1207 uint8_t *buf = ft[i].ft_buf;
1208 u_int len = ft[i].ft_len;
1209
1210 ND("slot %d frags %d", i, ft[i].ft_frags);
1211 /* Drop the packet if the virtio-net header is not into the first
1212 fragment nor at the very beginning of the second. */
1336
1337 ND("slot %d frags %d", i, ft[i].ft_frags);
1338 /* Drop the packet if the virtio-net header is not into the first
1339 fragment nor at the very beginning of the second. */
1213 if (unlikely(na->virt_hdr_len > len))
1340 if (unlikely(na->virt_hdr_len > ft[i].ft_len))
1214 continue;
1341 continue;
1215 if (len == na->virt_hdr_len) {
1216 buf = ft[i+1].ft_buf;
1217 len = ft[i+1].ft_len;
1218 } else {
1219 buf += na->virt_hdr_len;
1220 len -= na->virt_hdr_len;
1221 }
1222 dst_port = b->nm_bdg_lookup(buf, len, &dst_ring, na);
1342 dst_port = b->bdg_ops.lookup(&ft[i], &dst_ring, na);
1223 if (netmap_verbose > 255)
1224 RD(5, "slot %d port %d -> %d", i, me, dst_port);
1225 if (dst_port == NM_BDG_NOPORT)
1226 continue; /* this packet is identified to be dropped */
1227 else if (unlikely(dst_port > NM_BDG_MAXPORTS))
1228 continue;
1229 else if (dst_port == NM_BDG_BROADCAST)
1230 dst_ring = 0; /* broadcasts always go to ring 0 */

--- 34 unchanged lines hidden (view full) ---

1265 continue;
1266 d_i = i * NM_BDG_MAXRINGS;
1267 if (dst_ents[d_i].bq_head == NM_FT_NULL)
1268 dsts[num_dsts++] = d_i;
1269 }
1270 }
1271
1272 ND(5, "pass 1 done %d pkts %d dsts", n, num_dsts);
1343 if (netmap_verbose > 255)
1344 RD(5, "slot %d port %d -> %d", i, me, dst_port);
1345 if (dst_port == NM_BDG_NOPORT)
1346 continue; /* this packet is identified to be dropped */
1347 else if (unlikely(dst_port > NM_BDG_MAXPORTS))
1348 continue;
1349 else if (dst_port == NM_BDG_BROADCAST)
1350 dst_ring = 0; /* broadcasts always go to ring 0 */

--- 34 unchanged lines hidden (view full) ---

1385 continue;
1386 d_i = i * NM_BDG_MAXRINGS;
1387 if (dst_ents[d_i].bq_head == NM_FT_NULL)
1388 dsts[num_dsts++] = d_i;
1389 }
1390 }
1391
1392 ND(5, "pass 1 done %d pkts %d dsts", n, num_dsts);
1273 /* second pass: scan destinations (XXX will be modular somehow) */
1393 /* second pass: scan destinations */
1274 for (i = 0; i < num_dsts; i++) {
1394 for (i = 0; i < num_dsts; i++) {
1275 struct ifnet *dst_ifp;
1276 struct netmap_vp_adapter *dst_na;
1277 struct netmap_kring *kring;
1278 struct netmap_ring *ring;
1279 u_int dst_nr, lim, j, d_i, next, brd_next;
1280 u_int needed, howmany;
1281 int retry = netmap_txsync_retry;
1282 struct nm_bdg_q *d;
1283 uint32_t my_start = 0, lease_idx = 0;

--- 7 unchanged lines hidden (view full) ---

1291 dst_na = b->bdg_ports[d_i/NM_BDG_MAXRINGS];
1292 /* protect from the lookup function returning an inactive
1293 * destination port
1294 */
1295 if (unlikely(dst_na == NULL))
1296 goto cleanup;
1297 if (dst_na->up.na_flags & NAF_SW_ONLY)
1298 goto cleanup;
1395 struct netmap_vp_adapter *dst_na;
1396 struct netmap_kring *kring;
1397 struct netmap_ring *ring;
1398 u_int dst_nr, lim, j, d_i, next, brd_next;
1399 u_int needed, howmany;
1400 int retry = netmap_txsync_retry;
1401 struct nm_bdg_q *d;
1402 uint32_t my_start = 0, lease_idx = 0;

--- 7 unchanged lines hidden (view full) ---

1410 dst_na = b->bdg_ports[d_i/NM_BDG_MAXRINGS];
1411 /* protect from the lookup function returning an inactive
1412 * destination port
1413 */
1414 if (unlikely(dst_na == NULL))
1415 goto cleanup;
1416 if (dst_na->up.na_flags & NAF_SW_ONLY)
1417 goto cleanup;
1299 dst_ifp = dst_na->up.ifp;
1300 /*
1301 * The interface may be in !netmap mode in two cases:
1302 * - when na is attached but not activated yet;
1303 * - when na is being deactivated but is still attached.
1304 */
1418 /*
1419 * The interface may be in !netmap mode in two cases:
1420 * - when na is attached but not activated yet;
1421 * - when na is being deactivated but is still attached.
1422 */
1305 if (unlikely(!(dst_ifp->if_capenable & IFCAP_NETMAP))) {
1423 if (unlikely(!nm_netmap_on(&dst_na->up))) {
1306 ND("not in netmap mode!");
1307 goto cleanup;
1308 }
1309
1310 /* there is at least one either unicast or broadcast packet */
1311 brd_next = brddst->bq_head;
1312 next = d->bq_head;
1313 /* we need to reserve this many slots. If fewer are
1314 * available, some packets will be dropped.
1315 * Packets may have multiple fragments, so we may not use
1316 * there is a chance that we may not use all of the slots
1317 * we have claimed, so we will need to handle the leftover
1318 * ones when we regain the lock.
1319 */
1320 needed = d->bq_len + brddst->bq_len;
1321
1322 if (unlikely(dst_na->virt_hdr_len != na->virt_hdr_len)) {
1424 ND("not in netmap mode!");
1425 goto cleanup;
1426 }
1427
1428 /* there is at least one either unicast or broadcast packet */
1429 brd_next = brddst->bq_head;
1430 next = d->bq_head;
1431 /* we need to reserve this many slots. If fewer are
1432 * available, some packets will be dropped.
1433 * Packets may have multiple fragments, so we may not use
1434 * there is a chance that we may not use all of the slots
1435 * we have claimed, so we will need to handle the leftover
1436 * ones when we regain the lock.
1437 */
1438 needed = d->bq_len + brddst->bq_len;
1439
1440 if (unlikely(dst_na->virt_hdr_len != na->virt_hdr_len)) {
1323 RD(3, "virt_hdr_mismatch, src %d len %d", na->virt_hdr_len, dst_na->virt_hdr_len);
1441 RD(3, "virt_hdr_mismatch, src %d dst %d", na->virt_hdr_len, dst_na->virt_hdr_len);
1324 /* There is a virtio-net header/offloadings mismatch between
1325 * source and destination. The slower mismatch datapath will
1326 * be used to cope with all the mismatches.
1327 */
1328 virt_hdr_mismatch = 1;
1329 if (dst_na->mfs < na->mfs) {
1330 /* We may need to do segmentation offloadings, and so
1331 * we may need a number of destination slots greater

--- 21 unchanged lines hidden (view full) ---

1353 ring = kring->ring;
1354 lim = kring->nkr_num_slots - 1;
1355
1356retry:
1357
1358 if (dst_na->retry && retry) {
1359 /* try to get some free slot from the previous run */
1360 dst_na->up.nm_notify(&dst_na->up, dst_nr, NR_RX, 0);
1442 /* There is a virtio-net header/offloadings mismatch between
1443 * source and destination. The slower mismatch datapath will
1444 * be used to cope with all the mismatches.
1445 */
1446 virt_hdr_mismatch = 1;
1447 if (dst_na->mfs < na->mfs) {
1448 /* We may need to do segmentation offloadings, and so
1449 * we may need a number of destination slots greater

--- 21 unchanged lines hidden (view full) ---

1471 ring = kring->ring;
1472 lim = kring->nkr_num_slots - 1;
1473
1474retry:
1475
1476 if (dst_na->retry && retry) {
1477 /* try to get some free slot from the previous run */
1478 dst_na->up.nm_notify(&dst_na->up, dst_nr, NR_RX, 0);
1479 /* actually useful only for bwraps, since there
1480 * the notify will trigger a txsync on the hwna. VALE ports
1481 * have dst_na->retry == 0
1482 */
1361 }
1362 /* reserve the buffers in the queue and an entry
1363 * to report completion, and drop lock.
1364 * XXX this might become a helper function.
1365 */
1366 mtx_lock(&kring->q_lock);
1367 if (kring->nkr_stopped) {
1368 mtx_unlock(&kring->q_lock);

--- 39 unchanged lines hidden (view full) ---

1408 bdg_mismatch_datapath(na, dst_na, ft_p, ring, &j, lim, &howmany);
1409 } else {
1410 howmany -= cnt;
1411 do {
1412 char *dst, *src = ft_p->ft_buf;
1413 size_t copy_len = ft_p->ft_len, dst_len = copy_len;
1414
1415 slot = &ring->slot[j];
1483 }
1484 /* reserve the buffers in the queue and an entry
1485 * to report completion, and drop lock.
1486 * XXX this might become a helper function.
1487 */
1488 mtx_lock(&kring->q_lock);
1489 if (kring->nkr_stopped) {
1490 mtx_unlock(&kring->q_lock);

--- 39 unchanged lines hidden (view full) ---

1530 bdg_mismatch_datapath(na, dst_na, ft_p, ring, &j, lim, &howmany);
1531 } else {
1532 howmany -= cnt;
1533 do {
1534 char *dst, *src = ft_p->ft_buf;
1535 size_t copy_len = ft_p->ft_len, dst_len = copy_len;
1536
1537 slot = &ring->slot[j];
1416 dst = BDG_NMB(&dst_na->up, slot);
1538 dst = NMB(&dst_na->up, slot);
1417
1418 ND("send [%d] %d(%d) bytes at %s:%d",
1419 i, (int)copy_len, (int)dst_len,
1420 NM_IFPNAME(dst_ifp), j);
1421 /* round to a multiple of 64 */
1422 copy_len = (copy_len + 63) & ~63;
1423
1539
1540 ND("send [%d] %d(%d) bytes at %s:%d",
1541 i, (int)copy_len, (int)dst_len,
1542 NM_IFPNAME(dst_ifp), j);
1543 /* round to a multiple of 64 */
1544 copy_len = (copy_len + 63) & ~63;
1545
1424 if (unlikely(copy_len > NETMAP_BUF_SIZE ||
1425 copy_len > NETMAP_BUF_SIZE)) {
1546 if (unlikely(copy_len > NETMAP_BUF_SIZE(&dst_na->up) ||
1547 copy_len > NETMAP_BUF_SIZE(&na->up))) {
1426 RD(5, "invalid len %d, down to 64", (int)copy_len);
1427 copy_len = dst_len = 64; // XXX
1428 }
1429 if (ft_p->ft_flags & NS_INDIRECT) {
1430 if (copyin(src, dst, copy_len)) {
1431 // invalid user pointer, pretend len is 0
1432 dst_len = 0;
1433 }

--- 56 unchanged lines hidden (view full) ---

1490 /* j is the new 'write' position. j != my_start
1491 * means there are new buffers to report
1492 */
1493 if (likely(j != my_start)) {
1494 kring->nr_hwtail = j;
1495 still_locked = 0;
1496 mtx_unlock(&kring->q_lock);
1497 dst_na->up.nm_notify(&dst_na->up, dst_nr, NR_RX, 0);
1548 RD(5, "invalid len %d, down to 64", (int)copy_len);
1549 copy_len = dst_len = 64; // XXX
1550 }
1551 if (ft_p->ft_flags & NS_INDIRECT) {
1552 if (copyin(src, dst, copy_len)) {
1553 // invalid user pointer, pretend len is 0
1554 dst_len = 0;
1555 }

--- 56 unchanged lines hidden (view full) ---

1612 /* j is the new 'write' position. j != my_start
1613 * means there are new buffers to report
1614 */
1615 if (likely(j != my_start)) {
1616 kring->nr_hwtail = j;
1617 still_locked = 0;
1618 mtx_unlock(&kring->q_lock);
1619 dst_na->up.nm_notify(&dst_na->up, dst_nr, NR_RX, 0);
1498 if (dst_na->retry && retry--)
1620 /* this is netmap_notify for VALE ports and
1621 * netmap_bwrap_notify for bwrap. The latter will
1622 * trigger a txsync on the underlying hwna
1623 */
1624 if (dst_na->retry && retry--) {
1625 /* XXX this is going to call nm_notify again.
1626 * Only useful for bwrap in virtual machines
1627 */
1499 goto retry;
1628 goto retry;
1629 }
1500 }
1501 }
1502 if (still_locked)
1503 mtx_unlock(&kring->q_lock);
1504 }
1505cleanup:
1506 d->bq_head = d->bq_tail = NM_FT_NULL; /* cleanup */
1507 d->bq_len = 0;
1508 }
1509 brddst->bq_head = brddst->bq_tail = NM_FT_NULL; /* cleanup */
1510 brddst->bq_len = 0;
1511 return 0;
1512}
1513
1630 }
1631 }
1632 if (still_locked)
1633 mtx_unlock(&kring->q_lock);
1634 }
1635cleanup:
1636 d->bq_head = d->bq_tail = NM_FT_NULL; /* cleanup */
1637 d->bq_len = 0;
1638 }
1639 brddst->bq_head = brddst->bq_tail = NM_FT_NULL; /* cleanup */
1640 brddst->bq_len = 0;
1641 return 0;
1642}
1643
1514
1644/* nm_txsync callback for VALE ports */
1515static int
1645static int
1516netmap_vp_txsync(struct netmap_vp_adapter *na, u_int ring_nr, int flags)
1646netmap_vp_txsync(struct netmap_kring *kring, int flags)
1517{
1647{
1518 struct netmap_kring *kring = &na->up.tx_rings[ring_nr];
1648 struct netmap_vp_adapter *na =
1649 (struct netmap_vp_adapter *)kring->na;
1519 u_int done;
1520 u_int const lim = kring->nkr_num_slots - 1;
1521 u_int const cur = kring->rcur;
1522
1523 if (bridge_batch <= 0) { /* testing only */
1524 done = cur; // used all
1525 goto done;
1526 }
1650 u_int done;
1651 u_int const lim = kring->nkr_num_slots - 1;
1652 u_int const cur = kring->rcur;
1653
1654 if (bridge_batch <= 0) { /* testing only */
1655 done = cur; // used all
1656 goto done;
1657 }
1658 if (!na->na_bdg) {
1659 done = cur;
1660 goto done;
1661 }
1527 if (bridge_batch > NM_BDG_BATCH)
1528 bridge_batch = NM_BDG_BATCH;
1529
1662 if (bridge_batch > NM_BDG_BATCH)
1663 bridge_batch = NM_BDG_BATCH;
1664
1530 done = nm_bdg_preflush(na, ring_nr, kring, cur);
1665 done = nm_bdg_preflush(kring, cur);
1531done:
1532 if (done != cur)
1533 D("early break at %d/ %d, tail %d", done, cur, kring->nr_hwtail);
1534 /*
1535 * packets between 'done' and 'cur' are left unsent.
1536 */
1537 kring->nr_hwcur = done;
1538 kring->nr_hwtail = nm_prev(done, lim);
1539 nm_txsync_finalize(kring);
1540 if (netmap_verbose)
1666done:
1667 if (done != cur)
1668 D("early break at %d/ %d, tail %d", done, cur, kring->nr_hwtail);
1669 /*
1670 * packets between 'done' and 'cur' are left unsent.
1671 */
1672 kring->nr_hwcur = done;
1673 kring->nr_hwtail = nm_prev(done, lim);
1674 nm_txsync_finalize(kring);
1675 if (netmap_verbose)
1541 D("%s ring %d flags %d", NM_IFPNAME(na->up.ifp), ring_nr, flags);
1676 D("%s ring %d flags %d", na->up.name, kring->ring_id, flags);
1542 return 0;
1543}
1544
1545
1677 return 0;
1678}
1679
1680
1546/*
1547 * main dispatch routine for the bridge.
1548 * We already know that only one thread is running this.
1549 * we must run nm_bdg_preflush without lock.
1681/* rxsync code used by VALE ports nm_rxsync callback and also
1682 * internally by the brwap
1550 */
1551static int
1683 */
1684static int
1552bdg_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags)
1685netmap_vp_rxsync_locked(struct netmap_kring *kring, int flags)
1553{
1686{
1554 struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter*)na;
1555 return netmap_vp_txsync(vpna, ring_nr, flags);
1556}
1557
1558static int
1559netmap_vp_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags)
1560{
1561 struct netmap_kring *kring = &na->rx_rings[ring_nr];
1687 struct netmap_adapter *na = kring->na;
1562 struct netmap_ring *ring = kring->ring;
1563 u_int nm_i, lim = kring->nkr_num_slots - 1;
1564 u_int head = nm_rxsync_prologue(kring);
1565 int n;
1566
1567 if (head > lim) {
1568 D("ouch dangerous reset!!!");
1569 n = netmap_ring_reinit(kring);

--- 4 unchanged lines hidden (view full) ---

1574 /* actually nothing to do here, they are already in the kring */
1575
1576 /* Second part, skip past packets that userspace has released. */
1577 nm_i = kring->nr_hwcur;
1578 if (nm_i != head) {
1579 /* consistency check, but nothing really important here */
1580 for (n = 0; likely(nm_i != head); n++) {
1581 struct netmap_slot *slot = &ring->slot[nm_i];
1688 struct netmap_ring *ring = kring->ring;
1689 u_int nm_i, lim = kring->nkr_num_slots - 1;
1690 u_int head = nm_rxsync_prologue(kring);
1691 int n;
1692
1693 if (head > lim) {
1694 D("ouch dangerous reset!!!");
1695 n = netmap_ring_reinit(kring);

--- 4 unchanged lines hidden (view full) ---

1700 /* actually nothing to do here, they are already in the kring */
1701
1702 /* Second part, skip past packets that userspace has released. */
1703 nm_i = kring->nr_hwcur;
1704 if (nm_i != head) {
1705 /* consistency check, but nothing really important here */
1706 for (n = 0; likely(nm_i != head); n++) {
1707 struct netmap_slot *slot = &ring->slot[nm_i];
1582 void *addr = BDG_NMB(na, slot);
1708 void *addr = NMB(na, slot);
1583
1709
1584 if (addr == netmap_buffer_base) { /* bad buf */
1710 if (addr == NETMAP_BUF_BASE(kring->na)) { /* bad buf */
1585 D("bad buffer index %d, ignore ?",
1586 slot->buf_idx);
1587 }
1588 slot->flags &= ~NS_BUF_CHANGED;
1589 nm_i = nm_next(nm_i, lim);
1590 }
1591 kring->nr_hwcur = head;
1592 }
1593
1594 /* tell userspace that there are new packets */
1595 nm_rxsync_finalize(kring);
1596 n = 0;
1597done:
1598 return n;
1599}
1600
1601/*
1711 D("bad buffer index %d, ignore ?",
1712 slot->buf_idx);
1713 }
1714 slot->flags &= ~NS_BUF_CHANGED;
1715 nm_i = nm_next(nm_i, lim);
1716 }
1717 kring->nr_hwcur = head;
1718 }
1719
1720 /* tell userspace that there are new packets */
1721 nm_rxsync_finalize(kring);
1722 n = 0;
1723done:
1724 return n;
1725}
1726
1727/*
1728 * nm_rxsync callback for VALE ports
1602 * user process reading from a VALE switch.
1603 * Already protected against concurrent calls from userspace,
1604 * but we must acquire the queue's lock to protect against
1605 * writers on the same queue.
1606 */
1607static int
1729 * user process reading from a VALE switch.
1730 * Already protected against concurrent calls from userspace,
1731 * but we must acquire the queue's lock to protect against
1732 * writers on the same queue.
1733 */
1734static int
1608bdg_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags)
1735netmap_vp_rxsync(struct netmap_kring *kring, int flags)
1609{
1736{
1610 struct netmap_kring *kring = &na->rx_rings[ring_nr];
1611 int n;
1612
1613 mtx_lock(&kring->q_lock);
1737 int n;
1738
1739 mtx_lock(&kring->q_lock);
1614 n = netmap_vp_rxsync(na, ring_nr, flags);
1740 n = netmap_vp_rxsync_locked(kring, flags);
1615 mtx_unlock(&kring->q_lock);
1616 return n;
1617}
1618
1619
1741 mtx_unlock(&kring->q_lock);
1742 return n;
1743}
1744
1745
1746/* nm_bdg_attach callback for VALE ports
1747 * The na_vp port is this same netmap_adapter. There is no host port.
1748 */
1620static int
1749static int
1621bdg_netmap_attach(struct nmreq *nmr, struct ifnet *ifp)
1750netmap_vp_bdg_attach(const char *name, struct netmap_adapter *na)
1622{
1751{
1752 struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter *)na;
1753
1754 if (vpna->na_bdg)
1755 return EBUSY;
1756 na->na_vp = vpna;
1757 strncpy(na->name, name, sizeof(na->name));
1758 na->na_hostvp = NULL;
1759 return 0;
1760}
1761
1762/* create a netmap_vp_adapter that describes a VALE port.
1763 * Only persistent VALE ports have a non-null ifp.
1764 */
1765static int
1766netmap_vp_create(struct nmreq *nmr, struct ifnet *ifp, struct netmap_vp_adapter **ret)
1767{
1623 struct netmap_vp_adapter *vpna;
1624 struct netmap_adapter *na;
1625 int error;
1626 u_int npipes = 0;
1627
1628 vpna = malloc(sizeof(*vpna), M_DEVBUF, M_NOWAIT | M_ZERO);
1629 if (vpna == NULL)
1630 return ENOMEM;
1631
1632 na = &vpna->up;
1633
1634 na->ifp = ifp;
1768 struct netmap_vp_adapter *vpna;
1769 struct netmap_adapter *na;
1770 int error;
1771 u_int npipes = 0;
1772
1773 vpna = malloc(sizeof(*vpna), M_DEVBUF, M_NOWAIT | M_ZERO);
1774 if (vpna == NULL)
1775 return ENOMEM;
1776
1777 na = &vpna->up;
1778
1779 na->ifp = ifp;
1780 strncpy(na->name, nmr->nr_name, sizeof(na->name));
1635
1636 /* bound checking */
1637 na->num_tx_rings = nmr->nr_tx_rings;
1638 nm_bound_var(&na->num_tx_rings, 1, 1, NM_BDG_MAXRINGS, NULL);
1639 nmr->nr_tx_rings = na->num_tx_rings; // write back
1640 na->num_rx_rings = nmr->nr_rx_rings;
1641 nm_bound_var(&na->num_rx_rings, 1, 1, NM_BDG_MAXRINGS, NULL);
1642 nmr->nr_rx_rings = na->num_rx_rings; // write back

--- 16 unchanged lines hidden (view full) ---

1659 vpna->virt_hdr_len = 0;
1660 vpna->mfs = 1514;
1661 /*if (vpna->mfs > netmap_buf_size) TODO netmap_buf_size is zero??
1662 vpna->mfs = netmap_buf_size; */
1663 if (netmap_verbose)
1664 D("max frame size %u", vpna->mfs);
1665
1666 na->na_flags |= NAF_BDG_MAYSLEEP | NAF_MEM_OWNER;
1781
1782 /* bound checking */
1783 na->num_tx_rings = nmr->nr_tx_rings;
1784 nm_bound_var(&na->num_tx_rings, 1, 1, NM_BDG_MAXRINGS, NULL);
1785 nmr->nr_tx_rings = na->num_tx_rings; // write back
1786 na->num_rx_rings = nmr->nr_rx_rings;
1787 nm_bound_var(&na->num_rx_rings, 1, 1, NM_BDG_MAXRINGS, NULL);
1788 nmr->nr_rx_rings = na->num_rx_rings; // write back

--- 16 unchanged lines hidden (view full) ---

1805 vpna->virt_hdr_len = 0;
1806 vpna->mfs = 1514;
1807 /*if (vpna->mfs > netmap_buf_size) TODO netmap_buf_size is zero??
1808 vpna->mfs = netmap_buf_size; */
1809 if (netmap_verbose)
1810 D("max frame size %u", vpna->mfs);
1811
1812 na->na_flags |= NAF_BDG_MAYSLEEP | NAF_MEM_OWNER;
1667 na->nm_txsync = bdg_netmap_txsync;
1668 na->nm_rxsync = bdg_netmap_rxsync;
1669 na->nm_register = bdg_netmap_reg;
1670 na->nm_dtor = netmap_adapter_vp_dtor;
1813 na->nm_txsync = netmap_vp_txsync;
1814 na->nm_rxsync = netmap_vp_rxsync;
1815 na->nm_register = netmap_vp_reg;
1671 na->nm_krings_create = netmap_vp_krings_create;
1672 na->nm_krings_delete = netmap_vp_krings_delete;
1816 na->nm_krings_create = netmap_vp_krings_create;
1817 na->nm_krings_delete = netmap_vp_krings_delete;
1673 na->nm_mem = netmap_mem_private_new(NM_IFPNAME(na->ifp),
1818 na->nm_dtor = netmap_vp_dtor;
1819 na->nm_mem = netmap_mem_private_new(na->name,
1674 na->num_tx_rings, na->num_tx_desc,
1675 na->num_rx_rings, na->num_rx_desc,
1676 nmr->nr_arg3, npipes, &error);
1677 if (na->nm_mem == NULL)
1678 goto err;
1820 na->num_tx_rings, na->num_tx_desc,
1821 na->num_rx_rings, na->num_rx_desc,
1822 nmr->nr_arg3, npipes, &error);
1823 if (na->nm_mem == NULL)
1824 goto err;
1825 na->nm_bdg_attach = netmap_vp_bdg_attach;
1679 /* other nmd fields are set in the common routine */
1680 error = netmap_attach_common(na);
1681 if (error)
1682 goto err;
1826 /* other nmd fields are set in the common routine */
1827 error = netmap_attach_common(na);
1828 if (error)
1829 goto err;
1830 *ret = vpna;
1683 return 0;
1684
1685err:
1686 if (na->nm_mem != NULL)
1687 netmap_mem_private_delete(na->nm_mem);
1688 free(vpna, M_DEVBUF);
1689 return error;
1690}
1691
1831 return 0;
1832
1833err:
1834 if (na->nm_mem != NULL)
1835 netmap_mem_private_delete(na->nm_mem);
1836 free(vpna, M_DEVBUF);
1837 return error;
1838}
1839
1840/* Bridge wrapper code (bwrap).
1841 * This is used to connect a non-VALE-port netmap_adapter (hwna) to a
1842 * VALE switch.
1843 * The main task is to swap the meaning of tx and rx rings to match the
1844 * expectations of the VALE switch code (see nm_bdg_flush).
1845 *
1846 * The bwrap works by interposing a netmap_bwrap_adapter between the
1847 * rest of the system and the hwna. The netmap_bwrap_adapter looks like
1848 * a netmap_vp_adapter to the rest the system, but, internally, it
1849 * translates all callbacks to what the hwna expects.
1850 *
1851 * Note that we have to intercept callbacks coming from two sides:
1852 *
1853 * - callbacks coming from the netmap module are intercepted by
1854 * passing around the netmap_bwrap_adapter instead of the hwna
1855 *
1856 * - callbacks coming from outside of the netmap module only know
1857 * about the hwna. This, however, only happens in interrupt
1858 * handlers, where only the hwna->nm_notify callback is called.
1859 * What the bwrap does is to overwrite the hwna->nm_notify callback
1860 * with its own netmap_bwrap_intr_notify.
1861 * XXX This assumes that the hwna->nm_notify callback was the
1862 * standard netmap_notify(), as it is the case for nic adapters.
1863 * Any additional action performed by hwna->nm_notify will not be
1864 * performed by netmap_bwrap_intr_notify.
1865 *
1866 * Additionally, the bwrap can optionally attach the host rings pair
1867 * of the wrapped adapter to a different port of the switch.
1868 */
1692
1869
1870
1693static void
1694netmap_bwrap_dtor(struct netmap_adapter *na)
1695{
1696 struct netmap_bwrap_adapter *bna = (struct netmap_bwrap_adapter*)na;
1697 struct netmap_adapter *hwna = bna->hwna;
1871static void
1872netmap_bwrap_dtor(struct netmap_adapter *na)
1873{
1874 struct netmap_bwrap_adapter *bna = (struct netmap_bwrap_adapter*)na;
1875 struct netmap_adapter *hwna = bna->hwna;
1698 struct nm_bridge *b = bna->up.na_bdg,
1699 *bh = bna->host.na_bdg;
1700 struct ifnet *ifp = na->ifp;
1701
1702 ND("na %p", na);
1876
1877 ND("na %p", na);
1878 /* drop reference to hwna->ifp.
1879 * If we don't do this, netmap_detach_common(na)
1880 * will think it has set NA(na->ifp) to NULL
1881 */
1882 na->ifp = NULL;
1883 /* for safety, also drop the possible reference
1884 * in the hostna
1885 */
1886 bna->host.up.ifp = NULL;
1703
1887
1704 if (b) {
1705 netmap_bdg_detach_common(b, bna->up.bdg_port,
1706 (bh ? bna->host.bdg_port : -1));
1707 }
1708
1888 hwna->nm_mem = bna->save_nmd;
1709 hwna->na_private = NULL;
1889 hwna->na_private = NULL;
1890 hwna->na_vp = hwna->na_hostvp = NULL;
1891 hwna->na_flags &= ~NAF_BUSY;
1710 netmap_adapter_put(hwna);
1711
1892 netmap_adapter_put(hwna);
1893
1712 bzero(ifp, sizeof(*ifp));
1713 free(ifp, M_DEVBUF);
1714 na->ifp = NULL;
1715
1716}
1717
1718
1719/*
1720 * Intr callback for NICs connected to a bridge.
1721 * Simply ignore tx interrupts (maybe we could try to recover space ?)
1722 * and pass received packets from nic to the bridge.
1723 *

--- 8 unchanged lines hidden (view full) ---

1732 *
1733 * callback that overwrites the hwna notify callback.
1734 * Packets come from the outside or from the host stack and are put on an hwna rx ring.
1735 * The bridge wrapper then sends the packets through the bridge.
1736 */
1737static int
1738netmap_bwrap_intr_notify(struct netmap_adapter *na, u_int ring_nr, enum txrx tx, int flags)
1739{
1894}
1895
1896
1897/*
1898 * Intr callback for NICs connected to a bridge.
1899 * Simply ignore tx interrupts (maybe we could try to recover space ?)
1900 * and pass received packets from nic to the bridge.
1901 *

--- 8 unchanged lines hidden (view full) ---

1910 *
1911 * callback that overwrites the hwna notify callback.
1912 * Packets come from the outside or from the host stack and are put on an hwna rx ring.
1913 * The bridge wrapper then sends the packets through the bridge.
1914 */
1915static int
1916netmap_bwrap_intr_notify(struct netmap_adapter *na, u_int ring_nr, enum txrx tx, int flags)
1917{
1740 struct ifnet *ifp = na->ifp;
1741 struct netmap_bwrap_adapter *bna = na->na_private;
1742 struct netmap_vp_adapter *hostna = &bna->host;
1743 struct netmap_kring *kring, *bkring;
1744 struct netmap_ring *ring;
1745 int is_host_ring = ring_nr == na->num_rx_rings;
1746 struct netmap_vp_adapter *vpna = &bna->up;
1747 int error = 0;
1748
1749 if (netmap_verbose)
1918 struct netmap_bwrap_adapter *bna = na->na_private;
1919 struct netmap_vp_adapter *hostna = &bna->host;
1920 struct netmap_kring *kring, *bkring;
1921 struct netmap_ring *ring;
1922 int is_host_ring = ring_nr == na->num_rx_rings;
1923 struct netmap_vp_adapter *vpna = &bna->up;
1924 int error = 0;
1925
1926 if (netmap_verbose)
1750 D("%s %s%d 0x%x", NM_IFPNAME(ifp),
1927 D("%s %s%d 0x%x", na->name,
1751 (tx == NR_TX ? "TX" : "RX"), ring_nr, flags);
1752
1753 if (flags & NAF_DISABLE_NOTIFY) {
1928 (tx == NR_TX ? "TX" : "RX"), ring_nr, flags);
1929
1930 if (flags & NAF_DISABLE_NOTIFY) {
1754 kring = tx == NR_TX ? na->tx_rings : na->rx_rings;
1755 bkring = tx == NR_TX ? vpna->up.rx_rings : vpna->up.tx_rings;
1756 if (kring[ring_nr].nkr_stopped)
1757 netmap_disable_ring(&bkring[ring_nr]);
1758 else
1759 bkring[ring_nr].nkr_stopped = 0;
1931 /* the enabled/disabled state of the ring has changed,
1932 * propagate the info to the wrapper (with tx/rx swapped)
1933 */
1934 if (tx == NR_TX) {
1935 netmap_set_rxring(&vpna->up, ring_nr,
1936 na->tx_rings[ring_nr].nkr_stopped);
1937 } else {
1938 netmap_set_txring(&vpna->up, ring_nr,
1939 na->rx_rings[ring_nr].nkr_stopped);
1940 }
1760 return 0;
1761 }
1762
1941 return 0;
1942 }
1943
1763 if (ifp == NULL || !(ifp->if_capenable & IFCAP_NETMAP))
1944 if (!nm_netmap_on(na))
1764 return 0;
1765
1766 /* we only care about receive interrupts */
1767 if (tx == NR_TX)
1768 return 0;
1769
1770 kring = &na->rx_rings[ring_nr];
1771 ring = kring->ring;

--- 9 unchanged lines hidden (view full) ---

1781
1782 /* Here we expect ring->head = ring->cur = ring->tail
1783 * because everything has been released from the previous round.
1784 * However the ring is shared and we might have info from
1785 * the wrong side (the tx ring). Hence we overwrite with
1786 * the info from the rx kring.
1787 */
1788 if (netmap_verbose)
1945 return 0;
1946
1947 /* we only care about receive interrupts */
1948 if (tx == NR_TX)
1949 return 0;
1950
1951 kring = &na->rx_rings[ring_nr];
1952 ring = kring->ring;

--- 9 unchanged lines hidden (view full) ---

1962
1963 /* Here we expect ring->head = ring->cur = ring->tail
1964 * because everything has been released from the previous round.
1965 * However the ring is shared and we might have info from
1966 * the wrong side (the tx ring). Hence we overwrite with
1967 * the info from the rx kring.
1968 */
1969 if (netmap_verbose)
1789 D("%s head %d cur %d tail %d (kring %d %d %d)", NM_IFPNAME(ifp),
1970 D("%s head %d cur %d tail %d (kring %d %d %d)", na->name,
1790 ring->head, ring->cur, ring->tail,
1791 kring->rhead, kring->rcur, kring->rtail);
1792
1793 ring->head = kring->rhead;
1794 ring->cur = kring->rcur;
1795 ring->tail = kring->rtail;
1796
1797 if (is_host_ring) {

--- 4 unchanged lines hidden (view full) ---

1802 /* fetch packets that have arrived.
1803 * XXX maybe do this in a loop ?
1804 */
1805 error = kring->nm_sync(kring, 0);
1806 if (error)
1807 goto put_out;
1808 if (kring->nr_hwcur == kring->nr_hwtail && netmap_verbose) {
1809 D("how strange, interrupt with no packets on %s",
1971 ring->head, ring->cur, ring->tail,
1972 kring->rhead, kring->rcur, kring->rtail);
1973
1974 ring->head = kring->rhead;
1975 ring->cur = kring->rcur;
1976 ring->tail = kring->rtail;
1977
1978 if (is_host_ring) {

--- 4 unchanged lines hidden (view full) ---

1983 /* fetch packets that have arrived.
1984 * XXX maybe do this in a loop ?
1985 */
1986 error = kring->nm_sync(kring, 0);
1987 if (error)
1988 goto put_out;
1989 if (kring->nr_hwcur == kring->nr_hwtail && netmap_verbose) {
1990 D("how strange, interrupt with no packets on %s",
1810 NM_IFPNAME(ifp));
1991 na->name);
1811 goto put_out;
1812 }
1813
1814 /* new packets are ring->cur to ring->tail, and the bkring
1815 * had hwcur == ring->cur. So advance ring->cur to ring->tail
1816 * to push all packets out.
1817 */
1818 ring->head = ring->cur = ring->tail;
1819
1820 /* also set tail to what the bwrap expects */
1821 bkring = &vpna->up.tx_rings[ring_nr];
1822 ring->tail = bkring->nr_hwtail; // rtail too ?
1823
1824 /* pass packets to the switch */
1825 nm_txsync_prologue(bkring); // XXX error checking ?
1992 goto put_out;
1993 }
1994
1995 /* new packets are ring->cur to ring->tail, and the bkring
1996 * had hwcur == ring->cur. So advance ring->cur to ring->tail
1997 * to push all packets out.
1998 */
1999 ring->head = ring->cur = ring->tail;
2000
2001 /* also set tail to what the bwrap expects */
2002 bkring = &vpna->up.tx_rings[ring_nr];
2003 ring->tail = bkring->nr_hwtail; // rtail too ?
2004
2005 /* pass packets to the switch */
2006 nm_txsync_prologue(bkring); // XXX error checking ?
1826 netmap_vp_txsync(vpna, ring_nr, flags);
2007 netmap_vp_txsync(bkring, flags);
1827
1828 /* mark all buffers as released on this ring */
1829 ring->head = ring->cur = kring->nr_hwtail;
1830 ring->tail = kring->rtail;
1831 /* another call to actually release the buffers */
1832 if (!is_host_ring) {
1833 error = kring->nm_sync(kring, 0);
1834 } else {

--- 5 unchanged lines hidden (view full) ---

1840 }
1841
1842put_out:
1843 nm_kr_put(kring);
1844 return error;
1845}
1846
1847
2008
2009 /* mark all buffers as released on this ring */
2010 ring->head = ring->cur = kring->nr_hwtail;
2011 ring->tail = kring->rtail;
2012 /* another call to actually release the buffers */
2013 if (!is_host_ring) {
2014 error = kring->nm_sync(kring, 0);
2015 } else {

--- 5 unchanged lines hidden (view full) ---

2021 }
2022
2023put_out:
2024 nm_kr_put(kring);
2025 return error;
2026}
2027
2028
2029/* nm_register callback for bwrap */
1848static int
1849netmap_bwrap_register(struct netmap_adapter *na, int onoff)
1850{
1851 struct netmap_bwrap_adapter *bna =
1852 (struct netmap_bwrap_adapter *)na;
1853 struct netmap_adapter *hwna = bna->hwna;
1854 struct netmap_vp_adapter *hostna = &bna->host;
1855 int error;
1856
2030static int
2031netmap_bwrap_register(struct netmap_adapter *na, int onoff)
2032{
2033 struct netmap_bwrap_adapter *bna =
2034 (struct netmap_bwrap_adapter *)na;
2035 struct netmap_adapter *hwna = bna->hwna;
2036 struct netmap_vp_adapter *hostna = &bna->host;
2037 int error;
2038
1857 ND("%s %s", NM_IFPNAME(na->ifp), onoff ? "on" : "off");
2039 ND("%s %s", na->name, onoff ? "on" : "off");
1858
1859 if (onoff) {
1860 int i;
1861
2040
2041 if (onoff) {
2042 int i;
2043
2044 /* netmap_do_regif has been called on the bwrap na.
2045 * We need to pass the information about the
2046 * memory allocator down to the hwna before
2047 * putting it in netmap mode
2048 */
1862 hwna->na_lut = na->na_lut;
1863 hwna->na_lut_objtotal = na->na_lut_objtotal;
2049 hwna->na_lut = na->na_lut;
2050 hwna->na_lut_objtotal = na->na_lut_objtotal;
2051 hwna->na_lut_objsize = na->na_lut_objsize;
1864
1865 if (hostna->na_bdg) {
2052
2053 if (hostna->na_bdg) {
2054 /* if the host rings have been attached to switch,
2055 * we need to copy the memory allocator information
2056 * in the hostna also
2057 */
1866 hostna->up.na_lut = na->na_lut;
1867 hostna->up.na_lut_objtotal = na->na_lut_objtotal;
2058 hostna->up.na_lut = na->na_lut;
2059 hostna->up.na_lut_objtotal = na->na_lut_objtotal;
2060 hostna->up.na_lut_objsize = na->na_lut_objsize;
1868 }
1869
1870 /* cross-link the netmap rings
1871 * The original number of rings comes from hwna,
1872 * rx rings on one side equals tx rings on the other.
2061 }
2062
2063 /* cross-link the netmap rings
2064 * The original number of rings comes from hwna,
2065 * rx rings on one side equals tx rings on the other.
2066 * We need to do this now, after the initialization
2067 * of the kring->ring pointers
1873 */
1874 for (i = 0; i < na->num_rx_rings + 1; i++) {
1875 hwna->tx_rings[i].nkr_num_slots = na->rx_rings[i].nkr_num_slots;
1876 hwna->tx_rings[i].ring = na->rx_rings[i].ring;
1877 }
1878 for (i = 0; i < na->num_tx_rings + 1; i++) {
1879 hwna->rx_rings[i].nkr_num_slots = na->tx_rings[i].nkr_num_slots;
1880 hwna->rx_rings[i].ring = na->tx_rings[i].ring;
1881 }
1882 }
1883
2068 */
2069 for (i = 0; i < na->num_rx_rings + 1; i++) {
2070 hwna->tx_rings[i].nkr_num_slots = na->rx_rings[i].nkr_num_slots;
2071 hwna->tx_rings[i].ring = na->rx_rings[i].ring;
2072 }
2073 for (i = 0; i < na->num_tx_rings + 1; i++) {
2074 hwna->rx_rings[i].nkr_num_slots = na->tx_rings[i].nkr_num_slots;
2075 hwna->rx_rings[i].ring = na->tx_rings[i].ring;
2076 }
2077 }
2078
1884 if (hwna->ifp) {
1885 error = hwna->nm_register(hwna, onoff);
1886 if (error)
1887 return error;
1888 }
2079 /* forward the request to the hwna */
2080 error = hwna->nm_register(hwna, onoff);
2081 if (error)
2082 return error;
1889
2083
1890 bdg_netmap_reg(na, onoff);
2084 /* impersonate a netmap_vp_adapter */
2085 netmap_vp_reg(na, onoff);
2086 if (hostna->na_bdg)
2087 netmap_vp_reg(&hostna->up, onoff);
1891
1892 if (onoff) {
2088
2089 if (onoff) {
2090 /* intercept the hwna nm_nofify callback */
1893 bna->save_notify = hwna->nm_notify;
1894 hwna->nm_notify = netmap_bwrap_intr_notify;
1895 } else {
1896 hwna->nm_notify = bna->save_notify;
1897 hwna->na_lut = NULL;
1898 hwna->na_lut_objtotal = 0;
2091 bna->save_notify = hwna->nm_notify;
2092 hwna->nm_notify = netmap_bwrap_intr_notify;
2093 } else {
2094 hwna->nm_notify = bna->save_notify;
2095 hwna->na_lut = NULL;
2096 hwna->na_lut_objtotal = 0;
2097 hwna->na_lut_objsize = 0;
1899 }
1900
1901 return 0;
1902}
1903
2098 }
2099
2100 return 0;
2101}
2102
1904
2103/* nm_config callback for bwrap */
1905static int
1906netmap_bwrap_config(struct netmap_adapter *na, u_int *txr, u_int *txd,
1907 u_int *rxr, u_int *rxd)
1908{
1909 struct netmap_bwrap_adapter *bna =
1910 (struct netmap_bwrap_adapter *)na;
1911 struct netmap_adapter *hwna = bna->hwna;
1912

--- 4 unchanged lines hidden (view full) ---

1917 *txd = hwna->num_rx_desc;
1918 *rxr = hwna->num_tx_rings;
1919 *rxd = hwna->num_rx_desc;
1920
1921 return 0;
1922}
1923
1924
2104static int
2105netmap_bwrap_config(struct netmap_adapter *na, u_int *txr, u_int *txd,
2106 u_int *rxr, u_int *rxd)
2107{
2108 struct netmap_bwrap_adapter *bna =
2109 (struct netmap_bwrap_adapter *)na;
2110 struct netmap_adapter *hwna = bna->hwna;
2111

--- 4 unchanged lines hidden (view full) ---

2116 *txd = hwna->num_rx_desc;
2117 *rxr = hwna->num_tx_rings;
2118 *rxd = hwna->num_rx_desc;
2119
2120 return 0;
2121}
2122
2123
2124/* nm_krings_create callback for bwrap */
1925static int
1926netmap_bwrap_krings_create(struct netmap_adapter *na)
1927{
1928 struct netmap_bwrap_adapter *bna =
1929 (struct netmap_bwrap_adapter *)na;
1930 struct netmap_adapter *hwna = bna->hwna;
1931 struct netmap_adapter *hostna = &bna->host.up;
1932 int error;
1933
2125static int
2126netmap_bwrap_krings_create(struct netmap_adapter *na)
2127{
2128 struct netmap_bwrap_adapter *bna =
2129 (struct netmap_bwrap_adapter *)na;
2130 struct netmap_adapter *hwna = bna->hwna;
2131 struct netmap_adapter *hostna = &bna->host.up;
2132 int error;
2133
1934 ND("%s", NM_IFPNAME(na->ifp));
2134 ND("%s", na->name);
1935
2135
2136 /* impersonate a netmap_vp_adapter */
1936 error = netmap_vp_krings_create(na);
1937 if (error)
1938 return error;
1939
2137 error = netmap_vp_krings_create(na);
2138 if (error)
2139 return error;
2140
2141 /* also create the hwna krings */
1940 error = hwna->nm_krings_create(hwna);
1941 if (error) {
1942 netmap_vp_krings_delete(na);
1943 return error;
1944 }
2142 error = hwna->nm_krings_create(hwna);
2143 if (error) {
2144 netmap_vp_krings_delete(na);
2145 return error;
2146 }
2147 /* the connection between the bwrap krings and the hwna krings
2148 * will be perfomed later, in the nm_register callback, since
2149 * now the kring->ring pointers have not been initialized yet
2150 */
1945
1946 if (na->na_flags & NAF_HOST_RINGS) {
2151
2152 if (na->na_flags & NAF_HOST_RINGS) {
2153 /* the hostna rings are the host rings of the bwrap.
2154 * The corresponding krings must point back to the
2155 * hostna
2156 */
1947 hostna->tx_rings = na->tx_rings + na->num_tx_rings;
2157 hostna->tx_rings = na->tx_rings + na->num_tx_rings;
2158 hostna->tx_rings[0].na = hostna;
1948 hostna->rx_rings = na->rx_rings + na->num_rx_rings;
2159 hostna->rx_rings = na->rx_rings + na->num_rx_rings;
2160 hostna->rx_rings[0].na = hostna;
1949 }
1950
1951 return 0;
1952}
1953
1954
1955static void
1956netmap_bwrap_krings_delete(struct netmap_adapter *na)
1957{
1958 struct netmap_bwrap_adapter *bna =
1959 (struct netmap_bwrap_adapter *)na;
1960 struct netmap_adapter *hwna = bna->hwna;
1961
2161 }
2162
2163 return 0;
2164}
2165
2166
2167static void
2168netmap_bwrap_krings_delete(struct netmap_adapter *na)
2169{
2170 struct netmap_bwrap_adapter *bna =
2171 (struct netmap_bwrap_adapter *)na;
2172 struct netmap_adapter *hwna = bna->hwna;
2173
1962 ND("%s", NM_IFPNAME(na->ifp));
2174 ND("%s", na->name);
1963
1964 hwna->nm_krings_delete(hwna);
1965 netmap_vp_krings_delete(na);
1966}
1967
1968
1969/* notify method for the bridge-->hwna direction */
1970static int

--- 10 unchanged lines hidden (view full) ---

1981 if (tx == NR_TX)
1982 return EINVAL;
1983
1984 kring = &na->rx_rings[ring_n];
1985 hw_kring = &hwna->tx_rings[ring_n];
1986 ring = kring->ring;
1987 lim = kring->nkr_num_slots - 1;
1988
2175
2176 hwna->nm_krings_delete(hwna);
2177 netmap_vp_krings_delete(na);
2178}
2179
2180
2181/* notify method for the bridge-->hwna direction */
2182static int

--- 10 unchanged lines hidden (view full) ---

2193 if (tx == NR_TX)
2194 return EINVAL;
2195
2196 kring = &na->rx_rings[ring_n];
2197 hw_kring = &hwna->tx_rings[ring_n];
2198 ring = kring->ring;
2199 lim = kring->nkr_num_slots - 1;
2200
1989 if (hwna->ifp == NULL || !(hwna->ifp->if_capenable & IFCAP_NETMAP))
2201 if (!nm_netmap_on(hwna))
1990 return 0;
1991 mtx_lock(&kring->q_lock);
1992 /* first step: simulate a user wakeup on the rx ring */
2202 return 0;
2203 mtx_lock(&kring->q_lock);
2204 /* first step: simulate a user wakeup on the rx ring */
1993 netmap_vp_rxsync(na, ring_n, flags);
2205 netmap_vp_rxsync_locked(kring, flags);
1994 ND("%s[%d] PRE rx(c%3d t%3d l%3d) ring(h%3d c%3d t%3d) tx(c%3d ht%3d t%3d)",
2206 ND("%s[%d] PRE rx(c%3d t%3d l%3d) ring(h%3d c%3d t%3d) tx(c%3d ht%3d t%3d)",
1995 NM_IFPNAME(na->ifp), ring_n,
2207 na->name, ring_n,
1996 kring->nr_hwcur, kring->nr_hwtail, kring->nkr_hwlease,
1997 ring->head, ring->cur, ring->tail,
1998 hw_kring->nr_hwcur, hw_kring->nr_hwtail, hw_ring->rtail);
1999 /* second step: the simulated user consumes all new packets */
2000 ring->head = ring->cur = ring->tail;
2001
2002 /* third step: the new packets are sent on the tx ring
2003 * (which is actually the same ring)

--- 4 unchanged lines hidden (view full) ---

2008 error = hw_kring->nm_sync(hw_kring, flags);
2009
2010 /* fourth step: now we are back the rx ring */
2011 /* claim ownership on all hw owned bufs */
2012 ring->head = nm_next(ring->tail, lim); /* skip past reserved slot */
2013 ring->tail = kring->rtail; /* restore saved value of tail, for safety */
2014
2015 /* fifth step: the user goes to sleep again, causing another rxsync */
2208 kring->nr_hwcur, kring->nr_hwtail, kring->nkr_hwlease,
2209 ring->head, ring->cur, ring->tail,
2210 hw_kring->nr_hwcur, hw_kring->nr_hwtail, hw_ring->rtail);
2211 /* second step: the simulated user consumes all new packets */
2212 ring->head = ring->cur = ring->tail;
2213
2214 /* third step: the new packets are sent on the tx ring
2215 * (which is actually the same ring)

--- 4 unchanged lines hidden (view full) ---

2220 error = hw_kring->nm_sync(hw_kring, flags);
2221
2222 /* fourth step: now we are back the rx ring */
2223 /* claim ownership on all hw owned bufs */
2224 ring->head = nm_next(ring->tail, lim); /* skip past reserved slot */
2225 ring->tail = kring->rtail; /* restore saved value of tail, for safety */
2226
2227 /* fifth step: the user goes to sleep again, causing another rxsync */
2016 netmap_vp_rxsync(na, ring_n, flags);
2228 netmap_vp_rxsync_locked(kring, flags);
2017 ND("%s[%d] PST rx(c%3d t%3d l%3d) ring(h%3d c%3d t%3d) tx(c%3d ht%3d t%3d)",
2229 ND("%s[%d] PST rx(c%3d t%3d l%3d) ring(h%3d c%3d t%3d) tx(c%3d ht%3d t%3d)",
2018 NM_IFPNAME(na->ifp), ring_n,
2230 na->name, ring_n,
2019 kring->nr_hwcur, kring->nr_hwtail, kring->nkr_hwlease,
2020 ring->head, ring->cur, ring->tail,
2021 hw_kring->nr_hwcur, hw_kring->nr_hwtail, hw_kring->rtail);
2022 mtx_unlock(&kring->q_lock);
2023 return error;
2024}
2025
2026
2231 kring->nr_hwcur, kring->nr_hwtail, kring->nkr_hwlease,
2232 ring->head, ring->cur, ring->tail,
2233 hw_kring->nr_hwcur, hw_kring->nr_hwtail, hw_kring->rtail);
2234 mtx_unlock(&kring->q_lock);
2235 return error;
2236}
2237
2238
2239/* notify method for the bridge-->host-rings path */
2027static int
2028netmap_bwrap_host_notify(struct netmap_adapter *na, u_int ring_n, enum txrx tx, int flags)
2029{
2030 struct netmap_bwrap_adapter *bna = na->na_private;
2031 struct netmap_adapter *port_na = &bna->up.up;
2032 if (tx == NR_TX || ring_n != 0)
2033 return EINVAL;
2034 return netmap_bwrap_notify(port_na, port_na->num_rx_rings, NR_RX, flags);
2035}
2036
2037
2240static int
2241netmap_bwrap_host_notify(struct netmap_adapter *na, u_int ring_n, enum txrx tx, int flags)
2242{
2243 struct netmap_bwrap_adapter *bna = na->na_private;
2244 struct netmap_adapter *port_na = &bna->up.up;
2245 if (tx == NR_TX || ring_n != 0)
2246 return EINVAL;
2247 return netmap_bwrap_notify(port_na, port_na->num_rx_rings, NR_RX, flags);
2248}
2249
2250
2038/* attach a bridge wrapper to the 'real' device */
2251/* nm_bdg_ctl callback for the bwrap.
2252 * Called on bridge-attach and detach, as an effect of vale-ctl -[ahd].
2253 * On attach, it needs to provide a fake netmap_priv_d structure and
2254 * perform a netmap_do_regif() on the bwrap. This will put both the
2255 * bwrap and the hwna in netmap mode, with the netmap rings shared
2256 * and cross linked. Moroever, it will start intercepting interrupts
2257 * directed to hwna.
2258 */
2039static int
2259static int
2040netmap_bwrap_attach(struct ifnet *fake, struct ifnet *real)
2260netmap_bwrap_bdg_ctl(struct netmap_adapter *na, struct nmreq *nmr, int attach)
2041{
2261{
2262 struct netmap_priv_d *npriv;
2263 struct netmap_bwrap_adapter *bna = (struct netmap_bwrap_adapter*)na;
2264 struct netmap_if *nifp;
2265 int error = 0;
2266
2267 if (attach) {
2268 if (NETMAP_OWNED_BY_ANY(na)) {
2269 return EBUSY;
2270 }
2271 if (bna->na_kpriv) {
2272 /* nothing to do */
2273 return 0;
2274 }
2275 npriv = malloc(sizeof(*npriv), M_DEVBUF, M_NOWAIT|M_ZERO);
2276 if (npriv == NULL)
2277 return ENOMEM;
2278 nifp = netmap_do_regif(npriv, na, nmr->nr_ringid, nmr->nr_flags, &error);
2279 if (!nifp) {
2280 bzero(npriv, sizeof(*npriv));
2281 free(npriv, M_DEVBUF);
2282 return error;
2283 }
2284 bna->na_kpriv = npriv;
2285 na->na_flags |= NAF_BUSY;
2286 } else {
2287 int last_instance;
2288
2289 if (na->active_fds == 0) /* not registered */
2290 return EINVAL;
2291 last_instance = netmap_dtor_locked(bna->na_kpriv);
2292 if (!last_instance) {
2293 D("--- error, trying to detach an entry with active mmaps");
2294 error = EINVAL;
2295 } else {
2296 struct nm_bridge *b = bna->up.na_bdg,
2297 *bh = bna->host.na_bdg;
2298 npriv = bna->na_kpriv;
2299 bna->na_kpriv = NULL;
2300 D("deleting priv");
2301
2302 bzero(npriv, sizeof(*npriv));
2303 free(npriv, M_DEVBUF);
2304 if (b) {
2305 /* XXX the bwrap dtor should take care
2306 * of this (2014-06-16)
2307 */
2308 netmap_bdg_detach_common(b, bna->up.bdg_port,
2309 (bh ? bna->host.bdg_port : -1));
2310 }
2311 na->na_flags &= ~NAF_BUSY;
2312 }
2313 }
2314 return error;
2315
2316}
2317
2318/* attach a bridge wrapper to the 'real' device */
2319int
2320netmap_bwrap_attach(const char *nr_name, struct netmap_adapter *hwna)
2321{
2042 struct netmap_bwrap_adapter *bna;
2322 struct netmap_bwrap_adapter *bna;
2043 struct netmap_adapter *na;
2044 struct netmap_adapter *hwna = NA(real);
2045 struct netmap_adapter *hostna;
2046 int error;
2323 struct netmap_adapter *na = NULL;
2324 struct netmap_adapter *hostna = NULL;
2325 int error = 0;
2047
2326
2327 /* make sure the NIC is not already in use */
2328 if (NETMAP_OWNED_BY_ANY(hwna)) {
2329 D("NIC %s busy, cannot attach to bridge", hwna->name);
2330 return EBUSY;
2331 }
2048
2049 bna = malloc(sizeof(*bna), M_DEVBUF, M_NOWAIT | M_ZERO);
2332
2333 bna = malloc(sizeof(*bna), M_DEVBUF, M_NOWAIT | M_ZERO);
2050 if (bna == NULL)
2334 if (bna == NULL) {
2051 return ENOMEM;
2335 return ENOMEM;
2336 }
2052
2053 na = &bna->up.up;
2337
2338 na = &bna->up.up;
2054 na->ifp = fake;
2339 strncpy(na->name, nr_name, sizeof(na->name));
2055 /* fill the ring data for the bwrap adapter with rx/tx meanings
2056 * swapped. The real cross-linking will be done during register,
2057 * when all the krings will have been created.
2058 */
2059 na->num_rx_rings = hwna->num_tx_rings;
2060 na->num_tx_rings = hwna->num_rx_rings;
2061 na->num_tx_desc = hwna->num_rx_desc;
2062 na->num_rx_desc = hwna->num_tx_desc;
2063 na->nm_dtor = netmap_bwrap_dtor;
2064 na->nm_register = netmap_bwrap_register;
2065 // na->nm_txsync = netmap_bwrap_txsync;
2066 // na->nm_rxsync = netmap_bwrap_rxsync;
2067 na->nm_config = netmap_bwrap_config;
2068 na->nm_krings_create = netmap_bwrap_krings_create;
2069 na->nm_krings_delete = netmap_bwrap_krings_delete;
2070 na->nm_notify = netmap_bwrap_notify;
2340 /* fill the ring data for the bwrap adapter with rx/tx meanings
2341 * swapped. The real cross-linking will be done during register,
2342 * when all the krings will have been created.
2343 */
2344 na->num_rx_rings = hwna->num_tx_rings;
2345 na->num_tx_rings = hwna->num_rx_rings;
2346 na->num_tx_desc = hwna->num_rx_desc;
2347 na->num_rx_desc = hwna->num_tx_desc;
2348 na->nm_dtor = netmap_bwrap_dtor;
2349 na->nm_register = netmap_bwrap_register;
2350 // na->nm_txsync = netmap_bwrap_txsync;
2351 // na->nm_rxsync = netmap_bwrap_rxsync;
2352 na->nm_config = netmap_bwrap_config;
2353 na->nm_krings_create = netmap_bwrap_krings_create;
2354 na->nm_krings_delete = netmap_bwrap_krings_delete;
2355 na->nm_notify = netmap_bwrap_notify;
2071 na->nm_mem = hwna->nm_mem;
2072 na->na_private = na; /* prevent NIOCREGIF */
2356 na->nm_bdg_ctl = netmap_bwrap_bdg_ctl;
2357 na->pdev = hwna->pdev;
2358 na->nm_mem = netmap_mem_private_new(na->name,
2359 na->num_tx_rings, na->num_tx_desc,
2360 na->num_rx_rings, na->num_rx_desc,
2361 0, 0, &error);
2362 na->na_flags |= NAF_MEM_OWNER;
2363 if (na->nm_mem == NULL)
2364 goto err_put;
2073 bna->up.retry = 1; /* XXX maybe this should depend on the hwna */
2074
2075 bna->hwna = hwna;
2076 netmap_adapter_get(hwna);
2077 hwna->na_private = bna; /* weak reference */
2365 bna->up.retry = 1; /* XXX maybe this should depend on the hwna */
2366
2367 bna->hwna = hwna;
2368 netmap_adapter_get(hwna);
2369 hwna->na_private = bna; /* weak reference */
2078
2370 hwna->na_vp = &bna->up;
2371
2079 if (hwna->na_flags & NAF_HOST_RINGS) {
2372 if (hwna->na_flags & NAF_HOST_RINGS) {
2373 if (hwna->na_flags & NAF_SW_ONLY)
2374 na->na_flags |= NAF_SW_ONLY;
2080 na->na_flags |= NAF_HOST_RINGS;
2081 hostna = &bna->host.up;
2375 na->na_flags |= NAF_HOST_RINGS;
2376 hostna = &bna->host.up;
2377 snprintf(hostna->name, sizeof(hostna->name), "%s^", nr_name);
2082 hostna->ifp = hwna->ifp;
2083 hostna->num_tx_rings = 1;
2084 hostna->num_tx_desc = hwna->num_rx_desc;
2085 hostna->num_rx_rings = 1;
2086 hostna->num_rx_desc = hwna->num_tx_desc;
2087 // hostna->nm_txsync = netmap_bwrap_host_txsync;
2088 // hostna->nm_rxsync = netmap_bwrap_host_rxsync;
2089 hostna->nm_notify = netmap_bwrap_host_notify;
2090 hostna->nm_mem = na->nm_mem;
2091 hostna->na_private = bna;
2378 hostna->ifp = hwna->ifp;
2379 hostna->num_tx_rings = 1;
2380 hostna->num_tx_desc = hwna->num_rx_desc;
2381 hostna->num_rx_rings = 1;
2382 hostna->num_rx_desc = hwna->num_tx_desc;
2383 // hostna->nm_txsync = netmap_bwrap_host_txsync;
2384 // hostna->nm_rxsync = netmap_bwrap_host_rxsync;
2385 hostna->nm_notify = netmap_bwrap_host_notify;
2386 hostna->nm_mem = na->nm_mem;
2387 hostna->na_private = bna;
2388 hostna->na_vp = &bna->up;
2389 na->na_hostvp = hwna->na_hostvp =
2390 hostna->na_hostvp = &bna->host;
2391 hostna->na_flags = NAF_BUSY; /* prevent NIOCREGIF */
2092 }
2093
2094 ND("%s<->%s txr %d txd %d rxr %d rxd %d",
2392 }
2393
2394 ND("%s<->%s txr %d txd %d rxr %d rxd %d",
2095 fake->if_xname, real->if_xname,
2395 na->name, ifp->if_xname,
2096 na->num_tx_rings, na->num_tx_desc,
2097 na->num_rx_rings, na->num_rx_desc);
2098
2099 error = netmap_attach_common(na);
2100 if (error) {
2396 na->num_tx_rings, na->num_tx_desc,
2397 na->num_rx_rings, na->num_rx_desc);
2398
2399 error = netmap_attach_common(na);
2400 if (error) {
2101 netmap_adapter_put(hwna);
2102 free(bna, M_DEVBUF);
2103 return error;
2401 goto err_free;
2104 }
2402 }
2403 /* make bwrap ifp point to the real ifp
2404 * NOTE: netmap_attach_common() interprets a non-NULL na->ifp
2405 * as a request to make the ifp point to the na. Since we
2406 * do not want to change the na already pointed to by hwna->ifp,
2407 * the following assignment has to be delayed until now
2408 */
2409 na->ifp = hwna->ifp;
2410 hwna->na_flags |= NAF_BUSY;
2411 /* make hwna point to the allocator we are actually using,
2412 * so that monitors will be able to find it
2413 */
2414 bna->save_nmd = hwna->nm_mem;
2415 hwna->nm_mem = na->nm_mem;
2105 return 0;
2416 return 0;
2417
2418err_free:
2419 netmap_mem_private_delete(na->nm_mem);
2420err_put:
2421 hwna->na_vp = hwna->na_hostvp = NULL;
2422 netmap_adapter_put(hwna);
2423 free(bna, M_DEVBUF);
2424 return error;
2425
2106}
2107
2108
2109void
2110netmap_init_bridges(void)
2111{
2112 int i;
2113 bzero(nm_bridges, sizeof(struct nm_bridge) * NM_BRIDGES); /* safety */
2114 for (i = 0; i < NM_BRIDGES; i++)
2115 BDG_RWINIT(&nm_bridges[i]);
2116}
2117#endif /* WITH_VALE */
2426}
2427
2428
2429void
2430netmap_init_bridges(void)
2431{
2432 int i;
2433 bzero(nm_bridges, sizeof(struct nm_bridge) * NM_BRIDGES); /* safety */
2434 for (i = 0; i < NM_BRIDGES; i++)
2435 BDG_RWINIT(&nm_bridges[i]);
2436}
2437#endif /* WITH_VALE */