if_epair.c revision 215641
1194927Sbz/*- 2194927Sbz * Copyright (c) 2008 The FreeBSD Foundation 3204805Sbz * Copyright (c) 2009-2010 Bjoern A. Zeeb <bz@FreeBSD.org> 4194927Sbz * All rights reserved. 5194927Sbz * 6194927Sbz * This software was developed by CK Software GmbH under sponsorship 7194927Sbz * from the FreeBSD Foundation. 8194927Sbz * 9194927Sbz * Redistribution and use in source and binary forms, with or without 10194927Sbz * modification, are permitted provided that the following conditions 11194927Sbz * are met: 12194927Sbz * 1. Redistributions of source code must retain the above copyright 13194927Sbz * notice, this list of conditions and the following disclaimer. 14194927Sbz * 2. Redistributions in binary form must reproduce the above copyright 15194927Sbz * notice, this list of conditions and the following disclaimer in the 16194927Sbz * documentation and/or other materials provided with the distribution. 17194927Sbz * 18194927Sbz * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19194927Sbz * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20194927Sbz * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21194927Sbz * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22194927Sbz * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23194927Sbz * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24194927Sbz * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25194927Sbz * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26194927Sbz * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27194927Sbz * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28194927Sbz * SUCH DAMAGE. 29194927Sbz */ 30194927Sbz 31194927Sbz/* 32195892Sbz * A pair of virtual back-to-back connected ethernet like interfaces 33195892Sbz * (``two interfaces with a virtual cross-over cable''). 34195892Sbz * 35194927Sbz * This is mostly intended to be used to provide connectivity between 36194927Sbz * different virtual network stack instances. 37194927Sbz */ 38194927Sbz/* 39194927Sbz * Things to re-think once we have more experience: 40195892Sbz * - ifp->if_reassign function once we can test with vimage. Depending on 41196019Srwatson * how if_vmove() is going to be improved. 42195892Sbz * - Real random etheraddrs that are checked to be uniquish; we would need 43195892Sbz * to re-do them in case we move the interface between network stacks 44195892Sbz * in a private if_reassign function. 45195892Sbz * In case we bridge to a real interface/network or between indepedent 46195892Sbz * epairs on multiple stacks/machines, we may need this. 47195892Sbz * For now let the user handle that case. 48194927Sbz */ 49194927Sbz 50194927Sbz#include <sys/cdefs.h> 51194927Sbz__FBSDID("$FreeBSD: head/sys/net/if_epair.c 215641 2010-11-21 19:33:19Z bz $"); 52194927Sbz 53194927Sbz#include <sys/param.h> 54194927Sbz#include <sys/kernel.h> 55194927Sbz#include <sys/mbuf.h> 56194927Sbz#include <sys/module.h> 57194927Sbz#include <sys/refcount.h> 58194927Sbz#include <sys/queue.h> 59195892Sbz#include <sys/smp.h> 60194927Sbz#include <sys/socket.h> 61194927Sbz#include <sys/sockio.h> 62194927Sbz#include <sys/sysctl.h> 63194927Sbz#include <sys/types.h> 64194927Sbz 65194927Sbz#include <net/bpf.h> 66194927Sbz#include <net/ethernet.h> 67194927Sbz#include <net/if.h> 68194927Sbz#include <net/if_clone.h> 69194927Sbz#include <net/if_var.h> 70194927Sbz#include <net/if_types.h> 71194927Sbz#include <net/netisr.h> 72196019Srwatson#include <net/vnet.h> 73194927Sbz 74194927Sbz#define EPAIRNAME "epair" 75194927Sbz 76194927SbzSYSCTL_DECL(_net_link); 77194927SbzSYSCTL_NODE(_net_link, OID_AUTO, epair, CTLFLAG_RW, 0, "epair sysctl"); 78195892Sbz 79195892Sbz#ifdef EPAIR_DEBUG 80195892Sbzstatic int epair_debug = 0; 81215641SbzSYSCTL_INT(_net_link_epair, OID_AUTO, epair_debug, CTLFLAG_RW, 82194927Sbz &epair_debug, 0, "if_epair(4) debugging."); 83195892Sbz#define DPRINTF(fmt, arg...) \ 84195892Sbz if (epair_debug) \ 85195892Sbz printf("[%s:%d] " fmt, __func__, __LINE__, ##arg) 86194927Sbz#else 87194927Sbz#define DPRINTF(fmt, arg...) 88194927Sbz#endif 89194927Sbz 90195892Sbzstatic void epair_nh_sintr(struct mbuf *); 91195892Sbzstatic struct mbuf *epair_nh_m2cpuid(struct mbuf *, uintptr_t, u_int *); 92195892Sbzstatic void epair_nh_drainedcpu(u_int); 93195892Sbz 94195892Sbzstatic void epair_start_locked(struct ifnet *); 95195892Sbz 96195892Sbzstatic int epair_clone_match(struct if_clone *, const char *); 97195892Sbzstatic int epair_clone_create(struct if_clone *, char *, size_t, caddr_t); 98195892Sbzstatic int epair_clone_destroy(struct if_clone *, struct ifnet *); 99195892Sbz 100195892Sbz/* Netisr realted definitions and sysctl. */ 101195892Sbzstatic struct netisr_handler epair_nh = { 102195892Sbz .nh_name = EPAIRNAME, 103195892Sbz .nh_proto = NETISR_EPAIR, 104195892Sbz .nh_policy = NETISR_POLICY_CPU, 105195892Sbz .nh_handler = epair_nh_sintr, 106195892Sbz .nh_m2cpuid = epair_nh_m2cpuid, 107195892Sbz .nh_drainedcpu = epair_nh_drainedcpu, 108195892Sbz}; 109195892Sbz 110195892Sbzstatic int 111195892Sbzsysctl_epair_netisr_maxqlen(SYSCTL_HANDLER_ARGS) 112195892Sbz{ 113195892Sbz int error, qlimit; 114195892Sbz 115195892Sbz netisr_getqlimit(&epair_nh, &qlimit); 116195892Sbz error = sysctl_handle_int(oidp, &qlimit, 0, req); 117195892Sbz if (error || !req->newptr) 118195892Sbz return (error); 119195892Sbz if (qlimit < 1) 120195892Sbz return (EINVAL); 121195892Sbz return (netisr_setqlimit(&epair_nh, qlimit)); 122195892Sbz} 123195892SbzSYSCTL_PROC(_net_link_epair, OID_AUTO, netisr_maxqlen, CTLTYPE_INT|CTLFLAG_RW, 124195892Sbz 0, 0, sysctl_epair_netisr_maxqlen, "I", 125195892Sbz "Maximum if_epair(4) netisr \"hw\" queue length"); 126195892Sbz 127194927Sbzstruct epair_softc { 128195892Sbz struct ifnet *ifp; /* This ifp. */ 129195892Sbz struct ifnet *oifp; /* other ifp of pair. */ 130195892Sbz u_int refcount; /* # of mbufs in flight. */ 131195892Sbz u_int cpuid; /* CPU ID assigned upon creation. */ 132194927Sbz void (*if_qflush)(struct ifnet *); 133195892Sbz /* Original if_qflush routine. */ 134194927Sbz}; 135194927Sbz 136195892Sbz/* 137195892Sbz * Per-CPU list of ifps with data in the ifq that needs to be flushed 138195892Sbz * to the netisr ``hw'' queue before we allow any further direct queuing 139195892Sbz * to the ``hw'' queue. 140195892Sbz */ 141194927Sbzstruct epair_ifp_drain { 142194927Sbz STAILQ_ENTRY(epair_ifp_drain) ifp_next; 143194927Sbz struct ifnet *ifp; 144194927Sbz}; 145195892SbzSTAILQ_HEAD(eid_list, epair_ifp_drain); 146194927Sbz 147195892Sbz#define EPAIR_LOCK_INIT(dpcpu) mtx_init(&(dpcpu)->if_epair_mtx, \ 148195892Sbz "if_epair", NULL, MTX_DEF) 149195892Sbz#define EPAIR_LOCK_DESTROY(dpcpu) mtx_destroy(&(dpcpu)->if_epair_mtx) 150195892Sbz#define EPAIR_LOCK_ASSERT(dpcpu) mtx_assert(&(dpcpu)->if_epair_mtx, \ 151195892Sbz MA_OWNED) 152195892Sbz#define EPAIR_LOCK(dpcpu) mtx_lock(&(dpcpu)->if_epair_mtx) 153195892Sbz#define EPAIR_UNLOCK(dpcpu) mtx_unlock(&(dpcpu)->if_epair_mtx) 154194927Sbz 155195892Sbz#ifdef INVARIANTS 156195892Sbz#define EPAIR_REFCOUNT_INIT(r, v) refcount_init((r), (v)) 157195892Sbz#define EPAIR_REFCOUNT_AQUIRE(r) refcount_acquire((r)) 158195892Sbz#define EPAIR_REFCOUNT_RELEASE(r) refcount_release((r)) 159195892Sbz#define EPAIR_REFCOUNT_ASSERT(a, p) KASSERT(a, p) 160195892Sbz#else 161195892Sbz#define EPAIR_REFCOUNT_INIT(r, v) 162195892Sbz#define EPAIR_REFCOUNT_AQUIRE(r) 163195892Sbz#define EPAIR_REFCOUNT_RELEASE(r) 164195892Sbz#define EPAIR_REFCOUNT_ASSERT(a, p) 165195892Sbz#endif 166194927Sbz 167194927Sbzstatic MALLOC_DEFINE(M_EPAIR, EPAIRNAME, 168194927Sbz "Pair of virtual cross-over connected Ethernet-like interfaces"); 169194927Sbz 170194927Sbzstatic struct if_clone epair_cloner = IFC_CLONE_INITIALIZER( 171194927Sbz EPAIRNAME, NULL, IF_MAXUNIT, 172194927Sbz NULL, epair_clone_match, epair_clone_create, epair_clone_destroy); 173194927Sbz 174195892Sbz/* 175195892Sbz * DPCPU area and functions. 176195892Sbz */ 177195892Sbzstruct epair_dpcpu { 178195892Sbz struct mtx if_epair_mtx; /* Per-CPU locking. */ 179195892Sbz int epair_drv_flags; /* Per-CPU ``hw'' drv flags. */ 180195892Sbz struct eid_list epair_ifp_drain_list; /* Per-CPU list of ifps with 181195892Sbz * data in the ifq. */ 182195892Sbz}; 183195892SbzDPCPU_DEFINE(struct epair_dpcpu, epair_dpcpu); 184194927Sbz 185195892Sbzstatic void 186195892Sbzepair_dpcpu_init(void) 187195892Sbz{ 188195892Sbz struct epair_dpcpu *epair_dpcpu; 189195892Sbz struct eid_list *s; 190195892Sbz u_int cpuid; 191195892Sbz 192209059Sjhb CPU_FOREACH(cpuid) { 193195892Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 194195892Sbz 195195892Sbz /* Initialize per-cpu lock. */ 196195892Sbz EPAIR_LOCK_INIT(epair_dpcpu); 197195892Sbz 198195892Sbz /* Driver flags are per-cpu as are our netisr "hw" queues. */ 199195892Sbz epair_dpcpu->epair_drv_flags = 0; 200195892Sbz 201195892Sbz /* 202195892Sbz * Initialize per-cpu drain list. 203195892Sbz * Manually do what STAILQ_HEAD_INITIALIZER would do. 204195892Sbz */ 205195892Sbz s = &epair_dpcpu->epair_ifp_drain_list; 206195892Sbz s->stqh_first = NULL; 207195892Sbz s->stqh_last = &s->stqh_first; 208195892Sbz } 209195892Sbz} 210195892Sbz 211195892Sbzstatic void 212195892Sbzepair_dpcpu_detach(void) 213195892Sbz{ 214195892Sbz struct epair_dpcpu *epair_dpcpu; 215195892Sbz u_int cpuid; 216195892Sbz 217209059Sjhb CPU_FOREACH(cpuid) { 218195892Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 219195892Sbz 220195892Sbz /* Destroy per-cpu lock. */ 221195892Sbz EPAIR_LOCK_DESTROY(epair_dpcpu); 222195892Sbz } 223195892Sbz} 224195892Sbz 225194927Sbz/* 226195892Sbz * Helper functions. 227195892Sbz */ 228195892Sbzstatic u_int 229195892Sbzcpuid_from_ifp(struct ifnet *ifp) 230195892Sbz{ 231195892Sbz struct epair_softc *sc; 232195892Sbz 233195892Sbz if (ifp == NULL) 234195892Sbz return (0); 235195892Sbz sc = ifp->if_softc; 236195892Sbz 237195892Sbz return (sc->cpuid); 238195892Sbz} 239195892Sbz 240195892Sbz/* 241194927Sbz * Netisr handler functions. 242194927Sbz */ 243194927Sbzstatic void 244195892Sbzepair_nh_sintr(struct mbuf *m) 245194927Sbz{ 246194927Sbz struct ifnet *ifp; 247194927Sbz struct epair_softc *sc; 248194927Sbz 249194927Sbz ifp = m->m_pkthdr.rcvif; 250194927Sbz (*ifp->if_input)(ifp, m); 251194927Sbz sc = ifp->if_softc; 252195892Sbz EPAIR_REFCOUNT_RELEASE(&sc->refcount); 253204805Sbz EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 254204805Sbz ("%s: ifp=%p sc->refcount not >= 1: %d", 255204805Sbz __func__, ifp, sc->refcount)); 256194927Sbz DPRINTF("ifp=%p refcount=%u\n", ifp, sc->refcount); 257194927Sbz} 258194927Sbz 259195892Sbzstatic struct mbuf * 260195892Sbzepair_nh_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 261195892Sbz{ 262195892Sbz 263195892Sbz *cpuid = cpuid_from_ifp(m->m_pkthdr.rcvif); 264195892Sbz 265195892Sbz return (m); 266195892Sbz} 267195892Sbz 268194927Sbzstatic void 269195892Sbzepair_nh_drainedcpu(u_int cpuid) 270194927Sbz{ 271195892Sbz struct epair_dpcpu *epair_dpcpu; 272194927Sbz struct epair_ifp_drain *elm, *tvar; 273194927Sbz struct ifnet *ifp; 274194927Sbz 275195892Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 276195892Sbz EPAIR_LOCK(epair_dpcpu); 277194927Sbz /* 278194927Sbz * Assume our "hw" queue and possibly ifq will be emptied 279194927Sbz * again. In case we will overflow the "hw" queue while 280194927Sbz * draining, epair_start_locked will set IFF_DRV_OACTIVE 281194927Sbz * again and we will stop and return. 282194927Sbz */ 283195892Sbz STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, 284195892Sbz ifp_next, tvar) { 285194927Sbz ifp = elm->ifp; 286195892Sbz epair_dpcpu->epair_drv_flags &= ~IFF_DRV_OACTIVE; 287194927Sbz ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; 288194927Sbz epair_start_locked(ifp); 289194927Sbz 290194927Sbz IFQ_LOCK(&ifp->if_snd); 291194927Sbz if (IFQ_IS_EMPTY(&ifp->if_snd)) { 292204805Sbz struct epair_softc *sc; 293204805Sbz 294195892Sbz STAILQ_REMOVE(&epair_dpcpu->epair_ifp_drain_list, 295195892Sbz elm, epair_ifp_drain, ifp_next); 296204805Sbz /* The cached ifp goes off the list. */ 297204805Sbz sc = ifp->if_softc; 298204805Sbz EPAIR_REFCOUNT_RELEASE(&sc->refcount); 299204805Sbz EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 300204805Sbz ("%s: ifp=%p sc->refcount not >= 1: %d", 301204805Sbz __func__, ifp, sc->refcount)); 302194927Sbz free(elm, M_EPAIR); 303194927Sbz } 304194927Sbz IFQ_UNLOCK(&ifp->if_snd); 305194927Sbz 306194927Sbz if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) != 0) { 307194927Sbz /* Our "hw"q overflew again. */ 308215641Sbz epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 309194927Sbz DPRINTF("hw queue length overflow at %u\n", 310195892Sbz epair_nh.nh_qlimit); 311194927Sbz break; 312194927Sbz } 313194927Sbz } 314195892Sbz EPAIR_UNLOCK(epair_dpcpu); 315194927Sbz} 316194927Sbz 317194927Sbz/* 318194927Sbz * Network interface (`if') related functions. 319194927Sbz */ 320204805Sbzstatic void 321204805Sbzepair_remove_ifp_from_draining(struct ifnet *ifp) 322204805Sbz{ 323204805Sbz struct epair_dpcpu *epair_dpcpu; 324204805Sbz struct epair_ifp_drain *elm, *tvar; 325204805Sbz u_int cpuid; 326204805Sbz 327209059Sjhb CPU_FOREACH(cpuid) { 328204805Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid, epair_dpcpu); 329204805Sbz EPAIR_LOCK(epair_dpcpu); 330204805Sbz STAILQ_FOREACH_SAFE(elm, &epair_dpcpu->epair_ifp_drain_list, 331204805Sbz ifp_next, tvar) { 332204805Sbz if (ifp == elm->ifp) { 333204805Sbz struct epair_softc *sc; 334204805Sbz 335204805Sbz STAILQ_REMOVE( 336204805Sbz &epair_dpcpu->epair_ifp_drain_list, elm, 337204805Sbz epair_ifp_drain, ifp_next); 338204805Sbz /* The cached ifp goes off the list. */ 339204805Sbz sc = ifp->if_softc; 340204805Sbz EPAIR_REFCOUNT_RELEASE(&sc->refcount); 341204805Sbz EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 342204805Sbz ("%s: ifp=%p sc->refcount not >= 1: %d", 343204805Sbz __func__, ifp, sc->refcount)); 344204805Sbz free(elm, M_EPAIR); 345204805Sbz } 346204805Sbz } 347204805Sbz EPAIR_UNLOCK(epair_dpcpu); 348204805Sbz } 349204805Sbz} 350204805Sbz 351195892Sbzstatic int 352195892Sbzepair_add_ifp_for_draining(struct ifnet *ifp) 353195892Sbz{ 354195892Sbz struct epair_dpcpu *epair_dpcpu; 355204805Sbz struct epair_softc *sc; 356195892Sbz struct epair_ifp_drain *elm = NULL; 357195892Sbz 358204805Sbz sc = ifp->if_softc; 359195892Sbz epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 360204805Sbz EPAIR_LOCK_ASSERT(epair_dpcpu); 361195892Sbz STAILQ_FOREACH(elm, &epair_dpcpu->epair_ifp_drain_list, ifp_next) 362195892Sbz if (elm->ifp == ifp) 363195892Sbz break; 364201995Sbz /* If the ifp is there already, return success. */ 365195892Sbz if (elm != NULL) 366195892Sbz return (0); 367195892Sbz 368195892Sbz elm = malloc(sizeof(struct epair_ifp_drain), M_EPAIR, M_NOWAIT|M_ZERO); 369195892Sbz if (elm == NULL) 370195892Sbz return (ENOMEM); 371195892Sbz 372195892Sbz elm->ifp = ifp; 373204805Sbz /* Add a reference for the ifp pointer on the list. */ 374204805Sbz EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 375195892Sbz STAILQ_INSERT_TAIL(&epair_dpcpu->epair_ifp_drain_list, elm, ifp_next); 376195892Sbz 377195892Sbz return (0); 378195892Sbz} 379195892Sbz 380194927Sbzstatic void 381194927Sbzepair_start_locked(struct ifnet *ifp) 382194927Sbz{ 383195892Sbz struct epair_dpcpu *epair_dpcpu; 384194927Sbz struct mbuf *m; 385194927Sbz struct epair_softc *sc; 386194927Sbz struct ifnet *oifp; 387194927Sbz int error; 388194927Sbz 389194927Sbz DPRINTF("ifp=%p\n", ifp); 390195892Sbz sc = ifp->if_softc; 391195892Sbz epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 392195892Sbz EPAIR_LOCK_ASSERT(epair_dpcpu); 393194927Sbz 394194927Sbz if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 395194927Sbz return; 396194927Sbz if ((ifp->if_flags & IFF_UP) == 0) 397194927Sbz return; 398194927Sbz 399194927Sbz /* 400194927Sbz * We get patckets here from ether_output via if_handoff() 401194927Sbz * and ned to put them into the input queue of the oifp 402194927Sbz * and call oifp->if_input() via netisr/epair_sintr(). 403194927Sbz */ 404194927Sbz oifp = sc->oifp; 405194927Sbz sc = oifp->if_softc; 406194927Sbz for (;;) { 407194927Sbz IFQ_DEQUEUE(&ifp->if_snd, m); 408194927Sbz if (m == NULL) 409194927Sbz break; 410194927Sbz BPF_MTAP(ifp, m); 411194927Sbz 412194927Sbz /* 413194927Sbz * In case the outgoing interface is not usable, 414194927Sbz * drop the packet. 415194927Sbz */ 416194927Sbz if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 417194927Sbz (oifp->if_flags & IFF_UP) ==0) { 418194927Sbz ifp->if_oerrors++; 419194927Sbz m_freem(m); 420194927Sbz continue; 421194927Sbz } 422194927Sbz DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname); 423194927Sbz 424194927Sbz /* 425194927Sbz * Add a reference so the interface cannot go while the 426194927Sbz * packet is in transit as we rely on rcvif to stay valid. 427194927Sbz */ 428195892Sbz EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 429194927Sbz m->m_pkthdr.rcvif = oifp; 430194927Sbz CURVNET_SET_QUIET(oifp->if_vnet); 431194927Sbz error = netisr_queue(NETISR_EPAIR, m); 432194927Sbz CURVNET_RESTORE(); 433194927Sbz if (!error) { 434194927Sbz ifp->if_opackets++; 435194927Sbz /* Someone else received the packet. */ 436194927Sbz oifp->if_ipackets++; 437194927Sbz } else { 438204805Sbz /* The packet was freed already. */ 439195892Sbz epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 440194927Sbz ifp->if_drv_flags |= IFF_DRV_OACTIVE; 441204805Sbz (void) epair_add_ifp_for_draining(ifp); 442204805Sbz ifp->if_oerrors++; 443195892Sbz EPAIR_REFCOUNT_RELEASE(&sc->refcount); 444204805Sbz EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 445204805Sbz ("%s: ifp=%p sc->refcount not >= 1: %d", 446204805Sbz __func__, oifp, sc->refcount)); 447194927Sbz } 448194927Sbz } 449194927Sbz} 450194927Sbz 451194927Sbzstatic void 452194927Sbzepair_start(struct ifnet *ifp) 453194927Sbz{ 454195892Sbz struct epair_dpcpu *epair_dpcpu; 455194927Sbz 456195892Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu); 457195892Sbz EPAIR_LOCK(epair_dpcpu); 458194927Sbz epair_start_locked(ifp); 459195892Sbz EPAIR_UNLOCK(epair_dpcpu); 460194927Sbz} 461194927Sbz 462194927Sbzstatic int 463194927Sbzepair_transmit_locked(struct ifnet *ifp, struct mbuf *m) 464194927Sbz{ 465195892Sbz struct epair_dpcpu *epair_dpcpu; 466194927Sbz struct epair_softc *sc; 467194927Sbz struct ifnet *oifp; 468194927Sbz int error, len; 469194927Sbz short mflags; 470194927Sbz 471194927Sbz DPRINTF("ifp=%p m=%p\n", ifp, m); 472195892Sbz sc = ifp->if_softc; 473195892Sbz epair_dpcpu = DPCPU_ID_PTR(sc->cpuid, epair_dpcpu); 474195892Sbz EPAIR_LOCK_ASSERT(epair_dpcpu); 475194927Sbz 476194927Sbz if (m == NULL) 477194927Sbz return (0); 478194927Sbz 479194927Sbz /* 480194927Sbz * We are not going to use the interface en/dequeue mechanism 481194927Sbz * on the TX side. We are called from ether_output_frame() 482194927Sbz * and will put the packet into the incoming queue of the 483194927Sbz * other interface of our pair via the netsir. 484194927Sbz */ 485194927Sbz if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 486194927Sbz m_freem(m); 487194927Sbz return (ENXIO); 488194927Sbz } 489194927Sbz if ((ifp->if_flags & IFF_UP) == 0) { 490194927Sbz m_freem(m); 491194927Sbz return (ENETDOWN); 492194927Sbz } 493194927Sbz 494194927Sbz BPF_MTAP(ifp, m); 495194927Sbz 496194927Sbz /* 497194927Sbz * In case the outgoing interface is not usable, 498194927Sbz * drop the packet. 499194927Sbz */ 500194927Sbz oifp = sc->oifp; 501194927Sbz if ((oifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || 502194927Sbz (oifp->if_flags & IFF_UP) ==0) { 503194927Sbz ifp->if_oerrors++; 504194927Sbz m_freem(m); 505194927Sbz return (0); 506194927Sbz } 507194927Sbz len = m->m_pkthdr.len; 508194927Sbz mflags = m->m_flags; 509194927Sbz DPRINTF("packet %s -> %s\n", ifp->if_xname, oifp->if_xname); 510194927Sbz 511194927Sbz#ifdef ALTQ 512194927Sbz /* Support ALTQ via the clasic if_start() path. */ 513194927Sbz IF_LOCK(&ifp->if_snd); 514194927Sbz if (ALTQ_IS_ENABLED(&ifp->if_snd)) { 515194927Sbz ALTQ_ENQUEUE(&ifp->if_snd, m, NULL, error); 516194927Sbz if (error) 517194927Sbz ifp->if_snd.ifq_drops++; 518194927Sbz IF_UNLOCK(&ifp->if_snd); 519194927Sbz if (!error) { 520194927Sbz ifp->if_obytes += len; 521194927Sbz if (mflags & (M_BCAST|M_MCAST)) 522194927Sbz ifp->if_omcasts++; 523194927Sbz 524194927Sbz if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0) 525194927Sbz epair_start_locked(ifp); 526194927Sbz else 527195892Sbz (void)epair_add_ifp_for_draining(ifp); 528194927Sbz } 529194927Sbz return (error); 530194927Sbz } 531194927Sbz IF_UNLOCK(&ifp->if_snd); 532194927Sbz#endif 533194927Sbz 534195892Sbz if ((epair_dpcpu->epair_drv_flags & IFF_DRV_OACTIVE) != 0) { 535194927Sbz /* 536194927Sbz * Our hardware queue is full, try to fall back 537194927Sbz * queuing to the ifq but do not call ifp->if_start. 538194927Sbz * Either we are lucky or the packet is gone. 539194927Sbz */ 540194927Sbz IFQ_ENQUEUE(&ifp->if_snd, m, error); 541194927Sbz if (!error) 542195892Sbz (void)epair_add_ifp_for_draining(ifp); 543194927Sbz return (error); 544194927Sbz } 545194927Sbz sc = oifp->if_softc; 546194927Sbz /* 547194927Sbz * Add a reference so the interface cannot go while the 548194927Sbz * packet is in transit as we rely on rcvif to stay valid. 549194927Sbz */ 550195892Sbz EPAIR_REFCOUNT_AQUIRE(&sc->refcount); 551194927Sbz m->m_pkthdr.rcvif = oifp; 552194927Sbz CURVNET_SET_QUIET(oifp->if_vnet); 553194927Sbz error = netisr_queue(NETISR_EPAIR, m); 554194927Sbz CURVNET_RESTORE(); 555194927Sbz if (!error) { 556194927Sbz ifp->if_opackets++; 557194927Sbz /* 558194927Sbz * IFQ_HANDOFF_ADJ/ip_handoff() update statistics, 559194927Sbz * but as we bypass all this we have to duplicate 560194927Sbz * the logic another time. 561194927Sbz */ 562194927Sbz ifp->if_obytes += len; 563194927Sbz if (mflags & (M_BCAST|M_MCAST)) 564194927Sbz ifp->if_omcasts++; 565194927Sbz /* Someone else received the packet. */ 566194927Sbz oifp->if_ipackets++; 567194927Sbz } else { 568194927Sbz /* The packet was freed already. */ 569195892Sbz epair_dpcpu->epair_drv_flags |= IFF_DRV_OACTIVE; 570194927Sbz ifp->if_drv_flags |= IFF_DRV_OACTIVE; 571204805Sbz ifp->if_oerrors++; 572204805Sbz EPAIR_REFCOUNT_RELEASE(&sc->refcount); 573204805Sbz EPAIR_REFCOUNT_ASSERT((int)sc->refcount >= 1, 574204805Sbz ("%s: ifp=%p sc->refcount not >= 1: %d", 575204805Sbz __func__, oifp, sc->refcount)); 576194927Sbz } 577194927Sbz 578194927Sbz return (error); 579194927Sbz} 580194927Sbz 581194927Sbzstatic int 582194927Sbzepair_transmit(struct ifnet *ifp, struct mbuf *m) 583194927Sbz{ 584195892Sbz struct epair_dpcpu *epair_dpcpu; 585194927Sbz int error; 586194927Sbz 587195892Sbz epair_dpcpu = DPCPU_ID_PTR(cpuid_from_ifp(ifp), epair_dpcpu); 588195892Sbz EPAIR_LOCK(epair_dpcpu); 589194927Sbz error = epair_transmit_locked(ifp, m); 590195892Sbz EPAIR_UNLOCK(epair_dpcpu); 591194927Sbz return (error); 592194927Sbz} 593194927Sbz 594194927Sbzstatic void 595194927Sbzepair_qflush(struct ifnet *ifp) 596194927Sbz{ 597194927Sbz struct epair_softc *sc; 598194927Sbz 599194927Sbz sc = ifp->if_softc; 600204805Sbz KASSERT(sc != NULL, ("%s: ifp=%p, epair_softc gone? sc=%p\n", 601204805Sbz __func__, ifp, sc)); 602194927Sbz /* 603204805Sbz * Remove this ifp from all backpointer lists. The interface will not 604204805Sbz * usable for flushing anyway nor should it have anything to flush 605204805Sbz * after if_qflush(). 606194927Sbz */ 607204805Sbz epair_remove_ifp_from_draining(ifp); 608204805Sbz 609194927Sbz if (sc->if_qflush) 610194927Sbz sc->if_qflush(ifp); 611194927Sbz} 612194927Sbz 613194927Sbzstatic int 614194927Sbzepair_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data) 615194927Sbz{ 616194927Sbz struct ifreq *ifr; 617194927Sbz int error; 618194927Sbz 619194927Sbz ifr = (struct ifreq *)data; 620194927Sbz switch (cmd) { 621194927Sbz case SIOCSIFFLAGS: 622194927Sbz case SIOCADDMULTI: 623194927Sbz case SIOCDELMULTI: 624194927Sbz error = 0; 625194927Sbz break; 626194927Sbz 627195892Sbz case SIOCSIFMTU: 628195892Sbz /* We basically allow all kinds of MTUs. */ 629195892Sbz ifp->if_mtu = ifr->ifr_mtu; 630195892Sbz error = 0; 631195892Sbz break; 632195892Sbz 633194927Sbz default: 634194927Sbz /* Let the common ethernet handler process this. */ 635194927Sbz error = ether_ioctl(ifp, cmd, data); 636194927Sbz break; 637194927Sbz } 638194927Sbz 639194927Sbz return (error); 640194927Sbz} 641194927Sbz 642194927Sbzstatic void 643194927Sbzepair_init(void *dummy __unused) 644194927Sbz{ 645194927Sbz} 646194927Sbz 647194927Sbz 648194927Sbz/* 649194927Sbz * Interface cloning functions. 650194927Sbz * We use our private ones so that we can create/destroy our secondary 651194927Sbz * device along with the primary one. 652194927Sbz */ 653194927Sbzstatic int 654194927Sbzepair_clone_match(struct if_clone *ifc, const char *name) 655194927Sbz{ 656194927Sbz const char *cp; 657194927Sbz 658194927Sbz DPRINTF("name='%s'\n", name); 659194927Sbz 660194927Sbz /* 661194927Sbz * Our base name is epair. 662194927Sbz * Our interfaces will be named epair<n>[ab]. 663194927Sbz * So accept anything of the following list: 664194927Sbz * - epair 665194927Sbz * - epair<n> 666194927Sbz * but not the epair<n>[ab] versions. 667194927Sbz */ 668194927Sbz if (strncmp(EPAIRNAME, name, sizeof(EPAIRNAME)-1) != 0) 669194927Sbz return (0); 670194927Sbz 671194927Sbz for (cp = name + sizeof(EPAIRNAME) - 1; *cp != '\0'; cp++) { 672194927Sbz if (*cp < '0' || *cp > '9') 673194927Sbz return (0); 674194927Sbz } 675194927Sbz 676194927Sbz return (1); 677194927Sbz} 678194927Sbz 679194927Sbzstatic int 680194927Sbzepair_clone_create(struct if_clone *ifc, char *name, size_t len, caddr_t params) 681194927Sbz{ 682194927Sbz struct epair_softc *sca, *scb; 683194927Sbz struct ifnet *ifp; 684194927Sbz char *dp; 685194927Sbz int error, unit, wildcard; 686194927Sbz uint8_t eaddr[ETHER_ADDR_LEN]; /* 00:00:00:00:00:00 */ 687194927Sbz 688194927Sbz /* 689194927Sbz * We are abusing params to create our second interface. 690194927Sbz * Actually we already created it and called if_clone_createif() 691194927Sbz * for it to do the official insertion procedure the moment we knew 692194927Sbz * it cannot fail anymore. So just do attach it here. 693194927Sbz */ 694194927Sbz if (params) { 695194927Sbz scb = (struct epair_softc *)params; 696194927Sbz ifp = scb->ifp; 697194927Sbz /* Assign a hopefully unique, locally administered etheraddr. */ 698194927Sbz eaddr[0] = 0x02; 699194927Sbz eaddr[3] = (ifp->if_index >> 8) & 0xff; 700194927Sbz eaddr[4] = ifp->if_index & 0xff; 701194927Sbz eaddr[5] = 0x0b; 702194927Sbz ether_ifattach(ifp, eaddr); 703194927Sbz /* Correctly set the name for the cloner list. */ 704194927Sbz strlcpy(name, scb->ifp->if_xname, len); 705194927Sbz return (0); 706194927Sbz } 707194927Sbz 708194927Sbz /* Try to see if a special unit was requested. */ 709194927Sbz error = ifc_name2unit(name, &unit); 710194927Sbz if (error != 0) 711194927Sbz return (error); 712194927Sbz wildcard = (unit < 0); 713194927Sbz 714194927Sbz error = ifc_alloc_unit(ifc, &unit); 715194927Sbz if (error != 0) 716194927Sbz return (error); 717194927Sbz 718194927Sbz /* 719194927Sbz * If no unit had been given, we need to adjust the ifName. 720194927Sbz * Also make sure there is space for our extra [ab] suffix. 721194927Sbz */ 722194927Sbz for (dp = name; *dp != '\0'; dp++); 723194927Sbz if (wildcard) { 724194927Sbz error = snprintf(dp, len - (dp - name), "%d", unit); 725194927Sbz if (error > len - (dp - name) - 1) { 726194927Sbz /* ifName too long. */ 727194927Sbz ifc_free_unit(ifc, unit); 728194927Sbz return (ENOSPC); 729194927Sbz } 730194927Sbz dp += error; 731194927Sbz } 732194927Sbz if (len - (dp - name) - 1 < 1) { 733194927Sbz /* No space left for our [ab] suffix. */ 734194927Sbz ifc_free_unit(ifc, unit); 735194927Sbz return (ENOSPC); 736194927Sbz } 737194927Sbz *dp = 'a'; 738194927Sbz /* Must not change dp so we can replace 'a' by 'b' later. */ 739194927Sbz *(dp+1) = '\0'; 740194927Sbz 741194927Sbz /* Allocate memory for both [ab] interfaces */ 742194927Sbz sca = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); 743195892Sbz EPAIR_REFCOUNT_INIT(&sca->refcount, 1); 744194927Sbz sca->ifp = if_alloc(IFT_ETHER); 745194927Sbz if (sca->ifp == NULL) { 746194927Sbz free(sca, M_EPAIR); 747194927Sbz ifc_free_unit(ifc, unit); 748194927Sbz return (ENOSPC); 749194927Sbz } 750194927Sbz 751194927Sbz scb = malloc(sizeof(struct epair_softc), M_EPAIR, M_WAITOK | M_ZERO); 752195892Sbz EPAIR_REFCOUNT_INIT(&scb->refcount, 1); 753194927Sbz scb->ifp = if_alloc(IFT_ETHER); 754194927Sbz if (scb->ifp == NULL) { 755194927Sbz free(scb, M_EPAIR); 756194927Sbz if_free(sca->ifp); 757194927Sbz free(sca, M_EPAIR); 758194927Sbz ifc_free_unit(ifc, unit); 759194927Sbz return (ENOSPC); 760194927Sbz } 761194927Sbz 762194927Sbz /* 763194927Sbz * Cross-reference the interfaces so we will be able to free both. 764194927Sbz */ 765194927Sbz sca->oifp = scb->ifp; 766194927Sbz scb->oifp = sca->ifp; 767195892Sbz 768195892Sbz /* 769195892Sbz * Calculate the cpuid for netisr queueing based on the 770195892Sbz * ifIndex of the interfaces. As long as we cannot configure 771195892Sbz * this or use cpuset information easily we cannot guarantee 772195892Sbz * cache locality but we can at least allow parallelism. 773195892Sbz */ 774195892Sbz sca->cpuid = 775195892Sbz netisr_get_cpuid(sca->ifp->if_index % netisr_get_cpucount()); 776195892Sbz scb->cpuid = 777195892Sbz netisr_get_cpuid(scb->ifp->if_index % netisr_get_cpucount()); 778194927Sbz 779194927Sbz /* Finish initialization of interface <n>a. */ 780194927Sbz ifp = sca->ifp; 781194927Sbz ifp->if_softc = sca; 782194927Sbz strlcpy(ifp->if_xname, name, IFNAMSIZ); 783194927Sbz ifp->if_dname = ifc->ifc_name; 784194927Sbz ifp->if_dunit = unit; 785194927Sbz ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 786194927Sbz ifp->if_start = epair_start; 787194927Sbz ifp->if_ioctl = epair_ioctl; 788194927Sbz ifp->if_init = epair_init; 789194927Sbz ifp->if_snd.ifq_maxlen = ifqmaxlen; 790194927Sbz /* Assign a hopefully unique, locally administered etheraddr. */ 791194927Sbz eaddr[0] = 0x02; 792194927Sbz eaddr[3] = (ifp->if_index >> 8) & 0xff; 793194927Sbz eaddr[4] = ifp->if_index & 0xff; 794194927Sbz eaddr[5] = 0x0a; 795194927Sbz ether_ifattach(ifp, eaddr); 796194927Sbz sca->if_qflush = ifp->if_qflush; 797194927Sbz ifp->if_qflush = epair_qflush; 798194927Sbz ifp->if_transmit = epair_transmit; 799194927Sbz ifp->if_baudrate = IF_Gbps(10UL); /* arbitrary maximum */ 800194927Sbz 801194927Sbz /* Swap the name and finish initialization of interface <n>b. */ 802194927Sbz *dp = 'b'; 803194927Sbz 804194927Sbz ifp = scb->ifp; 805194927Sbz ifp->if_softc = scb; 806194927Sbz strlcpy(ifp->if_xname, name, IFNAMSIZ); 807194927Sbz ifp->if_dname = ifc->ifc_name; 808194927Sbz ifp->if_dunit = unit; 809194927Sbz ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; 810194927Sbz ifp->if_start = epair_start; 811194927Sbz ifp->if_ioctl = epair_ioctl; 812194927Sbz ifp->if_init = epair_init; 813194927Sbz ifp->if_snd.ifq_maxlen = ifqmaxlen; 814194927Sbz /* We need to play some tricks here for the second interface. */ 815194927Sbz strlcpy(name, EPAIRNAME, len); 816194927Sbz error = if_clone_create(name, len, (caddr_t)scb); 817194927Sbz if (error) 818194927Sbz panic("%s: if_clone_createif() for our 2nd iface failed: %d", 819194927Sbz __func__, error); 820194927Sbz scb->if_qflush = ifp->if_qflush; 821194927Sbz ifp->if_qflush = epair_qflush; 822194927Sbz ifp->if_transmit = epair_transmit; 823194927Sbz ifp->if_baudrate = IF_Gbps(10UL); /* arbitrary maximum */ 824194927Sbz 825194927Sbz /* 826194927Sbz * Restore name to <n>a as the ifp for this will go into the 827194927Sbz * cloner list for the initial call. 828194927Sbz */ 829194927Sbz strlcpy(name, sca->ifp->if_xname, len); 830194927Sbz DPRINTF("name='%s/%db' created sca=%p scb=%p\n", name, unit, sca, scb); 831194927Sbz 832194927Sbz /* Tell the world, that we are ready to rock. */ 833194927Sbz sca->ifp->if_drv_flags |= IFF_DRV_RUNNING; 834194927Sbz scb->ifp->if_drv_flags |= IFF_DRV_RUNNING; 835211904Sbz if_link_state_change(sca->ifp, LINK_STATE_UP); 836211904Sbz if_link_state_change(scb->ifp, LINK_STATE_UP); 837194927Sbz 838194927Sbz return (0); 839194927Sbz} 840194927Sbz 841194927Sbzstatic int 842194927Sbzepair_clone_destroy(struct if_clone *ifc, struct ifnet *ifp) 843194927Sbz{ 844194927Sbz struct ifnet *oifp; 845194927Sbz struct epair_softc *sca, *scb; 846194927Sbz int unit, error; 847194927Sbz 848194927Sbz DPRINTF("ifp=%p\n", ifp); 849194927Sbz 850194927Sbz /* 851194927Sbz * In case we called into if_clone_destroyif() ourselves 852194927Sbz * again to remove the second interface, the softc will be 853194927Sbz * NULL. In that case so not do anything but return success. 854194927Sbz */ 855194927Sbz if (ifp->if_softc == NULL) 856194927Sbz return (0); 857194927Sbz 858194927Sbz unit = ifp->if_dunit; 859194927Sbz sca = ifp->if_softc; 860194927Sbz oifp = sca->oifp; 861194927Sbz scb = oifp->if_softc; 862194927Sbz 863194927Sbz DPRINTF("ifp=%p oifp=%p\n", ifp, oifp); 864211904Sbz if_link_state_change(ifp, LINK_STATE_DOWN); 865211904Sbz if_link_state_change(oifp, LINK_STATE_DOWN); 866194927Sbz ifp->if_drv_flags &= ~IFF_DRV_RUNNING; 867194927Sbz oifp->if_drv_flags &= ~IFF_DRV_RUNNING; 868194927Sbz ether_ifdetach(oifp); 869194927Sbz ether_ifdetach(ifp); 870194927Sbz /* 871194927Sbz * Wait for all packets to be dispatched to if_input. 872194927Sbz * The numbers can only go down as the interfaces are 873194927Sbz * detached so there is no need to use atomics. 874194927Sbz */ 875194927Sbz DPRINTF("sca refcnt=%u scb refcnt=%u\n", sca->refcount, scb->refcount); 876195892Sbz EPAIR_REFCOUNT_ASSERT(sca->refcount == 1 && scb->refcount == 1, 877204805Sbz ("%s: ifp=%p sca->refcount!=1: %d || ifp=%p scb->refcount!=1: %d", 878204805Sbz __func__, ifp, sca->refcount, oifp, scb->refcount)); 879194927Sbz 880194927Sbz /* 881194927Sbz * Get rid of our second half. 882194927Sbz */ 883194927Sbz oifp->if_softc = NULL; 884194927Sbz error = if_clone_destroyif(ifc, oifp); 885194927Sbz if (error) 886194927Sbz panic("%s: if_clone_destroyif() for our 2nd iface failed: %d", 887194927Sbz __func__, error); 888194927Sbz 889195892Sbz /* 890195892Sbz * Finish cleaning up. Free them and release the unit. 891195892Sbz * As the other of the two interfaces my reside in a different vnet, 892195892Sbz * we need to switch before freeing them. 893195892Sbz */ 894195892Sbz CURVNET_SET_QUIET(oifp->if_vnet); 895212152Sbz if_free(oifp); 896195892Sbz CURVNET_RESTORE(); 897212152Sbz if_free(ifp); 898194927Sbz free(scb, M_EPAIR); 899194927Sbz free(sca, M_EPAIR); 900194927Sbz ifc_free_unit(ifc, unit); 901194927Sbz 902194927Sbz return (0); 903194927Sbz} 904194927Sbz 905194927Sbzstatic int 906194927Sbzepair_modevent(module_t mod, int type, void *data) 907194927Sbz{ 908195892Sbz int qlimit; 909194927Sbz 910194927Sbz switch (type) { 911194927Sbz case MOD_LOAD: 912194927Sbz /* For now limit us to one global mutex and one inq. */ 913195892Sbz epair_dpcpu_init(); 914195892Sbz epair_nh.nh_qlimit = 42 * ifqmaxlen; /* 42 shall be the number. */ 915195892Sbz if (TUNABLE_INT_FETCH("net.link.epair.netisr_maxqlen", &qlimit)) 916195892Sbz epair_nh.nh_qlimit = qlimit; 917195892Sbz netisr_register(&epair_nh); 918194927Sbz if_clone_attach(&epair_cloner); 919194927Sbz if (bootverbose) 920194927Sbz printf("%s initialized.\n", EPAIRNAME); 921194927Sbz break; 922194927Sbz case MOD_UNLOAD: 923194927Sbz if_clone_detach(&epair_cloner); 924195892Sbz netisr_unregister(&epair_nh); 925195892Sbz epair_dpcpu_detach(); 926194927Sbz if (bootverbose) 927194927Sbz printf("%s unloaded.\n", EPAIRNAME); 928194927Sbz break; 929194927Sbz default: 930194927Sbz return (EOPNOTSUPP); 931194927Sbz } 932194927Sbz return (0); 933194927Sbz} 934194927Sbz 935194927Sbzstatic moduledata_t epair_mod = { 936194927Sbz "if_epair", 937194927Sbz epair_modevent, 938194927Sbz 0 939194927Sbz}; 940194927Sbz 941194927SbzDECLARE_MODULE(if_epair, epair_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); 942194927SbzMODULE_VERSION(if_epair, 1); 943