netmap_kern.h revision 259412
1227614Sluigi/* 2249659Sluigi * Copyright (C) 2011-2013 Matteo Landi, Luigi Rizzo. All rights reserved. 3259412Sluigi * Copyright (C) 2013 Universita` di Pisa. All rights reserved. 4241719Sluigi * 5227614Sluigi * Redistribution and use in source and binary forms, with or without 6227614Sluigi * modification, are permitted provided that the following conditions 7227614Sluigi * are met: 8228276Sluigi * 1. Redistributions of source code must retain the above copyright 9228276Sluigi * notice, this list of conditions and the following disclaimer. 10228276Sluigi * 2. Redistributions in binary form must reproduce the above copyright 11228276Sluigi * notice, this list of conditions and the following disclaimer in the 12227614Sluigi * documentation and/or other materials provided with the distribution. 13241719Sluigi * 14227614Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15227614Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16227614Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17227614Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18227614Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19227614Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20227614Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21227614Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22227614Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23227614Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24227614Sluigi * SUCH DAMAGE. 25227614Sluigi */ 26227614Sluigi 27227614Sluigi/* 28227614Sluigi * $FreeBSD: head/sys/dev/netmap/netmap_kern.h 259412 2013-12-15 08:37:24Z luigi $ 29227614Sluigi * 30227614Sluigi * The header contains the definitions of constants and function 31227614Sluigi * prototypes used only in kernelspace. 32227614Sluigi */ 33227614Sluigi 34227614Sluigi#ifndef _NET_NETMAP_KERN_H_ 35227614Sluigi#define _NET_NETMAP_KERN_H_ 36227614Sluigi 37259412Sluigi#define WITH_VALE // comment out to disable VALE support 38259412Sluigi 39231594Sluigi#if defined(__FreeBSD__) 40250052Sluigi 41257529Sluigi#define likely(x) __builtin_expect((long)!!(x), 1L) 42257529Sluigi#define unlikely(x) __builtin_expect((long)!!(x), 0L) 43238812Sluigi 44231594Sluigi#define NM_LOCK_T struct mtx 45259412Sluigi#define NMG_LOCK_T struct mtx 46259412Sluigi#define NMG_LOCK_INIT() mtx_init(&netmap_global_lock, \ 47259412Sluigi "netmap global lock", NULL, MTX_DEF) 48259412Sluigi#define NMG_LOCK_DESTROY() mtx_destroy(&netmap_global_lock) 49259412Sluigi#define NMG_LOCK() mtx_lock(&netmap_global_lock) 50259412Sluigi#define NMG_UNLOCK() mtx_unlock(&netmap_global_lock) 51259412Sluigi#define NMG_LOCK_ASSERT() mtx_assert(&netmap_global_lock, MA_OWNED) 52259412Sluigi 53231594Sluigi#define NM_SELINFO_T struct selinfo 54231594Sluigi#define MBUF_LEN(m) ((m)->m_pkthdr.len) 55259412Sluigi#define MBUF_IFP(m) ((m)->m_pkthdr.rcvif) 56231594Sluigi#define NM_SEND_UP(ifp, m) ((ifp)->if_input)(ifp, m) 57250052Sluigi 58259412Sluigi#define NM_ATOMIC_T volatile int // XXX ? 59259412Sluigi/* atomic operations */ 60259412Sluigi#include <machine/atomic.h> 61259412Sluigi#define NM_ATOMIC_TEST_AND_SET(p) (!atomic_cmpset_acq_int((p), 0, 1)) 62259412Sluigi#define NM_ATOMIC_CLEAR(p) atomic_store_rel_int((p), 0) 63257529Sluigi 64259412Sluigi#define prefetch(x) __builtin_prefetch(x) 65259412Sluigi 66259412SluigiMALLOC_DECLARE(M_NETMAP); 67259412Sluigi 68259412Sluigi// XXX linux struct, not used in FreeBSD 69259412Sluigistruct net_device_ops { 70259412Sluigi}; 71259412Sluigistruct hrtimer { 72259412Sluigi}; 73259412Sluigi 74232238Sluigi#elif defined (linux) 75250052Sluigi 76250052Sluigi#define NM_LOCK_T safe_spinlock_t // see bsd_glue.h 77231594Sluigi#define NM_SELINFO_T wait_queue_head_t 78231594Sluigi#define MBUF_LEN(m) ((m)->len) 79259412Sluigi#define MBUF_IFP(m) ((m)->dev) 80231594Sluigi#define NM_SEND_UP(ifp, m) netif_rx(m) 81238812Sluigi 82257529Sluigi#define NM_ATOMIC_T volatile long unsigned int 83257529Sluigi 84259412Sluigi// XXX a mtx would suffice here too 20130404 gl 85259412Sluigi#define NMG_LOCK_T struct semaphore 86259412Sluigi#define NMG_LOCK_INIT() sema_init(&netmap_global_lock, 1) 87259412Sluigi#define NMG_LOCK_DESTROY() 88259412Sluigi#define NMG_LOCK() down(&netmap_global_lock) 89259412Sluigi#define NMG_UNLOCK() up(&netmap_global_lock) 90259412Sluigi#define NMG_LOCK_ASSERT() // XXX to be completed 91259412Sluigi 92238812Sluigi#ifndef DEV_NETMAP 93238812Sluigi#define DEV_NETMAP 94257529Sluigi#endif /* DEV_NETMAP */ 95238812Sluigi 96238812Sluigi/* 97241719Sluigi * IFCAP_NETMAP goes into net_device's priv_flags (if_capenable). 98241719Sluigi * This was 16 bits up to linux 2.6.36, so we need a 16 bit value on older 99238812Sluigi * platforms and tolerate the clash with IFF_DYNAMIC and IFF_BRIDGE_PORT. 100241719Sluigi * For the 32-bit value, 0x100000 has no clashes until at least 3.5.1 101238812Sluigi */ 102238812Sluigi#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37) 103238831Sluigi#define IFCAP_NETMAP 0x8000 104231594Sluigi#else 105251139Sluigi#define IFCAP_NETMAP 0x200000 106238812Sluigi#endif 107238812Sluigi 108238812Sluigi#elif defined (__APPLE__) 109250052Sluigi 110241719Sluigi#warning apple support is incomplete. 111238812Sluigi#define likely(x) __builtin_expect(!!(x), 1) 112238812Sluigi#define unlikely(x) __builtin_expect(!!(x), 0) 113238812Sluigi#define NM_LOCK_T IOLock * 114238812Sluigi#define NM_SELINFO_T struct selinfo 115238812Sluigi#define MBUF_LEN(m) ((m)->m_pkthdr.len) 116238812Sluigi#define NM_SEND_UP(ifp, m) ((ifp)->if_input)(ifp, m) 117238812Sluigi 118238812Sluigi#else 119250052Sluigi 120231594Sluigi#error unsupported platform 121231594Sluigi 122250052Sluigi#endif /* end - platform-specific code */ 123250052Sluigi 124227614Sluigi#define ND(format, ...) 125230058Sluigi#define D(format, ...) \ 126230058Sluigi do { \ 127230058Sluigi struct timeval __xxts; \ 128227614Sluigi microtime(&__xxts); \ 129230058Sluigi printf("%03d.%06d %s [%d] " format "\n", \ 130230058Sluigi (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \ 131230058Sluigi __FUNCTION__, __LINE__, ##__VA_ARGS__); \ 132227614Sluigi } while (0) 133241719Sluigi 134241719Sluigi/* rate limited, lps indicates how many per second */ 135241719Sluigi#define RD(lps, format, ...) \ 136241719Sluigi do { \ 137241719Sluigi static int t0, __cnt; \ 138241719Sluigi if (t0 != time_second) { \ 139241719Sluigi t0 = time_second; \ 140241719Sluigi __cnt = 0; \ 141241719Sluigi } \ 142241719Sluigi if (__cnt++ < lps) \ 143241719Sluigi D(format, ##__VA_ARGS__); \ 144241719Sluigi } while (0) 145241719Sluigi 146227614Sluigistruct netmap_adapter; 147251139Sluigistruct nm_bdg_fwd; 148251139Sluigistruct nm_bridge; 149251139Sluigistruct netmap_priv_d; 150227614Sluigi 151257529Sluigiconst char *nm_dump_buf(char *p, int len, int lim, char *dst); 152257529Sluigi 153259412Sluigi#include "netmap_mbq.h" 154259412Sluigi 155259412Sluigiextern NMG_LOCK_T netmap_global_lock; 156259412Sluigi 157227614Sluigi/* 158232238Sluigi * private, kernel view of a ring. Keeps track of the status of 159232238Sluigi * a ring across system calls. 160227614Sluigi * 161232238Sluigi * nr_hwcur index of the next buffer to refill. 162232238Sluigi * It corresponds to ring->cur - ring->reserved 163232238Sluigi * 164232238Sluigi * nr_hwavail the number of slots "owned" by userspace. 165232238Sluigi * nr_hwavail =:= ring->avail + ring->reserved 166232238Sluigi * 167231594Sluigi * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots. 168227614Sluigi * This is so that, on a reset, buffers owned by userspace are not 169227614Sluigi * modified by the kernel. In particular: 170231594Sluigi * RX rings: the next empty buffer (hwcur + hwavail + hwofs) coincides with 171227614Sluigi * the next empty buffer as known by the hardware (next_to_check or so). 172227614Sluigi * TX rings: hwcur + hwofs coincides with next_to_send 173245579Sluigi * 174257529Sluigi * Clients cannot issue concurrent syscall on a ring. The system 175257529Sluigi * detects this and reports an error using two flags, 176257529Sluigi * NKR_WBUSY and NKR_RBUSY 177245579Sluigi * For received packets, slot->flags is set to nkr_slot_flags 178245579Sluigi * so we can provide a proper initial value (e.g. set NS_FORWARD 179245579Sluigi * when operating in 'transparent' mode). 180257529Sluigi * 181257529Sluigi * The following fields are used to implement lock-free copy of packets 182257529Sluigi * from input to output ports in VALE switch: 183257529Sluigi * nkr_hwlease buffer after the last one being copied. 184257529Sluigi * A writer in nm_bdg_flush reserves N buffers 185257529Sluigi * from nr_hwlease, advances it, then does the 186257529Sluigi * copy outside the lock. 187257529Sluigi * In RX rings (used for VALE ports), 188257529Sluigi * nkr_hwcur + nkr_hwavail <= nkr_hwlease < nkr_hwcur+N-1 189257529Sluigi * In TX rings (used for NIC or host stack ports) 190257529Sluigi * nkr_hwcur <= nkr_hwlease < nkr_hwcur+ nkr_hwavail 191257529Sluigi * nkr_leases array of nkr_num_slots where writers can report 192257529Sluigi * completion of their block. NR_NOSLOT (~0) indicates 193257529Sluigi * that the writer has not finished yet 194259412Sluigi * nkr_lease_idx index of next free slot in nr_leases, to be assigned 195257529Sluigi * 196257529Sluigi * The kring is manipulated by txsync/rxsync and generic netmap function. 197257529Sluigi * q_lock is used to arbitrate access to the kring from within the netmap 198257529Sluigi * code, and this and other protections guarantee that there is never 199257529Sluigi * more than 1 concurrent call to txsync or rxsync. So we are free 200257529Sluigi * to manipulate the kring from within txsync/rxsync without any extra 201257529Sluigi * locks. 202227614Sluigi */ 203227614Sluigistruct netmap_kring { 204227614Sluigi struct netmap_ring *ring; 205257529Sluigi uint32_t nr_hwcur; 206257529Sluigi uint32_t nr_hwavail; 207257529Sluigi uint32_t nr_kflags; /* private driver flags */ 208259412Sluigi int32_t nr_hwreserved; 209234174Sluigi#define NKR_PENDINTR 0x1 // Pending interrupt. 210257529Sluigi uint32_t nkr_num_slots; 211257529Sluigi int32_t nkr_hwofs; /* offset between NIC and netmap ring */ 212227614Sluigi 213245579Sluigi uint16_t nkr_slot_flags; /* initial value for flags */ 214231594Sluigi struct netmap_adapter *na; 215251139Sluigi struct nm_bdg_fwd *nkr_ft; 216257529Sluigi uint32_t *nkr_leases; 217257529Sluigi#define NR_NOSLOT ((uint32_t)~0) 218257529Sluigi uint32_t nkr_hwlease; 219257529Sluigi uint32_t nkr_lease_idx; 220257529Sluigi 221231594Sluigi NM_SELINFO_T si; /* poll/select wait queue */ 222257529Sluigi NM_LOCK_T q_lock; /* protects kring and ring. */ 223257529Sluigi NM_ATOMIC_T nr_busy; /* prevent concurrent syscalls */ 224257529Sluigi 225257529Sluigi volatile int nkr_stopped; 226259412Sluigi 227259412Sluigi /* support for adapters without native netmap support. 228259412Sluigi * On tx rings we preallocate an array of tx buffers 229259412Sluigi * (same size as the netmap ring), on rx rings we 230259412Sluigi * store incoming packets in a queue. 231259412Sluigi * XXX who writes to the rx queue ? 232259412Sluigi */ 233259412Sluigi struct mbuf **tx_pool; 234259412Sluigi u_int nr_ntc; /* Emulation of a next-to-clean RX ring pointer. */ 235259412Sluigi struct mbq rx_queue; /* A queue for intercepted rx mbufs. */ 236259412Sluigi 237230572Sluigi} __attribute__((__aligned__(64))); 238227614Sluigi 239257529Sluigi 240257529Sluigi/* return the next index, with wraparound */ 241257529Sluigistatic inline uint32_t 242257529Sluiginm_next(uint32_t i, uint32_t lim) 243257529Sluigi{ 244257529Sluigi return unlikely (i == lim) ? 0 : i + 1; 245257529Sluigi} 246257529Sluigi 247227614Sluigi/* 248257529Sluigi * 249257529Sluigi * Here is the layout for the Rx and Tx rings. 250257529Sluigi 251257529Sluigi RxRING TxRING 252257529Sluigi 253257529Sluigi +-----------------+ +-----------------+ 254257529Sluigi | | | | 255257529Sluigi |XXX free slot XXX| |XXX free slot XXX| 256257529Sluigi +-----------------+ +-----------------+ 257257529Sluigi | |<-hwcur | |<-hwcur 258257529Sluigi | reserved h | | (ready | 259257529Sluigi +----------- w -+ | to be | 260257529Sluigi cur->| a | | sent) h | 261257529Sluigi | v | +---------- w | 262257529Sluigi | a | cur->| (being a | 263257529Sluigi | i | | prepared) v | 264257529Sluigi | avail l | | a | 265257529Sluigi +-----------------+ + a ------ i + 266257529Sluigi | | ... | v l |<-hwlease 267257529Sluigi | (being | ... | a | ... 268257529Sluigi | prepared) | ... | i | ... 269257529Sluigi +-----------------+ ... | l | ... 270257529Sluigi | |<-hwlease +-----------------+ 271257529Sluigi | | | | 272257529Sluigi | | | | 273257529Sluigi | | | | 274257529Sluigi | | | | 275257529Sluigi +-----------------+ +-----------------+ 276257529Sluigi 277257529Sluigi * The cur/avail (user view) and hwcur/hwavail (kernel view) 278257529Sluigi * are used in the normal operation of the card. 279257529Sluigi * 280257529Sluigi * When a ring is the output of a switch port (Rx ring for 281257529Sluigi * a VALE port, Tx ring for the host stack or NIC), slots 282257529Sluigi * are reserved in blocks through 'hwlease' which points 283257529Sluigi * to the next unused slot. 284257529Sluigi * On an Rx ring, hwlease is always after hwavail, 285257529Sluigi * and completions cause avail to advance. 286257529Sluigi * On a Tx ring, hwlease is always between cur and hwavail, 287257529Sluigi * and completions cause cur to advance. 288257529Sluigi * 289257529Sluigi * nm_kr_space() returns the maximum number of slots that 290257529Sluigi * can be assigned. 291257529Sluigi * nm_kr_lease() reserves the required number of buffers, 292257529Sluigi * advances nkr_hwlease and also returns an entry in 293257529Sluigi * a circular array where completions should be reported. 294257529Sluigi */ 295257529Sluigi 296257529Sluigi 297257529Sluigi 298257529Sluigi 299259412Sluigienum txrx { NR_RX = 0, NR_TX = 1 }; 300257529Sluigi 301257529Sluigi/* 302259412Sluigi * The "struct netmap_adapter" extends the "struct adapter" 303259412Sluigi * (or equivalent) device descriptor. 304259412Sluigi * It contains all base fields needed to support netmap operation. 305259412Sluigi * There are in fact different types of netmap adapters 306259412Sluigi * (native, generic, VALE switch...) so a netmap_adapter is 307259412Sluigi * just the first field in the derived type. 308227614Sluigi */ 309227614Sluigistruct netmap_adapter { 310241719Sluigi /* 311241719Sluigi * On linux we do not have a good way to tell if an interface 312259412Sluigi * is netmap-capable. So we always use the following trick: 313241719Sluigi * NA(ifp) points here, and the first entry (which hopefully 314241719Sluigi * always exists and is at least 32 bits) contains a magic 315241719Sluigi * value which we can use to detect that the interface is good. 316241719Sluigi */ 317241719Sluigi uint32_t magic; 318259412Sluigi uint32_t na_flags; /* enabled, and other flags */ 319241719Sluigi#define NAF_SKIP_INTR 1 /* use the regular interrupt handler. 320241719Sluigi * useful during initialization 321241719Sluigi */ 322251139Sluigi#define NAF_SW_ONLY 2 /* forward packets only to sw adapter */ 323257529Sluigi#define NAF_BDG_MAYSLEEP 4 /* the bridge is allowed to sleep when 324257529Sluigi * forwarding packets coming from this 325257529Sluigi * interface 326257529Sluigi */ 327257529Sluigi#define NAF_MEM_OWNER 8 /* the adapter is responsible for the 328257529Sluigi * deallocation of the memory allocator 329257529Sluigi */ 330259412Sluigi#define NAF_NATIVE_ON 16 /* the adapter is native and the attached 331259412Sluigi * interface is in netmap mode 332259412Sluigi */ 333259412Sluigi#define NAF_NETMAP_ON 32 /* netmap is active (either native or 334259412Sluigi * emulated. Where possible (e.g. FreeBSD) 335259412Sluigi * IFCAP_NETMAP also mirrors this flag. 336259412Sluigi */ 337259412Sluigi int active_fds; /* number of user-space descriptors using this 338227614Sluigi interface, which is equal to the number of 339227614Sluigi struct netmap_if objs in the mapped region. */ 340227614Sluigi 341239140Semaste u_int num_rx_rings; /* number of adapter receive rings */ 342239140Semaste u_int num_tx_rings; /* number of adapter transmit rings */ 343227614Sluigi 344227614Sluigi u_int num_tx_desc; /* number of descriptor in each queue */ 345227614Sluigi u_int num_rx_desc; 346227614Sluigi 347227614Sluigi /* tx_rings and rx_rings are private but allocated 348227614Sluigi * as a contiguous chunk of memory. Each array has 349227614Sluigi * N+1 entries, for the adapter queues and for the host queue. 350227614Sluigi */ 351227614Sluigi struct netmap_kring *tx_rings; /* array of TX rings. */ 352227614Sluigi struct netmap_kring *rx_rings; /* array of RX rings. */ 353259412Sluigi void *tailroom; /* space below the rings array */ 354259412Sluigi /* (used for leases) */ 355227614Sluigi 356259412Sluigi 357232238Sluigi NM_SELINFO_T tx_si, rx_si; /* global wait queues */ 358232238Sluigi 359227614Sluigi /* copy of if_qflush and if_transmit pointers, to intercept 360227614Sluigi * packets from the network stack when netmap is active. 361227614Sluigi */ 362227614Sluigi int (*if_transmit)(struct ifnet *, struct mbuf *); 363227614Sluigi 364227614Sluigi /* references to the ifnet and device routines, used by 365227614Sluigi * the generic netmap functions. 366227614Sluigi */ 367227614Sluigi struct ifnet *ifp; /* adapter is ifp->if_softc */ 368227614Sluigi 369259412Sluigi /* private cleanup */ 370259412Sluigi void (*nm_dtor)(struct netmap_adapter *); 371231594Sluigi 372259412Sluigi int (*nm_register)(struct netmap_adapter *, int onoff); 373257529Sluigi 374259412Sluigi int (*nm_txsync)(struct netmap_adapter *, u_int ring, int flags); 375259412Sluigi int (*nm_rxsync)(struct netmap_adapter *, u_int ring, int flags); 376257529Sluigi#define NAF_FORCE_READ 1 377257529Sluigi#define NAF_FORCE_RECLAIM 2 378245835Sluigi /* return configuration information */ 379259412Sluigi int (*nm_config)(struct netmap_adapter *, 380259412Sluigi u_int *txr, u_int *txd, u_int *rxr, u_int *rxd); 381259412Sluigi int (*nm_krings_create)(struct netmap_adapter *); 382259412Sluigi void (*nm_krings_delete)(struct netmap_adapter *); 383259412Sluigi int (*nm_notify)(struct netmap_adapter *, 384259412Sluigi u_int ring, enum txrx, int flags); 385259412Sluigi#define NAF_GLOBAL_NOTIFY 4 386259412Sluigi#define NAF_DISABLE_NOTIFY 8 387238812Sluigi 388259412Sluigi /* standard refcount to control the lifetime of the adapter 389259412Sluigi * (it should be equal to the lifetime of the corresponding ifp) 390259412Sluigi */ 391259412Sluigi int na_refcount; 392259412Sluigi 393259412Sluigi /* memory allocator (opaque) 394259412Sluigi * We also cache a pointer to the lut_entry for translating 395259412Sluigi * buffer addresses, and the total number of buffers. 396259412Sluigi */ 397259412Sluigi struct netmap_mem_d *nm_mem; 398259412Sluigi struct lut_entry *na_lut; 399259412Sluigi uint32_t na_lut_objtotal; /* max buffer index */ 400259412Sluigi 401259412Sluigi /* used internally. If non-null, the interface cannot be bound 402259412Sluigi * from userspace 403259412Sluigi */ 404259412Sluigi void *na_private; 405259412Sluigi}; 406259412Sluigi 407259412Sluigi/* 408259412Sluigi * If the NIC is owned by the kernel 409259412Sluigi * (i.e., bridge), neither another bridge nor user can use it; 410259412Sluigi * if the NIC is owned by a user, only users can share it. 411259412Sluigi * Evaluation must be done under NMG_LOCK(). 412259412Sluigi */ 413259412Sluigi#define NETMAP_OWNED_BY_KERN(na) (na->na_private) 414259412Sluigi#define NETMAP_OWNED_BY_ANY(na) \ 415259412Sluigi (NETMAP_OWNED_BY_KERN(na) || (na->active_fds > 0)) 416259412Sluigi 417259412Sluigi 418259412Sluigi/* 419259412Sluigi * derived netmap adapters for various types of ports 420259412Sluigi */ 421259412Sluigistruct netmap_vp_adapter { /* VALE software port */ 422259412Sluigi struct netmap_adapter up; 423259412Sluigi 424250107Sluigi /* 425250107Sluigi * Bridge support: 426250107Sluigi * 427250107Sluigi * bdg_port is the port number used in the bridge; 428251139Sluigi * na_bdg points to the bridge this NA is attached to. 429250107Sluigi */ 430238812Sluigi int bdg_port; 431251139Sluigi struct nm_bridge *na_bdg; 432259412Sluigi int retry; 433259412Sluigi 434259412Sluigi u_int offset; /* Offset of ethernet header for each packet. */ 435259412Sluigi}; 436259412Sluigi 437259412Sluigistruct netmap_hw_adapter { /* physical device */ 438259412Sluigi struct netmap_adapter up; 439259412Sluigi 440259412Sluigi struct net_device_ops nm_ndo; // XXX linux only 441259412Sluigi}; 442259412Sluigi 443259412Sluigistruct netmap_generic_adapter { /* non-native device */ 444259412Sluigi struct netmap_hw_adapter up; 445259412Sluigi 446259412Sluigi /* Pointer to a previously used netmap adapter. */ 447259412Sluigi struct netmap_adapter *prev; 448259412Sluigi 449259412Sluigi /* generic netmap adapters support: 450259412Sluigi * a net_device_ops struct overrides ndo_select_queue(), 451259412Sluigi * save_if_input saves the if_input hook (FreeBSD), 452259412Sluigi * mit_timer and mit_pending implement rx interrupt mitigation, 453259412Sluigi */ 454259412Sluigi struct net_device_ops generic_ndo; 455259412Sluigi void (*save_if_input)(struct ifnet *, struct mbuf *); 456259412Sluigi 457259412Sluigi struct hrtimer mit_timer; 458259412Sluigi int mit_pending; 459259412Sluigi}; 460259412Sluigi 461259412Sluigi#ifdef WITH_VALE 462259412Sluigi 463259412Sluigi/* bridge wrapper for non VALE ports. It is used to connect real devices to the bridge. 464259412Sluigi * 465259412Sluigi * The real device must already have its own netmap adapter (hwna). The 466259412Sluigi * bridge wrapper and the hwna adapter share the same set of netmap rings and 467259412Sluigi * buffers, but they have two separate sets of krings descriptors, with tx/rx 468259412Sluigi * meanings swapped: 469259412Sluigi * 470259412Sluigi * netmap 471259412Sluigi * bwrap krings rings krings hwna 472259412Sluigi * +------+ +------+ +-----+ +------+ +------+ 473259412Sluigi * |tx_rings->| |\ /| |----| |<-tx_rings| 474259412Sluigi * | | +------+ \ / +-----+ +------+ | | 475259412Sluigi * | | X | | 476259412Sluigi * | | / \ | | 477259412Sluigi * | | +------+/ \+-----+ +------+ | | 478259412Sluigi * |rx_rings->| | | |----| |<-rx_rings| 479259412Sluigi * | | +------+ +-----+ +------+ | | 480259412Sluigi * +------+ +------+ 481259412Sluigi * 482259412Sluigi * - packets coming from the bridge go to the brwap rx rings, which are also the 483259412Sluigi * hwna tx rings. The bwrap notify callback will then complete the hwna tx 484259412Sluigi * (see netmap_bwrap_notify). 485259412Sluigi * - packets coming from the outside go to the hwna rx rings, which are also the 486259412Sluigi * bwrap tx rings. The (overwritten) hwna notify method will then complete 487259412Sluigi * the bridge tx (see netmap_bwrap_intr_notify). 488259412Sluigi * 489259412Sluigi * The bridge wrapper may optionally connect the hwna 'host' rings to the 490259412Sluigi * bridge. This is done by using a second port in the bridge and connecting it 491259412Sluigi * to the 'host' netmap_vp_adapter contained in the netmap_bwrap_adapter. 492259412Sluigi * The brwap host adapter cross-links the hwna host rings in the same way as shown above. 493259412Sluigi * 494259412Sluigi * - packets coming from the bridge and directed to host stack are handled by the 495259412Sluigi * bwrap host notify callback (see netmap_bwrap_host_notify) 496259412Sluigi * - packets coming from the host stack are still handled by the overwritten 497259412Sluigi * hwna notify callback (netmap_bwrap_intr_notify), but are diverted to the 498259412Sluigi * host adapter depending on the ring number. 499259412Sluigi * 500259412Sluigi */ 501259412Sluigistruct netmap_bwrap_adapter { 502259412Sluigi struct netmap_vp_adapter up; 503259412Sluigi struct netmap_vp_adapter host; /* for host rings */ 504259412Sluigi struct netmap_adapter *hwna; /* the underlying device */ 505259412Sluigi 506259412Sluigi /* backup of the hwna notify callback */ 507259412Sluigi int (*save_notify)(struct netmap_adapter *, 508259412Sluigi u_int ring, enum txrx, int flags); 509251139Sluigi /* When we attach a physical interface to the bridge, we 510251139Sluigi * allow the controlling process to terminate, so we need 511251139Sluigi * a place to store the netmap_priv_d data structure. 512251139Sluigi * This is only done when physical interfaces are attached to a bridge. 513251139Sluigi */ 514251139Sluigi struct netmap_priv_d *na_kpriv; 515227614Sluigi}; 516227614Sluigi 517259412Sluigi 518227614Sluigi/* 519259412Sluigi * Available space in the ring. Only used in VALE code 520257529Sluigi */ 521257529Sluigistatic inline uint32_t 522257529Sluiginm_kr_space(struct netmap_kring *k, int is_rx) 523257529Sluigi{ 524257529Sluigi int space; 525257529Sluigi 526257529Sluigi if (is_rx) { 527259412Sluigi int busy = k->nkr_hwlease - k->nr_hwcur + k->nr_hwreserved; 528257529Sluigi if (busy < 0) 529257529Sluigi busy += k->nkr_num_slots; 530257529Sluigi space = k->nkr_num_slots - 1 - busy; 531257529Sluigi } else { 532257529Sluigi space = k->nr_hwcur + k->nr_hwavail - k->nkr_hwlease; 533257529Sluigi if (space < 0) 534257529Sluigi space += k->nkr_num_slots; 535257529Sluigi } 536257529Sluigi#if 0 537257529Sluigi // sanity check 538257529Sluigi if (k->nkr_hwlease >= k->nkr_num_slots || 539257529Sluigi k->nr_hwcur >= k->nkr_num_slots || 540257529Sluigi k->nr_hwavail >= k->nkr_num_slots || 541257529Sluigi busy < 0 || 542257529Sluigi busy >= k->nkr_num_slots) { 543257529Sluigi D("invalid kring, cur %d avail %d lease %d lease_idx %d lim %d", k->nr_hwcur, k->nr_hwavail, k->nkr_hwlease, 544257529Sluigi k->nkr_lease_idx, k->nkr_num_slots); 545257529Sluigi } 546257529Sluigi#endif 547257529Sluigi return space; 548257529Sluigi} 549257529Sluigi 550257529Sluigi 551257529Sluigi 552257529Sluigi 553257529Sluigi/* make a lease on the kring for N positions. return the 554257529Sluigi * lease index 555257529Sluigi */ 556257529Sluigistatic inline uint32_t 557257529Sluiginm_kr_lease(struct netmap_kring *k, u_int n, int is_rx) 558257529Sluigi{ 559257529Sluigi uint32_t lim = k->nkr_num_slots - 1; 560257529Sluigi uint32_t lease_idx = k->nkr_lease_idx; 561257529Sluigi 562257529Sluigi k->nkr_leases[lease_idx] = NR_NOSLOT; 563257529Sluigi k->nkr_lease_idx = nm_next(lease_idx, lim); 564257529Sluigi 565257529Sluigi if (n > nm_kr_space(k, is_rx)) { 566257529Sluigi D("invalid request for %d slots", n); 567257529Sluigi panic("x"); 568257529Sluigi } 569257529Sluigi /* XXX verify that there are n slots */ 570257529Sluigi k->nkr_hwlease += n; 571257529Sluigi if (k->nkr_hwlease > lim) 572257529Sluigi k->nkr_hwlease -= lim + 1; 573257529Sluigi 574257529Sluigi if (k->nkr_hwlease >= k->nkr_num_slots || 575257529Sluigi k->nr_hwcur >= k->nkr_num_slots || 576257529Sluigi k->nr_hwavail >= k->nkr_num_slots || 577257529Sluigi k->nkr_lease_idx >= k->nkr_num_slots) { 578257529Sluigi D("invalid kring %s, cur %d avail %d lease %d lease_idx %d lim %d", 579257529Sluigi k->na->ifp->if_xname, 580257529Sluigi k->nr_hwcur, k->nr_hwavail, k->nkr_hwlease, 581257529Sluigi k->nkr_lease_idx, k->nkr_num_slots); 582257529Sluigi } 583257529Sluigi return lease_idx; 584257529Sluigi} 585257529Sluigi 586259412Sluigi#endif /* WITH_VALE */ 587257529Sluigi 588259412Sluigi/* return update position */ 589259412Sluigistatic inline uint32_t 590259412Sluiginm_kr_rxpos(struct netmap_kring *k) 591259412Sluigi{ 592259412Sluigi uint32_t pos = k->nr_hwcur + k->nr_hwavail; 593259412Sluigi if (pos >= k->nkr_num_slots) 594259412Sluigi pos -= k->nkr_num_slots; 595259412Sluigi#if 0 596259412Sluigi if (pos >= k->nkr_num_slots || 597259412Sluigi k->nkr_hwlease >= k->nkr_num_slots || 598259412Sluigi k->nr_hwcur >= k->nkr_num_slots || 599259412Sluigi k->nr_hwavail >= k->nkr_num_slots || 600259412Sluigi k->nkr_lease_idx >= k->nkr_num_slots) { 601259412Sluigi D("invalid kring, cur %d avail %d lease %d lease_idx %d lim %d", k->nr_hwcur, k->nr_hwavail, k->nkr_hwlease, 602259412Sluigi k->nkr_lease_idx, k->nkr_num_slots); 603259412Sluigi } 604259412Sluigi#endif 605259412Sluigi return pos; 606259412Sluigi} 607259412Sluigi 608259412Sluigi 609257529Sluigi/* 610259412Sluigi * protect against multiple threads using the same ring. 611259412Sluigi * also check that the ring has not been stopped. 612259412Sluigi * We only care for 0 or !=0 as a return code. 613227614Sluigi */ 614259412Sluigi#define NM_KR_BUSY 1 615259412Sluigi#define NM_KR_STOPPED 2 616227614Sluigi 617259412Sluigistatic __inline void nm_kr_put(struct netmap_kring *kr) 618259412Sluigi{ 619259412Sluigi NM_ATOMIC_CLEAR(&kr->nr_busy); 620259412Sluigi} 621227614Sluigi 622259412Sluigistatic __inline int nm_kr_tryget(struct netmap_kring *kr) 623259412Sluigi{ 624259412Sluigi /* check a first time without taking the lock 625259412Sluigi * to avoid starvation for nm_kr_get() 626259412Sluigi */ 627259412Sluigi if (unlikely(kr->nkr_stopped)) { 628259412Sluigi ND("ring %p stopped (%d)", kr, kr->nkr_stopped); 629259412Sluigi return NM_KR_STOPPED; 630259412Sluigi } 631259412Sluigi if (unlikely(NM_ATOMIC_TEST_AND_SET(&kr->nr_busy))) 632259412Sluigi return NM_KR_BUSY; 633259412Sluigi /* check a second time with lock held */ 634259412Sluigi if (unlikely(kr->nkr_stopped)) { 635259412Sluigi ND("ring %p stopped (%d)", kr, kr->nkr_stopped); 636259412Sluigi nm_kr_put(kr); 637259412Sluigi return NM_KR_STOPPED; 638259412Sluigi } 639259412Sluigi return 0; 640259412Sluigi} 641227614Sluigi 642259412Sluigi 643227614Sluigi/* 644227614Sluigi * The following are support routines used by individual drivers to 645227614Sluigi * support netmap operation. 646227614Sluigi * 647227614Sluigi * netmap_attach() initializes a struct netmap_adapter, allocating the 648227614Sluigi * struct netmap_ring's and the struct selinfo. 649227614Sluigi * 650227614Sluigi * netmap_detach() frees the memory allocated by netmap_attach(). 651227614Sluigi * 652257529Sluigi * netmap_transmit() replaces the if_transmit routine of the interface, 653227614Sluigi * and is used to intercept packets coming from the stack. 654227614Sluigi * 655227614Sluigi * netmap_load_map/netmap_reload_map are helper routines to set/reset 656227614Sluigi * the dmamap for a packet buffer 657227614Sluigi * 658227614Sluigi * netmap_reset() is a helper routine to be called in the driver 659227614Sluigi * when reinitializing a ring. 660227614Sluigi */ 661259412Sluigiint netmap_attach(struct netmap_adapter *); 662259412Sluigiint netmap_attach_common(struct netmap_adapter *); 663259412Sluigivoid netmap_detach_common(struct netmap_adapter *na); 664227614Sluigivoid netmap_detach(struct ifnet *); 665257529Sluigiint netmap_transmit(struct ifnet *, struct mbuf *); 666227614Sluigistruct netmap_slot *netmap_reset(struct netmap_adapter *na, 667257529Sluigi enum txrx tx, u_int n, u_int new_cur); 668227614Sluigiint netmap_ring_reinit(struct netmap_kring *); 669227614Sluigi 670259412Sluigi/* set/clear native flags. XXX maybe also if_transmit ? */ 671259412Sluigistatic inline void 672259412Sluiginm_set_native_flags(struct netmap_adapter *na) 673259412Sluigi{ 674259412Sluigi struct ifnet *ifp = na->ifp; 675259412Sluigi 676259412Sluigi na->na_flags |= (NAF_NATIVE_ON | NAF_NETMAP_ON); 677259412Sluigi#ifdef IFCAP_NETMAP /* or FreeBSD ? */ 678259412Sluigi ifp->if_capenable |= IFCAP_NETMAP; 679259412Sluigi#endif 680259412Sluigi#ifdef __FreeBSD__ 681259412Sluigi na->if_transmit = ifp->if_transmit; 682259412Sluigi ifp->if_transmit = netmap_transmit; 683259412Sluigi#else 684259412Sluigi na->if_transmit = (void *)ifp->netdev_ops; 685259412Sluigi ifp->netdev_ops = &((struct netmap_hw_adapter *)na)->nm_ndo; 686259412Sluigi#endif 687259412Sluigi} 688259412Sluigi 689259412Sluigistatic inline void 690259412Sluiginm_clear_native_flags(struct netmap_adapter *na) 691259412Sluigi{ 692259412Sluigi struct ifnet *ifp = na->ifp; 693259412Sluigi 694259412Sluigi#ifdef __FreeBSD__ 695259412Sluigi ifp->if_transmit = na->if_transmit; 696259412Sluigi#else 697259412Sluigi ifp->netdev_ops = (void *)na->if_transmit; 698259412Sluigi#endif 699259412Sluigi na->na_flags &= ~(NAF_NATIVE_ON | NAF_NETMAP_ON); 700259412Sluigi#ifdef IFCAP_NETMAP /* or FreeBSD ? */ 701259412Sluigi ifp->if_capenable &= ~IFCAP_NETMAP; 702259412Sluigi#endif 703259412Sluigi} 704259412Sluigi 705259412Sluigi/* 706259412Sluigi * validates parameters in the ring/kring, returns a value for cur, 707259412Sluigi * and the 'new_slots' value in the argument. 708259412Sluigi * If any error, returns cur > lim to force a reinit. 709259412Sluigi */ 710259412Sluigiu_int nm_txsync_prologue(struct netmap_kring *, u_int *); 711259412Sluigi 712259412Sluigi/* 713259412Sluigi * validates parameters in the ring/kring, returns a value for cur, 714259412Sluigi * and the 'reserved' value in the argument. 715259412Sluigi * If any error, returns cur > lim to force a reinit. 716259412Sluigi */ 717259412Sluigiu_int nm_rxsync_prologue(struct netmap_kring *, u_int *); 718259412Sluigi 719259412Sluigi/* 720259412Sluigi * update kring and ring at the end of txsync 721259412Sluigi */ 722259412Sluigistatic inline void 723259412Sluiginm_txsync_finalize(struct netmap_kring *kring, u_int cur) 724259412Sluigi{ 725259412Sluigi /* recompute hwreserved */ 726259412Sluigi kring->nr_hwreserved = cur - kring->nr_hwcur; 727259412Sluigi if (kring->nr_hwreserved < 0) 728259412Sluigi kring->nr_hwreserved += kring->nkr_num_slots; 729259412Sluigi 730259412Sluigi /* update avail and reserved to what the kernel knows */ 731259412Sluigi kring->ring->avail = kring->nr_hwavail; 732259412Sluigi kring->ring->reserved = kring->nr_hwreserved; 733259412Sluigi} 734259412Sluigi 735259412Sluigi/* check/fix address and len in tx rings */ 736259412Sluigi#if 1 /* debug version */ 737259412Sluigi#define NM_CHECK_ADDR_LEN(_a, _l) do { \ 738259412Sluigi if (_a == netmap_buffer_base || _l > NETMAP_BUF_SIZE) { \ 739259412Sluigi RD(5, "bad addr/len ring %d slot %d idx %d len %d", \ 740259412Sluigi ring_nr, nm_i, slot->buf_idx, len); \ 741259412Sluigi if (_l > NETMAP_BUF_SIZE) \ 742259412Sluigi _l = NETMAP_BUF_SIZE; \ 743259412Sluigi } } while (0) 744259412Sluigi#else /* no debug version */ 745259412Sluigi#define NM_CHECK_ADDR_LEN(_a, _l) do { \ 746259412Sluigi if (_l > NETMAP_BUF_SIZE) \ 747259412Sluigi _l = NETMAP_BUF_SIZE; \ 748259412Sluigi } while (0) 749259412Sluigi#endif 750259412Sluigi 751259412Sluigi 752259412Sluigi/*---------------------------------------------------------------*/ 753259412Sluigi/* 754259412Sluigi * Support routines to be used with the VALE switch 755259412Sluigi */ 756259412Sluigiint netmap_update_config(struct netmap_adapter *na); 757259412Sluigiint netmap_krings_create(struct netmap_adapter *na, u_int ntx, u_int nrx, u_int tailroom); 758259412Sluigivoid netmap_krings_delete(struct netmap_adapter *na); 759259412Sluigi 760259412Sluigistruct netmap_if * 761259412Sluiginetmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na, 762259412Sluigi uint16_t ringid, int *err); 763259412Sluigi 764259412Sluigi 765259412Sluigi 766257529Sluigiu_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg); 767259412Sluigiint netmap_get_na(struct nmreq *nmr, struct netmap_adapter **na, int create); 768259412Sluigiint netmap_get_hw_na(struct ifnet *ifp, struct netmap_adapter **na); 769257529Sluigi 770259412Sluigi#ifdef WITH_VALE 771251139Sluigi/* 772251139Sluigi * The following bridge-related interfaces are used by other kernel modules 773251139Sluigi * In the version that only supports unicast or broadcast, the lookup 774251139Sluigi * function can return 0 .. NM_BDG_MAXPORTS-1 for regular ports, 775251139Sluigi * NM_BDG_MAXPORTS for broadcast, NM_BDG_MAXPORTS+1 for unknown. 776251139Sluigi * XXX in practice "unknown" might be handled same as broadcast. 777251139Sluigi */ 778259412Sluigitypedef u_int (*bdg_lookup_fn_t)(char *buf, u_int len, 779259412Sluigi uint8_t *ring_nr, struct netmap_vp_adapter *); 780259412Sluigiu_int netmap_bdg_learning(char *, u_int, uint8_t *, 781259412Sluigi struct netmap_vp_adapter *); 782259412Sluigi 783259412Sluigi#define NM_BDG_MAXPORTS 254 /* up to 254 */ 784251139Sluigi#define NM_BDG_BROADCAST NM_BDG_MAXPORTS 785251139Sluigi#define NM_BDG_NOPORT (NM_BDG_MAXPORTS+1) 786251139Sluigi 787259412Sluigi#define NM_NAME "vale" /* prefix for bridge port name */ 788259412Sluigi 789259412Sluigi 790259412Sluigi/* these are redefined in case of no VALE support */ 791259412Sluigiint netmap_get_bdg_na(struct nmreq *nmr, struct netmap_adapter **na, int create); 792259412Sluigivoid netmap_init_bridges(void); 793259412Sluigiint netmap_bdg_ctl(struct nmreq *nmr, bdg_lookup_fn_t func); 794259412Sluigi 795259412Sluigi#else /* !WITH_VALE */ 796259412Sluigi#define netmap_get_bdg_na(_1, _2, _3) 0 797259412Sluigi#define netmap_init_bridges(_1) 798259412Sluigi#define netmap_bdg_ctl(_1, _2) EINVAL 799259412Sluigi#endif /* !WITH_VALE */ 800259412Sluigi 801259412Sluigi/* Various prototypes */ 802259412Sluigiint netmap_poll(struct cdev *dev, int events, struct thread *td); 803259412Sluigi 804259412Sluigi 805259412Sluigiint netmap_init(void); 806259412Sluigivoid netmap_fini(void); 807259412Sluigiint netmap_get_memory(struct netmap_priv_d* p); 808259412Sluigivoid netmap_dtor(void *data); 809259412Sluigiint netmap_dtor_locked(struct netmap_priv_d *priv); 810259412Sluigi 811259412Sluigiint netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td); 812259412Sluigi 813259412Sluigi/* netmap_adapter creation/destruction */ 814259412Sluigi#define NM_IFPNAME(ifp) ((ifp) ? (ifp)->if_xname : "zombie") 815259412Sluigi#define NM_DEBUG_PUTGET 1 816259412Sluigi 817259412Sluigi#ifdef NM_DEBUG_PUTGET 818259412Sluigi 819259412Sluigi#define NM_DBG(f) __##f 820259412Sluigi 821259412Sluigivoid __netmap_adapter_get(struct netmap_adapter *na); 822259412Sluigi 823259412Sluigi#define netmap_adapter_get(na) \ 824259412Sluigi do { \ 825259412Sluigi struct netmap_adapter *__na = na; \ 826259412Sluigi D("getting %p:%s (%d)", __na, NM_IFPNAME(__na->ifp), __na->na_refcount); \ 827259412Sluigi __netmap_adapter_get(__na); \ 828259412Sluigi } while (0) 829259412Sluigi 830259412Sluigiint __netmap_adapter_put(struct netmap_adapter *na); 831259412Sluigi 832259412Sluigi#define netmap_adapter_put(na) \ 833259412Sluigi do { \ 834259412Sluigi struct netmap_adapter *__na = na; \ 835259412Sluigi D("putting %p:%s (%d)", __na, NM_IFPNAME(__na->ifp), __na->na_refcount); \ 836259412Sluigi __netmap_adapter_put(__na); \ 837259412Sluigi } while (0) 838259412Sluigi 839259412Sluigi#else /* !NM_DEBUG_PUTGET */ 840259412Sluigi 841259412Sluigi#define NM_DBG(f) f 842259412Sluigivoid netmap_adapter_get(struct netmap_adapter *na); 843259412Sluigiint netmap_adapter_put(struct netmap_adapter *na); 844259412Sluigi 845259412Sluigi#endif /* !NM_DEBUG_PUTGET */ 846259412Sluigi 847259412Sluigi 848238985Sluigiextern u_int netmap_buf_size; 849250107Sluigi#define NETMAP_BUF_SIZE netmap_buf_size // XXX remove 850230572Sluigiextern int netmap_mitigate; 851231198Sluigiextern int netmap_no_pendintr; 852227614Sluigiextern u_int netmap_total_buffers; 853227614Sluigiextern char *netmap_buffer_base; 854227614Sluigiextern int netmap_verbose; // XXX debugging 855227614Sluigienum { /* verbose flags */ 856227614Sluigi NM_VERB_ON = 1, /* generic verbose */ 857227614Sluigi NM_VERB_HOST = 0x2, /* verbose host stack */ 858227614Sluigi NM_VERB_RXSYNC = 0x10, /* verbose on rxsync/txsync */ 859227614Sluigi NM_VERB_TXSYNC = 0x20, 860227614Sluigi NM_VERB_RXINTR = 0x100, /* verbose on rx/tx intr (driver) */ 861227614Sluigi NM_VERB_TXINTR = 0x200, 862227614Sluigi NM_VERB_NIC_RXSYNC = 0x1000, /* verbose on rx/tx intr (driver) */ 863227614Sluigi NM_VERB_NIC_TXSYNC = 0x2000, 864227614Sluigi}; 865227614Sluigi 866259412Sluigiextern int netmap_txsync_retry; 867259412Sluigiextern int netmap_generic_mit; 868259412Sluigiextern int netmap_generic_ringsize; 869259412Sluigi 870227614Sluigi/* 871228845Sluigi * NA returns a pointer to the struct netmap adapter from the ifp, 872228845Sluigi * WNA is used to write it. 873227614Sluigi */ 874228845Sluigi#ifndef WNA 875228845Sluigi#define WNA(_ifp) (_ifp)->if_pspare[0] 876228845Sluigi#endif 877228845Sluigi#define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp)) 878227614Sluigi 879241719Sluigi/* 880241719Sluigi * Macros to determine if an interface is netmap capable or netmap enabled. 881241719Sluigi * See the magic field in struct netmap_adapter. 882241719Sluigi */ 883241719Sluigi#ifdef __FreeBSD__ 884241719Sluigi/* 885241719Sluigi * on FreeBSD just use if_capabilities and if_capenable. 886241719Sluigi */ 887241719Sluigi#define NETMAP_CAPABLE(ifp) (NA(ifp) && \ 888241719Sluigi (ifp)->if_capabilities & IFCAP_NETMAP ) 889227614Sluigi 890241719Sluigi#define NETMAP_SET_CAPABLE(ifp) \ 891241719Sluigi (ifp)->if_capabilities |= IFCAP_NETMAP 892241719Sluigi 893241719Sluigi#else /* linux */ 894241719Sluigi 895241719Sluigi/* 896241719Sluigi * on linux: 897241719Sluigi * we check if NA(ifp) is set and its first element has a related 898241719Sluigi * magic value. The capenable is within the struct netmap_adapter. 899241719Sluigi */ 900241719Sluigi#define NETMAP_MAGIC 0x52697a7a 901241719Sluigi 902241719Sluigi#define NETMAP_CAPABLE(ifp) (NA(ifp) && \ 903241719Sluigi ((uint32_t)(uintptr_t)NA(ifp) ^ NA(ifp)->magic) == NETMAP_MAGIC ) 904241719Sluigi 905241719Sluigi#define NETMAP_SET_CAPABLE(ifp) \ 906241719Sluigi NA(ifp)->magic = ((uint32_t)(uintptr_t)NA(ifp)) ^ NETMAP_MAGIC 907241719Sluigi 908241719Sluigi#endif /* linux */ 909241719Sluigi 910238812Sluigi#ifdef __FreeBSD__ 911259412Sluigi 912230052Sluigi/* Callback invoked by the dma machinery after a successfull dmamap_load */ 913230052Sluigistatic void netmap_dmamap_cb(__unused void *arg, 914230058Sluigi __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error) 915230052Sluigi{ 916230052Sluigi} 917230052Sluigi 918230052Sluigi/* bus_dmamap_load wrapper: call aforementioned function if map != NULL. 919230052Sluigi * XXX can we do it without a callback ? 920230052Sluigi */ 921230052Sluigistatic inline void 922230052Sluiginetmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 923230052Sluigi{ 924230052Sluigi if (map) 925230052Sluigi bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE, 926230058Sluigi netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 927230052Sluigi} 928230052Sluigi 929230052Sluigi/* update the map when a buffer changes. */ 930230052Sluigistatic inline void 931230052Sluiginetmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 932230052Sluigi{ 933230052Sluigi if (map) { 934230052Sluigi bus_dmamap_unload(tag, map); 935230052Sluigi bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE, 936230058Sluigi netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 937230052Sluigi } 938230052Sluigi} 939259412Sluigi 940238812Sluigi#else /* linux */ 941230052Sluigi 942231796Sluigi/* 943238812Sluigi * XXX How do we redefine these functions: 944238812Sluigi * 945238812Sluigi * on linux we need 946238831Sluigi * dma_map_single(&pdev->dev, virt_addr, len, direction) 947238831Sluigi * dma_unmap_single(&adapter->pdev->dev, phys_addr, len, direction 948238812Sluigi * The len can be implicit (on netmap it is NETMAP_BUF_SIZE) 949238812Sluigi * unfortunately the direction is not, so we need to change 950238812Sluigi * something to have a cross API 951238812Sluigi */ 952238812Sluigi#define netmap_load_map(_t, _m, _b) 953238812Sluigi#define netmap_reload_map(_t, _m, _b) 954238812Sluigi#if 0 955238812Sluigi struct e1000_buffer *buffer_info = &tx_ring->buffer_info[l]; 956238812Sluigi /* set time_stamp *before* dma to help avoid a possible race */ 957238812Sluigi buffer_info->time_stamp = jiffies; 958238812Sluigi buffer_info->mapped_as_page = false; 959238812Sluigi buffer_info->length = len; 960238812Sluigi //buffer_info->next_to_watch = l; 961238812Sluigi /* reload dma map */ 962238812Sluigi dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 963238831Sluigi NETMAP_BUF_SIZE, DMA_TO_DEVICE); 964238812Sluigi buffer_info->dma = dma_map_single(&adapter->pdev->dev, 965238831Sluigi addr, NETMAP_BUF_SIZE, DMA_TO_DEVICE); 966238812Sluigi 967238812Sluigi if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) { 968238812Sluigi D("dma mapping error"); 969238812Sluigi /* goto dma_error; See e1000_put_txbuf() */ 970238812Sluigi /* XXX reset */ 971238812Sluigi } 972238812Sluigi tx_desc->buffer_addr = htole64(buffer_info->dma); //XXX 973238812Sluigi 974238812Sluigi#endif 975238812Sluigi 976238812Sluigi/* 977238812Sluigi * The bus_dmamap_sync() can be one of wmb() or rmb() depending on direction. 978238812Sluigi */ 979238812Sluigi#define bus_dmamap_sync(_a, _b, _c) 980238812Sluigi 981238812Sluigi#endif /* linux */ 982238812Sluigi 983257529Sluigi 984238812Sluigi/* 985231796Sluigi * functions to map NIC to KRING indexes (n2k) and vice versa (k2n) 986231796Sluigi */ 987231796Sluigistatic inline int 988232238Sluiginetmap_idx_n2k(struct netmap_kring *kr, int idx) 989231796Sluigi{ 990232238Sluigi int n = kr->nkr_num_slots; 991232238Sluigi idx += kr->nkr_hwofs; 992232238Sluigi if (idx < 0) 993232238Sluigi return idx + n; 994232238Sluigi else if (idx < n) 995232238Sluigi return idx; 996231796Sluigi else 997232238Sluigi return idx - n; 998231796Sluigi} 999230052Sluigi 1000231796Sluigi 1001231796Sluigistatic inline int 1002232238Sluiginetmap_idx_k2n(struct netmap_kring *kr, int idx) 1003231796Sluigi{ 1004232238Sluigi int n = kr->nkr_num_slots; 1005232238Sluigi idx -= kr->nkr_hwofs; 1006232238Sluigi if (idx < 0) 1007232238Sluigi return idx + n; 1008232238Sluigi else if (idx < n) 1009232238Sluigi return idx; 1010231796Sluigi else 1011232238Sluigi return idx - n; 1012231796Sluigi} 1013231796Sluigi 1014231796Sluigi 1015234227Sluigi/* Entries of the look-up table. */ 1016234227Sluigistruct lut_entry { 1017234227Sluigi void *vaddr; /* virtual address. */ 1018250107Sluigi vm_paddr_t paddr; /* physical address. */ 1019234227Sluigi}; 1020234227Sluigi 1021234227Sluigistruct netmap_obj_pool; 1022234227Sluigiextern struct lut_entry *netmap_buffer_lut; 1023234227Sluigi#define NMB_VA(i) (netmap_buffer_lut[i].vaddr) 1024234227Sluigi#define NMB_PA(i) (netmap_buffer_lut[i].paddr) 1025234227Sluigi 1026227614Sluigi/* 1027229939Sluigi * NMB return the virtual address of a buffer (buffer 0 on bad index) 1028229939Sluigi * PNMB also fills the physical address 1029227614Sluigi */ 1030229939Sluigistatic inline void * 1031227614SluigiNMB(struct netmap_slot *slot) 1032227614Sluigi{ 1033227614Sluigi uint32_t i = slot->buf_idx; 1034238812Sluigi return (unlikely(i >= netmap_total_buffers)) ? NMB_VA(0) : NMB_VA(i); 1035227614Sluigi} 1036227614Sluigi 1037229939Sluigistatic inline void * 1038229939SluigiPNMB(struct netmap_slot *slot, uint64_t *pp) 1039229939Sluigi{ 1040229939Sluigi uint32_t i = slot->buf_idx; 1041234227Sluigi void *ret = (i >= netmap_total_buffers) ? NMB_VA(0) : NMB_VA(i); 1042249659Sluigi 1043234227Sluigi *pp = (i >= netmap_total_buffers) ? NMB_PA(0) : NMB_PA(i); 1044229939Sluigi return ret; 1045229939Sluigi} 1046229939Sluigi 1047259412Sluigi/* Generic version of NMB, which uses device-specific memory. */ 1048259412Sluigistatic inline void * 1049259412SluigiBDG_NMB(struct netmap_adapter *na, struct netmap_slot *slot) 1050259412Sluigi{ 1051259412Sluigi struct lut_entry *lut = na->na_lut; 1052259412Sluigi uint32_t i = slot->buf_idx; 1053259412Sluigi return (unlikely(i >= na->na_lut_objtotal)) ? 1054259412Sluigi lut[0].vaddr : lut[i].vaddr; 1055259412Sluigi} 1056259412Sluigi 1057231594Sluigi/* default functions to handle rx/tx interrupts */ 1058257529Sluigiint netmap_rx_irq(struct ifnet *, u_int, u_int *); 1059231594Sluigi#define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL) 1060259412Sluigivoid netmap_common_irq(struct ifnet *, u_int, u_int *work_done); 1061238812Sluigi 1062257529Sluigi 1063259412Sluigivoid netmap_txsync_to_host(struct netmap_adapter *na); 1064257529Sluigivoid netmap_disable_all_rings(struct ifnet *); 1065257529Sluigivoid netmap_enable_all_rings(struct ifnet *); 1066259412Sluigivoid netmap_disable_ring(struct netmap_kring *kr); 1067257529Sluigi 1068259412Sluigi 1069259412Sluigi/* Structure associated to each thread which registered an interface. 1070259412Sluigi * 1071259412Sluigi * The first 4 fields of this structure are written by NIOCREGIF and 1072259412Sluigi * read by poll() and NIOC?XSYNC. 1073259412Sluigi * There is low contention among writers (actually, a correct user program 1074259412Sluigi * should have no contention among writers) and among writers and readers, 1075259412Sluigi * so we use a single global lock to protect the structure initialization. 1076259412Sluigi * Since initialization involves the allocation of memory, we reuse the memory 1077259412Sluigi * allocator lock. 1078259412Sluigi * Read access to the structure is lock free. Readers must check that 1079259412Sluigi * np_nifp is not NULL before using the other fields. 1080259412Sluigi * If np_nifp is NULL initialization has not been performed, so they should 1081259412Sluigi * return an error to userlevel. 1082259412Sluigi * 1083259412Sluigi * The ref_done field is used to regulate access to the refcount in the 1084259412Sluigi * memory allocator. The refcount must be incremented at most once for 1085259412Sluigi * each open("/dev/netmap"). The increment is performed by the first 1086259412Sluigi * function that calls netmap_get_memory() (currently called by 1087259412Sluigi * mmap(), NIOCGINFO and NIOCREGIF). 1088259412Sluigi * If the refcount is incremented, it is then decremented when the 1089259412Sluigi * private structure is destroyed. 1090259412Sluigi */ 1091259412Sluigistruct netmap_priv_d { 1092259412Sluigi struct netmap_if * volatile np_nifp; /* netmap if descriptor. */ 1093259412Sluigi 1094259412Sluigi struct netmap_adapter *np_na; 1095259412Sluigi int np_ringid; /* from the ioctl */ 1096259412Sluigi u_int np_qfirst, np_qlast; /* range of rings to scan */ 1097259412Sluigi uint16_t np_txpoll; 1098259412Sluigi 1099259412Sluigi struct netmap_mem_d *np_mref; /* use with NMG_LOCK held */ 1100259412Sluigi /* np_refcount is only used on FreeBSD */ 1101259412Sluigi int np_refcount; /* use with NMG_LOCK held */ 1102259412Sluigi}; 1103259412Sluigi 1104259412Sluigi 1105259412Sluigi/* 1106259412Sluigi * generic netmap emulation for devices that do not have 1107259412Sluigi * native netmap support. 1108259412Sluigi * XXX generic_netmap_register() is only exported to implement 1109259412Sluigi * nma_is_generic(). 1110259412Sluigi */ 1111259412Sluigiint generic_netmap_register(struct netmap_adapter *na, int enable); 1112259412Sluigiint generic_netmap_attach(struct ifnet *ifp); 1113259412Sluigi 1114259412Sluigiint netmap_catch_rx(struct netmap_adapter *na, int intercept); 1115259412Sluigivoid generic_rx_handler(struct ifnet *ifp, struct mbuf *m);; 1116259412Sluigivoid netmap_catch_packet_steering(struct netmap_generic_adapter *na, int enable); 1117259412Sluigiint generic_xmit_frame(struct ifnet *ifp, struct mbuf *m, void *addr, u_int len, u_int ring_nr); 1118259412Sluigiint generic_find_num_desc(struct ifnet *ifp, u_int *tx, u_int *rx); 1119259412Sluigivoid generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq); 1120259412Sluigi 1121259412Sluigistatic __inline int 1122259412Sluiginma_is_generic(struct netmap_adapter *na) 1123259412Sluigi{ 1124259412Sluigi return na->nm_register == generic_netmap_register; 1125259412Sluigi} 1126259412Sluigi 1127259412Sluigi/* 1128259412Sluigi * netmap_mitigation API. This is used by the generic adapter 1129259412Sluigi * to reduce the number of interrupt requests/selwakeup 1130259412Sluigi * to clients on incoming packets. 1131259412Sluigi */ 1132259412Sluigivoid netmap_mitigation_init(struct netmap_generic_adapter *na); 1133259412Sluigivoid netmap_mitigation_start(struct netmap_generic_adapter *na); 1134259412Sluigivoid netmap_mitigation_restart(struct netmap_generic_adapter *na); 1135259412Sluigiint netmap_mitigation_active(struct netmap_generic_adapter *na); 1136259412Sluigivoid netmap_mitigation_cleanup(struct netmap_generic_adapter *na); 1137259412Sluigi 1138259412Sluigi// int generic_timer_handler(struct hrtimer *t); 1139259412Sluigi 1140227614Sluigi#endif /* _NET_NETMAP_KERN_H_ */ 1141