netmap_kern.h revision 344509
1331722Seadler/* 2341477Svmaffione * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo 3341477Svmaffione * Copyright (C) 2013-2016 Universita` di Pisa 4341477Svmaffione * All rights reserved. 5241719Sluigi * 6227614Sluigi * Redistribution and use in source and binary forms, with or without 7227614Sluigi * modification, are permitted provided that the following conditions 8227614Sluigi * are met: 9228276Sluigi * 1. Redistributions of source code must retain the above copyright 10228276Sluigi * notice, this list of conditions and the following disclaimer. 11228276Sluigi * 2. Redistributions in binary form must reproduce the above copyright 12228276Sluigi * notice, this list of conditions and the following disclaimer in the 13227614Sluigi * documentation and/or other materials provided with the distribution. 14241719Sluigi * 15227614Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16227614Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17227614Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18227614Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19227614Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20227614Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21227614Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22227614Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23227614Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24227614Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25227614Sluigi * SUCH DAMAGE. 26227614Sluigi */ 27227614Sluigi 28227614Sluigi/* 29227614Sluigi * $FreeBSD: stable/11/sys/dev/netmap/netmap_kern.h 344509 2019-02-25 09:13:33Z vmaffione $ 30227614Sluigi * 31227614Sluigi * The header contains the definitions of constants and function 32227614Sluigi * prototypes used only in kernelspace. 33227614Sluigi */ 34227614Sluigi 35227614Sluigi#ifndef _NET_NETMAP_KERN_H_ 36227614Sluigi#define _NET_NETMAP_KERN_H_ 37227614Sluigi 38285349Sluigi#if defined(linux) 39285349Sluigi 40341477Svmaffione#if defined(CONFIG_NETMAP_EXTMEM) 41341477Svmaffione#define WITH_EXTMEM 42341477Svmaffione#endif 43285349Sluigi#if defined(CONFIG_NETMAP_VALE) 44285349Sluigi#define WITH_VALE 45285349Sluigi#endif 46285349Sluigi#if defined(CONFIG_NETMAP_PIPE) 47285349Sluigi#define WITH_PIPES 48285349Sluigi#endif 49285349Sluigi#if defined(CONFIG_NETMAP_MONITOR) 50285349Sluigi#define WITH_MONITOR 51285349Sluigi#endif 52285349Sluigi#if defined(CONFIG_NETMAP_GENERIC) 53285349Sluigi#define WITH_GENERIC 54285349Sluigi#endif 55342033Svmaffione#if defined(CONFIG_NETMAP_PTNETMAP) 56342033Svmaffione#define WITH_PTNETMAP 57285349Sluigi#endif 58341477Svmaffione#if defined(CONFIG_NETMAP_SINK) 59341477Svmaffione#define WITH_SINK 60341477Svmaffione#endif 61342033Svmaffione#if defined(CONFIG_NETMAP_NULL) 62342033Svmaffione#define WITH_NMNULL 63342033Svmaffione#endif 64285349Sluigi 65341477Svmaffione#elif defined (_WIN32) 66341477Svmaffione#define WITH_VALE // comment out to disable VALE support 67341477Svmaffione#define WITH_PIPES 68341477Svmaffione#define WITH_MONITOR 69341477Svmaffione#define WITH_GENERIC 70342033Svmaffione#define WITH_NMNULL 71285349Sluigi 72341477Svmaffione#else /* neither linux nor windows */ 73259412Sluigi#define WITH_VALE // comment out to disable VALE support 74261909Sluigi#define WITH_PIPES 75270063Sluigi#define WITH_MONITOR 76274362Sluigi#define WITH_GENERIC 77342033Svmaffione#define WITH_PTNETMAP /* ptnetmap guest support */ 78341477Svmaffione#define WITH_EXTMEM 79342033Svmaffione#define WITH_NMNULL 80285349Sluigi#endif 81285349Sluigi 82231594Sluigi#if defined(__FreeBSD__) 83341477Svmaffione#include <sys/selinfo.h> 84250052Sluigi 85257529Sluigi#define likely(x) __builtin_expect((long)!!(x), 1L) 86257529Sluigi#define unlikely(x) __builtin_expect((long)!!(x), 0L) 87341477Svmaffione#define __user 88238812Sluigi 89285349Sluigi#define NM_LOCK_T struct mtx /* low level spinlock, used to protect queues */ 90274362Sluigi 91285349Sluigi#define NM_MTX_T struct sx /* OS-specific mutex (sleepable) */ 92285349Sluigi#define NM_MTX_INIT(m) sx_init(&(m), #m) 93285349Sluigi#define NM_MTX_DESTROY(m) sx_destroy(&(m)) 94285349Sluigi#define NM_MTX_LOCK(m) sx_xlock(&(m)) 95341477Svmaffione#define NM_MTX_SPINLOCK(m) while (!sx_try_xlock(&(m))) ; 96285349Sluigi#define NM_MTX_UNLOCK(m) sx_xunlock(&(m)) 97285349Sluigi#define NM_MTX_ASSERT(m) sx_assert(&(m), SA_XLOCKED) 98259412Sluigi 99274459Sluigi#define NM_SELINFO_T struct nm_selinfo 100341477Svmaffione#define NM_SELRECORD_T struct thread 101231594Sluigi#define MBUF_LEN(m) ((m)->m_pkthdr.len) 102341477Svmaffione#define MBUF_TXQ(m) ((m)->m_pkthdr.flowid) 103341477Svmaffione#define MBUF_TRANSMIT(na, ifp, m) ((na)->if_transmit(ifp, m)) 104341477Svmaffione#define GEN_TX_MBUF_IFP(m) ((m)->m_pkthdr.rcvif) 105250052Sluigi 106341477Svmaffione#define NM_ATOMIC_T volatile int /* required by atomic/bitops.h */ 107259412Sluigi/* atomic operations */ 108259412Sluigi#include <machine/atomic.h> 109259412Sluigi#define NM_ATOMIC_TEST_AND_SET(p) (!atomic_cmpset_acq_int((p), 0, 1)) 110259412Sluigi#define NM_ATOMIC_CLEAR(p) atomic_store_rel_int((p), 0) 111257529Sluigi 112272110Sluigi#if __FreeBSD_version >= 1100030 113272110Sluigi#define WNA(_ifp) (_ifp)->if_netmap 114272110Sluigi#else /* older FreeBSD */ 115272110Sluigi#define WNA(_ifp) (_ifp)->if_pspare[0] 116272110Sluigi#endif /* older FreeBSD */ 117272110Sluigi 118267283Sluigi#if __FreeBSD_version >= 1100005 119267283Sluigistruct netmap_adapter *netmap_getna(if_t ifp); 120267283Sluigi#endif 121259412Sluigi 122270063Sluigi#if __FreeBSD_version >= 1100027 123312783Sloos#define MBUF_REFCNT(m) ((m)->m_ext.ext_count) 124312783Sloos#define SET_MBUF_REFCNT(m, x) (m)->m_ext.ext_count = x 125270063Sluigi#else 126312783Sloos#define MBUF_REFCNT(m) ((m)->m_ext.ref_cnt ? *((m)->m_ext.ref_cnt) : -1) 127270063Sluigi#define SET_MBUF_REFCNT(m, x) *((m)->m_ext.ref_cnt) = x 128270063Sluigi#endif 129270063Sluigi 130341477Svmaffione#define MBUF_QUEUED(m) 1 131259412Sluigi 132274459Sluigistruct nm_selinfo { 133344509Svmaffione /* Support for select(2) and poll(2). */ 134274459Sluigi struct selinfo si; 135344509Svmaffione /* Support for kqueue(9). See comments in netmap_freebsd.c */ 136344509Svmaffione struct taskqueue *ntfytq; 137344509Svmaffione struct task ntfytask; 138274459Sluigi struct mtx m; 139344509Svmaffione char mtxname[32]; 140344509Svmaffione int kqueue_users; 141274459Sluigi}; 142274459Sluigi 143274459Sluigi 144259412Sluigistruct hrtimer { 145341477Svmaffione /* Not used in FreeBSD. */ 146259412Sluigi}; 147341477Svmaffione 148285349Sluigi#define NM_BNS_GET(b) 149285349Sluigi#define NM_BNS_PUT(b) 150259412Sluigi 151232238Sluigi#elif defined (linux) 152250052Sluigi 153250052Sluigi#define NM_LOCK_T safe_spinlock_t // see bsd_glue.h 154231594Sluigi#define NM_SELINFO_T wait_queue_head_t 155231594Sluigi#define MBUF_LEN(m) ((m)->len) 156341477Svmaffione#define MBUF_TRANSMIT(na, ifp, m) \ 157341477Svmaffione ({ \ 158341477Svmaffione /* Avoid infinite recursion with generic. */ \ 159341477Svmaffione m->priority = NM_MAGIC_PRIORITY_TX; \ 160341477Svmaffione (((struct net_device_ops *)(na)->if_transmit)->ndo_start_xmit(m, ifp)); \ 161341477Svmaffione 0; \ 162341477Svmaffione }) 163238812Sluigi 164341477Svmaffione/* See explanation in nm_os_generic_xmit_frame. */ 165341477Svmaffione#define GEN_TX_MBUF_IFP(m) ((struct ifnet *)skb_shinfo(m)->destructor_arg) 166341477Svmaffione 167257529Sluigi#define NM_ATOMIC_T volatile long unsigned int 168257529Sluigi 169285349Sluigi#define NM_MTX_T struct mutex /* OS-specific sleepable lock */ 170285349Sluigi#define NM_MTX_INIT(m) mutex_init(&(m)) 171285349Sluigi#define NM_MTX_DESTROY(m) do { (void)(m); } while (0) 172274362Sluigi#define NM_MTX_LOCK(m) mutex_lock(&(m)) 173274362Sluigi#define NM_MTX_UNLOCK(m) mutex_unlock(&(m)) 174285349Sluigi#define NM_MTX_ASSERT(m) mutex_is_locked(&(m)) 175259412Sluigi 176238812Sluigi#ifndef DEV_NETMAP 177238812Sluigi#define DEV_NETMAP 178257529Sluigi#endif /* DEV_NETMAP */ 179238812Sluigi 180238812Sluigi#elif defined (__APPLE__) 181250052Sluigi 182241719Sluigi#warning apple support is incomplete. 183238812Sluigi#define likely(x) __builtin_expect(!!(x), 1) 184238812Sluigi#define unlikely(x) __builtin_expect(!!(x), 0) 185238812Sluigi#define NM_LOCK_T IOLock * 186238812Sluigi#define NM_SELINFO_T struct selinfo 187238812Sluigi#define MBUF_LEN(m) ((m)->m_pkthdr.len) 188238812Sluigi 189341477Svmaffione#elif defined (_WIN32) 190341477Svmaffione#include "../../../WINDOWS/win_glue.h" 191341477Svmaffione 192341477Svmaffione#define NM_SELRECORD_T IO_STACK_LOCATION 193341477Svmaffione#define NM_SELINFO_T win_SELINFO // see win_glue.h 194341477Svmaffione#define NM_LOCK_T win_spinlock_t // see win_glue.h 195341477Svmaffione#define NM_MTX_T KGUARDED_MUTEX /* OS-specific mutex (sleepable) */ 196341477Svmaffione 197341477Svmaffione#define NM_MTX_INIT(m) KeInitializeGuardedMutex(&m); 198341477Svmaffione#define NM_MTX_DESTROY(m) do { (void)(m); } while (0) 199341477Svmaffione#define NM_MTX_LOCK(m) KeAcquireGuardedMutex(&(m)) 200341477Svmaffione#define NM_MTX_UNLOCK(m) KeReleaseGuardedMutex(&(m)) 201341477Svmaffione#define NM_MTX_ASSERT(m) assert(&m.Count>0) 202341477Svmaffione 203341477Svmaffione//These linknames are for the NDIS driver 204341477Svmaffione#define NETMAP_NDIS_LINKNAME_STRING L"\\DosDevices\\NMAPNDIS" 205341477Svmaffione#define NETMAP_NDIS_NTDEVICE_STRING L"\\Device\\NMAPNDIS" 206341477Svmaffione 207341477Svmaffione//Definition of internal driver-to-driver ioctl codes 208341477Svmaffione#define NETMAP_KERNEL_XCHANGE_POINTERS _IO('i', 180) 209341477Svmaffione#define NETMAP_KERNEL_SEND_SHUTDOWN_SIGNAL _IO_direct('i', 195) 210341477Svmaffione 211341477Svmaffionetypedef struct hrtimer{ 212341477Svmaffione KTIMER timer; 213341477Svmaffione BOOLEAN active; 214341477Svmaffione KDPC deferred_proc; 215341477Svmaffione}; 216341477Svmaffione 217341477Svmaffione/* MSVC does not have likely/unlikely support */ 218341477Svmaffione#ifdef _MSC_VER 219341477Svmaffione#define likely(x) (x) 220341477Svmaffione#define unlikely(x) (x) 221238812Sluigi#else 222341477Svmaffione#define likely(x) __builtin_expect((long)!!(x), 1L) 223341477Svmaffione#define unlikely(x) __builtin_expect((long)!!(x), 0L) 224341477Svmaffione#endif //_MSC_VER 225250052Sluigi 226341477Svmaffione#else 227341477Svmaffione 228231594Sluigi#error unsupported platform 229231594Sluigi 230250052Sluigi#endif /* end - platform-specific code */ 231250052Sluigi 232341477Svmaffione#ifndef _WIN32 /* support for emulated sysctl */ 233341477Svmaffione#define SYSBEGIN(x) 234341477Svmaffione#define SYSEND 235341477Svmaffione#endif /* _WIN32 */ 236341477Svmaffione 237341477Svmaffione#define NM_ACCESS_ONCE(x) (*(volatile __typeof__(x) *)&(x)) 238341477Svmaffione 239285349Sluigi#define NMG_LOCK_T NM_MTX_T 240285349Sluigi#define NMG_LOCK_INIT() NM_MTX_INIT(netmap_global_lock) 241285349Sluigi#define NMG_LOCK_DESTROY() NM_MTX_DESTROY(netmap_global_lock) 242285349Sluigi#define NMG_LOCK() NM_MTX_LOCK(netmap_global_lock) 243285349Sluigi#define NMG_UNLOCK() NM_MTX_UNLOCK(netmap_global_lock) 244285349Sluigi#define NMG_LOCK_ASSERT() NM_MTX_ASSERT(netmap_global_lock) 245285349Sluigi 246341477Svmaffione#if defined(__FreeBSD__) 247342033Svmaffione#define nm_prerr_int printf 248342033Svmaffione#define nm_prinf_int printf 249341477Svmaffione#elif defined (_WIN32) 250342033Svmaffione#define nm_prerr_int DbgPrint 251342033Svmaffione#define nm_prinf_int DbgPrint 252341477Svmaffione#elif defined(linux) 253342033Svmaffione#define nm_prerr_int(fmt, arg...) printk(KERN_ERR fmt, ##arg) 254342033Svmaffione#define nm_prinf_int(fmt, arg...) printk(KERN_INFO fmt, ##arg) 255341477Svmaffione#endif 256341477Svmaffione 257342033Svmaffione#define nm_prinf(format, ...) \ 258230058Sluigi do { \ 259230058Sluigi struct timeval __xxts; \ 260227614Sluigi microtime(&__xxts); \ 261342033Svmaffione nm_prinf_int("%03d.%06d [%4d] %-25s " format "\n",\ 262230058Sluigi (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \ 263260368Sluigi __LINE__, __FUNCTION__, ##__VA_ARGS__); \ 264227614Sluigi } while (0) 265241719Sluigi 266342033Svmaffione#define nm_prerr(format, ...) \ 267241719Sluigi do { \ 268342033Svmaffione struct timeval __xxts; \ 269342033Svmaffione microtime(&__xxts); \ 270342033Svmaffione nm_prerr_int("%03d.%06d [%4d] %-25s " format "\n",\ 271342033Svmaffione (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \ 272342033Svmaffione __LINE__, __FUNCTION__, ##__VA_ARGS__); \ 273342033Svmaffione } while (0) 274342033Svmaffione 275344047Svmaffione/* Disabled printf (used to be nm_prdis). */ 276342033Svmaffione#define nm_prdis(format, ...) 277342033Svmaffione 278342033Svmaffione/* Rate limited, lps indicates how many per second. */ 279342033Svmaffione#define nm_prlim(lps, format, ...) \ 280342033Svmaffione do { \ 281241719Sluigi static int t0, __cnt; \ 282241719Sluigi if (t0 != time_second) { \ 283241719Sluigi t0 = time_second; \ 284241719Sluigi __cnt = 0; \ 285241719Sluigi } \ 286241719Sluigi if (__cnt++ < lps) \ 287342033Svmaffione nm_prinf(format, ##__VA_ARGS__); \ 288241719Sluigi } while (0) 289241719Sluigi 290227614Sluigistruct netmap_adapter; 291251139Sluigistruct nm_bdg_fwd; 292251139Sluigistruct nm_bridge; 293251139Sluigistruct netmap_priv_d; 294341477Svmaffionestruct nm_bdg_args; 295227614Sluigi 296341477Svmaffione/* os-specific NM_SELINFO_T initialzation/destruction functions */ 297344509Svmaffioneint nm_os_selinfo_init(NM_SELINFO_T *, const char *name); 298341477Svmaffionevoid nm_os_selinfo_uninit(NM_SELINFO_T *); 299341477Svmaffione 300257529Sluigiconst char *nm_dump_buf(char *p, int len, int lim, char *dst); 301257529Sluigi 302341477Svmaffionevoid nm_os_selwakeup(NM_SELINFO_T *si); 303341477Svmaffionevoid nm_os_selrecord(NM_SELRECORD_T *sr, NM_SELINFO_T *si); 304341477Svmaffione 305341477Svmaffioneint nm_os_ifnet_init(void); 306341477Svmaffionevoid nm_os_ifnet_fini(void); 307341477Svmaffionevoid nm_os_ifnet_lock(void); 308341477Svmaffionevoid nm_os_ifnet_unlock(void); 309341477Svmaffione 310341477Svmaffioneunsigned nm_os_ifnet_mtu(struct ifnet *ifp); 311341477Svmaffione 312341477Svmaffionevoid nm_os_get_module(void); 313341477Svmaffionevoid nm_os_put_module(void); 314341477Svmaffione 315341477Svmaffionevoid netmap_make_zombie(struct ifnet *); 316341477Svmaffionevoid netmap_undo_zombie(struct ifnet *); 317341477Svmaffione 318341477Svmaffione/* os independent alloc/realloc/free */ 319341477Svmaffionevoid *nm_os_malloc(size_t); 320341477Svmaffionevoid *nm_os_vmalloc(size_t); 321341477Svmaffionevoid *nm_os_realloc(void *, size_t new_size, size_t old_size); 322341477Svmaffionevoid nm_os_free(void *); 323341477Svmaffionevoid nm_os_vfree(void *); 324341477Svmaffione 325341477Svmaffione/* os specific attach/detach enter/exit-netmap-mode routines */ 326341477Svmaffionevoid nm_os_onattach(struct ifnet *); 327341477Svmaffionevoid nm_os_ondetach(struct ifnet *); 328341477Svmaffionevoid nm_os_onenter(struct ifnet *); 329341477Svmaffionevoid nm_os_onexit(struct ifnet *); 330341477Svmaffione 331341477Svmaffione/* passes a packet up to the host stack. 332341477Svmaffione * If the packet is sent (or dropped) immediately it returns NULL, 333341477Svmaffione * otherwise it links the packet to prev and returns m. 334341477Svmaffione * In this case, a final call with m=NULL and prev != NULL will send up 335341477Svmaffione * the entire chain to the host stack. 336341477Svmaffione */ 337341477Svmaffionevoid *nm_os_send_up(struct ifnet *, struct mbuf *m, struct mbuf *prev); 338341477Svmaffione 339341477Svmaffioneint nm_os_mbuf_has_seg_offld(struct mbuf *m); 340341477Svmaffioneint nm_os_mbuf_has_csum_offld(struct mbuf *m); 341341477Svmaffione 342259412Sluigi#include "netmap_mbq.h" 343259412Sluigi 344259412Sluigiextern NMG_LOCK_T netmap_global_lock; 345259412Sluigi 346285349Sluigienum txrx { NR_RX = 0, NR_TX = 1, NR_TXRX }; 347285349Sluigi 348285349Sluigistatic __inline const char* 349285349Sluiginm_txrx2str(enum txrx t) 350285349Sluigi{ 351285349Sluigi return (t== NR_RX ? "RX" : "TX"); 352285349Sluigi} 353285349Sluigi 354285349Sluigistatic __inline enum txrx 355285349Sluiginm_txrx_swap(enum txrx t) 356285349Sluigi{ 357285349Sluigi return (t== NR_RX ? NR_TX : NR_RX); 358285349Sluigi} 359285349Sluigi 360285349Sluigi#define for_rx_tx(t) for ((t) = 0; (t) < NR_TXRX; (t)++) 361285349Sluigi 362341477Svmaffione#ifdef WITH_MONITOR 363341477Svmaffionestruct netmap_zmon_list { 364341477Svmaffione struct netmap_kring *next; 365341477Svmaffione struct netmap_kring *prev; 366341477Svmaffione}; 367341477Svmaffione#endif /* WITH_MONITOR */ 368285349Sluigi 369227614Sluigi/* 370232238Sluigi * private, kernel view of a ring. Keeps track of the status of 371232238Sluigi * a ring across system calls. 372227614Sluigi * 373232238Sluigi * nr_hwcur index of the next buffer to refill. 374260368Sluigi * It corresponds to ring->head 375260368Sluigi * at the time the system call returns. 376232238Sluigi * 377260368Sluigi * nr_hwtail index of the first buffer owned by the kernel. 378260368Sluigi * On RX, hwcur->hwtail are receive buffers 379260368Sluigi * not yet released. hwcur is advanced following 380260368Sluigi * ring->head, hwtail is advanced on incoming packets, 381260368Sluigi * and a wakeup is generated when hwtail passes ring->cur 382260368Sluigi * On TX, hwcur->rcur have been filled by the sender 383260368Sluigi * but not sent yet to the NIC; rcur->hwtail are available 384260368Sluigi * for new transmissions, and hwtail->hwcur-1 are pending 385260368Sluigi * transmissions not yet acknowledged. 386232238Sluigi * 387231594Sluigi * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots. 388227614Sluigi * This is so that, on a reset, buffers owned by userspace are not 389227614Sluigi * modified by the kernel. In particular: 390260368Sluigi * RX rings: the next empty buffer (hwtail + hwofs) coincides with 391227614Sluigi * the next empty buffer as known by the hardware (next_to_check or so). 392227614Sluigi * TX rings: hwcur + hwofs coincides with next_to_send 393245579Sluigi * 394257529Sluigi * The following fields are used to implement lock-free copy of packets 395257529Sluigi * from input to output ports in VALE switch: 396257529Sluigi * nkr_hwlease buffer after the last one being copied. 397257529Sluigi * A writer in nm_bdg_flush reserves N buffers 398257529Sluigi * from nr_hwlease, advances it, then does the 399257529Sluigi * copy outside the lock. 400257529Sluigi * In RX rings (used for VALE ports), 401260368Sluigi * nkr_hwtail <= nkr_hwlease < nkr_hwcur+N-1 402257529Sluigi * In TX rings (used for NIC or host stack ports) 403260368Sluigi * nkr_hwcur <= nkr_hwlease < nkr_hwtail 404257529Sluigi * nkr_leases array of nkr_num_slots where writers can report 405257529Sluigi * completion of their block. NR_NOSLOT (~0) indicates 406257529Sluigi * that the writer has not finished yet 407259412Sluigi * nkr_lease_idx index of next free slot in nr_leases, to be assigned 408257529Sluigi * 409257529Sluigi * The kring is manipulated by txsync/rxsync and generic netmap function. 410260368Sluigi * 411260368Sluigi * Concurrent rxsync or txsync on the same ring are prevented through 412267165Sluigi * by nm_kr_(try)lock() which in turn uses nr_busy. This is all we need 413260368Sluigi * for NIC rings, and for TX rings attached to the host stack. 414260368Sluigi * 415260368Sluigi * RX rings attached to the host stack use an mbq (rx_queue) on both 416260368Sluigi * rxsync_from_host() and netmap_transmit(). The mbq is protected 417260368Sluigi * by its internal lock. 418260368Sluigi * 419270063Sluigi * RX rings attached to the VALE switch are accessed by both senders 420260368Sluigi * and receiver. They are protected through the q_lock on the RX ring. 421227614Sluigi */ 422227614Sluigistruct netmap_kring { 423260368Sluigi struct netmap_ring *ring; 424227614Sluigi 425341477Svmaffione uint32_t nr_hwcur; /* should be nr_hwhead */ 426260368Sluigi uint32_t nr_hwtail; 427260368Sluigi 428260368Sluigi /* 429260368Sluigi * Copies of values in user rings, so we do not need to look 430260368Sluigi * at the ring (which could be modified). These are set in the 431260368Sluigi * *sync_prologue()/finalize() routines. 432260368Sluigi */ 433260368Sluigi uint32_t rhead; 434260368Sluigi uint32_t rcur; 435260368Sluigi uint32_t rtail; 436260368Sluigi 437260368Sluigi uint32_t nr_kflags; /* private driver flags */ 438260368Sluigi#define NKR_PENDINTR 0x1 // Pending interrupt. 439285349Sluigi#define NKR_EXCLUSIVE 0x2 /* exclusive binding */ 440341477Svmaffione#define NKR_FORWARD 0x4 /* (host ring only) there are 441341477Svmaffione packets to forward 442341477Svmaffione */ 443341477Svmaffione#define NKR_NEEDRING 0x8 /* ring needed even if users==0 444341477Svmaffione * (used internally by pipes and 445341477Svmaffione * by ptnetmap host ports) 446341477Svmaffione */ 447341477Svmaffione#define NKR_NOINTR 0x10 /* don't use interrupts on this ring */ 448341477Svmaffione#define NKR_FAKERING 0x20 /* don't allocate/free buffers */ 449341477Svmaffione 450341477Svmaffione uint32_t nr_mode; 451341477Svmaffione uint32_t nr_pending_mode; 452341477Svmaffione#define NKR_NETMAP_OFF 0x0 453341477Svmaffione#define NKR_NETMAP_ON 0x1 454341477Svmaffione 455260368Sluigi uint32_t nkr_num_slots; 456260368Sluigi 457260368Sluigi /* 458260368Sluigi * On a NIC reset, the NIC ring indexes may be reset but the 459260368Sluigi * indexes in the netmap rings remain the same. nkr_hwofs 460260368Sluigi * keeps track of the offset between the two. 461260368Sluigi */ 462260368Sluigi int32_t nkr_hwofs; 463260368Sluigi 464260368Sluigi /* last_reclaim is opaque marker to help reduce the frequency 465260368Sluigi * of operations such as reclaiming tx buffers. A possible use 466260368Sluigi * is set it to ticks and do the reclaim only once per tick. 467260368Sluigi */ 468260368Sluigi uint64_t last_reclaim; 469260368Sluigi 470260368Sluigi 471260368Sluigi NM_SELINFO_T si; /* poll/select wait queue */ 472260368Sluigi NM_LOCK_T q_lock; /* protects kring and ring. */ 473260368Sluigi NM_ATOMIC_T nr_busy; /* prevent concurrent syscalls */ 474260368Sluigi 475341477Svmaffione /* the adapter the owns this kring */ 476231594Sluigi struct netmap_adapter *na; 477260368Sluigi 478341477Svmaffione /* the adapter that wants to be notified when this kring has 479341477Svmaffione * new slots avaialable. This is usually the same as the above, 480341477Svmaffione * but wrappers may let it point to themselves 481341477Svmaffione */ 482341477Svmaffione struct netmap_adapter *notify_na; 483341477Svmaffione 484274355Sluigi /* The following fields are for VALE switch support */ 485251139Sluigi struct nm_bdg_fwd *nkr_ft; 486260368Sluigi uint32_t *nkr_leases; 487260368Sluigi#define NR_NOSLOT ((uint32_t)~0) /* used in nkr_*lease* */ 488260368Sluigi uint32_t nkr_hwlease; 489260368Sluigi uint32_t nkr_lease_idx; 490257529Sluigi 491270063Sluigi /* while nkr_stopped is set, no new [tr]xsync operations can 492270063Sluigi * be started on this kring. 493270063Sluigi * This is used by netmap_disable_all_rings() 494270063Sluigi * to find a synchronization point where critical data 495270063Sluigi * structures pointed to by the kring can be added or removed 496270063Sluigi */ 497270063Sluigi volatile int nkr_stopped; 498257529Sluigi 499261909Sluigi /* Support for adapters without native netmap support. 500259412Sluigi * On tx rings we preallocate an array of tx buffers 501259412Sluigi * (same size as the netmap ring), on rx rings we 502261909Sluigi * store incoming mbufs in a queue that is drained by 503261909Sluigi * a rxsync. 504259412Sluigi */ 505341477Svmaffione struct mbuf **tx_pool; 506341477Svmaffione struct mbuf *tx_event; /* TX event used as a notification */ 507341477Svmaffione NM_LOCK_T tx_event_lock; /* protects the tx_event mbuf */ 508341477Svmaffione struct mbq rx_queue; /* intercepted rx mbufs. */ 509259412Sluigi 510285349Sluigi uint32_t users; /* existing bindings for this ring */ 511285349Sluigi 512341477Svmaffione uint32_t ring_id; /* kring identifier */ 513285349Sluigi enum txrx tx; /* kind of ring (tx or rx) */ 514260368Sluigi char name[64]; /* diagnostic */ 515260368Sluigi 516270063Sluigi /* [tx]sync callback for this kring. 517270063Sluigi * The default nm_kring_create callback (netmap_krings_create) 518270063Sluigi * sets the nm_sync callback of each hardware tx(rx) kring to 519270063Sluigi * the corresponding nm_txsync(nm_rxsync) taken from the 520270063Sluigi * netmap_adapter; moreover, it sets the sync callback 521270063Sluigi * of the host tx(rx) ring to netmap_txsync_to_host 522270063Sluigi * (netmap_rxsync_from_host). 523270063Sluigi * 524270063Sluigi * Overrides: the above configuration is not changed by 525270063Sluigi * any of the nm_krings_create callbacks. 526270063Sluigi */ 527261909Sluigi int (*nm_sync)(struct netmap_kring *kring, int flags); 528285349Sluigi int (*nm_notify)(struct netmap_kring *kring, int flags); 529261909Sluigi 530261909Sluigi#ifdef WITH_PIPES 531270063Sluigi struct netmap_kring *pipe; /* if this is a pipe ring, 532270063Sluigi * pointer to the other end 533270063Sluigi */ 534341477Svmaffione uint32_t pipe_tail; /* hwtail updated by the other end */ 535261909Sluigi#endif /* WITH_PIPES */ 536261909Sluigi 537285349Sluigi int (*save_notify)(struct netmap_kring *kring, int flags); 538285349Sluigi 539270063Sluigi#ifdef WITH_MONITOR 540285349Sluigi /* array of krings that are monitoring this kring */ 541285349Sluigi struct netmap_kring **monitors; 542285349Sluigi uint32_t max_monitors; /* current size of the monitors array */ 543285349Sluigi uint32_t n_monitors; /* next unused entry in the monitor array */ 544341477Svmaffione uint32_t mon_pos[NR_TXRX]; /* index of this ring in the monitored ring array */ 545341477Svmaffione uint32_t mon_tail; /* last seen slot on rx */ 546341477Svmaffione 547341477Svmaffione /* circular list of zero-copy monitors */ 548341477Svmaffione struct netmap_zmon_list zmon_list[NR_TXRX]; 549341477Svmaffione 550270063Sluigi /* 551285349Sluigi * Monitors work by intercepting the sync and notify callbacks of the 552285349Sluigi * monitored krings. This is implemented by replacing the pointers 553285349Sluigi * above and saving the previous ones in mon_* pointers below 554270063Sluigi */ 555285349Sluigi int (*mon_sync)(struct netmap_kring *kring, int flags); 556285349Sluigi int (*mon_notify)(struct netmap_kring *kring, int flags); 557285349Sluigi 558270063Sluigi#endif 559341477Svmaffione} 560341477Svmaffione#ifdef _WIN32 561341477Svmaffione__declspec(align(64)); 562341477Svmaffione#else 563341477Svmaffione__attribute__((__aligned__(64))); 564341477Svmaffione#endif 565227614Sluigi 566341477Svmaffione/* return 1 iff the kring needs to be turned on */ 567341477Svmaffionestatic inline int 568341477Svmaffionenm_kring_pending_on(struct netmap_kring *kring) 569341477Svmaffione{ 570341477Svmaffione return kring->nr_pending_mode == NKR_NETMAP_ON && 571341477Svmaffione kring->nr_mode == NKR_NETMAP_OFF; 572341477Svmaffione} 573257529Sluigi 574341477Svmaffione/* return 1 iff the kring needs to be turned off */ 575341477Svmaffionestatic inline int 576341477Svmaffionenm_kring_pending_off(struct netmap_kring *kring) 577341477Svmaffione{ 578341477Svmaffione return kring->nr_pending_mode == NKR_NETMAP_OFF && 579341477Svmaffione kring->nr_mode == NKR_NETMAP_ON; 580341477Svmaffione} 581341477Svmaffione 582257529Sluigi/* return the next index, with wraparound */ 583257529Sluigistatic inline uint32_t 584257529Sluiginm_next(uint32_t i, uint32_t lim) 585257529Sluigi{ 586257529Sluigi return unlikely (i == lim) ? 0 : i + 1; 587257529Sluigi} 588257529Sluigi 589260368Sluigi 590260368Sluigi/* return the previous index, with wraparound */ 591260368Sluigistatic inline uint32_t 592260368Sluiginm_prev(uint32_t i, uint32_t lim) 593260368Sluigi{ 594260368Sluigi return unlikely (i == 0) ? lim : i - 1; 595260368Sluigi} 596260368Sluigi 597260368Sluigi 598227614Sluigi/* 599257529Sluigi * 600257529Sluigi * Here is the layout for the Rx and Tx rings. 601257529Sluigi 602257529Sluigi RxRING TxRING 603257529Sluigi 604257529Sluigi +-----------------+ +-----------------+ 605257529Sluigi | | | | 606341477Svmaffione | free | | free | 607257529Sluigi +-----------------+ +-----------------+ 608260368Sluigihead->| owned by user |<-hwcur | not sent to nic |<-hwcur 609260368Sluigi | | | yet | 610260368Sluigi +-----------------+ | | 611260368Sluigi cur->| available to | | | 612260368Sluigi | user, not read | +-----------------+ 613260368Sluigi | yet | cur->| (being | 614260368Sluigi | | | prepared) | 615260368Sluigi | | | | 616260368Sluigi +-----------------+ + ------ + 617260368Sluigitail->| |<-hwtail | |<-hwlease 618260368Sluigi | (being | ... | | ... 619260368Sluigi | prepared) | ... | | ... 620260368Sluigi +-----------------+ ... | | ... 621257529Sluigi | |<-hwlease +-----------------+ 622260368Sluigi | | tail->| |<-hwtail 623257529Sluigi | | | | 624257529Sluigi | | | | 625257529Sluigi | | | | 626257529Sluigi +-----------------+ +-----------------+ 627257529Sluigi 628260368Sluigi * The cur/tail (user view) and hwcur/hwtail (kernel view) 629257529Sluigi * are used in the normal operation of the card. 630257529Sluigi * 631257529Sluigi * When a ring is the output of a switch port (Rx ring for 632257529Sluigi * a VALE port, Tx ring for the host stack or NIC), slots 633257529Sluigi * are reserved in blocks through 'hwlease' which points 634257529Sluigi * to the next unused slot. 635260368Sluigi * On an Rx ring, hwlease is always after hwtail, 636260368Sluigi * and completions cause hwtail to advance. 637260368Sluigi * On a Tx ring, hwlease is always between cur and hwtail, 638257529Sluigi * and completions cause cur to advance. 639257529Sluigi * 640257529Sluigi * nm_kr_space() returns the maximum number of slots that 641257529Sluigi * can be assigned. 642257529Sluigi * nm_kr_lease() reserves the required number of buffers, 643257529Sluigi * advances nkr_hwlease and also returns an entry in 644257529Sluigi * a circular array where completions should be reported. 645257529Sluigi */ 646257529Sluigi 647341477Svmaffionestruct lut_entry; 648341477Svmaffione#ifdef __FreeBSD__ 649341477Svmaffione#define plut_entry lut_entry 650341477Svmaffione#endif 651257529Sluigi 652285349Sluigistruct netmap_lut { 653285349Sluigi struct lut_entry *lut; 654341477Svmaffione struct plut_entry *plut; 655285349Sluigi uint32_t objtotal; /* max buffer index */ 656285349Sluigi uint32_t objsize; /* buffer size */ 657285349Sluigi}; 658257529Sluigi 659270063Sluigistruct netmap_vp_adapter; // forward 660341477Svmaffionestruct nm_bridge; 661270063Sluigi 662341477Svmaffione/* Struct to be filled by nm_config callbacks. */ 663341477Svmaffionestruct nm_config_info { 664341477Svmaffione unsigned num_tx_rings; 665341477Svmaffione unsigned num_rx_rings; 666341477Svmaffione unsigned num_tx_descs; 667341477Svmaffione unsigned num_rx_descs; 668341477Svmaffione unsigned rx_buf_maxsize; 669341477Svmaffione}; 670341477Svmaffione 671257529Sluigi/* 672341477Svmaffione * default type for the magic field. 673341477Svmaffione * May be overriden in glue code. 674341477Svmaffione */ 675341477Svmaffione#ifndef NM_OS_MAGIC 676341477Svmaffione#define NM_OS_MAGIC uint32_t 677341477Svmaffione#endif /* !NM_OS_MAGIC */ 678341477Svmaffione 679341477Svmaffione/* 680259412Sluigi * The "struct netmap_adapter" extends the "struct adapter" 681259412Sluigi * (or equivalent) device descriptor. 682259412Sluigi * It contains all base fields needed to support netmap operation. 683259412Sluigi * There are in fact different types of netmap adapters 684259412Sluigi * (native, generic, VALE switch...) so a netmap_adapter is 685259412Sluigi * just the first field in the derived type. 686227614Sluigi */ 687227614Sluigistruct netmap_adapter { 688241719Sluigi /* 689241719Sluigi * On linux we do not have a good way to tell if an interface 690259412Sluigi * is netmap-capable. So we always use the following trick: 691241719Sluigi * NA(ifp) points here, and the first entry (which hopefully 692241719Sluigi * always exists and is at least 32 bits) contains a magic 693241719Sluigi * value which we can use to detect that the interface is good. 694241719Sluigi */ 695341477Svmaffione NM_OS_MAGIC magic; 696259412Sluigi uint32_t na_flags; /* enabled, and other flags */ 697241719Sluigi#define NAF_SKIP_INTR 1 /* use the regular interrupt handler. 698241719Sluigi * useful during initialization 699241719Sluigi */ 700251139Sluigi#define NAF_SW_ONLY 2 /* forward packets only to sw adapter */ 701257529Sluigi#define NAF_BDG_MAYSLEEP 4 /* the bridge is allowed to sleep when 702257529Sluigi * forwarding packets coming from this 703257529Sluigi * interface 704257529Sluigi */ 705285349Sluigi#define NAF_MEM_OWNER 8 /* the adapter uses its own memory area 706285349Sluigi * that cannot be changed 707257529Sluigi */ 708285349Sluigi#define NAF_NATIVE 16 /* the adapter is native. 709285695Sluigi * Virtual ports (non persistent vale ports, 710285695Sluigi * pipes, monitors...) should never use 711285695Sluigi * this flag. 712259412Sluigi */ 713259412Sluigi#define NAF_NETMAP_ON 32 /* netmap is active (either native or 714270063Sluigi * emulated). Where possible (e.g. FreeBSD) 715259412Sluigi * IFCAP_NETMAP also mirrors this flag. 716259412Sluigi */ 717261909Sluigi#define NAF_HOST_RINGS 64 /* the adapter supports the host rings */ 718270063Sluigi#define NAF_FORCE_NATIVE 128 /* the adapter is always NATIVE */ 719342033Svmaffione/* free */ 720341477Svmaffione#define NAF_MOREFRAG 512 /* the adapter supports NS_MOREFRAG */ 721341477Svmaffione#define NAF_ZOMBIE (1U<<30) /* the nic driver has been unloaded */ 722270063Sluigi#define NAF_BUSY (1U<<31) /* the adapter is used internally and 723270063Sluigi * cannot be registered from userspace 724270063Sluigi */ 725259412Sluigi int active_fds; /* number of user-space descriptors using this 726227614Sluigi interface, which is equal to the number of 727227614Sluigi struct netmap_if objs in the mapped region. */ 728227614Sluigi 729239140Semaste u_int num_rx_rings; /* number of adapter receive rings */ 730239140Semaste u_int num_tx_rings; /* number of adapter transmit rings */ 731341477Svmaffione u_int num_host_rx_rings; /* number of host receive rings */ 732341477Svmaffione u_int num_host_tx_rings; /* number of host transmit rings */ 733227614Sluigi 734285349Sluigi u_int num_tx_desc; /* number of descriptor in each queue */ 735227614Sluigi u_int num_rx_desc; 736227614Sluigi 737342033Svmaffione /* tx_rings and rx_rings are private but allocated as a 738342033Svmaffione * contiguous chunk of memory. Each array has N+K entries, 739342033Svmaffione * N for the hardware rings and K for the host rings. 740227614Sluigi */ 741341477Svmaffione struct netmap_kring **tx_rings; /* array of TX rings. */ 742341477Svmaffione struct netmap_kring **rx_rings; /* array of RX rings. */ 743260368Sluigi 744259412Sluigi void *tailroom; /* space below the rings array */ 745259412Sluigi /* (used for leases) */ 746227614Sluigi 747259412Sluigi 748285349Sluigi NM_SELINFO_T si[NR_TXRX]; /* global wait queues */ 749232238Sluigi 750261909Sluigi /* count users of the global wait queues */ 751285349Sluigi int si_users[NR_TXRX]; 752261909Sluigi 753270063Sluigi void *pdev; /* used to store pci device */ 754270063Sluigi 755227614Sluigi /* copy of if_qflush and if_transmit pointers, to intercept 756227614Sluigi * packets from the network stack when netmap is active. 757227614Sluigi */ 758227614Sluigi int (*if_transmit)(struct ifnet *, struct mbuf *); 759227614Sluigi 760260368Sluigi /* copy of if_input for netmap_send_up() */ 761260368Sluigi void (*if_input)(struct ifnet *, struct mbuf *); 762260368Sluigi 763341477Svmaffione /* Back reference to the parent ifnet struct. Used for 764341477Svmaffione * hardware ports (emulated netmap included). */ 765227614Sluigi struct ifnet *ifp; /* adapter is ifp->if_softc */ 766227614Sluigi 767260368Sluigi /*---- callbacks for this netmap adapter -----*/ 768260368Sluigi /* 769260368Sluigi * nm_dtor() is the cleanup routine called when destroying 770260368Sluigi * the adapter. 771267165Sluigi * Called with NMG_LOCK held. 772260368Sluigi * 773260368Sluigi * nm_register() is called on NIOCREGIF and close() to enter 774260368Sluigi * or exit netmap mode on the NIC 775270063Sluigi * Called with NNG_LOCK held. 776260368Sluigi * 777260368Sluigi * nm_txsync() pushes packets to the underlying hw/switch 778260368Sluigi * 779260368Sluigi * nm_rxsync() collects packets from the underlying hw/switch 780260368Sluigi * 781260368Sluigi * nm_config() returns configuration information from the OS 782267165Sluigi * Called with NMG_LOCK held. 783260368Sluigi * 784270063Sluigi * nm_krings_create() create and init the tx_rings and 785270063Sluigi * rx_rings arrays of kring structures. In particular, 786270063Sluigi * set the nm_sync callbacks for each ring. 787270063Sluigi * There is no need to also allocate the corresponding 788270063Sluigi * netmap_rings, since netmap_mem_rings_create() will always 789270063Sluigi * be called to provide the missing ones. 790270063Sluigi * Called with NNG_LOCK held. 791260368Sluigi * 792270063Sluigi * nm_krings_delete() cleanup and delete the tx_rings and rx_rings 793270063Sluigi * arrays 794270063Sluigi * Called with NMG_LOCK held. 795260368Sluigi * 796267165Sluigi * nm_notify() is used to act after data have become available 797270063Sluigi * (or the stopped state of the ring has changed) 798260368Sluigi * For hw devices this is typically a selwakeup(), 799260368Sluigi * but for NIC/host ports attached to a switch (or vice-versa) 800260368Sluigi * we also need to invoke the 'txsync' code downstream. 801341477Svmaffione * This callback pointer is actually used only to initialize 802341477Svmaffione * kring->nm_notify. 803341477Svmaffione * Return values are the same as for netmap_rx_irq(). 804260368Sluigi */ 805259412Sluigi void (*nm_dtor)(struct netmap_adapter *); 806231594Sluigi 807259412Sluigi int (*nm_register)(struct netmap_adapter *, int onoff); 808341477Svmaffione void (*nm_intr)(struct netmap_adapter *, int onoff); 809257529Sluigi 810270063Sluigi int (*nm_txsync)(struct netmap_kring *kring, int flags); 811270063Sluigi int (*nm_rxsync)(struct netmap_kring *kring, int flags); 812285349Sluigi int (*nm_notify)(struct netmap_kring *kring, int flags); 813341477Svmaffione#define NAF_FORCE_READ 1 814341477Svmaffione#define NAF_FORCE_RECLAIM 2 815341477Svmaffione#define NAF_CAN_FORWARD_DOWN 4 816245835Sluigi /* return configuration information */ 817341477Svmaffione int (*nm_config)(struct netmap_adapter *, struct nm_config_info *info); 818259412Sluigi int (*nm_krings_create)(struct netmap_adapter *); 819259412Sluigi void (*nm_krings_delete)(struct netmap_adapter *); 820270063Sluigi /* 821270063Sluigi * nm_bdg_attach() initializes the na_vp field to point 822270063Sluigi * to an adapter that can be attached to a VALE switch. If the 823270063Sluigi * current adapter is already a VALE port, na_vp is simply a cast; 824270063Sluigi * otherwise, na_vp points to a netmap_bwrap_adapter. 825270063Sluigi * If applicable, this callback also initializes na_hostvp, 826270063Sluigi * that can be used to connect the adapter host rings to the 827270063Sluigi * switch. 828270063Sluigi * Called with NMG_LOCK held. 829270063Sluigi * 830270063Sluigi * nm_bdg_ctl() is called on the actual attach/detach to/from 831270063Sluigi * to/from the switch, to perform adapter-specific 832270063Sluigi * initializations 833270063Sluigi * Called with NMG_LOCK held. 834270063Sluigi */ 835341477Svmaffione int (*nm_bdg_attach)(const char *bdg_name, struct netmap_adapter *, 836341477Svmaffione struct nm_bridge *); 837341477Svmaffione int (*nm_bdg_ctl)(struct nmreq_header *, struct netmap_adapter *); 838270063Sluigi 839270063Sluigi /* adapter used to attach this adapter to a VALE switch (if any) */ 840270063Sluigi struct netmap_vp_adapter *na_vp; 841270063Sluigi /* adapter used to attach the host rings of this adapter 842270063Sluigi * to a VALE switch (if any) */ 843270063Sluigi struct netmap_vp_adapter *na_hostvp; 844270063Sluigi 845259412Sluigi /* standard refcount to control the lifetime of the adapter 846259412Sluigi * (it should be equal to the lifetime of the corresponding ifp) 847259412Sluigi */ 848259412Sluigi int na_refcount; 849259412Sluigi 850259412Sluigi /* memory allocator (opaque) 851259412Sluigi * We also cache a pointer to the lut_entry for translating 852341477Svmaffione * buffer addresses, the total number of buffers and the buffer size. 853259412Sluigi */ 854259412Sluigi struct netmap_mem_d *nm_mem; 855341477Svmaffione struct netmap_mem_d *nm_mem_prev; 856285349Sluigi struct netmap_lut na_lut; 857259412Sluigi 858270063Sluigi /* additional information attached to this adapter 859270063Sluigi * by other netmap subsystems. Currently used by 860341477Svmaffione * bwrap, LINUX/v1000 and ptnetmap 861259412Sluigi */ 862259412Sluigi void *na_private; 863261909Sluigi 864270063Sluigi /* array of pipes that have this adapter as a parent */ 865261909Sluigi struct netmap_pipe_adapter **na_pipes; 866270063Sluigi int na_next_pipe; /* next free slot in the array */ 867270063Sluigi int na_max_pipes; /* size of the array */ 868270063Sluigi 869341477Svmaffione /* Offset of ethernet header for each packet. */ 870341477Svmaffione u_int virt_hdr_len; 871341477Svmaffione 872341477Svmaffione /* Max number of bytes that the NIC can store in the buffer 873341477Svmaffione * referenced by each RX descriptor. This translates to the maximum 874341477Svmaffione * bytes that a single netmap slot can reference. Larger packets 875341477Svmaffione * require NS_MOREFRAG support. */ 876341477Svmaffione unsigned rx_buf_maxsize; 877341477Svmaffione 878341477Svmaffione char name[NETMAP_REQ_IFNAMSIZ]; /* used at least by pipes */ 879341477Svmaffione 880341477Svmaffione#ifdef WITH_MONITOR 881341477Svmaffione unsigned long monitor_id; /* debugging */ 882341477Svmaffione#endif 883259412Sluigi}; 884259412Sluigi 885285349Sluigistatic __inline u_int 886285349Sluiginma_get_ndesc(struct netmap_adapter *na, enum txrx t) 887285349Sluigi{ 888285349Sluigi return (t == NR_TX ? na->num_tx_desc : na->num_rx_desc); 889285349Sluigi} 890260368Sluigi 891285349Sluigistatic __inline void 892285349Sluiginma_set_ndesc(struct netmap_adapter *na, enum txrx t, u_int v) 893285349Sluigi{ 894285349Sluigi if (t == NR_TX) 895285349Sluigi na->num_tx_desc = v; 896285349Sluigi else 897285349Sluigi na->num_rx_desc = v; 898285349Sluigi} 899285349Sluigi 900285349Sluigistatic __inline u_int 901285349Sluiginma_get_nrings(struct netmap_adapter *na, enum txrx t) 902285349Sluigi{ 903285349Sluigi return (t == NR_TX ? na->num_tx_rings : na->num_rx_rings); 904285349Sluigi} 905285349Sluigi 906341477Svmaffionestatic __inline u_int 907341477Svmaffionenma_get_host_nrings(struct netmap_adapter *na, enum txrx t) 908341477Svmaffione{ 909341477Svmaffione return (t == NR_TX ? na->num_host_tx_rings : na->num_host_rx_rings); 910341477Svmaffione} 911341477Svmaffione 912285349Sluigistatic __inline void 913285349Sluiginma_set_nrings(struct netmap_adapter *na, enum txrx t, u_int v) 914285349Sluigi{ 915285349Sluigi if (t == NR_TX) 916285349Sluigi na->num_tx_rings = v; 917285349Sluigi else 918285349Sluigi na->num_rx_rings = v; 919285349Sluigi} 920285349Sluigi 921341477Svmaffionestatic __inline void 922341477Svmaffionenma_set_host_nrings(struct netmap_adapter *na, enum txrx t, u_int v) 923341477Svmaffione{ 924341477Svmaffione if (t == NR_TX) 925341477Svmaffione na->num_host_tx_rings = v; 926341477Svmaffione else 927341477Svmaffione na->num_host_rx_rings = v; 928341477Svmaffione} 929341477Svmaffione 930341477Svmaffionestatic __inline struct netmap_kring** 931285349SluigiNMR(struct netmap_adapter *na, enum txrx t) 932285349Sluigi{ 933285349Sluigi return (t == NR_TX ? na->tx_rings : na->rx_rings); 934285349Sluigi} 935285349Sluigi 936341477Svmaffioneint nma_intr_enable(struct netmap_adapter *na, int onoff); 937341477Svmaffione 938259412Sluigi/* 939259412Sluigi * If the NIC is owned by the kernel 940259412Sluigi * (i.e., bridge), neither another bridge nor user can use it; 941259412Sluigi * if the NIC is owned by a user, only users can share it. 942259412Sluigi * Evaluation must be done under NMG_LOCK(). 943259412Sluigi */ 944270063Sluigi#define NETMAP_OWNED_BY_KERN(na) ((na)->na_flags & NAF_BUSY) 945259412Sluigi#define NETMAP_OWNED_BY_ANY(na) \ 946270063Sluigi (NETMAP_OWNED_BY_KERN(na) || ((na)->active_fds > 0)) 947259412Sluigi 948259412Sluigi/* 949259412Sluigi * derived netmap adapters for various types of ports 950259412Sluigi */ 951259412Sluigistruct netmap_vp_adapter { /* VALE software port */ 952259412Sluigi struct netmap_adapter up; 953259412Sluigi 954250107Sluigi /* 955250107Sluigi * Bridge support: 956250107Sluigi * 957250107Sluigi * bdg_port is the port number used in the bridge; 958251139Sluigi * na_bdg points to the bridge this NA is attached to. 959250107Sluigi */ 960238812Sluigi int bdg_port; 961251139Sluigi struct nm_bridge *na_bdg; 962259412Sluigi int retry; 963341477Svmaffione int autodelete; /* remove the ifp on last reference */ 964259412Sluigi 965261909Sluigi /* Maximum Frame Size, used in bdg_mismatch_datapath() */ 966261909Sluigi u_int mfs; 967285349Sluigi /* Last source MAC on this port */ 968285349Sluigi uint64_t last_smac; 969259412Sluigi}; 970259412Sluigi 971260368Sluigi 972259412Sluigistruct netmap_hw_adapter { /* physical device */ 973259412Sluigi struct netmap_adapter up; 974259412Sluigi 975341477Svmaffione#ifdef linux 976341477Svmaffione struct net_device_ops nm_ndo; 977341477Svmaffione struct ethtool_ops nm_eto; 978341477Svmaffione#endif 979270063Sluigi const struct ethtool_ops* save_ethtool; 980270063Sluigi 981270063Sluigi int (*nm_hw_register)(struct netmap_adapter *, int onoff); 982259412Sluigi}; 983259412Sluigi 984274362Sluigi#ifdef WITH_GENERIC 985261909Sluigi/* Mitigation support. */ 986261909Sluigistruct nm_generic_mit { 987261909Sluigi struct hrtimer mit_timer; 988261909Sluigi int mit_pending; 989270063Sluigi int mit_ring_idx; /* index of the ring being mitigated */ 990261909Sluigi struct netmap_adapter *mit_na; /* backpointer */ 991261909Sluigi}; 992260368Sluigi 993260368Sluigistruct netmap_generic_adapter { /* emulated device */ 994259412Sluigi struct netmap_hw_adapter up; 995259412Sluigi 996259412Sluigi /* Pointer to a previously used netmap adapter. */ 997259412Sluigi struct netmap_adapter *prev; 998259412Sluigi 999341477Svmaffione /* Emulated netmap adapters support: 1000341477Svmaffione * - save_if_input saves the if_input hook (FreeBSD); 1001341477Svmaffione * - mit implements rx interrupt mitigation; 1002259412Sluigi */ 1003259412Sluigi void (*save_if_input)(struct ifnet *, struct mbuf *); 1004259412Sluigi 1005261909Sluigi struct nm_generic_mit *mit; 1006260368Sluigi#ifdef linux 1007260368Sluigi netdev_tx_t (*save_start_xmit)(struct mbuf *, struct ifnet *); 1008260368Sluigi#endif 1009341477Svmaffione /* Is the adapter able to use multiple RX slots to scatter 1010341477Svmaffione * each packet pushed up by the driver? */ 1011341477Svmaffione int rxsg; 1012341477Svmaffione 1013341477Svmaffione /* Is the transmission path controlled by a netmap-aware 1014341477Svmaffione * device queue (i.e. qdisc on linux)? */ 1015341477Svmaffione int txqdisc; 1016259412Sluigi}; 1017274362Sluigi#endif /* WITH_GENERIC */ 1018259412Sluigi 1019341477Svmaffionestatic __inline u_int 1020285349Sluiginetmap_real_rings(struct netmap_adapter *na, enum txrx t) 1021261909Sluigi{ 1022341477Svmaffione return nma_get_nrings(na, t) + 1023341477Svmaffione !!(na->na_flags & NAF_HOST_RINGS) * nma_get_host_nrings(na, t); 1024261909Sluigi} 1025261909Sluigi 1026341477Svmaffione/* account for fake rings */ 1027341477Svmaffionestatic __inline u_int 1028341477Svmaffionenetmap_all_rings(struct netmap_adapter *na, enum txrx t) 1029341477Svmaffione{ 1030341477Svmaffione return max(nma_get_nrings(na, t) + 1, netmap_real_rings(na, t)); 1031341477Svmaffione} 1032259412Sluigi 1033341477Svmaffioneint netmap_default_bdg_attach(const char *name, struct netmap_adapter *na, 1034341477Svmaffione struct nm_bridge *); 1035341477Svmaffionestruct nm_bdg_polling_state; 1036260368Sluigi/* 1037260368Sluigi * Bridge wrapper for non VALE ports attached to a VALE switch. 1038259412Sluigi * 1039260368Sluigi * The real device must already have its own netmap adapter (hwna). 1040260368Sluigi * The bridge wrapper and the hwna adapter share the same set of 1041260368Sluigi * netmap rings and buffers, but they have two separate sets of 1042260368Sluigi * krings descriptors, with tx/rx meanings swapped: 1043259412Sluigi * 1044259412Sluigi * netmap 1045259412Sluigi * bwrap krings rings krings hwna 1046259412Sluigi * +------+ +------+ +-----+ +------+ +------+ 1047259412Sluigi * |tx_rings->| |\ /| |----| |<-tx_rings| 1048259412Sluigi * | | +------+ \ / +-----+ +------+ | | 1049259412Sluigi * | | X | | 1050259412Sluigi * | | / \ | | 1051259412Sluigi * | | +------+/ \+-----+ +------+ | | 1052259412Sluigi * |rx_rings->| | | |----| |<-rx_rings| 1053259412Sluigi * | | +------+ +-----+ +------+ | | 1054259412Sluigi * +------+ +------+ 1055259412Sluigi * 1056260368Sluigi * - packets coming from the bridge go to the brwap rx rings, 1057260368Sluigi * which are also the hwna tx rings. The bwrap notify callback 1058260368Sluigi * will then complete the hwna tx (see netmap_bwrap_notify). 1059259412Sluigi * 1060260368Sluigi * - packets coming from the outside go to the hwna rx rings, 1061260368Sluigi * which are also the bwrap tx rings. The (overwritten) hwna 1062260368Sluigi * notify method will then complete the bridge tx 1063260368Sluigi * (see netmap_bwrap_intr_notify). 1064259412Sluigi * 1065260368Sluigi * The bridge wrapper may optionally connect the hwna 'host' rings 1066260368Sluigi * to the bridge. This is done by using a second port in the 1067260368Sluigi * bridge and connecting it to the 'host' netmap_vp_adapter 1068260368Sluigi * contained in the netmap_bwrap_adapter. The brwap host adapter 1069260368Sluigi * cross-links the hwna host rings in the same way as shown above. 1070259412Sluigi * 1071260368Sluigi * - packets coming from the bridge and directed to the host stack 1072260368Sluigi * are handled by the bwrap host notify callback 1073260368Sluigi * (see netmap_bwrap_host_notify) 1074260368Sluigi * 1075260368Sluigi * - packets coming from the host stack are still handled by the 1076260368Sluigi * overwritten hwna notify callback (netmap_bwrap_intr_notify), 1077260368Sluigi * but are diverted to the host adapter depending on the ring number. 1078260368Sluigi * 1079259412Sluigi */ 1080259412Sluigistruct netmap_bwrap_adapter { 1081259412Sluigi struct netmap_vp_adapter up; 1082259412Sluigi struct netmap_vp_adapter host; /* for host rings */ 1083259412Sluigi struct netmap_adapter *hwna; /* the underlying device */ 1084259412Sluigi 1085260368Sluigi /* 1086260368Sluigi * When we attach a physical interface to the bridge, we 1087251139Sluigi * allow the controlling process to terminate, so we need 1088270063Sluigi * a place to store the n_detmap_priv_d data structure. 1089260368Sluigi * This is only done when physical interfaces 1090260368Sluigi * are attached to a bridge. 1091251139Sluigi */ 1092251139Sluigi struct netmap_priv_d *na_kpriv; 1093341477Svmaffione struct nm_bdg_polling_state *na_polling_state; 1094341477Svmaffione /* we overwrite the hwna->na_vp pointer, so we save 1095341477Svmaffione * here its original value, to be restored at detach 1096341477Svmaffione */ 1097341477Svmaffione struct netmap_vp_adapter *saved_na_vp; 1098227614Sluigi}; 1099341477Svmaffioneint nm_bdg_polling(struct nmreq_header *hdr); 1100227614Sluigi 1101341477Svmaffione#ifdef WITH_VALE 1102342033Svmaffioneint netmap_vale_attach(struct nmreq_header *hdr, void *auth_token); 1103342033Svmaffioneint netmap_vale_detach(struct nmreq_header *hdr, void *auth_token); 1104342033Svmaffioneint netmap_vale_list(struct nmreq_header *hdr); 1105341477Svmaffioneint netmap_vi_create(struct nmreq_header *hdr, int); 1106341477Svmaffioneint nm_vi_create(struct nmreq_header *); 1107341477Svmaffioneint nm_vi_destroy(const char *name); 1108341477Svmaffione#else /* !WITH_VALE */ 1109341477Svmaffione#define netmap_vi_create(hdr, a) (EOPNOTSUPP) 1110260368Sluigi#endif /* WITH_VALE */ 1111257529Sluigi 1112261909Sluigi#ifdef WITH_PIPES 1113257529Sluigi 1114261909Sluigi#define NM_MAXPIPES 64 /* max number of pipes per adapter */ 1115261909Sluigi 1116261909Sluigistruct netmap_pipe_adapter { 1117341477Svmaffione /* pipe identifier is up.name */ 1118261909Sluigi struct netmap_adapter up; 1119261909Sluigi 1120341477Svmaffione#define NM_PIPE_ROLE_MASTER 0x1 1121341477Svmaffione#define NM_PIPE_ROLE_SLAVE 0x2 1122341477Svmaffione int role; /* either NM_PIPE_ROLE_MASTER or NM_PIPE_ROLE_SLAVE */ 1123261909Sluigi 1124261909Sluigi struct netmap_adapter *parent; /* adapter that owns the memory */ 1125261909Sluigi struct netmap_pipe_adapter *peer; /* the other end of the pipe */ 1126261909Sluigi int peer_ref; /* 1 iff we are holding a ref to the peer */ 1127341477Svmaffione struct ifnet *parent_ifp; /* maybe null */ 1128261909Sluigi 1129261909Sluigi u_int parent_slot; /* index in the parent pipe array */ 1130261909Sluigi}; 1131261909Sluigi 1132261909Sluigi#endif /* WITH_PIPES */ 1133261909Sluigi 1134342033Svmaffione#ifdef WITH_NMNULL 1135342033Svmaffionestruct netmap_null_adapter { 1136342033Svmaffione struct netmap_adapter up; 1137342033Svmaffione}; 1138342033Svmaffione#endif /* WITH_NMNULL */ 1139261909Sluigi 1140342033Svmaffione 1141260368Sluigi/* return slots reserved to rx clients; used in drivers */ 1142257529Sluigistatic inline uint32_t 1143260368Sluiginm_kr_rxspace(struct netmap_kring *k) 1144257529Sluigi{ 1145260368Sluigi int space = k->nr_hwtail - k->nr_hwcur; 1146267128Sluigi if (space < 0) 1147260368Sluigi space += k->nkr_num_slots; 1148344047Svmaffione nm_prdis("preserving %d rx slots %d -> %d", space, k->nr_hwcur, k->nr_hwtail); 1149257529Sluigi 1150260368Sluigi return space; 1151257529Sluigi} 1152257529Sluigi 1153341477Svmaffione/* return slots reserved to tx clients */ 1154341477Svmaffione#define nm_kr_txspace(_k) nm_kr_rxspace(_k) 1155257529Sluigi 1156341477Svmaffione 1157341477Svmaffione/* True if no space in the tx ring, only valid after txsync_prologue */ 1158260368Sluigistatic inline int 1159260368Sluiginm_kr_txempty(struct netmap_kring *kring) 1160259412Sluigi{ 1161343832Svmaffione return kring->rhead == kring->nr_hwtail; 1162259412Sluigi} 1163259412Sluigi 1164341477Svmaffione/* True if no more completed slots in the rx ring, only valid after 1165341477Svmaffione * rxsync_prologue */ 1166341477Svmaffione#define nm_kr_rxempty(_k) nm_kr_txempty(_k) 1167259412Sluigi 1168343866Svmaffione/* True if the application needs to wait for more space on the ring 1169343866Svmaffione * (more received packets or more free tx slots). 1170343866Svmaffione * Only valid after *xsync_prologue. */ 1171343866Svmaffionestatic inline int 1172343866Svmaffionenm_kr_wouldblock(struct netmap_kring *kring) 1173343866Svmaffione{ 1174343866Svmaffione return kring->rcur == kring->nr_hwtail; 1175343866Svmaffione} 1176343866Svmaffione 1177257529Sluigi/* 1178259412Sluigi * protect against multiple threads using the same ring. 1179341477Svmaffione * also check that the ring has not been stopped or locked 1180227614Sluigi */ 1181341477Svmaffione#define NM_KR_BUSY 1 /* some other thread is syncing the ring */ 1182341477Svmaffione#define NM_KR_STOPPED 2 /* unbounded stop (ifconfig down or driver unload) */ 1183341477Svmaffione#define NM_KR_LOCKED 3 /* bounded, brief stop for mutual exclusion */ 1184227614Sluigi 1185260368Sluigi 1186341477Svmaffione/* release the previously acquired right to use the *sync() methods of the ring */ 1187259412Sluigistatic __inline void nm_kr_put(struct netmap_kring *kr) 1188259412Sluigi{ 1189259412Sluigi NM_ATOMIC_CLEAR(&kr->nr_busy); 1190259412Sluigi} 1191227614Sluigi 1192260368Sluigi 1193341477Svmaffione/* true if the ifp that backed the adapter has disappeared (e.g., the 1194341477Svmaffione * driver has been unloaded) 1195341477Svmaffione */ 1196341477Svmaffionestatic inline int nm_iszombie(struct netmap_adapter *na); 1197341477Svmaffione 1198341477Svmaffione/* try to obtain exclusive right to issue the *sync() operations on the ring. 1199341477Svmaffione * The right is obtained and must be later relinquished via nm_kr_put() if and 1200341477Svmaffione * only if nm_kr_tryget() returns 0. 1201341477Svmaffione * If can_sleep is 1 there are only two other possible outcomes: 1202341477Svmaffione * - the function returns NM_KR_BUSY 1203341477Svmaffione * - the function returns NM_KR_STOPPED and sets the POLLERR bit in *perr 1204341477Svmaffione * (if non-null) 1205341477Svmaffione * In both cases the caller will typically skip the ring, possibly collecting 1206341477Svmaffione * errors along the way. 1207341477Svmaffione * If the calling context does not allow sleeping, the caller must pass 0 in can_sleep. 1208341477Svmaffione * In the latter case, the function may also return NM_KR_LOCKED and leave *perr 1209341477Svmaffione * untouched: ideally, the caller should try again at a later time. 1210341477Svmaffione */ 1211341477Svmaffionestatic __inline int nm_kr_tryget(struct netmap_kring *kr, int can_sleep, int *perr) 1212259412Sluigi{ 1213341477Svmaffione int busy = 1, stopped; 1214259412Sluigi /* check a first time without taking the lock 1215259412Sluigi * to avoid starvation for nm_kr_get() 1216259412Sluigi */ 1217341477Svmaffioneretry: 1218341477Svmaffione stopped = kr->nkr_stopped; 1219341477Svmaffione if (unlikely(stopped)) { 1220341477Svmaffione goto stop; 1221259412Sluigi } 1222341477Svmaffione busy = NM_ATOMIC_TEST_AND_SET(&kr->nr_busy); 1223341477Svmaffione /* we should not return NM_KR_BUSY if the ring was 1224341477Svmaffione * actually stopped, so check another time after 1225341477Svmaffione * the barrier provided by the atomic operation 1226341477Svmaffione */ 1227341477Svmaffione stopped = kr->nkr_stopped; 1228341477Svmaffione if (unlikely(stopped)) { 1229341477Svmaffione goto stop; 1230341477Svmaffione } 1231341477Svmaffione 1232341477Svmaffione if (unlikely(nm_iszombie(kr->na))) { 1233341477Svmaffione stopped = NM_KR_STOPPED; 1234341477Svmaffione goto stop; 1235341477Svmaffione } 1236341477Svmaffione 1237341477Svmaffione return unlikely(busy) ? NM_KR_BUSY : 0; 1238341477Svmaffione 1239341477Svmaffionestop: 1240341477Svmaffione if (!busy) 1241259412Sluigi nm_kr_put(kr); 1242341477Svmaffione if (stopped == NM_KR_STOPPED) { 1243341477Svmaffione/* if POLLERR is defined we want to use it to simplify netmap_poll(). 1244341477Svmaffione * Otherwise, any non-zero value will do. 1245341477Svmaffione */ 1246341477Svmaffione#ifdef POLLERR 1247341477Svmaffione#define NM_POLLERR POLLERR 1248341477Svmaffione#else 1249341477Svmaffione#define NM_POLLERR 1 1250341477Svmaffione#endif /* POLLERR */ 1251341477Svmaffione if (perr) 1252341477Svmaffione *perr |= NM_POLLERR; 1253341477Svmaffione#undef NM_POLLERR 1254341477Svmaffione } else if (can_sleep) { 1255341477Svmaffione tsleep(kr, 0, "NM_KR_TRYGET", 4); 1256341477Svmaffione goto retry; 1257259412Sluigi } 1258341477Svmaffione return stopped; 1259259412Sluigi} 1260227614Sluigi 1261341477Svmaffione/* put the ring in the 'stopped' state and wait for the current user (if any) to 1262341477Svmaffione * notice. stopped must be either NM_KR_STOPPED or NM_KR_LOCKED 1263341477Svmaffione */ 1264341477Svmaffionestatic __inline void nm_kr_stop(struct netmap_kring *kr, int stopped) 1265285349Sluigi{ 1266341477Svmaffione kr->nkr_stopped = stopped; 1267285349Sluigi while (NM_ATOMIC_TEST_AND_SET(&kr->nr_busy)) 1268285349Sluigi tsleep(kr, 0, "NM_KR_GET", 4); 1269285349Sluigi} 1270259412Sluigi 1271341477Svmaffione/* restart a ring after a stop */ 1272341477Svmaffionestatic __inline void nm_kr_start(struct netmap_kring *kr) 1273341477Svmaffione{ 1274341477Svmaffione kr->nkr_stopped = 0; 1275341477Svmaffione nm_kr_put(kr); 1276341477Svmaffione} 1277285349Sluigi 1278341477Svmaffione 1279227614Sluigi/* 1280260368Sluigi * The following functions are used by individual drivers to 1281227614Sluigi * support netmap operation. 1282227614Sluigi * 1283227614Sluigi * netmap_attach() initializes a struct netmap_adapter, allocating the 1284227614Sluigi * struct netmap_ring's and the struct selinfo. 1285227614Sluigi * 1286227614Sluigi * netmap_detach() frees the memory allocated by netmap_attach(). 1287227614Sluigi * 1288257529Sluigi * netmap_transmit() replaces the if_transmit routine of the interface, 1289227614Sluigi * and is used to intercept packets coming from the stack. 1290227614Sluigi * 1291227614Sluigi * netmap_load_map/netmap_reload_map are helper routines to set/reset 1292227614Sluigi * the dmamap for a packet buffer 1293227614Sluigi * 1294270063Sluigi * netmap_reset() is a helper routine to be called in the hw driver 1295270063Sluigi * when reinitializing a ring. It should not be called by 1296270063Sluigi * virtual ports (vale, pipes, monitor) 1297227614Sluigi */ 1298259412Sluigiint netmap_attach(struct netmap_adapter *); 1299341477Svmaffioneint netmap_attach_ext(struct netmap_adapter *, size_t size, int override_reg); 1300227614Sluigivoid netmap_detach(struct ifnet *); 1301257529Sluigiint netmap_transmit(struct ifnet *, struct mbuf *); 1302227614Sluigistruct netmap_slot *netmap_reset(struct netmap_adapter *na, 1303257529Sluigi enum txrx tx, u_int n, u_int new_cur); 1304227614Sluigiint netmap_ring_reinit(struct netmap_kring *); 1305341477Svmaffioneint netmap_rings_config_get(struct netmap_adapter *, struct nm_config_info *); 1306227614Sluigi 1307341477Svmaffione/* Return codes for netmap_*x_irq. */ 1308341477Svmaffioneenum { 1309341477Svmaffione /* Driver should do normal interrupt processing, e.g. because 1310341477Svmaffione * the interface is not in netmap mode. */ 1311341477Svmaffione NM_IRQ_PASS = 0, 1312341477Svmaffione /* Port is in netmap mode, and the interrupt work has been 1313341477Svmaffione * completed. The driver does not have to notify netmap 1314341477Svmaffione * again before the next interrupt. */ 1315341477Svmaffione NM_IRQ_COMPLETED = -1, 1316341477Svmaffione /* Port is in netmap mode, but the interrupt work has not been 1317341477Svmaffione * completed. The driver has to make sure netmap will be 1318341477Svmaffione * notified again soon, even if no more interrupts come (e.g. 1319341477Svmaffione * on Linux the driver should not call napi_complete()). */ 1320341477Svmaffione NM_IRQ_RESCHED = -2, 1321341477Svmaffione}; 1322341477Svmaffione 1323260368Sluigi/* default functions to handle rx/tx interrupts */ 1324260368Sluigiint netmap_rx_irq(struct ifnet *, u_int, u_int *); 1325260368Sluigi#define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL) 1326341477Svmaffioneint netmap_common_irq(struct netmap_adapter *, u_int, u_int *work_done); 1327260368Sluigi 1328260368Sluigi 1329270063Sluigi#ifdef WITH_VALE 1330270063Sluigi/* functions used by external modules to interface with VALE */ 1331270063Sluigi#define netmap_vp_to_ifp(_vp) ((_vp)->up.ifp) 1332270063Sluigi#define netmap_ifp_to_vp(_ifp) (NA(_ifp)->na_vp) 1333270063Sluigi#define netmap_ifp_to_host_vp(_ifp) (NA(_ifp)->na_hostvp) 1334270063Sluigi#define netmap_bdg_idx(_vp) ((_vp)->bdg_port) 1335270063Sluigiconst char *netmap_bdg_name(struct netmap_vp_adapter *); 1336270063Sluigi#else /* !WITH_VALE */ 1337270063Sluigi#define netmap_vp_to_ifp(_vp) NULL 1338270063Sluigi#define netmap_ifp_to_vp(_ifp) NULL 1339270063Sluigi#define netmap_ifp_to_host_vp(_ifp) NULL 1340270063Sluigi#define netmap_bdg_idx(_vp) -1 1341270063Sluigi#endif /* WITH_VALE */ 1342260368Sluigi 1343270063Sluigistatic inline int 1344285349Sluiginm_netmap_on(struct netmap_adapter *na) 1345270063Sluigi{ 1346285349Sluigi return na && na->na_flags & NAF_NETMAP_ON; 1347270063Sluigi} 1348270063Sluigi 1349270063Sluigistatic inline int 1350285349Sluiginm_native_on(struct netmap_adapter *na) 1351270063Sluigi{ 1352285349Sluigi return nm_netmap_on(na) && (na->na_flags & NAF_NATIVE); 1353270063Sluigi} 1354270063Sluigi 1355341477Svmaffionestatic inline int 1356341477Svmaffionenm_iszombie(struct netmap_adapter *na) 1357259412Sluigi{ 1358341477Svmaffione return na == NULL || (na->na_flags & NAF_ZOMBIE); 1359259412Sluigi} 1360259412Sluigi 1361259412Sluigistatic inline void 1362341477Svmaffionenm_update_hostrings_mode(struct netmap_adapter *na) 1363259412Sluigi{ 1364341477Svmaffione /* Process nr_mode and nr_pending_mode for host rings. */ 1365341477Svmaffione na->tx_rings[na->num_tx_rings]->nr_mode = 1366341477Svmaffione na->tx_rings[na->num_tx_rings]->nr_pending_mode; 1367341477Svmaffione na->rx_rings[na->num_rx_rings]->nr_mode = 1368341477Svmaffione na->rx_rings[na->num_rx_rings]->nr_pending_mode; 1369259412Sluigi} 1370259412Sluigi 1371341477Svmaffionevoid nm_set_native_flags(struct netmap_adapter *); 1372341477Svmaffionevoid nm_clear_native_flags(struct netmap_adapter *); 1373260368Sluigi 1374344047Svmaffionevoid netmap_krings_mode_commit(struct netmap_adapter *na, int onoff); 1375344047Svmaffione 1376341477Svmaffione/* 1377341477Svmaffione * nm_*sync_prologue() functions are used in ioctl/poll and ptnetmap 1378341477Svmaffione * kthreads. 1379341477Svmaffione * We need netmap_ring* parameter, because in ptnetmap it is decoupled 1380341477Svmaffione * from host kring. 1381341477Svmaffione * The user-space ring pointers (head/cur/tail) are shared through 1382341477Svmaffione * CSB between host and guest. 1383341477Svmaffione */ 1384341477Svmaffione 1385341477Svmaffione/* 1386341477Svmaffione * validates parameters in the ring/kring, returns a value for head 1387341477Svmaffione * If any error, returns ring_size to force a reinit. 1388341477Svmaffione */ 1389341477Svmaffioneuint32_t nm_txsync_prologue(struct netmap_kring *, struct netmap_ring *); 1390341477Svmaffione 1391341477Svmaffione 1392341477Svmaffione/* 1393341477Svmaffione * validates parameters in the ring/kring, returns a value for head 1394341477Svmaffione * If any error, returns ring_size lim to force a reinit. 1395341477Svmaffione */ 1396341477Svmaffioneuint32_t nm_rxsync_prologue(struct netmap_kring *, struct netmap_ring *); 1397341477Svmaffione 1398341477Svmaffione 1399259412Sluigi/* check/fix address and len in tx rings */ 1400259412Sluigi#if 1 /* debug version */ 1401270063Sluigi#define NM_CHECK_ADDR_LEN(_na, _a, _l) do { \ 1402270063Sluigi if (_a == NETMAP_BUF_BASE(_na) || _l > NETMAP_BUF_SIZE(_na)) { \ 1403344047Svmaffione nm_prlim(5, "bad addr/len ring %d slot %d idx %d len %d", \ 1404270063Sluigi kring->ring_id, nm_i, slot->buf_idx, len); \ 1405270063Sluigi if (_l > NETMAP_BUF_SIZE(_na)) \ 1406270063Sluigi _l = NETMAP_BUF_SIZE(_na); \ 1407259412Sluigi } } while (0) 1408259412Sluigi#else /* no debug version */ 1409270063Sluigi#define NM_CHECK_ADDR_LEN(_na, _a, _l) do { \ 1410270063Sluigi if (_l > NETMAP_BUF_SIZE(_na)) \ 1411270063Sluigi _l = NETMAP_BUF_SIZE(_na); \ 1412259412Sluigi } while (0) 1413259412Sluigi#endif 1414259412Sluigi 1415259412Sluigi 1416259412Sluigi/*---------------------------------------------------------------*/ 1417259412Sluigi/* 1418270063Sluigi * Support routines used by netmap subsystems 1419270063Sluigi * (native drivers, VALE, generic, pipes, monitors, ...) 1420259412Sluigi */ 1421270063Sluigi 1422270063Sluigi 1423270063Sluigi/* common routine for all functions that create a netmap adapter. It performs 1424270063Sluigi * two main tasks: 1425270063Sluigi * - if the na points to an ifp, mark the ifp as netmap capable 1426270063Sluigi * using na as its native adapter; 1427270063Sluigi * - provide defaults for the setup callbacks and the memory allocator 1428270063Sluigi */ 1429270063Sluigiint netmap_attach_common(struct netmap_adapter *); 1430270063Sluigi/* fill priv->np_[tr]xq{first,last} using the ringid and flags information 1431341477Svmaffione * coming from a struct nmreq_register 1432270063Sluigi */ 1433341477Svmaffioneint netmap_interp_ringid(struct netmap_priv_d *priv, uint32_t nr_mode, 1434341477Svmaffione uint16_t nr_ringid, uint64_t nr_flags); 1435270063Sluigi/* update the ring parameters (number and size of tx and rx rings). 1436270063Sluigi * It calls the nm_config callback, if available. 1437270063Sluigi */ 1438259412Sluigiint netmap_update_config(struct netmap_adapter *na); 1439270063Sluigi/* create and initialize the common fields of the krings array. 1440270063Sluigi * using the information that must be already available in the na. 1441270063Sluigi * tailroom can be used to request the allocation of additional 1442270063Sluigi * tailroom bytes after the krings array. This is used by 1443270063Sluigi * netmap_vp_adapter's (i.e., VALE ports) to make room for 1444270063Sluigi * leasing-related data structures 1445270063Sluigi */ 1446261909Sluigiint netmap_krings_create(struct netmap_adapter *na, u_int tailroom); 1447270063Sluigi/* deletes the kring array of the adapter. The array must have 1448270063Sluigi * been created using netmap_krings_create 1449270063Sluigi */ 1450259412Sluigivoid netmap_krings_delete(struct netmap_adapter *na); 1451270063Sluigi 1452341477Svmaffioneint netmap_hw_krings_create(struct netmap_adapter *na); 1453341477Svmaffionevoid netmap_hw_krings_delete(struct netmap_adapter *na); 1454341477Svmaffione 1455270063Sluigi/* set the stopped/enabled status of ring 1456270063Sluigi * When stopping, they also wait for all current activity on the ring to 1457270063Sluigi * terminate. The status change is then notified using the na nm_notify 1458270063Sluigi * callback. 1459270063Sluigi */ 1460285349Sluigivoid netmap_set_ring(struct netmap_adapter *, u_int ring_id, enum txrx, int stopped); 1461270063Sluigi/* set the stopped/enabled status of all rings of the adapter. */ 1462270063Sluigivoid netmap_set_all_rings(struct netmap_adapter *, int stopped); 1463341477Svmaffione/* convenience wrappers for netmap_set_all_rings */ 1464270063Sluigivoid netmap_disable_all_rings(struct ifnet *); 1465270063Sluigivoid netmap_enable_all_rings(struct ifnet *); 1466270063Sluigi 1467342395Svmaffioneint netmap_buf_size_validate(const struct netmap_adapter *na, unsigned mtu); 1468285359Sluigiint netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na, 1469341477Svmaffione uint32_t nr_mode, uint16_t nr_ringid, uint64_t nr_flags); 1470341477Svmaffionevoid netmap_do_unregif(struct netmap_priv_d *priv); 1471259412Sluigi 1472257529Sluigiu_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg); 1473341477Svmaffioneint netmap_get_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1474341477Svmaffione struct ifnet **ifp, struct netmap_mem_d *nmd, int create); 1475341477Svmaffionevoid netmap_unget_na(struct netmap_adapter *na, struct ifnet *ifp); 1476341477Svmaffioneint netmap_get_hw_na(struct ifnet *ifp, 1477341477Svmaffione struct netmap_mem_d *nmd, struct netmap_adapter **na); 1478257529Sluigi 1479341477Svmaffione#ifdef WITH_VALE 1480342033Svmaffioneuint32_t netmap_vale_learning(struct nm_bdg_fwd *ft, uint8_t *dst_ring, 1481341477Svmaffione struct netmap_vp_adapter *, void *private_data); 1482341477Svmaffione 1483341477Svmaffione/* these are redefined in case of no VALE support */ 1484341477Svmaffioneint netmap_get_vale_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1485341477Svmaffione struct netmap_mem_d *nmd, int create); 1486341477Svmaffionevoid *netmap_vale_create(const char *bdg_name, int *return_status); 1487341477Svmaffioneint netmap_vale_destroy(const char *bdg_name, void *auth_token); 1488341477Svmaffione 1489259412Sluigi#else /* !WITH_VALE */ 1490341477Svmaffione#define netmap_bdg_learning(_1, _2, _3, _4) 0 1491341477Svmaffione#define netmap_get_vale_na(_1, _2, _3, _4) 0 1492341477Svmaffione#define netmap_bdg_create(_1, _2) NULL 1493341477Svmaffione#define netmap_bdg_destroy(_1, _2) 0 1494259412Sluigi#endif /* !WITH_VALE */ 1495259412Sluigi 1496261909Sluigi#ifdef WITH_PIPES 1497261909Sluigi/* max number of pipes per device */ 1498341477Svmaffione#define NM_MAXPIPES 64 /* XXX this should probably be a sysctl */ 1499261909Sluigivoid netmap_pipe_dealloc(struct netmap_adapter *); 1500341477Svmaffioneint netmap_get_pipe_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1501341477Svmaffione struct netmap_mem_d *nmd, int create); 1502261909Sluigi#else /* !WITH_PIPES */ 1503261909Sluigi#define NM_MAXPIPES 0 1504285349Sluigi#define netmap_pipe_alloc(_1, _2) 0 1505261909Sluigi#define netmap_pipe_dealloc(_1) 1506341477Svmaffione#define netmap_get_pipe_na(hdr, _2, _3, _4) \ 1507341477Svmaffione ((strchr(hdr->nr_name, '{') != NULL || strchr(hdr->nr_name, '}') != NULL) ? EOPNOTSUPP : 0) 1508261909Sluigi#endif 1509261909Sluigi 1510270063Sluigi#ifdef WITH_MONITOR 1511341477Svmaffioneint netmap_get_monitor_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1512341477Svmaffione struct netmap_mem_d *nmd, int create); 1513285349Sluigivoid netmap_monitor_stop(struct netmap_adapter *na); 1514270063Sluigi#else 1515341477Svmaffione#define netmap_get_monitor_na(hdr, _2, _3, _4) \ 1516341477Svmaffione (((struct nmreq_register *)(uintptr_t)hdr->nr_body)->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX) ? EOPNOTSUPP : 0) 1517270063Sluigi#endif 1518270063Sluigi 1519342033Svmaffione#ifdef WITH_NMNULL 1520342033Svmaffioneint netmap_get_null_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1521342033Svmaffione struct netmap_mem_d *nmd, int create); 1522342033Svmaffione#else /* !WITH_NMNULL */ 1523342033Svmaffione#define netmap_get_null_na(hdr, _2, _3, _4) \ 1524342033Svmaffione (((struct nmreq_register *)(uintptr_t)hdr->nr_body)->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX) ? EOPNOTSUPP : 0) 1525342033Svmaffione#endif /* WITH_NMNULL */ 1526342033Svmaffione 1527285349Sluigi#ifdef CONFIG_NET_NS 1528285349Sluigistruct net *netmap_bns_get(void); 1529285349Sluigivoid netmap_bns_put(struct net *); 1530285349Sluigivoid netmap_bns_getbridges(struct nm_bridge **, u_int *); 1531285349Sluigi#else 1532342033Svmaffioneextern struct nm_bridge *nm_bridges; 1533285349Sluigi#define netmap_bns_get() 1534285349Sluigi#define netmap_bns_put(_1) 1535285349Sluigi#define netmap_bns_getbridges(b, n) \ 1536285349Sluigi do { *b = nm_bridges; *n = NM_BRIDGES; } while (0) 1537285349Sluigi#endif 1538285349Sluigi 1539259412Sluigi/* Various prototypes */ 1540341477Svmaffioneint netmap_poll(struct netmap_priv_d *, int events, NM_SELRECORD_T *td); 1541259412Sluigiint netmap_init(void); 1542259412Sluigivoid netmap_fini(void); 1543259412Sluigiint netmap_get_memory(struct netmap_priv_d* p); 1544259412Sluigivoid netmap_dtor(void *data); 1545259412Sluigi 1546341477Svmaffioneint netmap_ioctl(struct netmap_priv_d *priv, u_long cmd, caddr_t data, 1547341477Svmaffione struct thread *, int nr_body_is_user); 1548341477Svmaffioneint netmap_ioctl_legacy(struct netmap_priv_d *priv, u_long cmd, caddr_t data, 1549341477Svmaffione struct thread *td); 1550341477Svmaffionesize_t nmreq_size_by_type(uint16_t nr_reqtype); 1551259412Sluigi 1552259412Sluigi/* netmap_adapter creation/destruction */ 1553259412Sluigi 1554260368Sluigi// #define NM_DEBUG_PUTGET 1 1555260368Sluigi 1556259412Sluigi#ifdef NM_DEBUG_PUTGET 1557259412Sluigi 1558259412Sluigi#define NM_DBG(f) __##f 1559259412Sluigi 1560259412Sluigivoid __netmap_adapter_get(struct netmap_adapter *na); 1561259412Sluigi 1562259412Sluigi#define netmap_adapter_get(na) \ 1563259412Sluigi do { \ 1564259412Sluigi struct netmap_adapter *__na = na; \ 1565344047Svmaffione nm_prinf("getting %p:%s (%d)", __na, (__na)->name, (__na)->na_refcount); \ 1566259412Sluigi __netmap_adapter_get(__na); \ 1567259412Sluigi } while (0) 1568259412Sluigi 1569259412Sluigiint __netmap_adapter_put(struct netmap_adapter *na); 1570259412Sluigi 1571259412Sluigi#define netmap_adapter_put(na) \ 1572260411Sluigi ({ \ 1573259412Sluigi struct netmap_adapter *__na = na; \ 1574344047Svmaffione nm_prinf("putting %p:%s (%d)", __na, (__na)->name, (__na)->na_refcount); \ 1575259412Sluigi __netmap_adapter_put(__na); \ 1576260411Sluigi }) 1577259412Sluigi 1578259412Sluigi#else /* !NM_DEBUG_PUTGET */ 1579259412Sluigi 1580259412Sluigi#define NM_DBG(f) f 1581259412Sluigivoid netmap_adapter_get(struct netmap_adapter *na); 1582259412Sluigiint netmap_adapter_put(struct netmap_adapter *na); 1583259412Sluigi 1584259412Sluigi#endif /* !NM_DEBUG_PUTGET */ 1585259412Sluigi 1586259412Sluigi 1587260368Sluigi/* 1588260368Sluigi * module variables 1589260368Sluigi */ 1590341477Svmaffione#define NETMAP_BUF_BASE(_na) ((_na)->na_lut.lut[0].vaddr) 1591341477Svmaffione#define NETMAP_BUF_SIZE(_na) ((_na)->na_lut.objsize) 1592231198Sluigiextern int netmap_no_pendintr; 1593341477Svmaffioneextern int netmap_mitigate; 1594342033Svmaffioneextern int netmap_verbose; 1595342033Svmaffione#ifdef CONFIG_NETMAP_DEBUG 1596342033Svmaffioneextern int netmap_debug; /* for debugging */ 1597342033Svmaffione#else /* !CONFIG_NETMAP_DEBUG */ 1598342033Svmaffione#define netmap_debug (0) 1599342033Svmaffione#endif /* !CONFIG_NETMAP_DEBUG */ 1600342033Svmaffioneenum { /* debug flags */ 1601342033Svmaffione NM_DEBUG_ON = 1, /* generic debug messsages */ 1602342033Svmaffione NM_DEBUG_HOST = 0x2, /* debug host stack */ 1603342033Svmaffione NM_DEBUG_RXSYNC = 0x10, /* debug on rxsync/txsync */ 1604342033Svmaffione NM_DEBUG_TXSYNC = 0x20, 1605342033Svmaffione NM_DEBUG_RXINTR = 0x100, /* debug on rx/tx intr (driver) */ 1606342033Svmaffione NM_DEBUG_TXINTR = 0x200, 1607342033Svmaffione NM_DEBUG_NIC_RXSYNC = 0x1000, /* debug on rx/tx intr (driver) */ 1608342033Svmaffione NM_DEBUG_NIC_TXSYNC = 0x2000, 1609342033Svmaffione NM_DEBUG_MEM = 0x4000, /* verbose memory allocations/deallocations */ 1610342033Svmaffione NM_DEBUG_VALE = 0x8000, /* debug messages from memory allocators */ 1611342033Svmaffione NM_DEBUG_BDG = NM_DEBUG_VALE, 1612227614Sluigi}; 1613227614Sluigi 1614259412Sluigiextern int netmap_txsync_retry; 1615341477Svmaffioneextern int netmap_flags; 1616341477Svmaffioneextern int netmap_generic_hwcsum; 1617259412Sluigiextern int netmap_generic_mit; 1618259412Sluigiextern int netmap_generic_ringsize; 1619261909Sluigiextern int netmap_generic_rings; 1620341477Svmaffione#ifdef linux 1621341477Svmaffioneextern int netmap_generic_txqdisc; 1622341477Svmaffione#endif 1623259412Sluigi 1624227614Sluigi/* 1625341477Svmaffione * NA returns a pointer to the struct netmap adapter from the ifp. 1626341477Svmaffione * WNA is os-specific and must be defined in glue code. 1627227614Sluigi */ 1628228845Sluigi#define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp)) 1629227614Sluigi 1630241719Sluigi/* 1631341477Svmaffione * we provide a default implementation of NM_ATTACH_NA/NM_DETACH_NA 1632341477Svmaffione * based on the WNA field. 1633341477Svmaffione * Glue code may override this by defining its own NM_ATTACH_NA 1634241719Sluigi */ 1635341477Svmaffione#ifndef NM_ATTACH_NA 1636241719Sluigi/* 1637341477Svmaffione * On old versions of FreeBSD, NA(ifp) is a pspare. On linux we 1638341477Svmaffione * overload another pointer in the netdev. 1639341477Svmaffione * 1640341477Svmaffione * We check if NA(ifp) is set and its first element has a related 1641241719Sluigi * magic value. The capenable is within the struct netmap_adapter. 1642241719Sluigi */ 1643241719Sluigi#define NETMAP_MAGIC 0x52697a7a 1644241719Sluigi 1645341477Svmaffione#define NM_NA_VALID(ifp) (NA(ifp) && \ 1646241719Sluigi ((uint32_t)(uintptr_t)NA(ifp) ^ NA(ifp)->magic) == NETMAP_MAGIC ) 1647241719Sluigi 1648341477Svmaffione#define NM_ATTACH_NA(ifp, na) do { \ 1649341477Svmaffione WNA(ifp) = na; \ 1650341477Svmaffione if (NA(ifp)) \ 1651341477Svmaffione NA(ifp)->magic = \ 1652341477Svmaffione ((uint32_t)(uintptr_t)NA(ifp)) ^ NETMAP_MAGIC; \ 1653341477Svmaffione} while(0) 1654341477Svmaffione#define NM_RESTORE_NA(ifp, na) WNA(ifp) = na; 1655241719Sluigi 1656341477Svmaffione#define NM_DETACH_NA(ifp) do { WNA(ifp) = NULL; } while (0) 1657341477Svmaffione#define NM_NA_CLASH(ifp) (NA(ifp) && !NM_NA_VALID(ifp)) 1658341477Svmaffione#endif /* !NM_ATTACH_NA */ 1659241719Sluigi 1660259412Sluigi 1661341477Svmaffione#define NM_IS_NATIVE(ifp) (NM_NA_VALID(ifp) && NA(ifp)->nm_dtor == netmap_hw_dtor) 1662341477Svmaffione 1663341477Svmaffione#if defined(__FreeBSD__) 1664341477Svmaffione 1665270063Sluigi/* Assigns the device IOMMU domain to an allocator. 1666270063Sluigi * Returns -ENOMEM in case the domain is different */ 1667270063Sluigi#define nm_iommu_group_id(dev) (0) 1668270063Sluigi 1669260368Sluigi/* Callback invoked by the dma machinery after a successful dmamap_load */ 1670230052Sluigistatic void netmap_dmamap_cb(__unused void *arg, 1671230058Sluigi __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error) 1672230052Sluigi{ 1673230052Sluigi} 1674230052Sluigi 1675230052Sluigi/* bus_dmamap_load wrapper: call aforementioned function if map != NULL. 1676230052Sluigi * XXX can we do it without a callback ? 1677230052Sluigi */ 1678341477Svmaffionestatic inline int 1679270063Sluiginetmap_load_map(struct netmap_adapter *na, 1680270063Sluigi bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1681230052Sluigi{ 1682230052Sluigi if (map) 1683270063Sluigi bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE(na), 1684230058Sluigi netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 1685341477Svmaffione return 0; 1686230052Sluigi} 1687230052Sluigi 1688270063Sluigistatic inline void 1689270063Sluiginetmap_unload_map(struct netmap_adapter *na, 1690270063Sluigi bus_dma_tag_t tag, bus_dmamap_t map) 1691270063Sluigi{ 1692270063Sluigi if (map) 1693270063Sluigi bus_dmamap_unload(tag, map); 1694270063Sluigi} 1695270063Sluigi 1696341477Svmaffione#define netmap_sync_map(na, tag, map, sz, t) 1697341477Svmaffione 1698230052Sluigi/* update the map when a buffer changes. */ 1699230052Sluigistatic inline void 1700270063Sluiginetmap_reload_map(struct netmap_adapter *na, 1701270063Sluigi bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1702230052Sluigi{ 1703230052Sluigi if (map) { 1704230052Sluigi bus_dmamap_unload(tag, map); 1705270063Sluigi bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE(na), 1706230058Sluigi netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 1707230052Sluigi } 1708230052Sluigi} 1709259412Sluigi 1710341477Svmaffione#elif defined(_WIN32) 1711341477Svmaffione 1712238812Sluigi#else /* linux */ 1713230052Sluigi 1714270063Sluigiint nm_iommu_group_id(bus_dma_tag_t dev); 1715270063Sluigi#include <linux/dma-mapping.h> 1716270063Sluigi 1717341477Svmaffione/* 1718341477Svmaffione * on linux we need 1719341477Svmaffione * dma_map_single(&pdev->dev, virt_addr, len, direction) 1720341477Svmaffione * dma_unmap_single(&adapter->pdev->dev, phys_addr, len, direction) 1721341477Svmaffione */ 1722341477Svmaffione#if 0 1723341477Svmaffione struct e1000_buffer *buffer_info = &tx_ring->buffer_info[l]; 1724341477Svmaffione /* set time_stamp *before* dma to help avoid a possible race */ 1725341477Svmaffione buffer_info->time_stamp = jiffies; 1726341477Svmaffione buffer_info->mapped_as_page = false; 1727341477Svmaffione buffer_info->length = len; 1728341477Svmaffione //buffer_info->next_to_watch = l; 1729341477Svmaffione /* reload dma map */ 1730341477Svmaffione dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 1731341477Svmaffione NETMAP_BUF_SIZE, DMA_TO_DEVICE); 1732341477Svmaffione buffer_info->dma = dma_map_single(&adapter->pdev->dev, 1733341477Svmaffione addr, NETMAP_BUF_SIZE, DMA_TO_DEVICE); 1734341477Svmaffione 1735341477Svmaffione if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) { 1736344047Svmaffione nm_prerr("dma mapping error"); 1737341477Svmaffione /* goto dma_error; See e1000_put_txbuf() */ 1738341477Svmaffione /* XXX reset */ 1739341477Svmaffione } 1740341477Svmaffione tx_desc->buffer_addr = htole64(buffer_info->dma); //XXX 1741341477Svmaffione 1742341477Svmaffione#endif 1743341477Svmaffione 1744341477Svmaffionestatic inline int 1745270063Sluiginetmap_load_map(struct netmap_adapter *na, 1746341477Svmaffione bus_dma_tag_t tag, bus_dmamap_t map, void *buf, u_int size) 1747270063Sluigi{ 1748341477Svmaffione if (map) { 1749341477Svmaffione *map = dma_map_single(na->pdev, buf, size, 1750341477Svmaffione DMA_BIDIRECTIONAL); 1751341477Svmaffione if (dma_mapping_error(na->pdev, *map)) { 1752341477Svmaffione *map = 0; 1753341477Svmaffione return ENOMEM; 1754341477Svmaffione } 1755270063Sluigi } 1756341477Svmaffione return 0; 1757270063Sluigi} 1758270063Sluigi 1759270063Sluigistatic inline void 1760270063Sluiginetmap_unload_map(struct netmap_adapter *na, 1761341477Svmaffione bus_dma_tag_t tag, bus_dmamap_t map, u_int sz) 1762270063Sluigi{ 1763270063Sluigi if (*map) { 1764270063Sluigi dma_unmap_single(na->pdev, *map, sz, 1765341477Svmaffione DMA_BIDIRECTIONAL); 1766270063Sluigi } 1767270063Sluigi} 1768270063Sluigi 1769341477Svmaffione#ifdef NETMAP_LINUX_HAVE_DMASYNC 1770270063Sluigistatic inline void 1771341477Svmaffionenetmap_sync_map_cpu(struct netmap_adapter *na, 1772341477Svmaffione bus_dma_tag_t tag, bus_dmamap_t map, u_int sz, enum txrx t) 1773341477Svmaffione{ 1774341477Svmaffione if (*map) { 1775341477Svmaffione dma_sync_single_for_cpu(na->pdev, *map, sz, 1776341477Svmaffione (t == NR_TX ? DMA_TO_DEVICE : DMA_FROM_DEVICE)); 1777341477Svmaffione } 1778341477Svmaffione} 1779341477Svmaffione 1780341477Svmaffionestatic inline void 1781341477Svmaffionenetmap_sync_map_dev(struct netmap_adapter *na, 1782341477Svmaffione bus_dma_tag_t tag, bus_dmamap_t map, u_int sz, enum txrx t) 1783341477Svmaffione{ 1784341477Svmaffione if (*map) { 1785341477Svmaffione dma_sync_single_for_device(na->pdev, *map, sz, 1786341477Svmaffione (t == NR_TX ? DMA_TO_DEVICE : DMA_FROM_DEVICE)); 1787341477Svmaffione } 1788341477Svmaffione} 1789341477Svmaffione 1790341477Svmaffionestatic inline void 1791270063Sluiginetmap_reload_map(struct netmap_adapter *na, 1792270063Sluigi bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1793270063Sluigi{ 1794341477Svmaffione u_int sz = NETMAP_BUF_SIZE(na); 1795270063Sluigi 1796270063Sluigi if (*map) { 1797270063Sluigi dma_unmap_single(na->pdev, *map, sz, 1798270063Sluigi DMA_BIDIRECTIONAL); 1799270063Sluigi } 1800270063Sluigi 1801270063Sluigi *map = dma_map_single(na->pdev, buf, sz, 1802270063Sluigi DMA_BIDIRECTIONAL); 1803270063Sluigi} 1804341477Svmaffione#else /* !NETMAP_LINUX_HAVE_DMASYNC */ 1805341477Svmaffione#define netmap_sync_map_cpu(na, tag, map, sz, t) 1806341477Svmaffione#define netmap_sync_map_dev(na, tag, map, sz, t) 1807341477Svmaffione#endif /* NETMAP_LINUX_HAVE_DMASYNC */ 1808270063Sluigi 1809238812Sluigi#endif /* linux */ 1810238812Sluigi 1811257529Sluigi 1812238812Sluigi/* 1813231796Sluigi * functions to map NIC to KRING indexes (n2k) and vice versa (k2n) 1814231796Sluigi */ 1815231796Sluigistatic inline int 1816232238Sluiginetmap_idx_n2k(struct netmap_kring *kr, int idx) 1817231796Sluigi{ 1818232238Sluigi int n = kr->nkr_num_slots; 1819342033Svmaffione 1820342033Svmaffione if (likely(kr->nkr_hwofs == 0)) { 1821342033Svmaffione return idx; 1822342033Svmaffione } 1823342033Svmaffione 1824232238Sluigi idx += kr->nkr_hwofs; 1825232238Sluigi if (idx < 0) 1826232238Sluigi return idx + n; 1827232238Sluigi else if (idx < n) 1828232238Sluigi return idx; 1829231796Sluigi else 1830232238Sluigi return idx - n; 1831231796Sluigi} 1832230052Sluigi 1833231796Sluigi 1834231796Sluigistatic inline int 1835232238Sluiginetmap_idx_k2n(struct netmap_kring *kr, int idx) 1836231796Sluigi{ 1837232238Sluigi int n = kr->nkr_num_slots; 1838342033Svmaffione 1839342033Svmaffione if (likely(kr->nkr_hwofs == 0)) { 1840342033Svmaffione return idx; 1841342033Svmaffione } 1842342033Svmaffione 1843232238Sluigi idx -= kr->nkr_hwofs; 1844232238Sluigi if (idx < 0) 1845232238Sluigi return idx + n; 1846232238Sluigi else if (idx < n) 1847232238Sluigi return idx; 1848231796Sluigi else 1849232238Sluigi return idx - n; 1850231796Sluigi} 1851231796Sluigi 1852231796Sluigi 1853234227Sluigi/* Entries of the look-up table. */ 1854341477Svmaffione#ifdef __FreeBSD__ 1855234227Sluigistruct lut_entry { 1856234227Sluigi void *vaddr; /* virtual address. */ 1857250107Sluigi vm_paddr_t paddr; /* physical address. */ 1858234227Sluigi}; 1859341477Svmaffione#else /* linux & _WIN32 */ 1860341477Svmaffione/* dma-mapping in linux can assign a buffer a different address 1861341477Svmaffione * depending on the device, so we need to have a separate 1862341477Svmaffione * physical-address look-up table for each na. 1863341477Svmaffione * We can still share the vaddrs, though, therefore we split 1864341477Svmaffione * the lut_entry structure. 1865341477Svmaffione */ 1866341477Svmaffionestruct lut_entry { 1867341477Svmaffione void *vaddr; /* virtual address. */ 1868341477Svmaffione}; 1869234227Sluigi 1870341477Svmaffionestruct plut_entry { 1871341477Svmaffione vm_paddr_t paddr; /* physical address. */ 1872341477Svmaffione}; 1873341477Svmaffione#endif /* linux & _WIN32 */ 1874341477Svmaffione 1875234227Sluigistruct netmap_obj_pool; 1876234227Sluigi 1877227614Sluigi/* 1878229939Sluigi * NMB return the virtual address of a buffer (buffer 0 on bad index) 1879229939Sluigi * PNMB also fills the physical address 1880227614Sluigi */ 1881229939Sluigistatic inline void * 1882270063SluigiNMB(struct netmap_adapter *na, struct netmap_slot *slot) 1883227614Sluigi{ 1884285349Sluigi struct lut_entry *lut = na->na_lut.lut; 1885227614Sluigi uint32_t i = slot->buf_idx; 1886285349Sluigi return (unlikely(i >= na->na_lut.objtotal)) ? 1887270063Sluigi lut[0].vaddr : lut[i].vaddr; 1888227614Sluigi} 1889227614Sluigi 1890229939Sluigistatic inline void * 1891270063SluigiPNMB(struct netmap_adapter *na, struct netmap_slot *slot, uint64_t *pp) 1892229939Sluigi{ 1893229939Sluigi uint32_t i = slot->buf_idx; 1894285349Sluigi struct lut_entry *lut = na->na_lut.lut; 1895341477Svmaffione struct plut_entry *plut = na->na_lut.plut; 1896285349Sluigi void *ret = (i >= na->na_lut.objtotal) ? lut[0].vaddr : lut[i].vaddr; 1897249659Sluigi 1898341477Svmaffione#ifdef _WIN32 1899341477Svmaffione *pp = (i >= na->na_lut.objtotal) ? (uint64_t)plut[0].paddr.QuadPart : (uint64_t)plut[i].paddr.QuadPart; 1900341477Svmaffione#else 1901341477Svmaffione *pp = (i >= na->na_lut.objtotal) ? plut[0].paddr : plut[i].paddr; 1902341477Svmaffione#endif 1903229939Sluigi return ret; 1904229939Sluigi} 1905229939Sluigi 1906259412Sluigi 1907260368Sluigi/* 1908285359Sluigi * Structure associated to each netmap file descriptor. 1909285359Sluigi * It is created on open and left unbound (np_nifp == NULL). 1910285359Sluigi * A successful NIOCREGIF will set np_nifp and the first few fields; 1911285359Sluigi * this is protected by a global lock (NMG_LOCK) due to low contention. 1912259412Sluigi * 1913285359Sluigi * np_refs counts the number of references to the structure: one for the fd, 1914285359Sluigi * plus (on FreeBSD) one for each active mmap which we track ourselves 1915285695Sluigi * (linux automatically tracks them, but FreeBSD does not). 1916285359Sluigi * np_refs is protected by NMG_LOCK. 1917260368Sluigi * 1918285359Sluigi * Read access to the structure is lock free, because ni_nifp once set 1919285359Sluigi * can only go to 0 when nobody is using the entry anymore. Readers 1920285359Sluigi * must check that np_nifp != NULL before using the other fields. 1921259412Sluigi */ 1922259412Sluigistruct netmap_priv_d { 1923259412Sluigi struct netmap_if * volatile np_nifp; /* netmap if descriptor. */ 1924259412Sluigi 1925259412Sluigi struct netmap_adapter *np_na; 1926341477Svmaffione struct ifnet *np_ifp; 1927261909Sluigi uint32_t np_flags; /* from the ioctl */ 1928341477Svmaffione u_int np_qfirst[NR_TXRX], 1929285349Sluigi np_qlast[NR_TXRX]; /* range of tx/rx rings to scan */ 1930341477Svmaffione uint16_t np_txpoll; 1931342033Svmaffione uint16_t np_kloop_state; /* use with NMG_LOCK held */ 1932342033Svmaffione#define NM_SYNC_KLOOP_RUNNING (1 << 0) 1933342033Svmaffione#define NM_SYNC_KLOOP_STOPPING (1 << 1) 1934341477Svmaffione int np_sync_flags; /* to be passed to nm_sync */ 1935259412Sluigi 1936285359Sluigi int np_refs; /* use with NMG_LOCK held */ 1937261909Sluigi 1938261909Sluigi /* pointers to the selinfo to be used for selrecord. 1939261909Sluigi * Either the local or the global one depending on the 1940261909Sluigi * number of rings. 1941261909Sluigi */ 1942285349Sluigi NM_SELINFO_T *np_si[NR_TXRX]; 1943342033Svmaffione 1944342033Svmaffione /* In the optional CSB mode, the user must specify the start address 1945342033Svmaffione * of two arrays of Communication Status Block (CSB) entries, for the 1946342033Svmaffione * two directions (kernel read application write, and kernel write 1947342033Svmaffione * application read). 1948342033Svmaffione * The number of entries must agree with the number of rings bound to 1949342033Svmaffione * the netmap file descriptor. The entries corresponding to the TX 1950342033Svmaffione * rings are laid out before the ones corresponding to the RX rings. 1951342033Svmaffione * 1952342033Svmaffione * Array of CSB entries for application --> kernel communication 1953342033Svmaffione * (N entries). */ 1954342033Svmaffione struct nm_csb_atok *np_csb_atok_base; 1955342033Svmaffione /* Array of CSB entries for kernel --> application communication 1956342033Svmaffione * (N entries). */ 1957342033Svmaffione struct nm_csb_ktoa *np_csb_ktoa_base; 1958342033Svmaffione 1959342033Svmaffione#ifdef linux 1960342033Svmaffione struct file *np_filp; /* used by sync kloop */ 1961342033Svmaffione#endif /* linux */ 1962259412Sluigi}; 1963259412Sluigi 1964341477Svmaffionestruct netmap_priv_d *netmap_priv_new(void); 1965341477Svmaffionevoid netmap_priv_delete(struct netmap_priv_d *); 1966341477Svmaffione 1967341477Svmaffionestatic inline int nm_kring_pending(struct netmap_priv_d *np) 1968341477Svmaffione{ 1969341477Svmaffione struct netmap_adapter *na = np->np_na; 1970341477Svmaffione enum txrx t; 1971341477Svmaffione int i; 1972341477Svmaffione 1973341477Svmaffione for_rx_tx(t) { 1974341477Svmaffione for (i = np->np_qfirst[t]; i < np->np_qlast[t]; i++) { 1975341477Svmaffione struct netmap_kring *kring = NMR(na, t)[i]; 1976341477Svmaffione if (kring->nr_mode != kring->nr_pending_mode) { 1977341477Svmaffione return 1; 1978341477Svmaffione } 1979341477Svmaffione } 1980341477Svmaffione } 1981341477Svmaffione return 0; 1982341477Svmaffione} 1983341477Svmaffione 1984342033Svmaffione/* call with NMG_LOCK held */ 1985342033Svmaffionestatic __inline int 1986342033Svmaffionenm_si_user(struct netmap_priv_d *priv, enum txrx t) 1987342033Svmaffione{ 1988342033Svmaffione return (priv->np_na != NULL && 1989342033Svmaffione (priv->np_qlast[t] - priv->np_qfirst[t] > 1)); 1990342033Svmaffione} 1991342033Svmaffione 1992341477Svmaffione#ifdef WITH_PIPES 1993341477Svmaffioneint netmap_pipe_txsync(struct netmap_kring *txkring, int flags); 1994341477Svmaffioneint netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags); 1995344047Svmaffioneint netmap_pipe_krings_create_both(struct netmap_adapter *na, 1996344047Svmaffione struct netmap_adapter *ona); 1997344047Svmaffionevoid netmap_pipe_krings_delete_both(struct netmap_adapter *na, 1998344047Svmaffione struct netmap_adapter *ona); 1999344047Svmaffioneint netmap_pipe_reg_both(struct netmap_adapter *na, 2000344047Svmaffione struct netmap_adapter *ona); 2001341477Svmaffione#endif /* WITH_PIPES */ 2002341477Svmaffione 2003270063Sluigi#ifdef WITH_MONITOR 2004259412Sluigi 2005270063Sluigistruct netmap_monitor_adapter { 2006270063Sluigi struct netmap_adapter up; 2007270063Sluigi 2008270063Sluigi struct netmap_priv_d priv; 2009270063Sluigi uint32_t flags; 2010270063Sluigi}; 2011270063Sluigi 2012270063Sluigi#endif /* WITH_MONITOR */ 2013270063Sluigi 2014270063Sluigi 2015274362Sluigi#ifdef WITH_GENERIC 2016259412Sluigi/* 2017259412Sluigi * generic netmap emulation for devices that do not have 2018259412Sluigi * native netmap support. 2019259412Sluigi */ 2020259412Sluigiint generic_netmap_attach(struct ifnet *ifp); 2021341477Svmaffioneint generic_rx_handler(struct ifnet *ifp, struct mbuf *m);; 2022259412Sluigi 2023341477Svmaffioneint nm_os_catch_rx(struct netmap_generic_adapter *gna, int intercept); 2024341477Svmaffioneint nm_os_catch_tx(struct netmap_generic_adapter *gna, int intercept); 2025341477Svmaffione 2026341477Svmaffioneint na_is_generic(struct netmap_adapter *na); 2027341477Svmaffione 2028341477Svmaffione/* 2029341477Svmaffione * the generic transmit routine is passed a structure to optionally 2030341477Svmaffione * build a queue of descriptors, in an OS-specific way. 2031341477Svmaffione * The payload is at addr, if non-null, and the routine should send or queue 2032341477Svmaffione * the packet, returning 0 if successful, 1 on failure. 2033341477Svmaffione * 2034341477Svmaffione * At the end, if head is non-null, there will be an additional call 2035341477Svmaffione * to the function with addr = NULL; this should tell the OS-specific 2036341477Svmaffione * routine to send the queue and free any resources. Failure is ignored. 2037341477Svmaffione */ 2038341477Svmaffionestruct nm_os_gen_arg { 2039341477Svmaffione struct ifnet *ifp; 2040341477Svmaffione void *m; /* os-specific mbuf-like object */ 2041341477Svmaffione void *head, *tail; /* tailq, if the OS-specific routine needs to build one */ 2042341477Svmaffione void *addr; /* payload of current packet */ 2043341477Svmaffione u_int len; /* packet length */ 2044341477Svmaffione u_int ring_nr; /* packet length */ 2045341477Svmaffione u_int qevent; /* in txqdisc mode, place an event on this mbuf */ 2046341477Svmaffione}; 2047341477Svmaffione 2048341477Svmaffioneint nm_os_generic_xmit_frame(struct nm_os_gen_arg *); 2049341477Svmaffioneint nm_os_generic_find_num_desc(struct ifnet *ifp, u_int *tx, u_int *rx); 2050341477Svmaffionevoid nm_os_generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq); 2051341477Svmaffionevoid nm_os_generic_set_features(struct netmap_generic_adapter *gna); 2052341477Svmaffione 2053285349Sluigistatic inline struct ifnet* 2054285349Sluiginetmap_generic_getifp(struct netmap_generic_adapter *gna) 2055285349Sluigi{ 2056285349Sluigi if (gna->prev) 2057285349Sluigi return gna->prev->ifp; 2058259412Sluigi 2059285349Sluigi return gna->up.up.ifp; 2060285349Sluigi} 2061285349Sluigi 2062341477Svmaffionevoid netmap_generic_irq(struct netmap_adapter *na, u_int q, u_int *work_done); 2063341477Svmaffione 2064270063Sluigi//#define RATE_GENERIC /* Enables communication statistics for generic. */ 2065270063Sluigi#ifdef RATE_GENERIC 2066270063Sluigivoid generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi); 2067270063Sluigi#else 2068270063Sluigi#define generic_rate(txp, txs, txi, rxp, rxs, rxi) 2069270063Sluigi#endif 2070270063Sluigi 2071259412Sluigi/* 2072259412Sluigi * netmap_mitigation API. This is used by the generic adapter 2073259412Sluigi * to reduce the number of interrupt requests/selwakeup 2074259412Sluigi * to clients on incoming packets. 2075259412Sluigi */ 2076341477Svmaffionevoid nm_os_mitigation_init(struct nm_generic_mit *mit, int idx, 2077270063Sluigi struct netmap_adapter *na); 2078341477Svmaffionevoid nm_os_mitigation_start(struct nm_generic_mit *mit); 2079341477Svmaffionevoid nm_os_mitigation_restart(struct nm_generic_mit *mit); 2080341477Svmaffioneint nm_os_mitigation_active(struct nm_generic_mit *mit); 2081341477Svmaffionevoid nm_os_mitigation_cleanup(struct nm_generic_mit *mit); 2082341477Svmaffione#else /* !WITH_GENERIC */ 2083341477Svmaffione#define generic_netmap_attach(ifp) (EOPNOTSUPP) 2084341477Svmaffione#define na_is_generic(na) (0) 2085274362Sluigi#endif /* WITH_GENERIC */ 2086259412Sluigi 2087261909Sluigi/* Shared declarations for the VALE switch. */ 2088261909Sluigi 2089261909Sluigi/* 2090261909Sluigi * Each transmit queue accumulates a batch of packets into 2091261909Sluigi * a structure before forwarding. Packets to the same 2092261909Sluigi * destination are put in a list using ft_next as a link field. 2093261909Sluigi * ft_frags and ft_next are valid only on the first fragment. 2094261909Sluigi */ 2095261909Sluigistruct nm_bdg_fwd { /* forwarding entry for a bridge */ 2096261909Sluigi void *ft_buf; /* netmap or indirect buffer */ 2097261909Sluigi uint8_t ft_frags; /* how many fragments (only on 1st frag) */ 2098341477Svmaffione uint16_t ft_offset; /* dst port (unused) */ 2099261909Sluigi uint16_t ft_flags; /* flags, e.g. indirect */ 2100261909Sluigi uint16_t ft_len; /* src fragment len */ 2101261909Sluigi uint16_t ft_next; /* next packet to same destination */ 2102261909Sluigi}; 2103261909Sluigi 2104261909Sluigi/* struct 'virtio_net_hdr' from linux. */ 2105261909Sluigistruct nm_vnet_hdr { 2106261909Sluigi#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start, csum_offset */ 2107261909Sluigi#define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */ 2108261909Sluigi uint8_t flags; 2109261909Sluigi#define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */ 2110261909Sluigi#define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */ 2111261909Sluigi#define VIRTIO_NET_HDR_GSO_UDP 3 /* GSO frame, IPv4 UDP (UFO) */ 2112261909Sluigi#define VIRTIO_NET_HDR_GSO_TCPV6 4 /* GSO frame, IPv6 TCP */ 2113261909Sluigi#define VIRTIO_NET_HDR_GSO_ECN 0x80 /* TCP has ECN set */ 2114261909Sluigi uint8_t gso_type; 2115261909Sluigi uint16_t hdr_len; 2116261909Sluigi uint16_t gso_size; 2117261909Sluigi uint16_t csum_start; 2118261909Sluigi uint16_t csum_offset; 2119261909Sluigi}; 2120261909Sluigi 2121261909Sluigi#define WORST_CASE_GSO_HEADER (14+40+60) /* IPv6 + TCP */ 2122261909Sluigi 2123261909Sluigi/* Private definitions for IPv4, IPv6, UDP and TCP headers. */ 2124261909Sluigi 2125261909Sluigistruct nm_iphdr { 2126261909Sluigi uint8_t version_ihl; 2127261909Sluigi uint8_t tos; 2128261909Sluigi uint16_t tot_len; 2129261909Sluigi uint16_t id; 2130261909Sluigi uint16_t frag_off; 2131261909Sluigi uint8_t ttl; 2132261909Sluigi uint8_t protocol; 2133261909Sluigi uint16_t check; 2134261909Sluigi uint32_t saddr; 2135261909Sluigi uint32_t daddr; 2136261909Sluigi /*The options start here. */ 2137261909Sluigi}; 2138261909Sluigi 2139261909Sluigistruct nm_tcphdr { 2140261909Sluigi uint16_t source; 2141261909Sluigi uint16_t dest; 2142261909Sluigi uint32_t seq; 2143261909Sluigi uint32_t ack_seq; 2144261909Sluigi uint8_t doff; /* Data offset + Reserved */ 2145261909Sluigi uint8_t flags; 2146261909Sluigi uint16_t window; 2147261909Sluigi uint16_t check; 2148261909Sluigi uint16_t urg_ptr; 2149261909Sluigi}; 2150261909Sluigi 2151261909Sluigistruct nm_udphdr { 2152261909Sluigi uint16_t source; 2153261909Sluigi uint16_t dest; 2154261909Sluigi uint16_t len; 2155261909Sluigi uint16_t check; 2156261909Sluigi}; 2157261909Sluigi 2158261909Sluigistruct nm_ipv6hdr { 2159261909Sluigi uint8_t priority_version; 2160261909Sluigi uint8_t flow_lbl[3]; 2161261909Sluigi 2162261909Sluigi uint16_t payload_len; 2163261909Sluigi uint8_t nexthdr; 2164261909Sluigi uint8_t hop_limit; 2165261909Sluigi 2166261909Sluigi uint8_t saddr[16]; 2167261909Sluigi uint8_t daddr[16]; 2168261909Sluigi}; 2169261909Sluigi 2170261909Sluigi/* Type used to store a checksum (in host byte order) that hasn't been 2171261909Sluigi * folded yet. 2172261909Sluigi */ 2173261909Sluigi#define rawsum_t uint32_t 2174261909Sluigi 2175341477Svmaffionerawsum_t nm_os_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum); 2176341477Svmaffioneuint16_t nm_os_csum_ipv4(struct nm_iphdr *iph); 2177341477Svmaffionevoid nm_os_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, 2178261909Sluigi size_t datalen, uint16_t *check); 2179341477Svmaffionevoid nm_os_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, 2180261909Sluigi size_t datalen, uint16_t *check); 2181341477Svmaffioneuint16_t nm_os_csum_fold(rawsum_t cur_sum); 2182261909Sluigi 2183261909Sluigivoid bdg_mismatch_datapath(struct netmap_vp_adapter *na, 2184261909Sluigi struct netmap_vp_adapter *dst_na, 2185341477Svmaffione const struct nm_bdg_fwd *ft_p, 2186341477Svmaffione struct netmap_ring *dst_ring, 2187261909Sluigi u_int *j, u_int lim, u_int *howmany); 2188270063Sluigi 2189270063Sluigi/* persistent virtual port routines */ 2190341477Svmaffioneint nm_os_vi_persist(const char *, struct ifnet **); 2191341477Svmaffionevoid nm_os_vi_detach(struct ifnet *); 2192341477Svmaffionevoid nm_os_vi_init_index(void); 2193270063Sluigi 2194341477Svmaffione/* 2195341477Svmaffione * kernel thread routines 2196341477Svmaffione */ 2197341477Svmaffionestruct nm_kctx; /* OS-specific kernel context - opaque */ 2198342033Svmaffionetypedef void (*nm_kctx_worker_fn_t)(void *data); 2199341477Svmaffione 2200341477Svmaffione/* kthread configuration */ 2201341477Svmaffionestruct nm_kctx_cfg { 2202341477Svmaffione long type; /* kthread type/identifier */ 2203341477Svmaffione nm_kctx_worker_fn_t worker_fn; /* worker function */ 2204341477Svmaffione void *worker_private;/* worker parameter */ 2205341477Svmaffione int attach_user; /* attach kthread to user process */ 2206341477Svmaffione}; 2207341477Svmaffione/* kthread configuration */ 2208341477Svmaffionestruct nm_kctx *nm_os_kctx_create(struct nm_kctx_cfg *cfg, 2209341477Svmaffione void *opaque); 2210341477Svmaffioneint nm_os_kctx_worker_start(struct nm_kctx *); 2211341477Svmaffionevoid nm_os_kctx_worker_stop(struct nm_kctx *); 2212341477Svmaffionevoid nm_os_kctx_destroy(struct nm_kctx *); 2213341477Svmaffionevoid nm_os_kctx_worker_setaff(struct nm_kctx *, int); 2214341477Svmaffioneu_int nm_os_ncpus(void); 2215341477Svmaffione 2216342033Svmaffioneint netmap_sync_kloop(struct netmap_priv_d *priv, 2217342033Svmaffione struct nmreq_header *hdr); 2218342033Svmaffioneint netmap_sync_kloop_stop(struct netmap_priv_d *priv); 2219342033Svmaffione 2220342033Svmaffione#ifdef WITH_PTNETMAP 2221342033Svmaffione/* ptnetmap guest routines */ 2222342033Svmaffione 2223341477Svmaffione/* 2224342033Svmaffione * ptnetmap_memdev routines used to talk with ptnetmap_memdev device driver 2225341477Svmaffione */ 2226342033Svmaffionestruct ptnetmap_memdev; 2227342033Svmaffioneint nm_os_pt_memdev_iomap(struct ptnetmap_memdev *, vm_paddr_t *, void **, 2228342033Svmaffione uint64_t *); 2229342033Svmaffionevoid nm_os_pt_memdev_iounmap(struct ptnetmap_memdev *); 2230342033Svmaffioneuint32_t nm_os_pt_memdev_ioread(struct ptnetmap_memdev *, unsigned int); 2231341477Svmaffione 2232341477Svmaffione/* 2233341477Svmaffione * netmap adapter for guest ptnetmap ports 2234341477Svmaffione */ 2235341477Svmaffionestruct netmap_pt_guest_adapter { 2236341477Svmaffione /* The netmap adapter to be used by netmap applications. 2237341477Svmaffione * This field must be the first, to allow upcast. */ 2238341477Svmaffione struct netmap_hw_adapter hwup; 2239341477Svmaffione 2240341477Svmaffione /* The netmap adapter to be used by the driver. */ 2241341477Svmaffione struct netmap_hw_adapter dr; 2242341477Svmaffione 2243341477Svmaffione /* Reference counter to track users of backend netmap port: the 2244341477Svmaffione * network stack and netmap clients. 2245341477Svmaffione * Used to decide when we need (de)allocate krings/rings and 2246341477Svmaffione * start (stop) ptnetmap kthreads. */ 2247342033Svmaffione int backend_users; 2248341477Svmaffione 2249341477Svmaffione}; 2250341477Svmaffione 2251341477Svmaffioneint netmap_pt_guest_attach(struct netmap_adapter *na, 2252341477Svmaffione unsigned int nifp_offset, 2253341477Svmaffione unsigned int memid); 2254342033Svmaffionebool netmap_pt_guest_txsync(struct nm_csb_atok *atok, 2255342033Svmaffione struct nm_csb_ktoa *ktoa, 2256341477Svmaffione struct netmap_kring *kring, int flags); 2257342033Svmaffionebool netmap_pt_guest_rxsync(struct nm_csb_atok *atok, 2258342033Svmaffione struct nm_csb_ktoa *ktoa, 2259342033Svmaffione struct netmap_kring *kring, int flags); 2260341477Svmaffioneint ptnet_nm_krings_create(struct netmap_adapter *na); 2261341477Svmaffionevoid ptnet_nm_krings_delete(struct netmap_adapter *na); 2262341477Svmaffionevoid ptnet_nm_dtor(struct netmap_adapter *na); 2263341477Svmaffione 2264343832Svmaffione/* Helper function wrapping nm_sync_kloop_appl_read(). */ 2265342033Svmaffionestatic inline void 2266342033Svmaffioneptnet_sync_tail(struct nm_csb_ktoa *ktoa, struct netmap_kring *kring) 2267342033Svmaffione{ 2268342033Svmaffione struct netmap_ring *ring = kring->ring; 2269342033Svmaffione 2270342033Svmaffione /* Update hwcur and hwtail as known by the host. */ 2271343832Svmaffione nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail, &kring->nr_hwcur); 2272342033Svmaffione 2273342033Svmaffione /* nm_sync_finalize */ 2274342033Svmaffione ring->tail = kring->rtail = kring->nr_hwtail; 2275342033Svmaffione} 2276342033Svmaffione#endif /* WITH_PTNETMAP */ 2277342033Svmaffione 2278341477Svmaffione#ifdef __FreeBSD__ 2279341477Svmaffione/* 2280341477Svmaffione * FreeBSD mbuf allocator/deallocator in emulation mode: 2281341477Svmaffione */ 2282341477Svmaffione#if __FreeBSD_version < 1100000 2283341477Svmaffione 2284341477Svmaffione/* 2285341477Svmaffione * For older versions of FreeBSD: 2286341477Svmaffione * 2287341477Svmaffione * We allocate EXT_PACKET mbuf+clusters, but need to set M_NOFREE 2288341477Svmaffione * so that the destructor, if invoked, will not free the packet. 2289341477Svmaffione * In principle we should set the destructor only on demand, 2290341477Svmaffione * but since there might be a race we better do it on allocation. 2291341477Svmaffione * As a consequence, we also need to set the destructor or we 2292341477Svmaffione * would leak buffers. 2293341477Svmaffione */ 2294341477Svmaffione 2295341477Svmaffione/* mbuf destructor, also need to change the type to EXT_EXTREF, 2296341477Svmaffione * add an M_NOFREE flag, and then clear the flag and 2297341477Svmaffione * chain into uma_zfree(zone_pack, mf) 2298341477Svmaffione * (or reinstall the buffer ?) 2299341477Svmaffione */ 2300341477Svmaffione#define SET_MBUF_DESTRUCTOR(m, fn) do { \ 2301341477Svmaffione (m)->m_ext.ext_free = (void *)fn; \ 2302341477Svmaffione (m)->m_ext.ext_type = EXT_EXTREF; \ 2303341477Svmaffione} while (0) 2304341477Svmaffione 2305341477Svmaffionestatic int 2306341477Svmaffionevoid_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) 2307341477Svmaffione{ 2308341477Svmaffione /* restore original mbuf */ 2309341477Svmaffione m->m_ext.ext_buf = m->m_data = m->m_ext.ext_arg1; 2310341477Svmaffione m->m_ext.ext_arg1 = NULL; 2311341477Svmaffione m->m_ext.ext_type = EXT_PACKET; 2312341477Svmaffione m->m_ext.ext_free = NULL; 2313341477Svmaffione if (MBUF_REFCNT(m) == 0) 2314341477Svmaffione SET_MBUF_REFCNT(m, 1); 2315341477Svmaffione uma_zfree(zone_pack, m); 2316341477Svmaffione 2317341477Svmaffione return 0; 2318341477Svmaffione} 2319341477Svmaffione 2320341477Svmaffionestatic inline struct mbuf * 2321341477Svmaffionenm_os_get_mbuf(struct ifnet *ifp, int len) 2322341477Svmaffione{ 2323341477Svmaffione struct mbuf *m; 2324341477Svmaffione 2325341477Svmaffione (void)ifp; 2326341477Svmaffione m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 2327341477Svmaffione if (m) { 2328341477Svmaffione /* m_getcl() (mb_ctor_mbuf) has an assert that checks that 2329341477Svmaffione * M_NOFREE flag is not specified as third argument, 2330341477Svmaffione * so we have to set M_NOFREE after m_getcl(). */ 2331341477Svmaffione m->m_flags |= M_NOFREE; 2332341477Svmaffione m->m_ext.ext_arg1 = m->m_ext.ext_buf; // XXX save 2333341477Svmaffione m->m_ext.ext_free = (void *)void_mbuf_dtor; 2334341477Svmaffione m->m_ext.ext_type = EXT_EXTREF; 2335344047Svmaffione nm_prdis(5, "create m %p refcnt %d", m, MBUF_REFCNT(m)); 2336341477Svmaffione } 2337341477Svmaffione return m; 2338341477Svmaffione} 2339341477Svmaffione 2340341477Svmaffione#else /* __FreeBSD_version >= 1100000 */ 2341341477Svmaffione 2342341477Svmaffione/* 2343341477Svmaffione * Newer versions of FreeBSD, using a straightforward scheme. 2344341477Svmaffione * 2345341477Svmaffione * We allocate mbufs with m_gethdr(), since the mbuf header is needed 2346341477Svmaffione * by the driver. We also attach a customly-provided external storage, 2347341477Svmaffione * which in this case is a netmap buffer. When calling m_extadd(), however 2348341477Svmaffione * we pass a NULL address, since the real address (and length) will be 2349341477Svmaffione * filled in by nm_os_generic_xmit_frame() right before calling 2350341477Svmaffione * if_transmit(). 2351341477Svmaffione * 2352341477Svmaffione * The dtor function does nothing, however we need it since mb_free_ext() 2353341477Svmaffione * has a KASSERT(), checking that the mbuf dtor function is not NULL. 2354341477Svmaffione */ 2355341477Svmaffione 2356341477Svmaffione#if __FreeBSD_version <= 1200050 2357341477Svmaffionestatic void void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) { } 2358341477Svmaffione#else /* __FreeBSD_version >= 1200051 */ 2359341477Svmaffione/* The arg1 and arg2 pointers argument were removed by r324446, which 2360341477Svmaffione * in included since version 1200051. */ 2361341477Svmaffionestatic void void_mbuf_dtor(struct mbuf *m) { } 2362341477Svmaffione#endif /* __FreeBSD_version >= 1200051 */ 2363341477Svmaffione 2364341477Svmaffione#define SET_MBUF_DESTRUCTOR(m, fn) do { \ 2365341477Svmaffione (m)->m_ext.ext_free = (fn != NULL) ? \ 2366341477Svmaffione (void *)fn : (void *)void_mbuf_dtor; \ 2367341477Svmaffione} while (0) 2368341477Svmaffione 2369341477Svmaffionestatic inline struct mbuf * 2370341477Svmaffionenm_os_get_mbuf(struct ifnet *ifp, int len) 2371341477Svmaffione{ 2372341477Svmaffione struct mbuf *m; 2373341477Svmaffione 2374341477Svmaffione (void)ifp; 2375341477Svmaffione (void)len; 2376341477Svmaffione 2377341477Svmaffione m = m_gethdr(M_NOWAIT, MT_DATA); 2378341477Svmaffione if (m == NULL) { 2379341477Svmaffione return m; 2380341477Svmaffione } 2381341477Svmaffione 2382341477Svmaffione m_extadd(m, NULL /* buf */, 0 /* size */, void_mbuf_dtor, 2383341477Svmaffione NULL, NULL, 0, EXT_NET_DRV); 2384341477Svmaffione 2385341477Svmaffione return m; 2386341477Svmaffione} 2387341477Svmaffione 2388341477Svmaffione#endif /* __FreeBSD_version >= 1100000 */ 2389341477Svmaffione#endif /* __FreeBSD__ */ 2390341477Svmaffione 2391341477Svmaffionestruct nmreq_option * nmreq_findoption(struct nmreq_option *, uint16_t); 2392341477Svmaffioneint nmreq_checkduplicate(struct nmreq_option *); 2393341477Svmaffione 2394342033Svmaffioneint netmap_init_bridges(void); 2395342033Svmaffionevoid netmap_uninit_bridges(void); 2396342033Svmaffione 2397342033Svmaffione/* Functions to read and write CSB fields from the kernel. */ 2398342033Svmaffione#if defined (linux) 2399342033Svmaffione#define CSB_READ(csb, field, r) (get_user(r, &csb->field)) 2400342033Svmaffione#define CSB_WRITE(csb, field, v) (put_user(v, &csb->field)) 2401342033Svmaffione#else /* ! linux */ 2402342033Svmaffione#define CSB_READ(csb, field, r) (r = fuword32(&csb->field)) 2403342033Svmaffione#define CSB_WRITE(csb, field, v) (suword32(&csb->field, v)) 2404342033Svmaffione#endif /* ! linux */ 2405342033Svmaffione 2406227614Sluigi#endif /* _NET_NETMAP_KERN_H_ */ 2407