1331722Seadler/* 2341477Svmaffione * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo 3341477Svmaffione * Copyright (C) 2013-2016 Universita` di Pisa 4341477Svmaffione * All rights reserved. 5241719Sluigi * 6227614Sluigi * Redistribution and use in source and binary forms, with or without 7227614Sluigi * modification, are permitted provided that the following conditions 8227614Sluigi * are met: 9228276Sluigi * 1. Redistributions of source code must retain the above copyright 10228276Sluigi * notice, this list of conditions and the following disclaimer. 11228276Sluigi * 2. Redistributions in binary form must reproduce the above copyright 12228276Sluigi * notice, this list of conditions and the following disclaimer in the 13227614Sluigi * documentation and/or other materials provided with the distribution. 14241719Sluigi * 15227614Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16227614Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17227614Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18227614Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19227614Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20227614Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21227614Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22227614Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23227614Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24227614Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25227614Sluigi * SUCH DAMAGE. 26227614Sluigi */ 27227614Sluigi 28227614Sluigi/* 29227614Sluigi * $FreeBSD: stable/11/sys/dev/netmap/netmap_kern.h 357961 2020-02-15 12:05:11Z vmaffione $ 30227614Sluigi * 31227614Sluigi * The header contains the definitions of constants and function 32227614Sluigi * prototypes used only in kernelspace. 33227614Sluigi */ 34227614Sluigi 35227614Sluigi#ifndef _NET_NETMAP_KERN_H_ 36227614Sluigi#define _NET_NETMAP_KERN_H_ 37227614Sluigi 38285349Sluigi#if defined(linux) 39285349Sluigi 40341477Svmaffione#if defined(CONFIG_NETMAP_EXTMEM) 41341477Svmaffione#define WITH_EXTMEM 42341477Svmaffione#endif 43285349Sluigi#if defined(CONFIG_NETMAP_VALE) 44285349Sluigi#define WITH_VALE 45285349Sluigi#endif 46285349Sluigi#if defined(CONFIG_NETMAP_PIPE) 47285349Sluigi#define WITH_PIPES 48285349Sluigi#endif 49285349Sluigi#if defined(CONFIG_NETMAP_MONITOR) 50285349Sluigi#define WITH_MONITOR 51285349Sluigi#endif 52285349Sluigi#if defined(CONFIG_NETMAP_GENERIC) 53285349Sluigi#define WITH_GENERIC 54285349Sluigi#endif 55342033Svmaffione#if defined(CONFIG_NETMAP_PTNETMAP) 56342033Svmaffione#define WITH_PTNETMAP 57285349Sluigi#endif 58341477Svmaffione#if defined(CONFIG_NETMAP_SINK) 59341477Svmaffione#define WITH_SINK 60341477Svmaffione#endif 61342033Svmaffione#if defined(CONFIG_NETMAP_NULL) 62342033Svmaffione#define WITH_NMNULL 63342033Svmaffione#endif 64285349Sluigi 65341477Svmaffione#elif defined (_WIN32) 66341477Svmaffione#define WITH_VALE // comment out to disable VALE support 67341477Svmaffione#define WITH_PIPES 68341477Svmaffione#define WITH_MONITOR 69341477Svmaffione#define WITH_GENERIC 70342033Svmaffione#define WITH_NMNULL 71285349Sluigi 72341477Svmaffione#else /* neither linux nor windows */ 73259412Sluigi#define WITH_VALE // comment out to disable VALE support 74261909Sluigi#define WITH_PIPES 75270063Sluigi#define WITH_MONITOR 76274362Sluigi#define WITH_GENERIC 77341477Svmaffione#define WITH_EXTMEM 78342033Svmaffione#define WITH_NMNULL 79285349Sluigi#endif 80285349Sluigi 81231594Sluigi#if defined(__FreeBSD__) 82341477Svmaffione#include <sys/selinfo.h> 83250052Sluigi 84257529Sluigi#define likely(x) __builtin_expect((long)!!(x), 1L) 85257529Sluigi#define unlikely(x) __builtin_expect((long)!!(x), 0L) 86341477Svmaffione#define __user 87238812Sluigi 88285349Sluigi#define NM_LOCK_T struct mtx /* low level spinlock, used to protect queues */ 89274362Sluigi 90285349Sluigi#define NM_MTX_T struct sx /* OS-specific mutex (sleepable) */ 91285349Sluigi#define NM_MTX_INIT(m) sx_init(&(m), #m) 92285349Sluigi#define NM_MTX_DESTROY(m) sx_destroy(&(m)) 93285349Sluigi#define NM_MTX_LOCK(m) sx_xlock(&(m)) 94341477Svmaffione#define NM_MTX_SPINLOCK(m) while (!sx_try_xlock(&(m))) ; 95285349Sluigi#define NM_MTX_UNLOCK(m) sx_xunlock(&(m)) 96285349Sluigi#define NM_MTX_ASSERT(m) sx_assert(&(m), SA_XLOCKED) 97259412Sluigi 98274459Sluigi#define NM_SELINFO_T struct nm_selinfo 99341477Svmaffione#define NM_SELRECORD_T struct thread 100231594Sluigi#define MBUF_LEN(m) ((m)->m_pkthdr.len) 101341477Svmaffione#define MBUF_TXQ(m) ((m)->m_pkthdr.flowid) 102341477Svmaffione#define MBUF_TRANSMIT(na, ifp, m) ((na)->if_transmit(ifp, m)) 103341477Svmaffione#define GEN_TX_MBUF_IFP(m) ((m)->m_pkthdr.rcvif) 104250052Sluigi 105341477Svmaffione#define NM_ATOMIC_T volatile int /* required by atomic/bitops.h */ 106259412Sluigi/* atomic operations */ 107259412Sluigi#include <machine/atomic.h> 108259412Sluigi#define NM_ATOMIC_TEST_AND_SET(p) (!atomic_cmpset_acq_int((p), 0, 1)) 109259412Sluigi#define NM_ATOMIC_CLEAR(p) atomic_store_rel_int((p), 0) 110257529Sluigi 111272110Sluigi#if __FreeBSD_version >= 1100030 112272110Sluigi#define WNA(_ifp) (_ifp)->if_netmap 113272110Sluigi#else /* older FreeBSD */ 114272110Sluigi#define WNA(_ifp) (_ifp)->if_pspare[0] 115272110Sluigi#endif /* older FreeBSD */ 116272110Sluigi 117267283Sluigi#if __FreeBSD_version >= 1100005 118267283Sluigistruct netmap_adapter *netmap_getna(if_t ifp); 119267283Sluigi#endif 120259412Sluigi 121270063Sluigi#if __FreeBSD_version >= 1100027 122312783Sloos#define MBUF_REFCNT(m) ((m)->m_ext.ext_count) 123312783Sloos#define SET_MBUF_REFCNT(m, x) (m)->m_ext.ext_count = x 124270063Sluigi#else 125312783Sloos#define MBUF_REFCNT(m) ((m)->m_ext.ref_cnt ? *((m)->m_ext.ref_cnt) : -1) 126270063Sluigi#define SET_MBUF_REFCNT(m, x) *((m)->m_ext.ref_cnt) = x 127270063Sluigi#endif 128270063Sluigi 129341477Svmaffione#define MBUF_QUEUED(m) 1 130259412Sluigi 131274459Sluigistruct nm_selinfo { 132344509Svmaffione /* Support for select(2) and poll(2). */ 133274459Sluigi struct selinfo si; 134344509Svmaffione /* Support for kqueue(9). See comments in netmap_freebsd.c */ 135344509Svmaffione struct taskqueue *ntfytq; 136344509Svmaffione struct task ntfytask; 137274459Sluigi struct mtx m; 138344509Svmaffione char mtxname[32]; 139344509Svmaffione int kqueue_users; 140274459Sluigi}; 141274459Sluigi 142274459Sluigi 143259412Sluigistruct hrtimer { 144341477Svmaffione /* Not used in FreeBSD. */ 145259412Sluigi}; 146341477Svmaffione 147285349Sluigi#define NM_BNS_GET(b) 148285349Sluigi#define NM_BNS_PUT(b) 149259412Sluigi 150232238Sluigi#elif defined (linux) 151250052Sluigi 152250052Sluigi#define NM_LOCK_T safe_spinlock_t // see bsd_glue.h 153231594Sluigi#define NM_SELINFO_T wait_queue_head_t 154231594Sluigi#define MBUF_LEN(m) ((m)->len) 155341477Svmaffione#define MBUF_TRANSMIT(na, ifp, m) \ 156341477Svmaffione ({ \ 157341477Svmaffione /* Avoid infinite recursion with generic. */ \ 158341477Svmaffione m->priority = NM_MAGIC_PRIORITY_TX; \ 159341477Svmaffione (((struct net_device_ops *)(na)->if_transmit)->ndo_start_xmit(m, ifp)); \ 160341477Svmaffione 0; \ 161341477Svmaffione }) 162238812Sluigi 163341477Svmaffione/* See explanation in nm_os_generic_xmit_frame. */ 164341477Svmaffione#define GEN_TX_MBUF_IFP(m) ((struct ifnet *)skb_shinfo(m)->destructor_arg) 165341477Svmaffione 166257529Sluigi#define NM_ATOMIC_T volatile long unsigned int 167257529Sluigi 168285349Sluigi#define NM_MTX_T struct mutex /* OS-specific sleepable lock */ 169285349Sluigi#define NM_MTX_INIT(m) mutex_init(&(m)) 170285349Sluigi#define NM_MTX_DESTROY(m) do { (void)(m); } while (0) 171274362Sluigi#define NM_MTX_LOCK(m) mutex_lock(&(m)) 172274362Sluigi#define NM_MTX_UNLOCK(m) mutex_unlock(&(m)) 173285349Sluigi#define NM_MTX_ASSERT(m) mutex_is_locked(&(m)) 174259412Sluigi 175238812Sluigi#ifndef DEV_NETMAP 176238812Sluigi#define DEV_NETMAP 177257529Sluigi#endif /* DEV_NETMAP */ 178238812Sluigi 179238812Sluigi#elif defined (__APPLE__) 180250052Sluigi 181241719Sluigi#warning apple support is incomplete. 182238812Sluigi#define likely(x) __builtin_expect(!!(x), 1) 183238812Sluigi#define unlikely(x) __builtin_expect(!!(x), 0) 184238812Sluigi#define NM_LOCK_T IOLock * 185238812Sluigi#define NM_SELINFO_T struct selinfo 186238812Sluigi#define MBUF_LEN(m) ((m)->m_pkthdr.len) 187238812Sluigi 188341477Svmaffione#elif defined (_WIN32) 189341477Svmaffione#include "../../../WINDOWS/win_glue.h" 190341477Svmaffione 191341477Svmaffione#define NM_SELRECORD_T IO_STACK_LOCATION 192341477Svmaffione#define NM_SELINFO_T win_SELINFO // see win_glue.h 193341477Svmaffione#define NM_LOCK_T win_spinlock_t // see win_glue.h 194341477Svmaffione#define NM_MTX_T KGUARDED_MUTEX /* OS-specific mutex (sleepable) */ 195341477Svmaffione 196341477Svmaffione#define NM_MTX_INIT(m) KeInitializeGuardedMutex(&m); 197341477Svmaffione#define NM_MTX_DESTROY(m) do { (void)(m); } while (0) 198341477Svmaffione#define NM_MTX_LOCK(m) KeAcquireGuardedMutex(&(m)) 199341477Svmaffione#define NM_MTX_UNLOCK(m) KeReleaseGuardedMutex(&(m)) 200341477Svmaffione#define NM_MTX_ASSERT(m) assert(&m.Count>0) 201341477Svmaffione 202341477Svmaffione//These linknames are for the NDIS driver 203341477Svmaffione#define NETMAP_NDIS_LINKNAME_STRING L"\\DosDevices\\NMAPNDIS" 204341477Svmaffione#define NETMAP_NDIS_NTDEVICE_STRING L"\\Device\\NMAPNDIS" 205341477Svmaffione 206341477Svmaffione//Definition of internal driver-to-driver ioctl codes 207341477Svmaffione#define NETMAP_KERNEL_XCHANGE_POINTERS _IO('i', 180) 208341477Svmaffione#define NETMAP_KERNEL_SEND_SHUTDOWN_SIGNAL _IO_direct('i', 195) 209341477Svmaffione 210341477Svmaffionetypedef struct hrtimer{ 211341477Svmaffione KTIMER timer; 212341477Svmaffione BOOLEAN active; 213341477Svmaffione KDPC deferred_proc; 214341477Svmaffione}; 215341477Svmaffione 216341477Svmaffione/* MSVC does not have likely/unlikely support */ 217341477Svmaffione#ifdef _MSC_VER 218341477Svmaffione#define likely(x) (x) 219341477Svmaffione#define unlikely(x) (x) 220238812Sluigi#else 221341477Svmaffione#define likely(x) __builtin_expect((long)!!(x), 1L) 222341477Svmaffione#define unlikely(x) __builtin_expect((long)!!(x), 0L) 223341477Svmaffione#endif //_MSC_VER 224250052Sluigi 225341477Svmaffione#else 226341477Svmaffione 227231594Sluigi#error unsupported platform 228231594Sluigi 229250052Sluigi#endif /* end - platform-specific code */ 230250052Sluigi 231341477Svmaffione#ifndef _WIN32 /* support for emulated sysctl */ 232341477Svmaffione#define SYSBEGIN(x) 233341477Svmaffione#define SYSEND 234341477Svmaffione#endif /* _WIN32 */ 235341477Svmaffione 236341477Svmaffione#define NM_ACCESS_ONCE(x) (*(volatile __typeof__(x) *)&(x)) 237341477Svmaffione 238285349Sluigi#define NMG_LOCK_T NM_MTX_T 239285349Sluigi#define NMG_LOCK_INIT() NM_MTX_INIT(netmap_global_lock) 240285349Sluigi#define NMG_LOCK_DESTROY() NM_MTX_DESTROY(netmap_global_lock) 241285349Sluigi#define NMG_LOCK() NM_MTX_LOCK(netmap_global_lock) 242285349Sluigi#define NMG_UNLOCK() NM_MTX_UNLOCK(netmap_global_lock) 243285349Sluigi#define NMG_LOCK_ASSERT() NM_MTX_ASSERT(netmap_global_lock) 244285349Sluigi 245341477Svmaffione#if defined(__FreeBSD__) 246342033Svmaffione#define nm_prerr_int printf 247342033Svmaffione#define nm_prinf_int printf 248341477Svmaffione#elif defined (_WIN32) 249342033Svmaffione#define nm_prerr_int DbgPrint 250342033Svmaffione#define nm_prinf_int DbgPrint 251341477Svmaffione#elif defined(linux) 252342033Svmaffione#define nm_prerr_int(fmt, arg...) printk(KERN_ERR fmt, ##arg) 253342033Svmaffione#define nm_prinf_int(fmt, arg...) printk(KERN_INFO fmt, ##arg) 254341477Svmaffione#endif 255341477Svmaffione 256342033Svmaffione#define nm_prinf(format, ...) \ 257230058Sluigi do { \ 258230058Sluigi struct timeval __xxts; \ 259227614Sluigi microtime(&__xxts); \ 260342033Svmaffione nm_prinf_int("%03d.%06d [%4d] %-25s " format "\n",\ 261230058Sluigi (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \ 262260368Sluigi __LINE__, __FUNCTION__, ##__VA_ARGS__); \ 263227614Sluigi } while (0) 264241719Sluigi 265342033Svmaffione#define nm_prerr(format, ...) \ 266241719Sluigi do { \ 267342033Svmaffione struct timeval __xxts; \ 268342033Svmaffione microtime(&__xxts); \ 269342033Svmaffione nm_prerr_int("%03d.%06d [%4d] %-25s " format "\n",\ 270342033Svmaffione (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \ 271342033Svmaffione __LINE__, __FUNCTION__, ##__VA_ARGS__); \ 272342033Svmaffione } while (0) 273342033Svmaffione 274344047Svmaffione/* Disabled printf (used to be nm_prdis). */ 275342033Svmaffione#define nm_prdis(format, ...) 276342033Svmaffione 277342033Svmaffione/* Rate limited, lps indicates how many per second. */ 278342033Svmaffione#define nm_prlim(lps, format, ...) \ 279342033Svmaffione do { \ 280241719Sluigi static int t0, __cnt; \ 281241719Sluigi if (t0 != time_second) { \ 282241719Sluigi t0 = time_second; \ 283241719Sluigi __cnt = 0; \ 284241719Sluigi } \ 285241719Sluigi if (__cnt++ < lps) \ 286342033Svmaffione nm_prinf(format, ##__VA_ARGS__); \ 287241719Sluigi } while (0) 288241719Sluigi 289227614Sluigistruct netmap_adapter; 290251139Sluigistruct nm_bdg_fwd; 291251139Sluigistruct nm_bridge; 292251139Sluigistruct netmap_priv_d; 293341477Svmaffionestruct nm_bdg_args; 294227614Sluigi 295341477Svmaffione/* os-specific NM_SELINFO_T initialzation/destruction functions */ 296344509Svmaffioneint nm_os_selinfo_init(NM_SELINFO_T *, const char *name); 297341477Svmaffionevoid nm_os_selinfo_uninit(NM_SELINFO_T *); 298341477Svmaffione 299257529Sluigiconst char *nm_dump_buf(char *p, int len, int lim, char *dst); 300257529Sluigi 301341477Svmaffionevoid nm_os_selwakeup(NM_SELINFO_T *si); 302341477Svmaffionevoid nm_os_selrecord(NM_SELRECORD_T *sr, NM_SELINFO_T *si); 303341477Svmaffione 304341477Svmaffioneint nm_os_ifnet_init(void); 305341477Svmaffionevoid nm_os_ifnet_fini(void); 306341477Svmaffionevoid nm_os_ifnet_lock(void); 307341477Svmaffionevoid nm_os_ifnet_unlock(void); 308341477Svmaffione 309341477Svmaffioneunsigned nm_os_ifnet_mtu(struct ifnet *ifp); 310341477Svmaffione 311341477Svmaffionevoid nm_os_get_module(void); 312341477Svmaffionevoid nm_os_put_module(void); 313341477Svmaffione 314341477Svmaffionevoid netmap_make_zombie(struct ifnet *); 315341477Svmaffionevoid netmap_undo_zombie(struct ifnet *); 316341477Svmaffione 317341477Svmaffione/* os independent alloc/realloc/free */ 318341477Svmaffionevoid *nm_os_malloc(size_t); 319341477Svmaffionevoid *nm_os_vmalloc(size_t); 320341477Svmaffionevoid *nm_os_realloc(void *, size_t new_size, size_t old_size); 321341477Svmaffionevoid nm_os_free(void *); 322341477Svmaffionevoid nm_os_vfree(void *); 323341477Svmaffione 324341477Svmaffione/* os specific attach/detach enter/exit-netmap-mode routines */ 325341477Svmaffionevoid nm_os_onattach(struct ifnet *); 326341477Svmaffionevoid nm_os_ondetach(struct ifnet *); 327341477Svmaffionevoid nm_os_onenter(struct ifnet *); 328341477Svmaffionevoid nm_os_onexit(struct ifnet *); 329341477Svmaffione 330341477Svmaffione/* passes a packet up to the host stack. 331341477Svmaffione * If the packet is sent (or dropped) immediately it returns NULL, 332341477Svmaffione * otherwise it links the packet to prev and returns m. 333341477Svmaffione * In this case, a final call with m=NULL and prev != NULL will send up 334341477Svmaffione * the entire chain to the host stack. 335341477Svmaffione */ 336341477Svmaffionevoid *nm_os_send_up(struct ifnet *, struct mbuf *m, struct mbuf *prev); 337341477Svmaffione 338341477Svmaffioneint nm_os_mbuf_has_seg_offld(struct mbuf *m); 339341477Svmaffioneint nm_os_mbuf_has_csum_offld(struct mbuf *m); 340341477Svmaffione 341259412Sluigi#include "netmap_mbq.h" 342259412Sluigi 343259412Sluigiextern NMG_LOCK_T netmap_global_lock; 344259412Sluigi 345285349Sluigienum txrx { NR_RX = 0, NR_TX = 1, NR_TXRX }; 346285349Sluigi 347285349Sluigistatic __inline const char* 348285349Sluiginm_txrx2str(enum txrx t) 349285349Sluigi{ 350285349Sluigi return (t== NR_RX ? "RX" : "TX"); 351285349Sluigi} 352285349Sluigi 353285349Sluigistatic __inline enum txrx 354285349Sluiginm_txrx_swap(enum txrx t) 355285349Sluigi{ 356285349Sluigi return (t== NR_RX ? NR_TX : NR_RX); 357285349Sluigi} 358285349Sluigi 359285349Sluigi#define for_rx_tx(t) for ((t) = 0; (t) < NR_TXRX; (t)++) 360285349Sluigi 361341477Svmaffione#ifdef WITH_MONITOR 362341477Svmaffionestruct netmap_zmon_list { 363341477Svmaffione struct netmap_kring *next; 364341477Svmaffione struct netmap_kring *prev; 365341477Svmaffione}; 366341477Svmaffione#endif /* WITH_MONITOR */ 367285349Sluigi 368227614Sluigi/* 369232238Sluigi * private, kernel view of a ring. Keeps track of the status of 370232238Sluigi * a ring across system calls. 371227614Sluigi * 372232238Sluigi * nr_hwcur index of the next buffer to refill. 373260368Sluigi * It corresponds to ring->head 374260368Sluigi * at the time the system call returns. 375232238Sluigi * 376260368Sluigi * nr_hwtail index of the first buffer owned by the kernel. 377260368Sluigi * On RX, hwcur->hwtail are receive buffers 378260368Sluigi * not yet released. hwcur is advanced following 379260368Sluigi * ring->head, hwtail is advanced on incoming packets, 380260368Sluigi * and a wakeup is generated when hwtail passes ring->cur 381260368Sluigi * On TX, hwcur->rcur have been filled by the sender 382260368Sluigi * but not sent yet to the NIC; rcur->hwtail are available 383260368Sluigi * for new transmissions, and hwtail->hwcur-1 are pending 384260368Sluigi * transmissions not yet acknowledged. 385232238Sluigi * 386231594Sluigi * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots. 387227614Sluigi * This is so that, on a reset, buffers owned by userspace are not 388227614Sluigi * modified by the kernel. In particular: 389260368Sluigi * RX rings: the next empty buffer (hwtail + hwofs) coincides with 390227614Sluigi * the next empty buffer as known by the hardware (next_to_check or so). 391227614Sluigi * TX rings: hwcur + hwofs coincides with next_to_send 392245579Sluigi * 393257529Sluigi * The following fields are used to implement lock-free copy of packets 394257529Sluigi * from input to output ports in VALE switch: 395257529Sluigi * nkr_hwlease buffer after the last one being copied. 396257529Sluigi * A writer in nm_bdg_flush reserves N buffers 397257529Sluigi * from nr_hwlease, advances it, then does the 398257529Sluigi * copy outside the lock. 399257529Sluigi * In RX rings (used for VALE ports), 400260368Sluigi * nkr_hwtail <= nkr_hwlease < nkr_hwcur+N-1 401257529Sluigi * In TX rings (used for NIC or host stack ports) 402260368Sluigi * nkr_hwcur <= nkr_hwlease < nkr_hwtail 403257529Sluigi * nkr_leases array of nkr_num_slots where writers can report 404257529Sluigi * completion of their block. NR_NOSLOT (~0) indicates 405257529Sluigi * that the writer has not finished yet 406259412Sluigi * nkr_lease_idx index of next free slot in nr_leases, to be assigned 407257529Sluigi * 408257529Sluigi * The kring is manipulated by txsync/rxsync and generic netmap function. 409260368Sluigi * 410260368Sluigi * Concurrent rxsync or txsync on the same ring are prevented through 411267165Sluigi * by nm_kr_(try)lock() which in turn uses nr_busy. This is all we need 412260368Sluigi * for NIC rings, and for TX rings attached to the host stack. 413260368Sluigi * 414260368Sluigi * RX rings attached to the host stack use an mbq (rx_queue) on both 415260368Sluigi * rxsync_from_host() and netmap_transmit(). The mbq is protected 416260368Sluigi * by its internal lock. 417260368Sluigi * 418270063Sluigi * RX rings attached to the VALE switch are accessed by both senders 419260368Sluigi * and receiver. They are protected through the q_lock on the RX ring. 420227614Sluigi */ 421227614Sluigistruct netmap_kring { 422260368Sluigi struct netmap_ring *ring; 423227614Sluigi 424341477Svmaffione uint32_t nr_hwcur; /* should be nr_hwhead */ 425260368Sluigi uint32_t nr_hwtail; 426260368Sluigi 427260368Sluigi /* 428260368Sluigi * Copies of values in user rings, so we do not need to look 429260368Sluigi * at the ring (which could be modified). These are set in the 430260368Sluigi * *sync_prologue()/finalize() routines. 431260368Sluigi */ 432260368Sluigi uint32_t rhead; 433260368Sluigi uint32_t rcur; 434260368Sluigi uint32_t rtail; 435260368Sluigi 436260368Sluigi uint32_t nr_kflags; /* private driver flags */ 437260368Sluigi#define NKR_PENDINTR 0x1 // Pending interrupt. 438285349Sluigi#define NKR_EXCLUSIVE 0x2 /* exclusive binding */ 439341477Svmaffione#define NKR_FORWARD 0x4 /* (host ring only) there are 440341477Svmaffione packets to forward 441341477Svmaffione */ 442341477Svmaffione#define NKR_NEEDRING 0x8 /* ring needed even if users==0 443341477Svmaffione * (used internally by pipes and 444341477Svmaffione * by ptnetmap host ports) 445341477Svmaffione */ 446341477Svmaffione#define NKR_NOINTR 0x10 /* don't use interrupts on this ring */ 447341477Svmaffione#define NKR_FAKERING 0x20 /* don't allocate/free buffers */ 448341477Svmaffione 449341477Svmaffione uint32_t nr_mode; 450341477Svmaffione uint32_t nr_pending_mode; 451341477Svmaffione#define NKR_NETMAP_OFF 0x0 452341477Svmaffione#define NKR_NETMAP_ON 0x1 453341477Svmaffione 454260368Sluigi uint32_t nkr_num_slots; 455260368Sluigi 456260368Sluigi /* 457260368Sluigi * On a NIC reset, the NIC ring indexes may be reset but the 458260368Sluigi * indexes in the netmap rings remain the same. nkr_hwofs 459260368Sluigi * keeps track of the offset between the two. 460260368Sluigi */ 461260368Sluigi int32_t nkr_hwofs; 462260368Sluigi 463260368Sluigi /* last_reclaim is opaque marker to help reduce the frequency 464260368Sluigi * of operations such as reclaiming tx buffers. A possible use 465260368Sluigi * is set it to ticks and do the reclaim only once per tick. 466260368Sluigi */ 467260368Sluigi uint64_t last_reclaim; 468260368Sluigi 469260368Sluigi 470260368Sluigi NM_SELINFO_T si; /* poll/select wait queue */ 471260368Sluigi NM_LOCK_T q_lock; /* protects kring and ring. */ 472260368Sluigi NM_ATOMIC_T nr_busy; /* prevent concurrent syscalls */ 473260368Sluigi 474341477Svmaffione /* the adapter the owns this kring */ 475231594Sluigi struct netmap_adapter *na; 476260368Sluigi 477341477Svmaffione /* the adapter that wants to be notified when this kring has 478341477Svmaffione * new slots avaialable. This is usually the same as the above, 479341477Svmaffione * but wrappers may let it point to themselves 480341477Svmaffione */ 481341477Svmaffione struct netmap_adapter *notify_na; 482341477Svmaffione 483274355Sluigi /* The following fields are for VALE switch support */ 484251139Sluigi struct nm_bdg_fwd *nkr_ft; 485260368Sluigi uint32_t *nkr_leases; 486260368Sluigi#define NR_NOSLOT ((uint32_t)~0) /* used in nkr_*lease* */ 487260368Sluigi uint32_t nkr_hwlease; 488260368Sluigi uint32_t nkr_lease_idx; 489257529Sluigi 490270063Sluigi /* while nkr_stopped is set, no new [tr]xsync operations can 491270063Sluigi * be started on this kring. 492270063Sluigi * This is used by netmap_disable_all_rings() 493270063Sluigi * to find a synchronization point where critical data 494270063Sluigi * structures pointed to by the kring can be added or removed 495270063Sluigi */ 496270063Sluigi volatile int nkr_stopped; 497257529Sluigi 498261909Sluigi /* Support for adapters without native netmap support. 499259412Sluigi * On tx rings we preallocate an array of tx buffers 500259412Sluigi * (same size as the netmap ring), on rx rings we 501261909Sluigi * store incoming mbufs in a queue that is drained by 502261909Sluigi * a rxsync. 503259412Sluigi */ 504341477Svmaffione struct mbuf **tx_pool; 505341477Svmaffione struct mbuf *tx_event; /* TX event used as a notification */ 506341477Svmaffione NM_LOCK_T tx_event_lock; /* protects the tx_event mbuf */ 507341477Svmaffione struct mbq rx_queue; /* intercepted rx mbufs. */ 508259412Sluigi 509285349Sluigi uint32_t users; /* existing bindings for this ring */ 510285349Sluigi 511341477Svmaffione uint32_t ring_id; /* kring identifier */ 512285349Sluigi enum txrx tx; /* kind of ring (tx or rx) */ 513260368Sluigi char name[64]; /* diagnostic */ 514260368Sluigi 515270063Sluigi /* [tx]sync callback for this kring. 516270063Sluigi * The default nm_kring_create callback (netmap_krings_create) 517270063Sluigi * sets the nm_sync callback of each hardware tx(rx) kring to 518270063Sluigi * the corresponding nm_txsync(nm_rxsync) taken from the 519270063Sluigi * netmap_adapter; moreover, it sets the sync callback 520270063Sluigi * of the host tx(rx) ring to netmap_txsync_to_host 521270063Sluigi * (netmap_rxsync_from_host). 522270063Sluigi * 523270063Sluigi * Overrides: the above configuration is not changed by 524270063Sluigi * any of the nm_krings_create callbacks. 525270063Sluigi */ 526261909Sluigi int (*nm_sync)(struct netmap_kring *kring, int flags); 527285349Sluigi int (*nm_notify)(struct netmap_kring *kring, int flags); 528261909Sluigi 529261909Sluigi#ifdef WITH_PIPES 530270063Sluigi struct netmap_kring *pipe; /* if this is a pipe ring, 531270063Sluigi * pointer to the other end 532270063Sluigi */ 533341477Svmaffione uint32_t pipe_tail; /* hwtail updated by the other end */ 534261909Sluigi#endif /* WITH_PIPES */ 535261909Sluigi 536285349Sluigi int (*save_notify)(struct netmap_kring *kring, int flags); 537285349Sluigi 538270063Sluigi#ifdef WITH_MONITOR 539285349Sluigi /* array of krings that are monitoring this kring */ 540285349Sluigi struct netmap_kring **monitors; 541285349Sluigi uint32_t max_monitors; /* current size of the monitors array */ 542285349Sluigi uint32_t n_monitors; /* next unused entry in the monitor array */ 543341477Svmaffione uint32_t mon_pos[NR_TXRX]; /* index of this ring in the monitored ring array */ 544341477Svmaffione uint32_t mon_tail; /* last seen slot on rx */ 545341477Svmaffione 546341477Svmaffione /* circular list of zero-copy monitors */ 547341477Svmaffione struct netmap_zmon_list zmon_list[NR_TXRX]; 548341477Svmaffione 549270063Sluigi /* 550285349Sluigi * Monitors work by intercepting the sync and notify callbacks of the 551285349Sluigi * monitored krings. This is implemented by replacing the pointers 552285349Sluigi * above and saving the previous ones in mon_* pointers below 553270063Sluigi */ 554285349Sluigi int (*mon_sync)(struct netmap_kring *kring, int flags); 555285349Sluigi int (*mon_notify)(struct netmap_kring *kring, int flags); 556285349Sluigi 557270063Sluigi#endif 558341477Svmaffione} 559341477Svmaffione#ifdef _WIN32 560341477Svmaffione__declspec(align(64)); 561341477Svmaffione#else 562341477Svmaffione__attribute__((__aligned__(64))); 563341477Svmaffione#endif 564227614Sluigi 565341477Svmaffione/* return 1 iff the kring needs to be turned on */ 566341477Svmaffionestatic inline int 567341477Svmaffionenm_kring_pending_on(struct netmap_kring *kring) 568341477Svmaffione{ 569341477Svmaffione return kring->nr_pending_mode == NKR_NETMAP_ON && 570341477Svmaffione kring->nr_mode == NKR_NETMAP_OFF; 571341477Svmaffione} 572257529Sluigi 573341477Svmaffione/* return 1 iff the kring needs to be turned off */ 574341477Svmaffionestatic inline int 575341477Svmaffionenm_kring_pending_off(struct netmap_kring *kring) 576341477Svmaffione{ 577341477Svmaffione return kring->nr_pending_mode == NKR_NETMAP_OFF && 578341477Svmaffione kring->nr_mode == NKR_NETMAP_ON; 579341477Svmaffione} 580341477Svmaffione 581257529Sluigi/* return the next index, with wraparound */ 582257529Sluigistatic inline uint32_t 583257529Sluiginm_next(uint32_t i, uint32_t lim) 584257529Sluigi{ 585257529Sluigi return unlikely (i == lim) ? 0 : i + 1; 586257529Sluigi} 587257529Sluigi 588260368Sluigi 589260368Sluigi/* return the previous index, with wraparound */ 590260368Sluigistatic inline uint32_t 591260368Sluiginm_prev(uint32_t i, uint32_t lim) 592260368Sluigi{ 593260368Sluigi return unlikely (i == 0) ? lim : i - 1; 594260368Sluigi} 595260368Sluigi 596260368Sluigi 597227614Sluigi/* 598257529Sluigi * 599257529Sluigi * Here is the layout for the Rx and Tx rings. 600257529Sluigi 601257529Sluigi RxRING TxRING 602257529Sluigi 603257529Sluigi +-----------------+ +-----------------+ 604257529Sluigi | | | | 605341477Svmaffione | free | | free | 606257529Sluigi +-----------------+ +-----------------+ 607260368Sluigihead->| owned by user |<-hwcur | not sent to nic |<-hwcur 608260368Sluigi | | | yet | 609260368Sluigi +-----------------+ | | 610260368Sluigi cur->| available to | | | 611260368Sluigi | user, not read | +-----------------+ 612260368Sluigi | yet | cur->| (being | 613260368Sluigi | | | prepared) | 614260368Sluigi | | | | 615260368Sluigi +-----------------+ + ------ + 616260368Sluigitail->| |<-hwtail | |<-hwlease 617260368Sluigi | (being | ... | | ... 618260368Sluigi | prepared) | ... | | ... 619260368Sluigi +-----------------+ ... | | ... 620257529Sluigi | |<-hwlease +-----------------+ 621260368Sluigi | | tail->| |<-hwtail 622257529Sluigi | | | | 623257529Sluigi | | | | 624257529Sluigi | | | | 625257529Sluigi +-----------------+ +-----------------+ 626257529Sluigi 627260368Sluigi * The cur/tail (user view) and hwcur/hwtail (kernel view) 628257529Sluigi * are used in the normal operation of the card. 629257529Sluigi * 630257529Sluigi * When a ring is the output of a switch port (Rx ring for 631257529Sluigi * a VALE port, Tx ring for the host stack or NIC), slots 632257529Sluigi * are reserved in blocks through 'hwlease' which points 633257529Sluigi * to the next unused slot. 634260368Sluigi * On an Rx ring, hwlease is always after hwtail, 635260368Sluigi * and completions cause hwtail to advance. 636260368Sluigi * On a Tx ring, hwlease is always between cur and hwtail, 637257529Sluigi * and completions cause cur to advance. 638257529Sluigi * 639257529Sluigi * nm_kr_space() returns the maximum number of slots that 640257529Sluigi * can be assigned. 641257529Sluigi * nm_kr_lease() reserves the required number of buffers, 642257529Sluigi * advances nkr_hwlease and also returns an entry in 643257529Sluigi * a circular array where completions should be reported. 644257529Sluigi */ 645257529Sluigi 646341477Svmaffionestruct lut_entry; 647341477Svmaffione#ifdef __FreeBSD__ 648341477Svmaffione#define plut_entry lut_entry 649341477Svmaffione#endif 650257529Sluigi 651285349Sluigistruct netmap_lut { 652285349Sluigi struct lut_entry *lut; 653341477Svmaffione struct plut_entry *plut; 654285349Sluigi uint32_t objtotal; /* max buffer index */ 655285349Sluigi uint32_t objsize; /* buffer size */ 656285349Sluigi}; 657257529Sluigi 658270063Sluigistruct netmap_vp_adapter; // forward 659341477Svmaffionestruct nm_bridge; 660270063Sluigi 661341477Svmaffione/* Struct to be filled by nm_config callbacks. */ 662341477Svmaffionestruct nm_config_info { 663341477Svmaffione unsigned num_tx_rings; 664341477Svmaffione unsigned num_rx_rings; 665341477Svmaffione unsigned num_tx_descs; 666341477Svmaffione unsigned num_rx_descs; 667341477Svmaffione unsigned rx_buf_maxsize; 668341477Svmaffione}; 669341477Svmaffione 670257529Sluigi/* 671341477Svmaffione * default type for the magic field. 672341477Svmaffione * May be overriden in glue code. 673341477Svmaffione */ 674341477Svmaffione#ifndef NM_OS_MAGIC 675341477Svmaffione#define NM_OS_MAGIC uint32_t 676341477Svmaffione#endif /* !NM_OS_MAGIC */ 677341477Svmaffione 678341477Svmaffione/* 679259412Sluigi * The "struct netmap_adapter" extends the "struct adapter" 680259412Sluigi * (or equivalent) device descriptor. 681259412Sluigi * It contains all base fields needed to support netmap operation. 682259412Sluigi * There are in fact different types of netmap adapters 683259412Sluigi * (native, generic, VALE switch...) so a netmap_adapter is 684259412Sluigi * just the first field in the derived type. 685227614Sluigi */ 686227614Sluigistruct netmap_adapter { 687241719Sluigi /* 688241719Sluigi * On linux we do not have a good way to tell if an interface 689259412Sluigi * is netmap-capable. So we always use the following trick: 690241719Sluigi * NA(ifp) points here, and the first entry (which hopefully 691241719Sluigi * always exists and is at least 32 bits) contains a magic 692241719Sluigi * value which we can use to detect that the interface is good. 693241719Sluigi */ 694341477Svmaffione NM_OS_MAGIC magic; 695259412Sluigi uint32_t na_flags; /* enabled, and other flags */ 696241719Sluigi#define NAF_SKIP_INTR 1 /* use the regular interrupt handler. 697241719Sluigi * useful during initialization 698241719Sluigi */ 699251139Sluigi#define NAF_SW_ONLY 2 /* forward packets only to sw adapter */ 700257529Sluigi#define NAF_BDG_MAYSLEEP 4 /* the bridge is allowed to sleep when 701257529Sluigi * forwarding packets coming from this 702257529Sluigi * interface 703257529Sluigi */ 704285349Sluigi#define NAF_MEM_OWNER 8 /* the adapter uses its own memory area 705285349Sluigi * that cannot be changed 706257529Sluigi */ 707285349Sluigi#define NAF_NATIVE 16 /* the adapter is native. 708285695Sluigi * Virtual ports (non persistent vale ports, 709285695Sluigi * pipes, monitors...) should never use 710285695Sluigi * this flag. 711259412Sluigi */ 712259412Sluigi#define NAF_NETMAP_ON 32 /* netmap is active (either native or 713270063Sluigi * emulated). Where possible (e.g. FreeBSD) 714259412Sluigi * IFCAP_NETMAP also mirrors this flag. 715259412Sluigi */ 716261909Sluigi#define NAF_HOST_RINGS 64 /* the adapter supports the host rings */ 717270063Sluigi#define NAF_FORCE_NATIVE 128 /* the adapter is always NATIVE */ 718342033Svmaffione/* free */ 719341477Svmaffione#define NAF_MOREFRAG 512 /* the adapter supports NS_MOREFRAG */ 720341477Svmaffione#define NAF_ZOMBIE (1U<<30) /* the nic driver has been unloaded */ 721270063Sluigi#define NAF_BUSY (1U<<31) /* the adapter is used internally and 722270063Sluigi * cannot be registered from userspace 723270063Sluigi */ 724259412Sluigi int active_fds; /* number of user-space descriptors using this 725227614Sluigi interface, which is equal to the number of 726227614Sluigi struct netmap_if objs in the mapped region. */ 727227614Sluigi 728239140Semaste u_int num_rx_rings; /* number of adapter receive rings */ 729239140Semaste u_int num_tx_rings; /* number of adapter transmit rings */ 730341477Svmaffione u_int num_host_rx_rings; /* number of host receive rings */ 731341477Svmaffione u_int num_host_tx_rings; /* number of host transmit rings */ 732227614Sluigi 733285349Sluigi u_int num_tx_desc; /* number of descriptor in each queue */ 734227614Sluigi u_int num_rx_desc; 735227614Sluigi 736342033Svmaffione /* tx_rings and rx_rings are private but allocated as a 737342033Svmaffione * contiguous chunk of memory. Each array has N+K entries, 738342033Svmaffione * N for the hardware rings and K for the host rings. 739227614Sluigi */ 740341477Svmaffione struct netmap_kring **tx_rings; /* array of TX rings. */ 741341477Svmaffione struct netmap_kring **rx_rings; /* array of RX rings. */ 742260368Sluigi 743259412Sluigi void *tailroom; /* space below the rings array */ 744259412Sluigi /* (used for leases) */ 745227614Sluigi 746259412Sluigi 747285349Sluigi NM_SELINFO_T si[NR_TXRX]; /* global wait queues */ 748232238Sluigi 749261909Sluigi /* count users of the global wait queues */ 750285349Sluigi int si_users[NR_TXRX]; 751261909Sluigi 752270063Sluigi void *pdev; /* used to store pci device */ 753270063Sluigi 754227614Sluigi /* copy of if_qflush and if_transmit pointers, to intercept 755227614Sluigi * packets from the network stack when netmap is active. 756227614Sluigi */ 757227614Sluigi int (*if_transmit)(struct ifnet *, struct mbuf *); 758227614Sluigi 759260368Sluigi /* copy of if_input for netmap_send_up() */ 760260368Sluigi void (*if_input)(struct ifnet *, struct mbuf *); 761260368Sluigi 762341477Svmaffione /* Back reference to the parent ifnet struct. Used for 763341477Svmaffione * hardware ports (emulated netmap included). */ 764227614Sluigi struct ifnet *ifp; /* adapter is ifp->if_softc */ 765227614Sluigi 766260368Sluigi /*---- callbacks for this netmap adapter -----*/ 767260368Sluigi /* 768260368Sluigi * nm_dtor() is the cleanup routine called when destroying 769260368Sluigi * the adapter. 770267165Sluigi * Called with NMG_LOCK held. 771260368Sluigi * 772260368Sluigi * nm_register() is called on NIOCREGIF and close() to enter 773260368Sluigi * or exit netmap mode on the NIC 774270063Sluigi * Called with NNG_LOCK held. 775260368Sluigi * 776260368Sluigi * nm_txsync() pushes packets to the underlying hw/switch 777260368Sluigi * 778260368Sluigi * nm_rxsync() collects packets from the underlying hw/switch 779260368Sluigi * 780260368Sluigi * nm_config() returns configuration information from the OS 781267165Sluigi * Called with NMG_LOCK held. 782260368Sluigi * 783270063Sluigi * nm_krings_create() create and init the tx_rings and 784270063Sluigi * rx_rings arrays of kring structures. In particular, 785270063Sluigi * set the nm_sync callbacks for each ring. 786270063Sluigi * There is no need to also allocate the corresponding 787270063Sluigi * netmap_rings, since netmap_mem_rings_create() will always 788270063Sluigi * be called to provide the missing ones. 789270063Sluigi * Called with NNG_LOCK held. 790260368Sluigi * 791270063Sluigi * nm_krings_delete() cleanup and delete the tx_rings and rx_rings 792270063Sluigi * arrays 793270063Sluigi * Called with NMG_LOCK held. 794260368Sluigi * 795267165Sluigi * nm_notify() is used to act after data have become available 796270063Sluigi * (or the stopped state of the ring has changed) 797260368Sluigi * For hw devices this is typically a selwakeup(), 798260368Sluigi * but for NIC/host ports attached to a switch (or vice-versa) 799260368Sluigi * we also need to invoke the 'txsync' code downstream. 800341477Svmaffione * This callback pointer is actually used only to initialize 801341477Svmaffione * kring->nm_notify. 802341477Svmaffione * Return values are the same as for netmap_rx_irq(). 803260368Sluigi */ 804259412Sluigi void (*nm_dtor)(struct netmap_adapter *); 805231594Sluigi 806259412Sluigi int (*nm_register)(struct netmap_adapter *, int onoff); 807341477Svmaffione void (*nm_intr)(struct netmap_adapter *, int onoff); 808257529Sluigi 809270063Sluigi int (*nm_txsync)(struct netmap_kring *kring, int flags); 810270063Sluigi int (*nm_rxsync)(struct netmap_kring *kring, int flags); 811285349Sluigi int (*nm_notify)(struct netmap_kring *kring, int flags); 812341477Svmaffione#define NAF_FORCE_READ 1 813341477Svmaffione#define NAF_FORCE_RECLAIM 2 814341477Svmaffione#define NAF_CAN_FORWARD_DOWN 4 815245835Sluigi /* return configuration information */ 816341477Svmaffione int (*nm_config)(struct netmap_adapter *, struct nm_config_info *info); 817259412Sluigi int (*nm_krings_create)(struct netmap_adapter *); 818259412Sluigi void (*nm_krings_delete)(struct netmap_adapter *); 819270063Sluigi /* 820270063Sluigi * nm_bdg_attach() initializes the na_vp field to point 821270063Sluigi * to an adapter that can be attached to a VALE switch. If the 822270063Sluigi * current adapter is already a VALE port, na_vp is simply a cast; 823270063Sluigi * otherwise, na_vp points to a netmap_bwrap_adapter. 824270063Sluigi * If applicable, this callback also initializes na_hostvp, 825270063Sluigi * that can be used to connect the adapter host rings to the 826270063Sluigi * switch. 827270063Sluigi * Called with NMG_LOCK held. 828270063Sluigi * 829270063Sluigi * nm_bdg_ctl() is called on the actual attach/detach to/from 830270063Sluigi * to/from the switch, to perform adapter-specific 831270063Sluigi * initializations 832270063Sluigi * Called with NMG_LOCK held. 833270063Sluigi */ 834341477Svmaffione int (*nm_bdg_attach)(const char *bdg_name, struct netmap_adapter *, 835341477Svmaffione struct nm_bridge *); 836341477Svmaffione int (*nm_bdg_ctl)(struct nmreq_header *, struct netmap_adapter *); 837270063Sluigi 838270063Sluigi /* adapter used to attach this adapter to a VALE switch (if any) */ 839270063Sluigi struct netmap_vp_adapter *na_vp; 840270063Sluigi /* adapter used to attach the host rings of this adapter 841270063Sluigi * to a VALE switch (if any) */ 842270063Sluigi struct netmap_vp_adapter *na_hostvp; 843270063Sluigi 844259412Sluigi /* standard refcount to control the lifetime of the adapter 845259412Sluigi * (it should be equal to the lifetime of the corresponding ifp) 846259412Sluigi */ 847259412Sluigi int na_refcount; 848259412Sluigi 849259412Sluigi /* memory allocator (opaque) 850259412Sluigi * We also cache a pointer to the lut_entry for translating 851341477Svmaffione * buffer addresses, the total number of buffers and the buffer size. 852259412Sluigi */ 853259412Sluigi struct netmap_mem_d *nm_mem; 854341477Svmaffione struct netmap_mem_d *nm_mem_prev; 855285349Sluigi struct netmap_lut na_lut; 856259412Sluigi 857270063Sluigi /* additional information attached to this adapter 858270063Sluigi * by other netmap subsystems. Currently used by 859341477Svmaffione * bwrap, LINUX/v1000 and ptnetmap 860259412Sluigi */ 861259412Sluigi void *na_private; 862261909Sluigi 863270063Sluigi /* array of pipes that have this adapter as a parent */ 864261909Sluigi struct netmap_pipe_adapter **na_pipes; 865270063Sluigi int na_next_pipe; /* next free slot in the array */ 866270063Sluigi int na_max_pipes; /* size of the array */ 867270063Sluigi 868341477Svmaffione /* Offset of ethernet header for each packet. */ 869341477Svmaffione u_int virt_hdr_len; 870341477Svmaffione 871341477Svmaffione /* Max number of bytes that the NIC can store in the buffer 872341477Svmaffione * referenced by each RX descriptor. This translates to the maximum 873341477Svmaffione * bytes that a single netmap slot can reference. Larger packets 874341477Svmaffione * require NS_MOREFRAG support. */ 875341477Svmaffione unsigned rx_buf_maxsize; 876341477Svmaffione 877341477Svmaffione char name[NETMAP_REQ_IFNAMSIZ]; /* used at least by pipes */ 878341477Svmaffione 879341477Svmaffione#ifdef WITH_MONITOR 880341477Svmaffione unsigned long monitor_id; /* debugging */ 881341477Svmaffione#endif 882259412Sluigi}; 883259412Sluigi 884285349Sluigistatic __inline u_int 885285349Sluiginma_get_ndesc(struct netmap_adapter *na, enum txrx t) 886285349Sluigi{ 887285349Sluigi return (t == NR_TX ? na->num_tx_desc : na->num_rx_desc); 888285349Sluigi} 889260368Sluigi 890285349Sluigistatic __inline void 891285349Sluiginma_set_ndesc(struct netmap_adapter *na, enum txrx t, u_int v) 892285349Sluigi{ 893285349Sluigi if (t == NR_TX) 894285349Sluigi na->num_tx_desc = v; 895285349Sluigi else 896285349Sluigi na->num_rx_desc = v; 897285349Sluigi} 898285349Sluigi 899285349Sluigistatic __inline u_int 900285349Sluiginma_get_nrings(struct netmap_adapter *na, enum txrx t) 901285349Sluigi{ 902285349Sluigi return (t == NR_TX ? na->num_tx_rings : na->num_rx_rings); 903285349Sluigi} 904285349Sluigi 905341477Svmaffionestatic __inline u_int 906341477Svmaffionenma_get_host_nrings(struct netmap_adapter *na, enum txrx t) 907341477Svmaffione{ 908341477Svmaffione return (t == NR_TX ? na->num_host_tx_rings : na->num_host_rx_rings); 909341477Svmaffione} 910341477Svmaffione 911285349Sluigistatic __inline void 912285349Sluiginma_set_nrings(struct netmap_adapter *na, enum txrx t, u_int v) 913285349Sluigi{ 914285349Sluigi if (t == NR_TX) 915285349Sluigi na->num_tx_rings = v; 916285349Sluigi else 917285349Sluigi na->num_rx_rings = v; 918285349Sluigi} 919285349Sluigi 920341477Svmaffionestatic __inline void 921341477Svmaffionenma_set_host_nrings(struct netmap_adapter *na, enum txrx t, u_int v) 922341477Svmaffione{ 923341477Svmaffione if (t == NR_TX) 924341477Svmaffione na->num_host_tx_rings = v; 925341477Svmaffione else 926341477Svmaffione na->num_host_rx_rings = v; 927341477Svmaffione} 928341477Svmaffione 929341477Svmaffionestatic __inline struct netmap_kring** 930285349SluigiNMR(struct netmap_adapter *na, enum txrx t) 931285349Sluigi{ 932285349Sluigi return (t == NR_TX ? na->tx_rings : na->rx_rings); 933285349Sluigi} 934285349Sluigi 935341477Svmaffioneint nma_intr_enable(struct netmap_adapter *na, int onoff); 936341477Svmaffione 937259412Sluigi/* 938259412Sluigi * If the NIC is owned by the kernel 939259412Sluigi * (i.e., bridge), neither another bridge nor user can use it; 940259412Sluigi * if the NIC is owned by a user, only users can share it. 941259412Sluigi * Evaluation must be done under NMG_LOCK(). 942259412Sluigi */ 943270063Sluigi#define NETMAP_OWNED_BY_KERN(na) ((na)->na_flags & NAF_BUSY) 944259412Sluigi#define NETMAP_OWNED_BY_ANY(na) \ 945270063Sluigi (NETMAP_OWNED_BY_KERN(na) || ((na)->active_fds > 0)) 946259412Sluigi 947259412Sluigi/* 948259412Sluigi * derived netmap adapters for various types of ports 949259412Sluigi */ 950259412Sluigistruct netmap_vp_adapter { /* VALE software port */ 951259412Sluigi struct netmap_adapter up; 952259412Sluigi 953250107Sluigi /* 954250107Sluigi * Bridge support: 955250107Sluigi * 956250107Sluigi * bdg_port is the port number used in the bridge; 957251139Sluigi * na_bdg points to the bridge this NA is attached to. 958250107Sluigi */ 959238812Sluigi int bdg_port; 960251139Sluigi struct nm_bridge *na_bdg; 961259412Sluigi int retry; 962341477Svmaffione int autodelete; /* remove the ifp on last reference */ 963259412Sluigi 964261909Sluigi /* Maximum Frame Size, used in bdg_mismatch_datapath() */ 965261909Sluigi u_int mfs; 966285349Sluigi /* Last source MAC on this port */ 967285349Sluigi uint64_t last_smac; 968259412Sluigi}; 969259412Sluigi 970260368Sluigi 971259412Sluigistruct netmap_hw_adapter { /* physical device */ 972259412Sluigi struct netmap_adapter up; 973259412Sluigi 974341477Svmaffione#ifdef linux 975341477Svmaffione struct net_device_ops nm_ndo; 976341477Svmaffione struct ethtool_ops nm_eto; 977341477Svmaffione#endif 978270063Sluigi const struct ethtool_ops* save_ethtool; 979270063Sluigi 980270063Sluigi int (*nm_hw_register)(struct netmap_adapter *, int onoff); 981259412Sluigi}; 982259412Sluigi 983274362Sluigi#ifdef WITH_GENERIC 984261909Sluigi/* Mitigation support. */ 985261909Sluigistruct nm_generic_mit { 986261909Sluigi struct hrtimer mit_timer; 987261909Sluigi int mit_pending; 988270063Sluigi int mit_ring_idx; /* index of the ring being mitigated */ 989261909Sluigi struct netmap_adapter *mit_na; /* backpointer */ 990261909Sluigi}; 991260368Sluigi 992260368Sluigistruct netmap_generic_adapter { /* emulated device */ 993259412Sluigi struct netmap_hw_adapter up; 994259412Sluigi 995259412Sluigi /* Pointer to a previously used netmap adapter. */ 996259412Sluigi struct netmap_adapter *prev; 997259412Sluigi 998341477Svmaffione /* Emulated netmap adapters support: 999341477Svmaffione * - save_if_input saves the if_input hook (FreeBSD); 1000341477Svmaffione * - mit implements rx interrupt mitigation; 1001259412Sluigi */ 1002259412Sluigi void (*save_if_input)(struct ifnet *, struct mbuf *); 1003259412Sluigi 1004261909Sluigi struct nm_generic_mit *mit; 1005260368Sluigi#ifdef linux 1006260368Sluigi netdev_tx_t (*save_start_xmit)(struct mbuf *, struct ifnet *); 1007260368Sluigi#endif 1008341477Svmaffione /* Is the adapter able to use multiple RX slots to scatter 1009341477Svmaffione * each packet pushed up by the driver? */ 1010341477Svmaffione int rxsg; 1011341477Svmaffione 1012341477Svmaffione /* Is the transmission path controlled by a netmap-aware 1013341477Svmaffione * device queue (i.e. qdisc on linux)? */ 1014341477Svmaffione int txqdisc; 1015259412Sluigi}; 1016274362Sluigi#endif /* WITH_GENERIC */ 1017259412Sluigi 1018341477Svmaffionestatic __inline u_int 1019285349Sluiginetmap_real_rings(struct netmap_adapter *na, enum txrx t) 1020261909Sluigi{ 1021341477Svmaffione return nma_get_nrings(na, t) + 1022341477Svmaffione !!(na->na_flags & NAF_HOST_RINGS) * nma_get_host_nrings(na, t); 1023261909Sluigi} 1024261909Sluigi 1025341477Svmaffione/* account for fake rings */ 1026341477Svmaffionestatic __inline u_int 1027341477Svmaffionenetmap_all_rings(struct netmap_adapter *na, enum txrx t) 1028341477Svmaffione{ 1029341477Svmaffione return max(nma_get_nrings(na, t) + 1, netmap_real_rings(na, t)); 1030341477Svmaffione} 1031259412Sluigi 1032341477Svmaffioneint netmap_default_bdg_attach(const char *name, struct netmap_adapter *na, 1033341477Svmaffione struct nm_bridge *); 1034341477Svmaffionestruct nm_bdg_polling_state; 1035260368Sluigi/* 1036260368Sluigi * Bridge wrapper for non VALE ports attached to a VALE switch. 1037259412Sluigi * 1038260368Sluigi * The real device must already have its own netmap adapter (hwna). 1039260368Sluigi * The bridge wrapper and the hwna adapter share the same set of 1040260368Sluigi * netmap rings and buffers, but they have two separate sets of 1041260368Sluigi * krings descriptors, with tx/rx meanings swapped: 1042259412Sluigi * 1043259412Sluigi * netmap 1044259412Sluigi * bwrap krings rings krings hwna 1045259412Sluigi * +------+ +------+ +-----+ +------+ +------+ 1046259412Sluigi * |tx_rings->| |\ /| |----| |<-tx_rings| 1047259412Sluigi * | | +------+ \ / +-----+ +------+ | | 1048259412Sluigi * | | X | | 1049259412Sluigi * | | / \ | | 1050259412Sluigi * | | +------+/ \+-----+ +------+ | | 1051259412Sluigi * |rx_rings->| | | |----| |<-rx_rings| 1052259412Sluigi * | | +------+ +-----+ +------+ | | 1053259412Sluigi * +------+ +------+ 1054259412Sluigi * 1055260368Sluigi * - packets coming from the bridge go to the brwap rx rings, 1056260368Sluigi * which are also the hwna tx rings. The bwrap notify callback 1057260368Sluigi * will then complete the hwna tx (see netmap_bwrap_notify). 1058259412Sluigi * 1059260368Sluigi * - packets coming from the outside go to the hwna rx rings, 1060260368Sluigi * which are also the bwrap tx rings. The (overwritten) hwna 1061260368Sluigi * notify method will then complete the bridge tx 1062260368Sluigi * (see netmap_bwrap_intr_notify). 1063259412Sluigi * 1064260368Sluigi * The bridge wrapper may optionally connect the hwna 'host' rings 1065260368Sluigi * to the bridge. This is done by using a second port in the 1066260368Sluigi * bridge and connecting it to the 'host' netmap_vp_adapter 1067260368Sluigi * contained in the netmap_bwrap_adapter. The brwap host adapter 1068260368Sluigi * cross-links the hwna host rings in the same way as shown above. 1069259412Sluigi * 1070260368Sluigi * - packets coming from the bridge and directed to the host stack 1071260368Sluigi * are handled by the bwrap host notify callback 1072260368Sluigi * (see netmap_bwrap_host_notify) 1073260368Sluigi * 1074260368Sluigi * - packets coming from the host stack are still handled by the 1075260368Sluigi * overwritten hwna notify callback (netmap_bwrap_intr_notify), 1076260368Sluigi * but are diverted to the host adapter depending on the ring number. 1077260368Sluigi * 1078259412Sluigi */ 1079259412Sluigistruct netmap_bwrap_adapter { 1080259412Sluigi struct netmap_vp_adapter up; 1081259412Sluigi struct netmap_vp_adapter host; /* for host rings */ 1082259412Sluigi struct netmap_adapter *hwna; /* the underlying device */ 1083259412Sluigi 1084260368Sluigi /* 1085260368Sluigi * When we attach a physical interface to the bridge, we 1086251139Sluigi * allow the controlling process to terminate, so we need 1087270063Sluigi * a place to store the n_detmap_priv_d data structure. 1088260368Sluigi * This is only done when physical interfaces 1089260368Sluigi * are attached to a bridge. 1090251139Sluigi */ 1091251139Sluigi struct netmap_priv_d *na_kpriv; 1092341477Svmaffione struct nm_bdg_polling_state *na_polling_state; 1093341477Svmaffione /* we overwrite the hwna->na_vp pointer, so we save 1094341477Svmaffione * here its original value, to be restored at detach 1095341477Svmaffione */ 1096341477Svmaffione struct netmap_vp_adapter *saved_na_vp; 1097227614Sluigi}; 1098341477Svmaffioneint nm_bdg_polling(struct nmreq_header *hdr); 1099227614Sluigi 1100341477Svmaffione#ifdef WITH_VALE 1101342033Svmaffioneint netmap_vale_attach(struct nmreq_header *hdr, void *auth_token); 1102342033Svmaffioneint netmap_vale_detach(struct nmreq_header *hdr, void *auth_token); 1103342033Svmaffioneint netmap_vale_list(struct nmreq_header *hdr); 1104341477Svmaffioneint netmap_vi_create(struct nmreq_header *hdr, int); 1105341477Svmaffioneint nm_vi_create(struct nmreq_header *); 1106341477Svmaffioneint nm_vi_destroy(const char *name); 1107341477Svmaffione#else /* !WITH_VALE */ 1108341477Svmaffione#define netmap_vi_create(hdr, a) (EOPNOTSUPP) 1109260368Sluigi#endif /* WITH_VALE */ 1110257529Sluigi 1111261909Sluigi#ifdef WITH_PIPES 1112257529Sluigi 1113261909Sluigi#define NM_MAXPIPES 64 /* max number of pipes per adapter */ 1114261909Sluigi 1115261909Sluigistruct netmap_pipe_adapter { 1116341477Svmaffione /* pipe identifier is up.name */ 1117261909Sluigi struct netmap_adapter up; 1118261909Sluigi 1119341477Svmaffione#define NM_PIPE_ROLE_MASTER 0x1 1120341477Svmaffione#define NM_PIPE_ROLE_SLAVE 0x2 1121341477Svmaffione int role; /* either NM_PIPE_ROLE_MASTER or NM_PIPE_ROLE_SLAVE */ 1122261909Sluigi 1123261909Sluigi struct netmap_adapter *parent; /* adapter that owns the memory */ 1124261909Sluigi struct netmap_pipe_adapter *peer; /* the other end of the pipe */ 1125261909Sluigi int peer_ref; /* 1 iff we are holding a ref to the peer */ 1126341477Svmaffione struct ifnet *parent_ifp; /* maybe null */ 1127261909Sluigi 1128261909Sluigi u_int parent_slot; /* index in the parent pipe array */ 1129261909Sluigi}; 1130261909Sluigi 1131261909Sluigi#endif /* WITH_PIPES */ 1132261909Sluigi 1133342033Svmaffione#ifdef WITH_NMNULL 1134342033Svmaffionestruct netmap_null_adapter { 1135342033Svmaffione struct netmap_adapter up; 1136342033Svmaffione}; 1137342033Svmaffione#endif /* WITH_NMNULL */ 1138261909Sluigi 1139342033Svmaffione 1140260368Sluigi/* return slots reserved to rx clients; used in drivers */ 1141257529Sluigistatic inline uint32_t 1142260368Sluiginm_kr_rxspace(struct netmap_kring *k) 1143257529Sluigi{ 1144260368Sluigi int space = k->nr_hwtail - k->nr_hwcur; 1145267128Sluigi if (space < 0) 1146260368Sluigi space += k->nkr_num_slots; 1147344047Svmaffione nm_prdis("preserving %d rx slots %d -> %d", space, k->nr_hwcur, k->nr_hwtail); 1148257529Sluigi 1149260368Sluigi return space; 1150257529Sluigi} 1151257529Sluigi 1152341477Svmaffione/* return slots reserved to tx clients */ 1153341477Svmaffione#define nm_kr_txspace(_k) nm_kr_rxspace(_k) 1154257529Sluigi 1155341477Svmaffione 1156341477Svmaffione/* True if no space in the tx ring, only valid after txsync_prologue */ 1157260368Sluigistatic inline int 1158260368Sluiginm_kr_txempty(struct netmap_kring *kring) 1159259412Sluigi{ 1160343832Svmaffione return kring->rhead == kring->nr_hwtail; 1161259412Sluigi} 1162259412Sluigi 1163341477Svmaffione/* True if no more completed slots in the rx ring, only valid after 1164341477Svmaffione * rxsync_prologue */ 1165341477Svmaffione#define nm_kr_rxempty(_k) nm_kr_txempty(_k) 1166259412Sluigi 1167343866Svmaffione/* True if the application needs to wait for more space on the ring 1168343866Svmaffione * (more received packets or more free tx slots). 1169343866Svmaffione * Only valid after *xsync_prologue. */ 1170343866Svmaffionestatic inline int 1171343866Svmaffionenm_kr_wouldblock(struct netmap_kring *kring) 1172343866Svmaffione{ 1173343866Svmaffione return kring->rcur == kring->nr_hwtail; 1174343866Svmaffione} 1175343866Svmaffione 1176257529Sluigi/* 1177259412Sluigi * protect against multiple threads using the same ring. 1178341477Svmaffione * also check that the ring has not been stopped or locked 1179227614Sluigi */ 1180341477Svmaffione#define NM_KR_BUSY 1 /* some other thread is syncing the ring */ 1181341477Svmaffione#define NM_KR_STOPPED 2 /* unbounded stop (ifconfig down or driver unload) */ 1182341477Svmaffione#define NM_KR_LOCKED 3 /* bounded, brief stop for mutual exclusion */ 1183227614Sluigi 1184260368Sluigi 1185341477Svmaffione/* release the previously acquired right to use the *sync() methods of the ring */ 1186259412Sluigistatic __inline void nm_kr_put(struct netmap_kring *kr) 1187259412Sluigi{ 1188259412Sluigi NM_ATOMIC_CLEAR(&kr->nr_busy); 1189259412Sluigi} 1190227614Sluigi 1191260368Sluigi 1192341477Svmaffione/* true if the ifp that backed the adapter has disappeared (e.g., the 1193341477Svmaffione * driver has been unloaded) 1194341477Svmaffione */ 1195341477Svmaffionestatic inline int nm_iszombie(struct netmap_adapter *na); 1196341477Svmaffione 1197341477Svmaffione/* try to obtain exclusive right to issue the *sync() operations on the ring. 1198341477Svmaffione * The right is obtained and must be later relinquished via nm_kr_put() if and 1199341477Svmaffione * only if nm_kr_tryget() returns 0. 1200341477Svmaffione * If can_sleep is 1 there are only two other possible outcomes: 1201341477Svmaffione * - the function returns NM_KR_BUSY 1202341477Svmaffione * - the function returns NM_KR_STOPPED and sets the POLLERR bit in *perr 1203341477Svmaffione * (if non-null) 1204341477Svmaffione * In both cases the caller will typically skip the ring, possibly collecting 1205341477Svmaffione * errors along the way. 1206341477Svmaffione * If the calling context does not allow sleeping, the caller must pass 0 in can_sleep. 1207341477Svmaffione * In the latter case, the function may also return NM_KR_LOCKED and leave *perr 1208341477Svmaffione * untouched: ideally, the caller should try again at a later time. 1209341477Svmaffione */ 1210341477Svmaffionestatic __inline int nm_kr_tryget(struct netmap_kring *kr, int can_sleep, int *perr) 1211259412Sluigi{ 1212341477Svmaffione int busy = 1, stopped; 1213259412Sluigi /* check a first time without taking the lock 1214259412Sluigi * to avoid starvation for nm_kr_get() 1215259412Sluigi */ 1216341477Svmaffioneretry: 1217341477Svmaffione stopped = kr->nkr_stopped; 1218341477Svmaffione if (unlikely(stopped)) { 1219341477Svmaffione goto stop; 1220259412Sluigi } 1221341477Svmaffione busy = NM_ATOMIC_TEST_AND_SET(&kr->nr_busy); 1222341477Svmaffione /* we should not return NM_KR_BUSY if the ring was 1223341477Svmaffione * actually stopped, so check another time after 1224341477Svmaffione * the barrier provided by the atomic operation 1225341477Svmaffione */ 1226341477Svmaffione stopped = kr->nkr_stopped; 1227341477Svmaffione if (unlikely(stopped)) { 1228341477Svmaffione goto stop; 1229341477Svmaffione } 1230341477Svmaffione 1231341477Svmaffione if (unlikely(nm_iszombie(kr->na))) { 1232341477Svmaffione stopped = NM_KR_STOPPED; 1233341477Svmaffione goto stop; 1234341477Svmaffione } 1235341477Svmaffione 1236341477Svmaffione return unlikely(busy) ? NM_KR_BUSY : 0; 1237341477Svmaffione 1238341477Svmaffionestop: 1239341477Svmaffione if (!busy) 1240259412Sluigi nm_kr_put(kr); 1241341477Svmaffione if (stopped == NM_KR_STOPPED) { 1242341477Svmaffione/* if POLLERR is defined we want to use it to simplify netmap_poll(). 1243341477Svmaffione * Otherwise, any non-zero value will do. 1244341477Svmaffione */ 1245341477Svmaffione#ifdef POLLERR 1246341477Svmaffione#define NM_POLLERR POLLERR 1247341477Svmaffione#else 1248341477Svmaffione#define NM_POLLERR 1 1249341477Svmaffione#endif /* POLLERR */ 1250341477Svmaffione if (perr) 1251341477Svmaffione *perr |= NM_POLLERR; 1252341477Svmaffione#undef NM_POLLERR 1253341477Svmaffione } else if (can_sleep) { 1254341477Svmaffione tsleep(kr, 0, "NM_KR_TRYGET", 4); 1255341477Svmaffione goto retry; 1256259412Sluigi } 1257341477Svmaffione return stopped; 1258259412Sluigi} 1259227614Sluigi 1260341477Svmaffione/* put the ring in the 'stopped' state and wait for the current user (if any) to 1261341477Svmaffione * notice. stopped must be either NM_KR_STOPPED or NM_KR_LOCKED 1262341477Svmaffione */ 1263341477Svmaffionestatic __inline void nm_kr_stop(struct netmap_kring *kr, int stopped) 1264285349Sluigi{ 1265341477Svmaffione kr->nkr_stopped = stopped; 1266285349Sluigi while (NM_ATOMIC_TEST_AND_SET(&kr->nr_busy)) 1267285349Sluigi tsleep(kr, 0, "NM_KR_GET", 4); 1268285349Sluigi} 1269259412Sluigi 1270341477Svmaffione/* restart a ring after a stop */ 1271341477Svmaffionestatic __inline void nm_kr_start(struct netmap_kring *kr) 1272341477Svmaffione{ 1273341477Svmaffione kr->nkr_stopped = 0; 1274341477Svmaffione nm_kr_put(kr); 1275341477Svmaffione} 1276285349Sluigi 1277341477Svmaffione 1278227614Sluigi/* 1279260368Sluigi * The following functions are used by individual drivers to 1280227614Sluigi * support netmap operation. 1281227614Sluigi * 1282227614Sluigi * netmap_attach() initializes a struct netmap_adapter, allocating the 1283227614Sluigi * struct netmap_ring's and the struct selinfo. 1284227614Sluigi * 1285227614Sluigi * netmap_detach() frees the memory allocated by netmap_attach(). 1286227614Sluigi * 1287257529Sluigi * netmap_transmit() replaces the if_transmit routine of the interface, 1288227614Sluigi * and is used to intercept packets coming from the stack. 1289227614Sluigi * 1290227614Sluigi * netmap_load_map/netmap_reload_map are helper routines to set/reset 1291227614Sluigi * the dmamap for a packet buffer 1292227614Sluigi * 1293270063Sluigi * netmap_reset() is a helper routine to be called in the hw driver 1294270063Sluigi * when reinitializing a ring. It should not be called by 1295270063Sluigi * virtual ports (vale, pipes, monitor) 1296227614Sluigi */ 1297259412Sluigiint netmap_attach(struct netmap_adapter *); 1298341477Svmaffioneint netmap_attach_ext(struct netmap_adapter *, size_t size, int override_reg); 1299227614Sluigivoid netmap_detach(struct ifnet *); 1300257529Sluigiint netmap_transmit(struct ifnet *, struct mbuf *); 1301227614Sluigistruct netmap_slot *netmap_reset(struct netmap_adapter *na, 1302257529Sluigi enum txrx tx, u_int n, u_int new_cur); 1303227614Sluigiint netmap_ring_reinit(struct netmap_kring *); 1304341477Svmaffioneint netmap_rings_config_get(struct netmap_adapter *, struct nm_config_info *); 1305227614Sluigi 1306341477Svmaffione/* Return codes for netmap_*x_irq. */ 1307341477Svmaffioneenum { 1308341477Svmaffione /* Driver should do normal interrupt processing, e.g. because 1309341477Svmaffione * the interface is not in netmap mode. */ 1310341477Svmaffione NM_IRQ_PASS = 0, 1311341477Svmaffione /* Port is in netmap mode, and the interrupt work has been 1312341477Svmaffione * completed. The driver does not have to notify netmap 1313341477Svmaffione * again before the next interrupt. */ 1314341477Svmaffione NM_IRQ_COMPLETED = -1, 1315341477Svmaffione /* Port is in netmap mode, but the interrupt work has not been 1316341477Svmaffione * completed. The driver has to make sure netmap will be 1317341477Svmaffione * notified again soon, even if no more interrupts come (e.g. 1318341477Svmaffione * on Linux the driver should not call napi_complete()). */ 1319341477Svmaffione NM_IRQ_RESCHED = -2, 1320341477Svmaffione}; 1321341477Svmaffione 1322260368Sluigi/* default functions to handle rx/tx interrupts */ 1323260368Sluigiint netmap_rx_irq(struct ifnet *, u_int, u_int *); 1324260368Sluigi#define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL) 1325341477Svmaffioneint netmap_common_irq(struct netmap_adapter *, u_int, u_int *work_done); 1326260368Sluigi 1327260368Sluigi 1328270063Sluigi#ifdef WITH_VALE 1329270063Sluigi/* functions used by external modules to interface with VALE */ 1330270063Sluigi#define netmap_vp_to_ifp(_vp) ((_vp)->up.ifp) 1331270063Sluigi#define netmap_ifp_to_vp(_ifp) (NA(_ifp)->na_vp) 1332270063Sluigi#define netmap_ifp_to_host_vp(_ifp) (NA(_ifp)->na_hostvp) 1333270063Sluigi#define netmap_bdg_idx(_vp) ((_vp)->bdg_port) 1334270063Sluigiconst char *netmap_bdg_name(struct netmap_vp_adapter *); 1335270063Sluigi#else /* !WITH_VALE */ 1336270063Sluigi#define netmap_vp_to_ifp(_vp) NULL 1337270063Sluigi#define netmap_ifp_to_vp(_ifp) NULL 1338270063Sluigi#define netmap_ifp_to_host_vp(_ifp) NULL 1339270063Sluigi#define netmap_bdg_idx(_vp) -1 1340270063Sluigi#endif /* WITH_VALE */ 1341260368Sluigi 1342270063Sluigistatic inline int 1343285349Sluiginm_netmap_on(struct netmap_adapter *na) 1344270063Sluigi{ 1345285349Sluigi return na && na->na_flags & NAF_NETMAP_ON; 1346270063Sluigi} 1347270063Sluigi 1348270063Sluigistatic inline int 1349285349Sluiginm_native_on(struct netmap_adapter *na) 1350270063Sluigi{ 1351285349Sluigi return nm_netmap_on(na) && (na->na_flags & NAF_NATIVE); 1352270063Sluigi} 1353270063Sluigi 1354341477Svmaffionestatic inline int 1355341477Svmaffionenm_iszombie(struct netmap_adapter *na) 1356259412Sluigi{ 1357341477Svmaffione return na == NULL || (na->na_flags & NAF_ZOMBIE); 1358259412Sluigi} 1359259412Sluigi 1360259412Sluigistatic inline void 1361341477Svmaffionenm_update_hostrings_mode(struct netmap_adapter *na) 1362259412Sluigi{ 1363341477Svmaffione /* Process nr_mode and nr_pending_mode for host rings. */ 1364341477Svmaffione na->tx_rings[na->num_tx_rings]->nr_mode = 1365341477Svmaffione na->tx_rings[na->num_tx_rings]->nr_pending_mode; 1366341477Svmaffione na->rx_rings[na->num_rx_rings]->nr_mode = 1367341477Svmaffione na->rx_rings[na->num_rx_rings]->nr_pending_mode; 1368259412Sluigi} 1369259412Sluigi 1370341477Svmaffionevoid nm_set_native_flags(struct netmap_adapter *); 1371341477Svmaffionevoid nm_clear_native_flags(struct netmap_adapter *); 1372260368Sluigi 1373344047Svmaffionevoid netmap_krings_mode_commit(struct netmap_adapter *na, int onoff); 1374344047Svmaffione 1375341477Svmaffione/* 1376341477Svmaffione * nm_*sync_prologue() functions are used in ioctl/poll and ptnetmap 1377341477Svmaffione * kthreads. 1378341477Svmaffione * We need netmap_ring* parameter, because in ptnetmap it is decoupled 1379341477Svmaffione * from host kring. 1380341477Svmaffione * The user-space ring pointers (head/cur/tail) are shared through 1381341477Svmaffione * CSB between host and guest. 1382341477Svmaffione */ 1383341477Svmaffione 1384341477Svmaffione/* 1385341477Svmaffione * validates parameters in the ring/kring, returns a value for head 1386341477Svmaffione * If any error, returns ring_size to force a reinit. 1387341477Svmaffione */ 1388341477Svmaffioneuint32_t nm_txsync_prologue(struct netmap_kring *, struct netmap_ring *); 1389341477Svmaffione 1390341477Svmaffione 1391341477Svmaffione/* 1392341477Svmaffione * validates parameters in the ring/kring, returns a value for head 1393341477Svmaffione * If any error, returns ring_size lim to force a reinit. 1394341477Svmaffione */ 1395341477Svmaffioneuint32_t nm_rxsync_prologue(struct netmap_kring *, struct netmap_ring *); 1396341477Svmaffione 1397341477Svmaffione 1398259412Sluigi/* check/fix address and len in tx rings */ 1399259412Sluigi#if 1 /* debug version */ 1400270063Sluigi#define NM_CHECK_ADDR_LEN(_na, _a, _l) do { \ 1401270063Sluigi if (_a == NETMAP_BUF_BASE(_na) || _l > NETMAP_BUF_SIZE(_na)) { \ 1402344047Svmaffione nm_prlim(5, "bad addr/len ring %d slot %d idx %d len %d", \ 1403270063Sluigi kring->ring_id, nm_i, slot->buf_idx, len); \ 1404270063Sluigi if (_l > NETMAP_BUF_SIZE(_na)) \ 1405270063Sluigi _l = NETMAP_BUF_SIZE(_na); \ 1406259412Sluigi } } while (0) 1407259412Sluigi#else /* no debug version */ 1408270063Sluigi#define NM_CHECK_ADDR_LEN(_na, _a, _l) do { \ 1409270063Sluigi if (_l > NETMAP_BUF_SIZE(_na)) \ 1410270063Sluigi _l = NETMAP_BUF_SIZE(_na); \ 1411259412Sluigi } while (0) 1412259412Sluigi#endif 1413259412Sluigi 1414259412Sluigi 1415259412Sluigi/*---------------------------------------------------------------*/ 1416259412Sluigi/* 1417270063Sluigi * Support routines used by netmap subsystems 1418270063Sluigi * (native drivers, VALE, generic, pipes, monitors, ...) 1419259412Sluigi */ 1420270063Sluigi 1421270063Sluigi 1422270063Sluigi/* common routine for all functions that create a netmap adapter. It performs 1423270063Sluigi * two main tasks: 1424270063Sluigi * - if the na points to an ifp, mark the ifp as netmap capable 1425270063Sluigi * using na as its native adapter; 1426270063Sluigi * - provide defaults for the setup callbacks and the memory allocator 1427270063Sluigi */ 1428270063Sluigiint netmap_attach_common(struct netmap_adapter *); 1429270063Sluigi/* fill priv->np_[tr]xq{first,last} using the ringid and flags information 1430341477Svmaffione * coming from a struct nmreq_register 1431270063Sluigi */ 1432341477Svmaffioneint netmap_interp_ringid(struct netmap_priv_d *priv, uint32_t nr_mode, 1433341477Svmaffione uint16_t nr_ringid, uint64_t nr_flags); 1434270063Sluigi/* update the ring parameters (number and size of tx and rx rings). 1435270063Sluigi * It calls the nm_config callback, if available. 1436270063Sluigi */ 1437259412Sluigiint netmap_update_config(struct netmap_adapter *na); 1438270063Sluigi/* create and initialize the common fields of the krings array. 1439270063Sluigi * using the information that must be already available in the na. 1440270063Sluigi * tailroom can be used to request the allocation of additional 1441270063Sluigi * tailroom bytes after the krings array. This is used by 1442270063Sluigi * netmap_vp_adapter's (i.e., VALE ports) to make room for 1443270063Sluigi * leasing-related data structures 1444270063Sluigi */ 1445261909Sluigiint netmap_krings_create(struct netmap_adapter *na, u_int tailroom); 1446270063Sluigi/* deletes the kring array of the adapter. The array must have 1447270063Sluigi * been created using netmap_krings_create 1448270063Sluigi */ 1449259412Sluigivoid netmap_krings_delete(struct netmap_adapter *na); 1450270063Sluigi 1451341477Svmaffioneint netmap_hw_krings_create(struct netmap_adapter *na); 1452341477Svmaffionevoid netmap_hw_krings_delete(struct netmap_adapter *na); 1453341477Svmaffione 1454270063Sluigi/* set the stopped/enabled status of ring 1455270063Sluigi * When stopping, they also wait for all current activity on the ring to 1456270063Sluigi * terminate. The status change is then notified using the na nm_notify 1457270063Sluigi * callback. 1458270063Sluigi */ 1459285349Sluigivoid netmap_set_ring(struct netmap_adapter *, u_int ring_id, enum txrx, int stopped); 1460270063Sluigi/* set the stopped/enabled status of all rings of the adapter. */ 1461270063Sluigivoid netmap_set_all_rings(struct netmap_adapter *, int stopped); 1462341477Svmaffione/* convenience wrappers for netmap_set_all_rings */ 1463270063Sluigivoid netmap_disable_all_rings(struct ifnet *); 1464270063Sluigivoid netmap_enable_all_rings(struct ifnet *); 1465270063Sluigi 1466342395Svmaffioneint netmap_buf_size_validate(const struct netmap_adapter *na, unsigned mtu); 1467285359Sluigiint netmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na, 1468341477Svmaffione uint32_t nr_mode, uint16_t nr_ringid, uint64_t nr_flags); 1469341477Svmaffionevoid netmap_do_unregif(struct netmap_priv_d *priv); 1470259412Sluigi 1471257529Sluigiu_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg); 1472341477Svmaffioneint netmap_get_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1473341477Svmaffione struct ifnet **ifp, struct netmap_mem_d *nmd, int create); 1474341477Svmaffionevoid netmap_unget_na(struct netmap_adapter *na, struct ifnet *ifp); 1475341477Svmaffioneint netmap_get_hw_na(struct ifnet *ifp, 1476341477Svmaffione struct netmap_mem_d *nmd, struct netmap_adapter **na); 1477257529Sluigi 1478341477Svmaffione#ifdef WITH_VALE 1479342033Svmaffioneuint32_t netmap_vale_learning(struct nm_bdg_fwd *ft, uint8_t *dst_ring, 1480341477Svmaffione struct netmap_vp_adapter *, void *private_data); 1481341477Svmaffione 1482341477Svmaffione/* these are redefined in case of no VALE support */ 1483341477Svmaffioneint netmap_get_vale_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1484341477Svmaffione struct netmap_mem_d *nmd, int create); 1485341477Svmaffionevoid *netmap_vale_create(const char *bdg_name, int *return_status); 1486341477Svmaffioneint netmap_vale_destroy(const char *bdg_name, void *auth_token); 1487341477Svmaffione 1488259412Sluigi#else /* !WITH_VALE */ 1489341477Svmaffione#define netmap_bdg_learning(_1, _2, _3, _4) 0 1490341477Svmaffione#define netmap_get_vale_na(_1, _2, _3, _4) 0 1491341477Svmaffione#define netmap_bdg_create(_1, _2) NULL 1492341477Svmaffione#define netmap_bdg_destroy(_1, _2) 0 1493259412Sluigi#endif /* !WITH_VALE */ 1494259412Sluigi 1495261909Sluigi#ifdef WITH_PIPES 1496261909Sluigi/* max number of pipes per device */ 1497341477Svmaffione#define NM_MAXPIPES 64 /* XXX this should probably be a sysctl */ 1498261909Sluigivoid netmap_pipe_dealloc(struct netmap_adapter *); 1499341477Svmaffioneint netmap_get_pipe_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1500341477Svmaffione struct netmap_mem_d *nmd, int create); 1501261909Sluigi#else /* !WITH_PIPES */ 1502261909Sluigi#define NM_MAXPIPES 0 1503285349Sluigi#define netmap_pipe_alloc(_1, _2) 0 1504261909Sluigi#define netmap_pipe_dealloc(_1) 1505341477Svmaffione#define netmap_get_pipe_na(hdr, _2, _3, _4) \ 1506341477Svmaffione ((strchr(hdr->nr_name, '{') != NULL || strchr(hdr->nr_name, '}') != NULL) ? EOPNOTSUPP : 0) 1507261909Sluigi#endif 1508261909Sluigi 1509270063Sluigi#ifdef WITH_MONITOR 1510341477Svmaffioneint netmap_get_monitor_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1511341477Svmaffione struct netmap_mem_d *nmd, int create); 1512285349Sluigivoid netmap_monitor_stop(struct netmap_adapter *na); 1513270063Sluigi#else 1514341477Svmaffione#define netmap_get_monitor_na(hdr, _2, _3, _4) \ 1515341477Svmaffione (((struct nmreq_register *)(uintptr_t)hdr->nr_body)->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX) ? EOPNOTSUPP : 0) 1516270063Sluigi#endif 1517270063Sluigi 1518342033Svmaffione#ifdef WITH_NMNULL 1519342033Svmaffioneint netmap_get_null_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1520342033Svmaffione struct netmap_mem_d *nmd, int create); 1521342033Svmaffione#else /* !WITH_NMNULL */ 1522342033Svmaffione#define netmap_get_null_na(hdr, _2, _3, _4) \ 1523342033Svmaffione (((struct nmreq_register *)(uintptr_t)hdr->nr_body)->nr_flags & (NR_MONITOR_TX | NR_MONITOR_RX) ? EOPNOTSUPP : 0) 1524342033Svmaffione#endif /* WITH_NMNULL */ 1525342033Svmaffione 1526285349Sluigi#ifdef CONFIG_NET_NS 1527285349Sluigistruct net *netmap_bns_get(void); 1528285349Sluigivoid netmap_bns_put(struct net *); 1529285349Sluigivoid netmap_bns_getbridges(struct nm_bridge **, u_int *); 1530285349Sluigi#else 1531342033Svmaffioneextern struct nm_bridge *nm_bridges; 1532285349Sluigi#define netmap_bns_get() 1533285349Sluigi#define netmap_bns_put(_1) 1534285349Sluigi#define netmap_bns_getbridges(b, n) \ 1535285349Sluigi do { *b = nm_bridges; *n = NM_BRIDGES; } while (0) 1536285349Sluigi#endif 1537285349Sluigi 1538259412Sluigi/* Various prototypes */ 1539341477Svmaffioneint netmap_poll(struct netmap_priv_d *, int events, NM_SELRECORD_T *td); 1540259412Sluigiint netmap_init(void); 1541259412Sluigivoid netmap_fini(void); 1542259412Sluigiint netmap_get_memory(struct netmap_priv_d* p); 1543259412Sluigivoid netmap_dtor(void *data); 1544259412Sluigi 1545341477Svmaffioneint netmap_ioctl(struct netmap_priv_d *priv, u_long cmd, caddr_t data, 1546341477Svmaffione struct thread *, int nr_body_is_user); 1547341477Svmaffioneint netmap_ioctl_legacy(struct netmap_priv_d *priv, u_long cmd, caddr_t data, 1548341477Svmaffione struct thread *td); 1549341477Svmaffionesize_t nmreq_size_by_type(uint16_t nr_reqtype); 1550259412Sluigi 1551259412Sluigi/* netmap_adapter creation/destruction */ 1552259412Sluigi 1553260368Sluigi// #define NM_DEBUG_PUTGET 1 1554260368Sluigi 1555259412Sluigi#ifdef NM_DEBUG_PUTGET 1556259412Sluigi 1557259412Sluigi#define NM_DBG(f) __##f 1558259412Sluigi 1559259412Sluigivoid __netmap_adapter_get(struct netmap_adapter *na); 1560259412Sluigi 1561259412Sluigi#define netmap_adapter_get(na) \ 1562259412Sluigi do { \ 1563259412Sluigi struct netmap_adapter *__na = na; \ 1564344047Svmaffione nm_prinf("getting %p:%s (%d)", __na, (__na)->name, (__na)->na_refcount); \ 1565259412Sluigi __netmap_adapter_get(__na); \ 1566259412Sluigi } while (0) 1567259412Sluigi 1568259412Sluigiint __netmap_adapter_put(struct netmap_adapter *na); 1569259412Sluigi 1570259412Sluigi#define netmap_adapter_put(na) \ 1571260411Sluigi ({ \ 1572259412Sluigi struct netmap_adapter *__na = na; \ 1573344047Svmaffione nm_prinf("putting %p:%s (%d)", __na, (__na)->name, (__na)->na_refcount); \ 1574259412Sluigi __netmap_adapter_put(__na); \ 1575260411Sluigi }) 1576259412Sluigi 1577259412Sluigi#else /* !NM_DEBUG_PUTGET */ 1578259412Sluigi 1579259412Sluigi#define NM_DBG(f) f 1580259412Sluigivoid netmap_adapter_get(struct netmap_adapter *na); 1581259412Sluigiint netmap_adapter_put(struct netmap_adapter *na); 1582259412Sluigi 1583259412Sluigi#endif /* !NM_DEBUG_PUTGET */ 1584259412Sluigi 1585259412Sluigi 1586260368Sluigi/* 1587260368Sluigi * module variables 1588260368Sluigi */ 1589341477Svmaffione#define NETMAP_BUF_BASE(_na) ((_na)->na_lut.lut[0].vaddr) 1590341477Svmaffione#define NETMAP_BUF_SIZE(_na) ((_na)->na_lut.objsize) 1591231198Sluigiextern int netmap_no_pendintr; 1592341477Svmaffioneextern int netmap_mitigate; 1593342033Svmaffioneextern int netmap_verbose; 1594342033Svmaffione#ifdef CONFIG_NETMAP_DEBUG 1595342033Svmaffioneextern int netmap_debug; /* for debugging */ 1596342033Svmaffione#else /* !CONFIG_NETMAP_DEBUG */ 1597342033Svmaffione#define netmap_debug (0) 1598342033Svmaffione#endif /* !CONFIG_NETMAP_DEBUG */ 1599342033Svmaffioneenum { /* debug flags */ 1600342033Svmaffione NM_DEBUG_ON = 1, /* generic debug messsages */ 1601342033Svmaffione NM_DEBUG_HOST = 0x2, /* debug host stack */ 1602342033Svmaffione NM_DEBUG_RXSYNC = 0x10, /* debug on rxsync/txsync */ 1603342033Svmaffione NM_DEBUG_TXSYNC = 0x20, 1604342033Svmaffione NM_DEBUG_RXINTR = 0x100, /* debug on rx/tx intr (driver) */ 1605342033Svmaffione NM_DEBUG_TXINTR = 0x200, 1606342033Svmaffione NM_DEBUG_NIC_RXSYNC = 0x1000, /* debug on rx/tx intr (driver) */ 1607342033Svmaffione NM_DEBUG_NIC_TXSYNC = 0x2000, 1608342033Svmaffione NM_DEBUG_MEM = 0x4000, /* verbose memory allocations/deallocations */ 1609342033Svmaffione NM_DEBUG_VALE = 0x8000, /* debug messages from memory allocators */ 1610342033Svmaffione NM_DEBUG_BDG = NM_DEBUG_VALE, 1611227614Sluigi}; 1612227614Sluigi 1613259412Sluigiextern int netmap_txsync_retry; 1614341477Svmaffioneextern int netmap_generic_hwcsum; 1615259412Sluigiextern int netmap_generic_mit; 1616259412Sluigiextern int netmap_generic_ringsize; 1617261909Sluigiextern int netmap_generic_rings; 1618341477Svmaffione#ifdef linux 1619341477Svmaffioneextern int netmap_generic_txqdisc; 1620341477Svmaffione#endif 1621259412Sluigi 1622227614Sluigi/* 1623341477Svmaffione * NA returns a pointer to the struct netmap adapter from the ifp. 1624341477Svmaffione * WNA is os-specific and must be defined in glue code. 1625227614Sluigi */ 1626228845Sluigi#define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp)) 1627227614Sluigi 1628241719Sluigi/* 1629341477Svmaffione * we provide a default implementation of NM_ATTACH_NA/NM_DETACH_NA 1630341477Svmaffione * based on the WNA field. 1631341477Svmaffione * Glue code may override this by defining its own NM_ATTACH_NA 1632241719Sluigi */ 1633341477Svmaffione#ifndef NM_ATTACH_NA 1634241719Sluigi/* 1635341477Svmaffione * On old versions of FreeBSD, NA(ifp) is a pspare. On linux we 1636341477Svmaffione * overload another pointer in the netdev. 1637341477Svmaffione * 1638341477Svmaffione * We check if NA(ifp) is set and its first element has a related 1639241719Sluigi * magic value. The capenable is within the struct netmap_adapter. 1640241719Sluigi */ 1641241719Sluigi#define NETMAP_MAGIC 0x52697a7a 1642241719Sluigi 1643341477Svmaffione#define NM_NA_VALID(ifp) (NA(ifp) && \ 1644241719Sluigi ((uint32_t)(uintptr_t)NA(ifp) ^ NA(ifp)->magic) == NETMAP_MAGIC ) 1645241719Sluigi 1646341477Svmaffione#define NM_ATTACH_NA(ifp, na) do { \ 1647341477Svmaffione WNA(ifp) = na; \ 1648341477Svmaffione if (NA(ifp)) \ 1649341477Svmaffione NA(ifp)->magic = \ 1650341477Svmaffione ((uint32_t)(uintptr_t)NA(ifp)) ^ NETMAP_MAGIC; \ 1651341477Svmaffione} while(0) 1652341477Svmaffione#define NM_RESTORE_NA(ifp, na) WNA(ifp) = na; 1653241719Sluigi 1654341477Svmaffione#define NM_DETACH_NA(ifp) do { WNA(ifp) = NULL; } while (0) 1655341477Svmaffione#define NM_NA_CLASH(ifp) (NA(ifp) && !NM_NA_VALID(ifp)) 1656341477Svmaffione#endif /* !NM_ATTACH_NA */ 1657241719Sluigi 1658259412Sluigi 1659341477Svmaffione#define NM_IS_NATIVE(ifp) (NM_NA_VALID(ifp) && NA(ifp)->nm_dtor == netmap_hw_dtor) 1660341477Svmaffione 1661341477Svmaffione#if defined(__FreeBSD__) 1662341477Svmaffione 1663270063Sluigi/* Assigns the device IOMMU domain to an allocator. 1664270063Sluigi * Returns -ENOMEM in case the domain is different */ 1665270063Sluigi#define nm_iommu_group_id(dev) (0) 1666270063Sluigi 1667260368Sluigi/* Callback invoked by the dma machinery after a successful dmamap_load */ 1668230052Sluigistatic void netmap_dmamap_cb(__unused void *arg, 1669230058Sluigi __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error) 1670230052Sluigi{ 1671230052Sluigi} 1672230052Sluigi 1673230052Sluigi/* bus_dmamap_load wrapper: call aforementioned function if map != NULL. 1674230052Sluigi * XXX can we do it without a callback ? 1675230052Sluigi */ 1676341477Svmaffionestatic inline int 1677270063Sluiginetmap_load_map(struct netmap_adapter *na, 1678270063Sluigi bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1679230052Sluigi{ 1680230052Sluigi if (map) 1681270063Sluigi bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE(na), 1682230058Sluigi netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 1683341477Svmaffione return 0; 1684230052Sluigi} 1685230052Sluigi 1686270063Sluigistatic inline void 1687270063Sluiginetmap_unload_map(struct netmap_adapter *na, 1688270063Sluigi bus_dma_tag_t tag, bus_dmamap_t map) 1689270063Sluigi{ 1690270063Sluigi if (map) 1691270063Sluigi bus_dmamap_unload(tag, map); 1692270063Sluigi} 1693270063Sluigi 1694341477Svmaffione#define netmap_sync_map(na, tag, map, sz, t) 1695341477Svmaffione 1696230052Sluigi/* update the map when a buffer changes. */ 1697230052Sluigistatic inline void 1698270063Sluiginetmap_reload_map(struct netmap_adapter *na, 1699270063Sluigi bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1700230052Sluigi{ 1701230052Sluigi if (map) { 1702230052Sluigi bus_dmamap_unload(tag, map); 1703270063Sluigi bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE(na), 1704230058Sluigi netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 1705230052Sluigi } 1706230052Sluigi} 1707259412Sluigi 1708341477Svmaffione#elif defined(_WIN32) 1709341477Svmaffione 1710238812Sluigi#else /* linux */ 1711230052Sluigi 1712270063Sluigiint nm_iommu_group_id(bus_dma_tag_t dev); 1713270063Sluigi#include <linux/dma-mapping.h> 1714270063Sluigi 1715341477Svmaffione/* 1716341477Svmaffione * on linux we need 1717341477Svmaffione * dma_map_single(&pdev->dev, virt_addr, len, direction) 1718341477Svmaffione * dma_unmap_single(&adapter->pdev->dev, phys_addr, len, direction) 1719341477Svmaffione */ 1720341477Svmaffione#if 0 1721341477Svmaffione struct e1000_buffer *buffer_info = &tx_ring->buffer_info[l]; 1722341477Svmaffione /* set time_stamp *before* dma to help avoid a possible race */ 1723341477Svmaffione buffer_info->time_stamp = jiffies; 1724341477Svmaffione buffer_info->mapped_as_page = false; 1725341477Svmaffione buffer_info->length = len; 1726341477Svmaffione //buffer_info->next_to_watch = l; 1727341477Svmaffione /* reload dma map */ 1728341477Svmaffione dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 1729341477Svmaffione NETMAP_BUF_SIZE, DMA_TO_DEVICE); 1730341477Svmaffione buffer_info->dma = dma_map_single(&adapter->pdev->dev, 1731341477Svmaffione addr, NETMAP_BUF_SIZE, DMA_TO_DEVICE); 1732341477Svmaffione 1733341477Svmaffione if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) { 1734344047Svmaffione nm_prerr("dma mapping error"); 1735341477Svmaffione /* goto dma_error; See e1000_put_txbuf() */ 1736341477Svmaffione /* XXX reset */ 1737341477Svmaffione } 1738341477Svmaffione tx_desc->buffer_addr = htole64(buffer_info->dma); //XXX 1739341477Svmaffione 1740341477Svmaffione#endif 1741341477Svmaffione 1742341477Svmaffionestatic inline int 1743270063Sluiginetmap_load_map(struct netmap_adapter *na, 1744341477Svmaffione bus_dma_tag_t tag, bus_dmamap_t map, void *buf, u_int size) 1745270063Sluigi{ 1746341477Svmaffione if (map) { 1747341477Svmaffione *map = dma_map_single(na->pdev, buf, size, 1748341477Svmaffione DMA_BIDIRECTIONAL); 1749341477Svmaffione if (dma_mapping_error(na->pdev, *map)) { 1750341477Svmaffione *map = 0; 1751341477Svmaffione return ENOMEM; 1752341477Svmaffione } 1753270063Sluigi } 1754341477Svmaffione return 0; 1755270063Sluigi} 1756270063Sluigi 1757270063Sluigistatic inline void 1758270063Sluiginetmap_unload_map(struct netmap_adapter *na, 1759341477Svmaffione bus_dma_tag_t tag, bus_dmamap_t map, u_int sz) 1760270063Sluigi{ 1761270063Sluigi if (*map) { 1762270063Sluigi dma_unmap_single(na->pdev, *map, sz, 1763341477Svmaffione DMA_BIDIRECTIONAL); 1764270063Sluigi } 1765270063Sluigi} 1766270063Sluigi 1767341477Svmaffione#ifdef NETMAP_LINUX_HAVE_DMASYNC 1768270063Sluigistatic inline void 1769341477Svmaffionenetmap_sync_map_cpu(struct netmap_adapter *na, 1770341477Svmaffione bus_dma_tag_t tag, bus_dmamap_t map, u_int sz, enum txrx t) 1771341477Svmaffione{ 1772341477Svmaffione if (*map) { 1773341477Svmaffione dma_sync_single_for_cpu(na->pdev, *map, sz, 1774341477Svmaffione (t == NR_TX ? DMA_TO_DEVICE : DMA_FROM_DEVICE)); 1775341477Svmaffione } 1776341477Svmaffione} 1777341477Svmaffione 1778341477Svmaffionestatic inline void 1779341477Svmaffionenetmap_sync_map_dev(struct netmap_adapter *na, 1780341477Svmaffione bus_dma_tag_t tag, bus_dmamap_t map, u_int sz, enum txrx t) 1781341477Svmaffione{ 1782341477Svmaffione if (*map) { 1783341477Svmaffione dma_sync_single_for_device(na->pdev, *map, sz, 1784341477Svmaffione (t == NR_TX ? DMA_TO_DEVICE : DMA_FROM_DEVICE)); 1785341477Svmaffione } 1786341477Svmaffione} 1787341477Svmaffione 1788341477Svmaffionestatic inline void 1789270063Sluiginetmap_reload_map(struct netmap_adapter *na, 1790270063Sluigi bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1791270063Sluigi{ 1792341477Svmaffione u_int sz = NETMAP_BUF_SIZE(na); 1793270063Sluigi 1794270063Sluigi if (*map) { 1795270063Sluigi dma_unmap_single(na->pdev, *map, sz, 1796270063Sluigi DMA_BIDIRECTIONAL); 1797270063Sluigi } 1798270063Sluigi 1799270063Sluigi *map = dma_map_single(na->pdev, buf, sz, 1800270063Sluigi DMA_BIDIRECTIONAL); 1801270063Sluigi} 1802341477Svmaffione#else /* !NETMAP_LINUX_HAVE_DMASYNC */ 1803341477Svmaffione#define netmap_sync_map_cpu(na, tag, map, sz, t) 1804341477Svmaffione#define netmap_sync_map_dev(na, tag, map, sz, t) 1805341477Svmaffione#endif /* NETMAP_LINUX_HAVE_DMASYNC */ 1806270063Sluigi 1807238812Sluigi#endif /* linux */ 1808238812Sluigi 1809257529Sluigi 1810238812Sluigi/* 1811231796Sluigi * functions to map NIC to KRING indexes (n2k) and vice versa (k2n) 1812231796Sluigi */ 1813231796Sluigistatic inline int 1814232238Sluiginetmap_idx_n2k(struct netmap_kring *kr, int idx) 1815231796Sluigi{ 1816232238Sluigi int n = kr->nkr_num_slots; 1817342033Svmaffione 1818342033Svmaffione if (likely(kr->nkr_hwofs == 0)) { 1819342033Svmaffione return idx; 1820342033Svmaffione } 1821342033Svmaffione 1822232238Sluigi idx += kr->nkr_hwofs; 1823232238Sluigi if (idx < 0) 1824232238Sluigi return idx + n; 1825232238Sluigi else if (idx < n) 1826232238Sluigi return idx; 1827231796Sluigi else 1828232238Sluigi return idx - n; 1829231796Sluigi} 1830230052Sluigi 1831231796Sluigi 1832231796Sluigistatic inline int 1833232238Sluiginetmap_idx_k2n(struct netmap_kring *kr, int idx) 1834231796Sluigi{ 1835232238Sluigi int n = kr->nkr_num_slots; 1836342033Svmaffione 1837342033Svmaffione if (likely(kr->nkr_hwofs == 0)) { 1838342033Svmaffione return idx; 1839342033Svmaffione } 1840342033Svmaffione 1841232238Sluigi idx -= kr->nkr_hwofs; 1842232238Sluigi if (idx < 0) 1843232238Sluigi return idx + n; 1844232238Sluigi else if (idx < n) 1845232238Sluigi return idx; 1846231796Sluigi else 1847232238Sluigi return idx - n; 1848231796Sluigi} 1849231796Sluigi 1850231796Sluigi 1851234227Sluigi/* Entries of the look-up table. */ 1852341477Svmaffione#ifdef __FreeBSD__ 1853234227Sluigistruct lut_entry { 1854234227Sluigi void *vaddr; /* virtual address. */ 1855250107Sluigi vm_paddr_t paddr; /* physical address. */ 1856234227Sluigi}; 1857341477Svmaffione#else /* linux & _WIN32 */ 1858341477Svmaffione/* dma-mapping in linux can assign a buffer a different address 1859341477Svmaffione * depending on the device, so we need to have a separate 1860341477Svmaffione * physical-address look-up table for each na. 1861341477Svmaffione * We can still share the vaddrs, though, therefore we split 1862341477Svmaffione * the lut_entry structure. 1863341477Svmaffione */ 1864341477Svmaffionestruct lut_entry { 1865341477Svmaffione void *vaddr; /* virtual address. */ 1866341477Svmaffione}; 1867234227Sluigi 1868341477Svmaffionestruct plut_entry { 1869341477Svmaffione vm_paddr_t paddr; /* physical address. */ 1870341477Svmaffione}; 1871341477Svmaffione#endif /* linux & _WIN32 */ 1872341477Svmaffione 1873234227Sluigistruct netmap_obj_pool; 1874234227Sluigi 1875227614Sluigi/* 1876229939Sluigi * NMB return the virtual address of a buffer (buffer 0 on bad index) 1877229939Sluigi * PNMB also fills the physical address 1878227614Sluigi */ 1879229939Sluigistatic inline void * 1880270063SluigiNMB(struct netmap_adapter *na, struct netmap_slot *slot) 1881227614Sluigi{ 1882285349Sluigi struct lut_entry *lut = na->na_lut.lut; 1883227614Sluigi uint32_t i = slot->buf_idx; 1884285349Sluigi return (unlikely(i >= na->na_lut.objtotal)) ? 1885270063Sluigi lut[0].vaddr : lut[i].vaddr; 1886227614Sluigi} 1887227614Sluigi 1888229939Sluigistatic inline void * 1889270063SluigiPNMB(struct netmap_adapter *na, struct netmap_slot *slot, uint64_t *pp) 1890229939Sluigi{ 1891229939Sluigi uint32_t i = slot->buf_idx; 1892285349Sluigi struct lut_entry *lut = na->na_lut.lut; 1893341477Svmaffione struct plut_entry *plut = na->na_lut.plut; 1894285349Sluigi void *ret = (i >= na->na_lut.objtotal) ? lut[0].vaddr : lut[i].vaddr; 1895249659Sluigi 1896341477Svmaffione#ifdef _WIN32 1897341477Svmaffione *pp = (i >= na->na_lut.objtotal) ? (uint64_t)plut[0].paddr.QuadPart : (uint64_t)plut[i].paddr.QuadPart; 1898341477Svmaffione#else 1899341477Svmaffione *pp = (i >= na->na_lut.objtotal) ? plut[0].paddr : plut[i].paddr; 1900341477Svmaffione#endif 1901229939Sluigi return ret; 1902229939Sluigi} 1903229939Sluigi 1904259412Sluigi 1905260368Sluigi/* 1906285359Sluigi * Structure associated to each netmap file descriptor. 1907285359Sluigi * It is created on open and left unbound (np_nifp == NULL). 1908285359Sluigi * A successful NIOCREGIF will set np_nifp and the first few fields; 1909285359Sluigi * this is protected by a global lock (NMG_LOCK) due to low contention. 1910259412Sluigi * 1911285359Sluigi * np_refs counts the number of references to the structure: one for the fd, 1912285359Sluigi * plus (on FreeBSD) one for each active mmap which we track ourselves 1913285695Sluigi * (linux automatically tracks them, but FreeBSD does not). 1914285359Sluigi * np_refs is protected by NMG_LOCK. 1915260368Sluigi * 1916285359Sluigi * Read access to the structure is lock free, because ni_nifp once set 1917285359Sluigi * can only go to 0 when nobody is using the entry anymore. Readers 1918285359Sluigi * must check that np_nifp != NULL before using the other fields. 1919259412Sluigi */ 1920259412Sluigistruct netmap_priv_d { 1921259412Sluigi struct netmap_if * volatile np_nifp; /* netmap if descriptor. */ 1922259412Sluigi 1923259412Sluigi struct netmap_adapter *np_na; 1924341477Svmaffione struct ifnet *np_ifp; 1925261909Sluigi uint32_t np_flags; /* from the ioctl */ 1926341477Svmaffione u_int np_qfirst[NR_TXRX], 1927285349Sluigi np_qlast[NR_TXRX]; /* range of tx/rx rings to scan */ 1928341477Svmaffione uint16_t np_txpoll; 1929342033Svmaffione uint16_t np_kloop_state; /* use with NMG_LOCK held */ 1930342033Svmaffione#define NM_SYNC_KLOOP_RUNNING (1 << 0) 1931342033Svmaffione#define NM_SYNC_KLOOP_STOPPING (1 << 1) 1932341477Svmaffione int np_sync_flags; /* to be passed to nm_sync */ 1933259412Sluigi 1934285359Sluigi int np_refs; /* use with NMG_LOCK held */ 1935261909Sluigi 1936261909Sluigi /* pointers to the selinfo to be used for selrecord. 1937261909Sluigi * Either the local or the global one depending on the 1938261909Sluigi * number of rings. 1939261909Sluigi */ 1940285349Sluigi NM_SELINFO_T *np_si[NR_TXRX]; 1941342033Svmaffione 1942342033Svmaffione /* In the optional CSB mode, the user must specify the start address 1943342033Svmaffione * of two arrays of Communication Status Block (CSB) entries, for the 1944342033Svmaffione * two directions (kernel read application write, and kernel write 1945342033Svmaffione * application read). 1946342033Svmaffione * The number of entries must agree with the number of rings bound to 1947342033Svmaffione * the netmap file descriptor. The entries corresponding to the TX 1948342033Svmaffione * rings are laid out before the ones corresponding to the RX rings. 1949342033Svmaffione * 1950342033Svmaffione * Array of CSB entries for application --> kernel communication 1951342033Svmaffione * (N entries). */ 1952342033Svmaffione struct nm_csb_atok *np_csb_atok_base; 1953342033Svmaffione /* Array of CSB entries for kernel --> application communication 1954342033Svmaffione * (N entries). */ 1955342033Svmaffione struct nm_csb_ktoa *np_csb_ktoa_base; 1956342033Svmaffione 1957342033Svmaffione#ifdef linux 1958342033Svmaffione struct file *np_filp; /* used by sync kloop */ 1959342033Svmaffione#endif /* linux */ 1960259412Sluigi}; 1961259412Sluigi 1962341477Svmaffionestruct netmap_priv_d *netmap_priv_new(void); 1963341477Svmaffionevoid netmap_priv_delete(struct netmap_priv_d *); 1964341477Svmaffione 1965341477Svmaffionestatic inline int nm_kring_pending(struct netmap_priv_d *np) 1966341477Svmaffione{ 1967341477Svmaffione struct netmap_adapter *na = np->np_na; 1968341477Svmaffione enum txrx t; 1969341477Svmaffione int i; 1970341477Svmaffione 1971341477Svmaffione for_rx_tx(t) { 1972341477Svmaffione for (i = np->np_qfirst[t]; i < np->np_qlast[t]; i++) { 1973341477Svmaffione struct netmap_kring *kring = NMR(na, t)[i]; 1974341477Svmaffione if (kring->nr_mode != kring->nr_pending_mode) { 1975341477Svmaffione return 1; 1976341477Svmaffione } 1977341477Svmaffione } 1978341477Svmaffione } 1979341477Svmaffione return 0; 1980341477Svmaffione} 1981341477Svmaffione 1982342033Svmaffione/* call with NMG_LOCK held */ 1983342033Svmaffionestatic __inline int 1984342033Svmaffionenm_si_user(struct netmap_priv_d *priv, enum txrx t) 1985342033Svmaffione{ 1986342033Svmaffione return (priv->np_na != NULL && 1987342033Svmaffione (priv->np_qlast[t] - priv->np_qfirst[t] > 1)); 1988342033Svmaffione} 1989342033Svmaffione 1990341477Svmaffione#ifdef WITH_PIPES 1991341477Svmaffioneint netmap_pipe_txsync(struct netmap_kring *txkring, int flags); 1992341477Svmaffioneint netmap_pipe_rxsync(struct netmap_kring *rxkring, int flags); 1993344047Svmaffioneint netmap_pipe_krings_create_both(struct netmap_adapter *na, 1994344047Svmaffione struct netmap_adapter *ona); 1995344047Svmaffionevoid netmap_pipe_krings_delete_both(struct netmap_adapter *na, 1996344047Svmaffione struct netmap_adapter *ona); 1997344047Svmaffioneint netmap_pipe_reg_both(struct netmap_adapter *na, 1998344047Svmaffione struct netmap_adapter *ona); 1999341477Svmaffione#endif /* WITH_PIPES */ 2000341477Svmaffione 2001270063Sluigi#ifdef WITH_MONITOR 2002259412Sluigi 2003270063Sluigistruct netmap_monitor_adapter { 2004270063Sluigi struct netmap_adapter up; 2005270063Sluigi 2006270063Sluigi struct netmap_priv_d priv; 2007270063Sluigi uint32_t flags; 2008270063Sluigi}; 2009270063Sluigi 2010270063Sluigi#endif /* WITH_MONITOR */ 2011270063Sluigi 2012270063Sluigi 2013274362Sluigi#ifdef WITH_GENERIC 2014259412Sluigi/* 2015259412Sluigi * generic netmap emulation for devices that do not have 2016259412Sluigi * native netmap support. 2017259412Sluigi */ 2018259412Sluigiint generic_netmap_attach(struct ifnet *ifp); 2019341477Svmaffioneint generic_rx_handler(struct ifnet *ifp, struct mbuf *m);; 2020259412Sluigi 2021341477Svmaffioneint nm_os_catch_rx(struct netmap_generic_adapter *gna, int intercept); 2022341477Svmaffioneint nm_os_catch_tx(struct netmap_generic_adapter *gna, int intercept); 2023341477Svmaffione 2024341477Svmaffioneint na_is_generic(struct netmap_adapter *na); 2025341477Svmaffione 2026341477Svmaffione/* 2027341477Svmaffione * the generic transmit routine is passed a structure to optionally 2028341477Svmaffione * build a queue of descriptors, in an OS-specific way. 2029341477Svmaffione * The payload is at addr, if non-null, and the routine should send or queue 2030341477Svmaffione * the packet, returning 0 if successful, 1 on failure. 2031341477Svmaffione * 2032341477Svmaffione * At the end, if head is non-null, there will be an additional call 2033341477Svmaffione * to the function with addr = NULL; this should tell the OS-specific 2034341477Svmaffione * routine to send the queue and free any resources. Failure is ignored. 2035341477Svmaffione */ 2036341477Svmaffionestruct nm_os_gen_arg { 2037341477Svmaffione struct ifnet *ifp; 2038341477Svmaffione void *m; /* os-specific mbuf-like object */ 2039341477Svmaffione void *head, *tail; /* tailq, if the OS-specific routine needs to build one */ 2040341477Svmaffione void *addr; /* payload of current packet */ 2041341477Svmaffione u_int len; /* packet length */ 2042341477Svmaffione u_int ring_nr; /* packet length */ 2043341477Svmaffione u_int qevent; /* in txqdisc mode, place an event on this mbuf */ 2044341477Svmaffione}; 2045341477Svmaffione 2046341477Svmaffioneint nm_os_generic_xmit_frame(struct nm_os_gen_arg *); 2047341477Svmaffioneint nm_os_generic_find_num_desc(struct ifnet *ifp, u_int *tx, u_int *rx); 2048341477Svmaffionevoid nm_os_generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq); 2049341477Svmaffionevoid nm_os_generic_set_features(struct netmap_generic_adapter *gna); 2050341477Svmaffione 2051285349Sluigistatic inline struct ifnet* 2052285349Sluiginetmap_generic_getifp(struct netmap_generic_adapter *gna) 2053285349Sluigi{ 2054285349Sluigi if (gna->prev) 2055285349Sluigi return gna->prev->ifp; 2056259412Sluigi 2057285349Sluigi return gna->up.up.ifp; 2058285349Sluigi} 2059285349Sluigi 2060341477Svmaffionevoid netmap_generic_irq(struct netmap_adapter *na, u_int q, u_int *work_done); 2061341477Svmaffione 2062270063Sluigi//#define RATE_GENERIC /* Enables communication statistics for generic. */ 2063270063Sluigi#ifdef RATE_GENERIC 2064270063Sluigivoid generic_rate(int txp, int txs, int txi, int rxp, int rxs, int rxi); 2065270063Sluigi#else 2066270063Sluigi#define generic_rate(txp, txs, txi, rxp, rxs, rxi) 2067270063Sluigi#endif 2068270063Sluigi 2069259412Sluigi/* 2070259412Sluigi * netmap_mitigation API. This is used by the generic adapter 2071259412Sluigi * to reduce the number of interrupt requests/selwakeup 2072259412Sluigi * to clients on incoming packets. 2073259412Sluigi */ 2074341477Svmaffionevoid nm_os_mitigation_init(struct nm_generic_mit *mit, int idx, 2075270063Sluigi struct netmap_adapter *na); 2076341477Svmaffionevoid nm_os_mitigation_start(struct nm_generic_mit *mit); 2077341477Svmaffionevoid nm_os_mitigation_restart(struct nm_generic_mit *mit); 2078341477Svmaffioneint nm_os_mitigation_active(struct nm_generic_mit *mit); 2079341477Svmaffionevoid nm_os_mitigation_cleanup(struct nm_generic_mit *mit); 2080341477Svmaffione#else /* !WITH_GENERIC */ 2081341477Svmaffione#define generic_netmap_attach(ifp) (EOPNOTSUPP) 2082341477Svmaffione#define na_is_generic(na) (0) 2083274362Sluigi#endif /* WITH_GENERIC */ 2084259412Sluigi 2085261909Sluigi/* Shared declarations for the VALE switch. */ 2086261909Sluigi 2087261909Sluigi/* 2088261909Sluigi * Each transmit queue accumulates a batch of packets into 2089261909Sluigi * a structure before forwarding. Packets to the same 2090261909Sluigi * destination are put in a list using ft_next as a link field. 2091261909Sluigi * ft_frags and ft_next are valid only on the first fragment. 2092261909Sluigi */ 2093261909Sluigistruct nm_bdg_fwd { /* forwarding entry for a bridge */ 2094261909Sluigi void *ft_buf; /* netmap or indirect buffer */ 2095261909Sluigi uint8_t ft_frags; /* how many fragments (only on 1st frag) */ 2096341477Svmaffione uint16_t ft_offset; /* dst port (unused) */ 2097261909Sluigi uint16_t ft_flags; /* flags, e.g. indirect */ 2098261909Sluigi uint16_t ft_len; /* src fragment len */ 2099261909Sluigi uint16_t ft_next; /* next packet to same destination */ 2100261909Sluigi}; 2101261909Sluigi 2102261909Sluigi/* struct 'virtio_net_hdr' from linux. */ 2103261909Sluigistruct nm_vnet_hdr { 2104261909Sluigi#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start, csum_offset */ 2105261909Sluigi#define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */ 2106261909Sluigi uint8_t flags; 2107261909Sluigi#define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */ 2108261909Sluigi#define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */ 2109261909Sluigi#define VIRTIO_NET_HDR_GSO_UDP 3 /* GSO frame, IPv4 UDP (UFO) */ 2110261909Sluigi#define VIRTIO_NET_HDR_GSO_TCPV6 4 /* GSO frame, IPv6 TCP */ 2111261909Sluigi#define VIRTIO_NET_HDR_GSO_ECN 0x80 /* TCP has ECN set */ 2112261909Sluigi uint8_t gso_type; 2113261909Sluigi uint16_t hdr_len; 2114261909Sluigi uint16_t gso_size; 2115261909Sluigi uint16_t csum_start; 2116261909Sluigi uint16_t csum_offset; 2117261909Sluigi}; 2118261909Sluigi 2119261909Sluigi#define WORST_CASE_GSO_HEADER (14+40+60) /* IPv6 + TCP */ 2120261909Sluigi 2121261909Sluigi/* Private definitions for IPv4, IPv6, UDP and TCP headers. */ 2122261909Sluigi 2123261909Sluigistruct nm_iphdr { 2124261909Sluigi uint8_t version_ihl; 2125261909Sluigi uint8_t tos; 2126261909Sluigi uint16_t tot_len; 2127261909Sluigi uint16_t id; 2128261909Sluigi uint16_t frag_off; 2129261909Sluigi uint8_t ttl; 2130261909Sluigi uint8_t protocol; 2131261909Sluigi uint16_t check; 2132261909Sluigi uint32_t saddr; 2133261909Sluigi uint32_t daddr; 2134261909Sluigi /*The options start here. */ 2135261909Sluigi}; 2136261909Sluigi 2137261909Sluigistruct nm_tcphdr { 2138261909Sluigi uint16_t source; 2139261909Sluigi uint16_t dest; 2140261909Sluigi uint32_t seq; 2141261909Sluigi uint32_t ack_seq; 2142261909Sluigi uint8_t doff; /* Data offset + Reserved */ 2143261909Sluigi uint8_t flags; 2144261909Sluigi uint16_t window; 2145261909Sluigi uint16_t check; 2146261909Sluigi uint16_t urg_ptr; 2147261909Sluigi}; 2148261909Sluigi 2149261909Sluigistruct nm_udphdr { 2150261909Sluigi uint16_t source; 2151261909Sluigi uint16_t dest; 2152261909Sluigi uint16_t len; 2153261909Sluigi uint16_t check; 2154261909Sluigi}; 2155261909Sluigi 2156261909Sluigistruct nm_ipv6hdr { 2157261909Sluigi uint8_t priority_version; 2158261909Sluigi uint8_t flow_lbl[3]; 2159261909Sluigi 2160261909Sluigi uint16_t payload_len; 2161261909Sluigi uint8_t nexthdr; 2162261909Sluigi uint8_t hop_limit; 2163261909Sluigi 2164261909Sluigi uint8_t saddr[16]; 2165261909Sluigi uint8_t daddr[16]; 2166261909Sluigi}; 2167261909Sluigi 2168261909Sluigi/* Type used to store a checksum (in host byte order) that hasn't been 2169261909Sluigi * folded yet. 2170261909Sluigi */ 2171261909Sluigi#define rawsum_t uint32_t 2172261909Sluigi 2173341477Svmaffionerawsum_t nm_os_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum); 2174341477Svmaffioneuint16_t nm_os_csum_ipv4(struct nm_iphdr *iph); 2175341477Svmaffionevoid nm_os_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, 2176261909Sluigi size_t datalen, uint16_t *check); 2177341477Svmaffionevoid nm_os_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, 2178261909Sluigi size_t datalen, uint16_t *check); 2179341477Svmaffioneuint16_t nm_os_csum_fold(rawsum_t cur_sum); 2180261909Sluigi 2181261909Sluigivoid bdg_mismatch_datapath(struct netmap_vp_adapter *na, 2182261909Sluigi struct netmap_vp_adapter *dst_na, 2183341477Svmaffione const struct nm_bdg_fwd *ft_p, 2184341477Svmaffione struct netmap_ring *dst_ring, 2185261909Sluigi u_int *j, u_int lim, u_int *howmany); 2186270063Sluigi 2187270063Sluigi/* persistent virtual port routines */ 2188341477Svmaffioneint nm_os_vi_persist(const char *, struct ifnet **); 2189341477Svmaffionevoid nm_os_vi_detach(struct ifnet *); 2190341477Svmaffionevoid nm_os_vi_init_index(void); 2191270063Sluigi 2192341477Svmaffione/* 2193341477Svmaffione * kernel thread routines 2194341477Svmaffione */ 2195341477Svmaffionestruct nm_kctx; /* OS-specific kernel context - opaque */ 2196342033Svmaffionetypedef void (*nm_kctx_worker_fn_t)(void *data); 2197341477Svmaffione 2198341477Svmaffione/* kthread configuration */ 2199341477Svmaffionestruct nm_kctx_cfg { 2200341477Svmaffione long type; /* kthread type/identifier */ 2201341477Svmaffione nm_kctx_worker_fn_t worker_fn; /* worker function */ 2202341477Svmaffione void *worker_private;/* worker parameter */ 2203341477Svmaffione int attach_user; /* attach kthread to user process */ 2204341477Svmaffione}; 2205341477Svmaffione/* kthread configuration */ 2206341477Svmaffionestruct nm_kctx *nm_os_kctx_create(struct nm_kctx_cfg *cfg, 2207341477Svmaffione void *opaque); 2208341477Svmaffioneint nm_os_kctx_worker_start(struct nm_kctx *); 2209341477Svmaffionevoid nm_os_kctx_worker_stop(struct nm_kctx *); 2210341477Svmaffionevoid nm_os_kctx_destroy(struct nm_kctx *); 2211341477Svmaffionevoid nm_os_kctx_worker_setaff(struct nm_kctx *, int); 2212341477Svmaffioneu_int nm_os_ncpus(void); 2213341477Svmaffione 2214342033Svmaffioneint netmap_sync_kloop(struct netmap_priv_d *priv, 2215342033Svmaffione struct nmreq_header *hdr); 2216342033Svmaffioneint netmap_sync_kloop_stop(struct netmap_priv_d *priv); 2217342033Svmaffione 2218342033Svmaffione#ifdef WITH_PTNETMAP 2219342033Svmaffione/* ptnetmap guest routines */ 2220342033Svmaffione 2221341477Svmaffione/* 2222342033Svmaffione * ptnetmap_memdev routines used to talk with ptnetmap_memdev device driver 2223341477Svmaffione */ 2224342033Svmaffionestruct ptnetmap_memdev; 2225342033Svmaffioneint nm_os_pt_memdev_iomap(struct ptnetmap_memdev *, vm_paddr_t *, void **, 2226342033Svmaffione uint64_t *); 2227342033Svmaffionevoid nm_os_pt_memdev_iounmap(struct ptnetmap_memdev *); 2228342033Svmaffioneuint32_t nm_os_pt_memdev_ioread(struct ptnetmap_memdev *, unsigned int); 2229341477Svmaffione 2230341477Svmaffione/* 2231341477Svmaffione * netmap adapter for guest ptnetmap ports 2232341477Svmaffione */ 2233341477Svmaffionestruct netmap_pt_guest_adapter { 2234341477Svmaffione /* The netmap adapter to be used by netmap applications. 2235341477Svmaffione * This field must be the first, to allow upcast. */ 2236341477Svmaffione struct netmap_hw_adapter hwup; 2237341477Svmaffione 2238341477Svmaffione /* The netmap adapter to be used by the driver. */ 2239341477Svmaffione struct netmap_hw_adapter dr; 2240341477Svmaffione 2241341477Svmaffione /* Reference counter to track users of backend netmap port: the 2242341477Svmaffione * network stack and netmap clients. 2243341477Svmaffione * Used to decide when we need (de)allocate krings/rings and 2244341477Svmaffione * start (stop) ptnetmap kthreads. */ 2245342033Svmaffione int backend_users; 2246341477Svmaffione 2247341477Svmaffione}; 2248341477Svmaffione 2249341477Svmaffioneint netmap_pt_guest_attach(struct netmap_adapter *na, 2250341477Svmaffione unsigned int nifp_offset, 2251341477Svmaffione unsigned int memid); 2252342033Svmaffionebool netmap_pt_guest_txsync(struct nm_csb_atok *atok, 2253342033Svmaffione struct nm_csb_ktoa *ktoa, 2254341477Svmaffione struct netmap_kring *kring, int flags); 2255342033Svmaffionebool netmap_pt_guest_rxsync(struct nm_csb_atok *atok, 2256342033Svmaffione struct nm_csb_ktoa *ktoa, 2257342033Svmaffione struct netmap_kring *kring, int flags); 2258341477Svmaffioneint ptnet_nm_krings_create(struct netmap_adapter *na); 2259341477Svmaffionevoid ptnet_nm_krings_delete(struct netmap_adapter *na); 2260341477Svmaffionevoid ptnet_nm_dtor(struct netmap_adapter *na); 2261341477Svmaffione 2262343832Svmaffione/* Helper function wrapping nm_sync_kloop_appl_read(). */ 2263342033Svmaffionestatic inline void 2264342033Svmaffioneptnet_sync_tail(struct nm_csb_ktoa *ktoa, struct netmap_kring *kring) 2265342033Svmaffione{ 2266342033Svmaffione struct netmap_ring *ring = kring->ring; 2267342033Svmaffione 2268342033Svmaffione /* Update hwcur and hwtail as known by the host. */ 2269343832Svmaffione nm_sync_kloop_appl_read(ktoa, &kring->nr_hwtail, &kring->nr_hwcur); 2270342033Svmaffione 2271342033Svmaffione /* nm_sync_finalize */ 2272342033Svmaffione ring->tail = kring->rtail = kring->nr_hwtail; 2273342033Svmaffione} 2274342033Svmaffione#endif /* WITH_PTNETMAP */ 2275342033Svmaffione 2276341477Svmaffione#ifdef __FreeBSD__ 2277341477Svmaffione/* 2278341477Svmaffione * FreeBSD mbuf allocator/deallocator in emulation mode: 2279341477Svmaffione */ 2280341477Svmaffione#if __FreeBSD_version < 1100000 2281341477Svmaffione 2282341477Svmaffione/* 2283341477Svmaffione * For older versions of FreeBSD: 2284341477Svmaffione * 2285341477Svmaffione * We allocate EXT_PACKET mbuf+clusters, but need to set M_NOFREE 2286341477Svmaffione * so that the destructor, if invoked, will not free the packet. 2287341477Svmaffione * In principle we should set the destructor only on demand, 2288341477Svmaffione * but since there might be a race we better do it on allocation. 2289341477Svmaffione * As a consequence, we also need to set the destructor or we 2290341477Svmaffione * would leak buffers. 2291341477Svmaffione */ 2292341477Svmaffione 2293341477Svmaffione/* mbuf destructor, also need to change the type to EXT_EXTREF, 2294341477Svmaffione * add an M_NOFREE flag, and then clear the flag and 2295341477Svmaffione * chain into uma_zfree(zone_pack, mf) 2296341477Svmaffione * (or reinstall the buffer ?) 2297341477Svmaffione */ 2298341477Svmaffione#define SET_MBUF_DESTRUCTOR(m, fn) do { \ 2299341477Svmaffione (m)->m_ext.ext_free = (void *)fn; \ 2300341477Svmaffione (m)->m_ext.ext_type = EXT_EXTREF; \ 2301341477Svmaffione} while (0) 2302341477Svmaffione 2303341477Svmaffionestatic int 2304341477Svmaffionevoid_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) 2305341477Svmaffione{ 2306341477Svmaffione /* restore original mbuf */ 2307341477Svmaffione m->m_ext.ext_buf = m->m_data = m->m_ext.ext_arg1; 2308341477Svmaffione m->m_ext.ext_arg1 = NULL; 2309341477Svmaffione m->m_ext.ext_type = EXT_PACKET; 2310341477Svmaffione m->m_ext.ext_free = NULL; 2311341477Svmaffione if (MBUF_REFCNT(m) == 0) 2312341477Svmaffione SET_MBUF_REFCNT(m, 1); 2313341477Svmaffione uma_zfree(zone_pack, m); 2314341477Svmaffione 2315341477Svmaffione return 0; 2316341477Svmaffione} 2317341477Svmaffione 2318341477Svmaffionestatic inline struct mbuf * 2319341477Svmaffionenm_os_get_mbuf(struct ifnet *ifp, int len) 2320341477Svmaffione{ 2321341477Svmaffione struct mbuf *m; 2322341477Svmaffione 2323341477Svmaffione (void)ifp; 2324341477Svmaffione m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 2325341477Svmaffione if (m) { 2326341477Svmaffione /* m_getcl() (mb_ctor_mbuf) has an assert that checks that 2327341477Svmaffione * M_NOFREE flag is not specified as third argument, 2328341477Svmaffione * so we have to set M_NOFREE after m_getcl(). */ 2329341477Svmaffione m->m_flags |= M_NOFREE; 2330341477Svmaffione m->m_ext.ext_arg1 = m->m_ext.ext_buf; // XXX save 2331341477Svmaffione m->m_ext.ext_free = (void *)void_mbuf_dtor; 2332341477Svmaffione m->m_ext.ext_type = EXT_EXTREF; 2333344047Svmaffione nm_prdis(5, "create m %p refcnt %d", m, MBUF_REFCNT(m)); 2334341477Svmaffione } 2335341477Svmaffione return m; 2336341477Svmaffione} 2337341477Svmaffione 2338341477Svmaffione#else /* __FreeBSD_version >= 1100000 */ 2339341477Svmaffione 2340341477Svmaffione/* 2341341477Svmaffione * Newer versions of FreeBSD, using a straightforward scheme. 2342341477Svmaffione * 2343341477Svmaffione * We allocate mbufs with m_gethdr(), since the mbuf header is needed 2344341477Svmaffione * by the driver. We also attach a customly-provided external storage, 2345341477Svmaffione * which in this case is a netmap buffer. When calling m_extadd(), however 2346341477Svmaffione * we pass a NULL address, since the real address (and length) will be 2347341477Svmaffione * filled in by nm_os_generic_xmit_frame() right before calling 2348341477Svmaffione * if_transmit(). 2349341477Svmaffione * 2350341477Svmaffione * The dtor function does nothing, however we need it since mb_free_ext() 2351341477Svmaffione * has a KASSERT(), checking that the mbuf dtor function is not NULL. 2352341477Svmaffione */ 2353341477Svmaffione 2354341477Svmaffione#if __FreeBSD_version <= 1200050 2355341477Svmaffionestatic void void_mbuf_dtor(struct mbuf *m, void *arg1, void *arg2) { } 2356341477Svmaffione#else /* __FreeBSD_version >= 1200051 */ 2357341477Svmaffione/* The arg1 and arg2 pointers argument were removed by r324446, which 2358341477Svmaffione * in included since version 1200051. */ 2359341477Svmaffionestatic void void_mbuf_dtor(struct mbuf *m) { } 2360341477Svmaffione#endif /* __FreeBSD_version >= 1200051 */ 2361341477Svmaffione 2362341477Svmaffione#define SET_MBUF_DESTRUCTOR(m, fn) do { \ 2363341477Svmaffione (m)->m_ext.ext_free = (fn != NULL) ? \ 2364341477Svmaffione (void *)fn : (void *)void_mbuf_dtor; \ 2365341477Svmaffione} while (0) 2366341477Svmaffione 2367341477Svmaffionestatic inline struct mbuf * 2368341477Svmaffionenm_os_get_mbuf(struct ifnet *ifp, int len) 2369341477Svmaffione{ 2370341477Svmaffione struct mbuf *m; 2371341477Svmaffione 2372341477Svmaffione (void)ifp; 2373341477Svmaffione (void)len; 2374341477Svmaffione 2375341477Svmaffione m = m_gethdr(M_NOWAIT, MT_DATA); 2376341477Svmaffione if (m == NULL) { 2377341477Svmaffione return m; 2378341477Svmaffione } 2379341477Svmaffione 2380341477Svmaffione m_extadd(m, NULL /* buf */, 0 /* size */, void_mbuf_dtor, 2381341477Svmaffione NULL, NULL, 0, EXT_NET_DRV); 2382341477Svmaffione 2383341477Svmaffione return m; 2384341477Svmaffione} 2385341477Svmaffione 2386341477Svmaffione#endif /* __FreeBSD_version >= 1100000 */ 2387341477Svmaffione#endif /* __FreeBSD__ */ 2388341477Svmaffione 2389341477Svmaffionestruct nmreq_option * nmreq_findoption(struct nmreq_option *, uint16_t); 2390341477Svmaffioneint nmreq_checkduplicate(struct nmreq_option *); 2391341477Svmaffione 2392342033Svmaffioneint netmap_init_bridges(void); 2393342033Svmaffionevoid netmap_uninit_bridges(void); 2394342033Svmaffione 2395342033Svmaffione/* Functions to read and write CSB fields from the kernel. */ 2396342033Svmaffione#if defined (linux) 2397342033Svmaffione#define CSB_READ(csb, field, r) (get_user(r, &csb->field)) 2398342033Svmaffione#define CSB_WRITE(csb, field, v) (put_user(v, &csb->field)) 2399342033Svmaffione#else /* ! linux */ 2400342033Svmaffione#define CSB_READ(csb, field, r) (r = fuword32(&csb->field)) 2401342033Svmaffione#define CSB_WRITE(csb, field, v) (suword32(&csb->field, v)) 2402342033Svmaffione#endif /* ! linux */ 2403342033Svmaffione 2404227614Sluigi#endif /* _NET_NETMAP_KERN_H_ */ 2405