netmap_kern.h revision 267128
1227614Sluigi/* 2260368Sluigi * Copyright (C) 2011-2014 Matteo Landi, Luigi Rizzo. All rights reserved. 3260368Sluigi * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved. 4241719Sluigi * 5227614Sluigi * Redistribution and use in source and binary forms, with or without 6227614Sluigi * modification, are permitted provided that the following conditions 7227614Sluigi * are met: 8228276Sluigi * 1. Redistributions of source code must retain the above copyright 9228276Sluigi * notice, this list of conditions and the following disclaimer. 10228276Sluigi * 2. Redistributions in binary form must reproduce the above copyright 11228276Sluigi * notice, this list of conditions and the following disclaimer in the 12227614Sluigi * documentation and/or other materials provided with the distribution. 13241719Sluigi * 14227614Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15227614Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16227614Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17227614Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18227614Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19227614Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20227614Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21227614Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22227614Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23227614Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24227614Sluigi * SUCH DAMAGE. 25227614Sluigi */ 26227614Sluigi 27227614Sluigi/* 28227614Sluigi * $FreeBSD: head/sys/dev/netmap/netmap_kern.h 267128 2014-06-05 21:12:41Z luigi $ 29227614Sluigi * 30227614Sluigi * The header contains the definitions of constants and function 31227614Sluigi * prototypes used only in kernelspace. 32227614Sluigi */ 33227614Sluigi 34227614Sluigi#ifndef _NET_NETMAP_KERN_H_ 35227614Sluigi#define _NET_NETMAP_KERN_H_ 36227614Sluigi 37259412Sluigi#define WITH_VALE // comment out to disable VALE support 38261909Sluigi#define WITH_PIPES 39259412Sluigi 40231594Sluigi#if defined(__FreeBSD__) 41250052Sluigi 42257529Sluigi#define likely(x) __builtin_expect((long)!!(x), 1L) 43257529Sluigi#define unlikely(x) __builtin_expect((long)!!(x), 0L) 44238812Sluigi 45231594Sluigi#define NM_LOCK_T struct mtx 46259412Sluigi#define NMG_LOCK_T struct mtx 47259412Sluigi#define NMG_LOCK_INIT() mtx_init(&netmap_global_lock, \ 48259412Sluigi "netmap global lock", NULL, MTX_DEF) 49259412Sluigi#define NMG_LOCK_DESTROY() mtx_destroy(&netmap_global_lock) 50259412Sluigi#define NMG_LOCK() mtx_lock(&netmap_global_lock) 51259412Sluigi#define NMG_UNLOCK() mtx_unlock(&netmap_global_lock) 52259412Sluigi#define NMG_LOCK_ASSERT() mtx_assert(&netmap_global_lock, MA_OWNED) 53259412Sluigi 54231594Sluigi#define NM_SELINFO_T struct selinfo 55231594Sluigi#define MBUF_LEN(m) ((m)->m_pkthdr.len) 56259412Sluigi#define MBUF_IFP(m) ((m)->m_pkthdr.rcvif) 57260368Sluigi#define NM_SEND_UP(ifp, m) ((NA(ifp))->if_input)(ifp, m) 58250052Sluigi 59259412Sluigi#define NM_ATOMIC_T volatile int // XXX ? 60259412Sluigi/* atomic operations */ 61259412Sluigi#include <machine/atomic.h> 62259412Sluigi#define NM_ATOMIC_TEST_AND_SET(p) (!atomic_cmpset_acq_int((p), 0, 1)) 63259412Sluigi#define NM_ATOMIC_CLEAR(p) atomic_store_rel_int((p), 0) 64257529Sluigi 65259412Sluigi 66259412SluigiMALLOC_DECLARE(M_NETMAP); 67259412Sluigi 68259412Sluigi// XXX linux struct, not used in FreeBSD 69259412Sluigistruct net_device_ops { 70259412Sluigi}; 71259412Sluigistruct hrtimer { 72259412Sluigi}; 73259412Sluigi 74232238Sluigi#elif defined (linux) 75250052Sluigi 76250052Sluigi#define NM_LOCK_T safe_spinlock_t // see bsd_glue.h 77231594Sluigi#define NM_SELINFO_T wait_queue_head_t 78231594Sluigi#define MBUF_LEN(m) ((m)->len) 79259412Sluigi#define MBUF_IFP(m) ((m)->dev) 80260368Sluigi#define NM_SEND_UP(ifp, m) \ 81260368Sluigi do { \ 82260368Sluigi m->priority = NM_MAGIC_PRIORITY; \ 83260368Sluigi netif_rx(m); \ 84260368Sluigi } while (0) 85238812Sluigi 86257529Sluigi#define NM_ATOMIC_T volatile long unsigned int 87257529Sluigi 88259412Sluigi// XXX a mtx would suffice here too 20130404 gl 89259412Sluigi#define NMG_LOCK_T struct semaphore 90259412Sluigi#define NMG_LOCK_INIT() sema_init(&netmap_global_lock, 1) 91259412Sluigi#define NMG_LOCK_DESTROY() 92259412Sluigi#define NMG_LOCK() down(&netmap_global_lock) 93259412Sluigi#define NMG_UNLOCK() up(&netmap_global_lock) 94259412Sluigi#define NMG_LOCK_ASSERT() // XXX to be completed 95259412Sluigi 96238812Sluigi#ifndef DEV_NETMAP 97238812Sluigi#define DEV_NETMAP 98257529Sluigi#endif /* DEV_NETMAP */ 99238812Sluigi 100238812Sluigi/* 101241719Sluigi * IFCAP_NETMAP goes into net_device's priv_flags (if_capenable). 102241719Sluigi * This was 16 bits up to linux 2.6.36, so we need a 16 bit value on older 103238812Sluigi * platforms and tolerate the clash with IFF_DYNAMIC and IFF_BRIDGE_PORT. 104241719Sluigi * For the 32-bit value, 0x100000 has no clashes until at least 3.5.1 105238812Sluigi */ 106238812Sluigi#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,37) 107238831Sluigi#define IFCAP_NETMAP 0x8000 108231594Sluigi#else 109251139Sluigi#define IFCAP_NETMAP 0x200000 110238812Sluigi#endif 111238812Sluigi 112238812Sluigi#elif defined (__APPLE__) 113250052Sluigi 114241719Sluigi#warning apple support is incomplete. 115238812Sluigi#define likely(x) __builtin_expect(!!(x), 1) 116238812Sluigi#define unlikely(x) __builtin_expect(!!(x), 0) 117238812Sluigi#define NM_LOCK_T IOLock * 118238812Sluigi#define NM_SELINFO_T struct selinfo 119238812Sluigi#define MBUF_LEN(m) ((m)->m_pkthdr.len) 120238812Sluigi#define NM_SEND_UP(ifp, m) ((ifp)->if_input)(ifp, m) 121238812Sluigi 122238812Sluigi#else 123250052Sluigi 124231594Sluigi#error unsupported platform 125231594Sluigi 126250052Sluigi#endif /* end - platform-specific code */ 127250052Sluigi 128227614Sluigi#define ND(format, ...) 129230058Sluigi#define D(format, ...) \ 130230058Sluigi do { \ 131230058Sluigi struct timeval __xxts; \ 132227614Sluigi microtime(&__xxts); \ 133260368Sluigi printf("%03d.%06d [%4d] %-25s " format "\n", \ 134230058Sluigi (int)__xxts.tv_sec % 1000, (int)__xxts.tv_usec, \ 135260368Sluigi __LINE__, __FUNCTION__, ##__VA_ARGS__); \ 136227614Sluigi } while (0) 137241719Sluigi 138241719Sluigi/* rate limited, lps indicates how many per second */ 139241719Sluigi#define RD(lps, format, ...) \ 140241719Sluigi do { \ 141241719Sluigi static int t0, __cnt; \ 142241719Sluigi if (t0 != time_second) { \ 143241719Sluigi t0 = time_second; \ 144241719Sluigi __cnt = 0; \ 145241719Sluigi } \ 146241719Sluigi if (__cnt++ < lps) \ 147241719Sluigi D(format, ##__VA_ARGS__); \ 148241719Sluigi } while (0) 149241719Sluigi 150227614Sluigistruct netmap_adapter; 151251139Sluigistruct nm_bdg_fwd; 152251139Sluigistruct nm_bridge; 153251139Sluigistruct netmap_priv_d; 154227614Sluigi 155257529Sluigiconst char *nm_dump_buf(char *p, int len, int lim, char *dst); 156257529Sluigi 157259412Sluigi#include "netmap_mbq.h" 158259412Sluigi 159259412Sluigiextern NMG_LOCK_T netmap_global_lock; 160259412Sluigi 161227614Sluigi/* 162232238Sluigi * private, kernel view of a ring. Keeps track of the status of 163232238Sluigi * a ring across system calls. 164227614Sluigi * 165232238Sluigi * nr_hwcur index of the next buffer to refill. 166260368Sluigi * It corresponds to ring->head 167260368Sluigi * at the time the system call returns. 168232238Sluigi * 169260368Sluigi * nr_hwtail index of the first buffer owned by the kernel. 170260368Sluigi * On RX, hwcur->hwtail are receive buffers 171260368Sluigi * not yet released. hwcur is advanced following 172260368Sluigi * ring->head, hwtail is advanced on incoming packets, 173260368Sluigi * and a wakeup is generated when hwtail passes ring->cur 174260368Sluigi * On TX, hwcur->rcur have been filled by the sender 175260368Sluigi * but not sent yet to the NIC; rcur->hwtail are available 176260368Sluigi * for new transmissions, and hwtail->hwcur-1 are pending 177260368Sluigi * transmissions not yet acknowledged. 178232238Sluigi * 179231594Sluigi * The indexes in the NIC and netmap rings are offset by nkr_hwofs slots. 180227614Sluigi * This is so that, on a reset, buffers owned by userspace are not 181227614Sluigi * modified by the kernel. In particular: 182260368Sluigi * RX rings: the next empty buffer (hwtail + hwofs) coincides with 183227614Sluigi * the next empty buffer as known by the hardware (next_to_check or so). 184227614Sluigi * TX rings: hwcur + hwofs coincides with next_to_send 185245579Sluigi * 186257529Sluigi * Clients cannot issue concurrent syscall on a ring. The system 187257529Sluigi * detects this and reports an error using two flags, 188257529Sluigi * NKR_WBUSY and NKR_RBUSY 189245579Sluigi * For received packets, slot->flags is set to nkr_slot_flags 190245579Sluigi * so we can provide a proper initial value (e.g. set NS_FORWARD 191245579Sluigi * when operating in 'transparent' mode). 192257529Sluigi * 193257529Sluigi * The following fields are used to implement lock-free copy of packets 194257529Sluigi * from input to output ports in VALE switch: 195257529Sluigi * nkr_hwlease buffer after the last one being copied. 196257529Sluigi * A writer in nm_bdg_flush reserves N buffers 197257529Sluigi * from nr_hwlease, advances it, then does the 198257529Sluigi * copy outside the lock. 199257529Sluigi * In RX rings (used for VALE ports), 200260368Sluigi * nkr_hwtail <= nkr_hwlease < nkr_hwcur+N-1 201257529Sluigi * In TX rings (used for NIC or host stack ports) 202260368Sluigi * nkr_hwcur <= nkr_hwlease < nkr_hwtail 203257529Sluigi * nkr_leases array of nkr_num_slots where writers can report 204257529Sluigi * completion of their block. NR_NOSLOT (~0) indicates 205257529Sluigi * that the writer has not finished yet 206259412Sluigi * nkr_lease_idx index of next free slot in nr_leases, to be assigned 207257529Sluigi * 208257529Sluigi * The kring is manipulated by txsync/rxsync and generic netmap function. 209260368Sluigi * 210260368Sluigi * Concurrent rxsync or txsync on the same ring are prevented through 211260368Sluigi * by nm_kr_lock() which in turn uses nr_busy. This is all we need 212260368Sluigi * for NIC rings, and for TX rings attached to the host stack. 213260368Sluigi * 214260368Sluigi * RX rings attached to the host stack use an mbq (rx_queue) on both 215260368Sluigi * rxsync_from_host() and netmap_transmit(). The mbq is protected 216260368Sluigi * by its internal lock. 217260368Sluigi * 218260368Sluigi * RX rings attached to the VALE switch are accessed by both sender 219260368Sluigi * and receiver. They are protected through the q_lock on the RX ring. 220227614Sluigi */ 221227614Sluigistruct netmap_kring { 222260368Sluigi struct netmap_ring *ring; 223227614Sluigi 224260368Sluigi uint32_t nr_hwcur; 225260368Sluigi uint32_t nr_hwtail; 226260368Sluigi 227260368Sluigi /* 228260368Sluigi * Copies of values in user rings, so we do not need to look 229260368Sluigi * at the ring (which could be modified). These are set in the 230260368Sluigi * *sync_prologue()/finalize() routines. 231260368Sluigi */ 232260368Sluigi uint32_t rhead; 233260368Sluigi uint32_t rcur; 234260368Sluigi uint32_t rtail; 235260368Sluigi 236260368Sluigi uint32_t nr_kflags; /* private driver flags */ 237260368Sluigi#define NKR_PENDINTR 0x1 // Pending interrupt. 238260368Sluigi uint32_t nkr_num_slots; 239260368Sluigi 240260368Sluigi /* 241260368Sluigi * On a NIC reset, the NIC ring indexes may be reset but the 242260368Sluigi * indexes in the netmap rings remain the same. nkr_hwofs 243260368Sluigi * keeps track of the offset between the two. 244260368Sluigi */ 245260368Sluigi int32_t nkr_hwofs; 246260368Sluigi 247245579Sluigi uint16_t nkr_slot_flags; /* initial value for flags */ 248260368Sluigi 249260368Sluigi /* last_reclaim is opaque marker to help reduce the frequency 250260368Sluigi * of operations such as reclaiming tx buffers. A possible use 251260368Sluigi * is set it to ticks and do the reclaim only once per tick. 252260368Sluigi */ 253260368Sluigi uint64_t last_reclaim; 254260368Sluigi 255260368Sluigi 256260368Sluigi NM_SELINFO_T si; /* poll/select wait queue */ 257260368Sluigi NM_LOCK_T q_lock; /* protects kring and ring. */ 258260368Sluigi NM_ATOMIC_T nr_busy; /* prevent concurrent syscalls */ 259260368Sluigi 260231594Sluigi struct netmap_adapter *na; 261260368Sluigi 262260368Sluigi /* The folloiwing fields are for VALE switch support */ 263251139Sluigi struct nm_bdg_fwd *nkr_ft; 264260368Sluigi uint32_t *nkr_leases; 265260368Sluigi#define NR_NOSLOT ((uint32_t)~0) /* used in nkr_*lease* */ 266260368Sluigi uint32_t nkr_hwlease; 267260368Sluigi uint32_t nkr_lease_idx; 268257529Sluigi 269260368Sluigi volatile int nkr_stopped; // XXX what for ? 270257529Sluigi 271261909Sluigi /* Support for adapters without native netmap support. 272259412Sluigi * On tx rings we preallocate an array of tx buffers 273259412Sluigi * (same size as the netmap ring), on rx rings we 274261909Sluigi * store incoming mbufs in a queue that is drained by 275261909Sluigi * a rxsync. 276259412Sluigi */ 277259412Sluigi struct mbuf **tx_pool; 278260368Sluigi // u_int nr_ntc; /* Emulation of a next-to-clean RX ring pointer. */ 279260368Sluigi struct mbq rx_queue; /* intercepted rx mbufs. */ 280259412Sluigi 281260368Sluigi uint32_t ring_id; /* debugging */ 282260368Sluigi char name[64]; /* diagnostic */ 283260368Sluigi 284261909Sluigi int (*nm_sync)(struct netmap_kring *kring, int flags); 285261909Sluigi 286261909Sluigi#ifdef WITH_PIPES 287261909Sluigi struct netmap_kring *pipe; 288261909Sluigi struct netmap_ring *save_ring; 289261909Sluigi#endif /* WITH_PIPES */ 290261909Sluigi 291230572Sluigi} __attribute__((__aligned__(64))); 292227614Sluigi 293257529Sluigi 294257529Sluigi/* return the next index, with wraparound */ 295257529Sluigistatic inline uint32_t 296257529Sluiginm_next(uint32_t i, uint32_t lim) 297257529Sluigi{ 298257529Sluigi return unlikely (i == lim) ? 0 : i + 1; 299257529Sluigi} 300257529Sluigi 301260368Sluigi 302260368Sluigi/* return the previous index, with wraparound */ 303260368Sluigistatic inline uint32_t 304260368Sluiginm_prev(uint32_t i, uint32_t lim) 305260368Sluigi{ 306260368Sluigi return unlikely (i == 0) ? lim : i - 1; 307260368Sluigi} 308260368Sluigi 309260368Sluigi 310227614Sluigi/* 311257529Sluigi * 312257529Sluigi * Here is the layout for the Rx and Tx rings. 313257529Sluigi 314257529Sluigi RxRING TxRING 315257529Sluigi 316257529Sluigi +-----------------+ +-----------------+ 317257529Sluigi | | | | 318257529Sluigi |XXX free slot XXX| |XXX free slot XXX| 319257529Sluigi +-----------------+ +-----------------+ 320260368Sluigihead->| owned by user |<-hwcur | not sent to nic |<-hwcur 321260368Sluigi | | | yet | 322260368Sluigi +-----------------+ | | 323260368Sluigi cur->| available to | | | 324260368Sluigi | user, not read | +-----------------+ 325260368Sluigi | yet | cur->| (being | 326260368Sluigi | | | prepared) | 327260368Sluigi | | | | 328260368Sluigi +-----------------+ + ------ + 329260368Sluigitail->| |<-hwtail | |<-hwlease 330260368Sluigi | (being | ... | | ... 331260368Sluigi | prepared) | ... | | ... 332260368Sluigi +-----------------+ ... | | ... 333257529Sluigi | |<-hwlease +-----------------+ 334260368Sluigi | | tail->| |<-hwtail 335257529Sluigi | | | | 336257529Sluigi | | | | 337257529Sluigi | | | | 338257529Sluigi +-----------------+ +-----------------+ 339257529Sluigi 340260368Sluigi * The cur/tail (user view) and hwcur/hwtail (kernel view) 341257529Sluigi * are used in the normal operation of the card. 342257529Sluigi * 343257529Sluigi * When a ring is the output of a switch port (Rx ring for 344257529Sluigi * a VALE port, Tx ring for the host stack or NIC), slots 345257529Sluigi * are reserved in blocks through 'hwlease' which points 346257529Sluigi * to the next unused slot. 347260368Sluigi * On an Rx ring, hwlease is always after hwtail, 348260368Sluigi * and completions cause hwtail to advance. 349260368Sluigi * On a Tx ring, hwlease is always between cur and hwtail, 350257529Sluigi * and completions cause cur to advance. 351257529Sluigi * 352257529Sluigi * nm_kr_space() returns the maximum number of slots that 353257529Sluigi * can be assigned. 354257529Sluigi * nm_kr_lease() reserves the required number of buffers, 355257529Sluigi * advances nkr_hwlease and also returns an entry in 356257529Sluigi * a circular array where completions should be reported. 357257529Sluigi */ 358257529Sluigi 359257529Sluigi 360257529Sluigi 361259412Sluigienum txrx { NR_RX = 0, NR_TX = 1 }; 362257529Sluigi 363257529Sluigi/* 364259412Sluigi * The "struct netmap_adapter" extends the "struct adapter" 365259412Sluigi * (or equivalent) device descriptor. 366259412Sluigi * It contains all base fields needed to support netmap operation. 367259412Sluigi * There are in fact different types of netmap adapters 368259412Sluigi * (native, generic, VALE switch...) so a netmap_adapter is 369259412Sluigi * just the first field in the derived type. 370227614Sluigi */ 371227614Sluigistruct netmap_adapter { 372241719Sluigi /* 373241719Sluigi * On linux we do not have a good way to tell if an interface 374259412Sluigi * is netmap-capable. So we always use the following trick: 375241719Sluigi * NA(ifp) points here, and the first entry (which hopefully 376241719Sluigi * always exists and is at least 32 bits) contains a magic 377241719Sluigi * value which we can use to detect that the interface is good. 378241719Sluigi */ 379241719Sluigi uint32_t magic; 380259412Sluigi uint32_t na_flags; /* enabled, and other flags */ 381241719Sluigi#define NAF_SKIP_INTR 1 /* use the regular interrupt handler. 382241719Sluigi * useful during initialization 383241719Sluigi */ 384251139Sluigi#define NAF_SW_ONLY 2 /* forward packets only to sw adapter */ 385257529Sluigi#define NAF_BDG_MAYSLEEP 4 /* the bridge is allowed to sleep when 386257529Sluigi * forwarding packets coming from this 387257529Sluigi * interface 388257529Sluigi */ 389257529Sluigi#define NAF_MEM_OWNER 8 /* the adapter is responsible for the 390257529Sluigi * deallocation of the memory allocator 391257529Sluigi */ 392259412Sluigi#define NAF_NATIVE_ON 16 /* the adapter is native and the attached 393259412Sluigi * interface is in netmap mode 394259412Sluigi */ 395259412Sluigi#define NAF_NETMAP_ON 32 /* netmap is active (either native or 396259412Sluigi * emulated. Where possible (e.g. FreeBSD) 397259412Sluigi * IFCAP_NETMAP also mirrors this flag. 398259412Sluigi */ 399261909Sluigi#define NAF_HOST_RINGS 64 /* the adapter supports the host rings */ 400259412Sluigi int active_fds; /* number of user-space descriptors using this 401227614Sluigi interface, which is equal to the number of 402227614Sluigi struct netmap_if objs in the mapped region. */ 403227614Sluigi 404239140Semaste u_int num_rx_rings; /* number of adapter receive rings */ 405239140Semaste u_int num_tx_rings; /* number of adapter transmit rings */ 406227614Sluigi 407227614Sluigi u_int num_tx_desc; /* number of descriptor in each queue */ 408227614Sluigi u_int num_rx_desc; 409227614Sluigi 410227614Sluigi /* tx_rings and rx_rings are private but allocated 411227614Sluigi * as a contiguous chunk of memory. Each array has 412227614Sluigi * N+1 entries, for the adapter queues and for the host queue. 413227614Sluigi */ 414227614Sluigi struct netmap_kring *tx_rings; /* array of TX rings. */ 415227614Sluigi struct netmap_kring *rx_rings; /* array of RX rings. */ 416260368Sluigi 417259412Sluigi void *tailroom; /* space below the rings array */ 418259412Sluigi /* (used for leases) */ 419227614Sluigi 420259412Sluigi 421232238Sluigi NM_SELINFO_T tx_si, rx_si; /* global wait queues */ 422232238Sluigi 423261909Sluigi /* count users of the global wait queues */ 424261909Sluigi int tx_si_users, rx_si_users; 425261909Sluigi 426227614Sluigi /* copy of if_qflush and if_transmit pointers, to intercept 427227614Sluigi * packets from the network stack when netmap is active. 428227614Sluigi */ 429227614Sluigi int (*if_transmit)(struct ifnet *, struct mbuf *); 430227614Sluigi 431260368Sluigi /* copy of if_input for netmap_send_up() */ 432260368Sluigi void (*if_input)(struct ifnet *, struct mbuf *); 433260368Sluigi 434227614Sluigi /* references to the ifnet and device routines, used by 435227614Sluigi * the generic netmap functions. 436227614Sluigi */ 437227614Sluigi struct ifnet *ifp; /* adapter is ifp->if_softc */ 438227614Sluigi 439260368Sluigi /*---- callbacks for this netmap adapter -----*/ 440260368Sluigi /* 441260368Sluigi * nm_dtor() is the cleanup routine called when destroying 442260368Sluigi * the adapter. 443260368Sluigi * 444260368Sluigi * nm_register() is called on NIOCREGIF and close() to enter 445260368Sluigi * or exit netmap mode on the NIC 446260368Sluigi * 447260368Sluigi * nm_txsync() pushes packets to the underlying hw/switch 448260368Sluigi * 449260368Sluigi * nm_rxsync() collects packets from the underlying hw/switch 450260368Sluigi * 451260368Sluigi * nm_config() returns configuration information from the OS 452260368Sluigi * 453261909Sluigi * nm_krings_create() create and init the krings array 454261909Sluigi * (the array layout must conform to the description 455261909Sluigi * found above the definition of netmap_krings_create) 456260368Sluigi * 457261909Sluigi * nm_krings_delete() cleanup and delete the kring array 458260368Sluigi * 459260368Sluigi * nm_notify() is used to act after data have become available. 460260368Sluigi * For hw devices this is typically a selwakeup(), 461260368Sluigi * but for NIC/host ports attached to a switch (or vice-versa) 462260368Sluigi * we also need to invoke the 'txsync' code downstream. 463260368Sluigi */ 464260368Sluigi 465259412Sluigi /* private cleanup */ 466259412Sluigi void (*nm_dtor)(struct netmap_adapter *); 467231594Sluigi 468259412Sluigi int (*nm_register)(struct netmap_adapter *, int onoff); 469257529Sluigi 470259412Sluigi int (*nm_txsync)(struct netmap_adapter *, u_int ring, int flags); 471259412Sluigi int (*nm_rxsync)(struct netmap_adapter *, u_int ring, int flags); 472257529Sluigi#define NAF_FORCE_READ 1 473257529Sluigi#define NAF_FORCE_RECLAIM 2 474245835Sluigi /* return configuration information */ 475259412Sluigi int (*nm_config)(struct netmap_adapter *, 476259412Sluigi u_int *txr, u_int *txd, u_int *rxr, u_int *rxd); 477259412Sluigi int (*nm_krings_create)(struct netmap_adapter *); 478259412Sluigi void (*nm_krings_delete)(struct netmap_adapter *); 479259412Sluigi int (*nm_notify)(struct netmap_adapter *, 480259412Sluigi u_int ring, enum txrx, int flags); 481259412Sluigi#define NAF_DISABLE_NOTIFY 8 482238812Sluigi 483259412Sluigi /* standard refcount to control the lifetime of the adapter 484259412Sluigi * (it should be equal to the lifetime of the corresponding ifp) 485259412Sluigi */ 486259412Sluigi int na_refcount; 487259412Sluigi 488259412Sluigi /* memory allocator (opaque) 489259412Sluigi * We also cache a pointer to the lut_entry for translating 490259412Sluigi * buffer addresses, and the total number of buffers. 491259412Sluigi */ 492259412Sluigi struct netmap_mem_d *nm_mem; 493259412Sluigi struct lut_entry *na_lut; 494259412Sluigi uint32_t na_lut_objtotal; /* max buffer index */ 495259412Sluigi 496259412Sluigi /* used internally. If non-null, the interface cannot be bound 497259412Sluigi * from userspace 498259412Sluigi */ 499259412Sluigi void *na_private; 500261909Sluigi 501261909Sluigi#ifdef WITH_PIPES 502261909Sluigi struct netmap_pipe_adapter **na_pipes; 503261909Sluigi int na_next_pipe; 504261909Sluigi int na_max_pipes; 505261909Sluigi#endif /* WITH_PIPES */ 506259412Sluigi}; 507259412Sluigi 508260368Sluigi 509259412Sluigi/* 510259412Sluigi * If the NIC is owned by the kernel 511259412Sluigi * (i.e., bridge), neither another bridge nor user can use it; 512259412Sluigi * if the NIC is owned by a user, only users can share it. 513259412Sluigi * Evaluation must be done under NMG_LOCK(). 514259412Sluigi */ 515259412Sluigi#define NETMAP_OWNED_BY_KERN(na) (na->na_private) 516259412Sluigi#define NETMAP_OWNED_BY_ANY(na) \ 517259412Sluigi (NETMAP_OWNED_BY_KERN(na) || (na->active_fds > 0)) 518259412Sluigi 519259412Sluigi 520259412Sluigi/* 521259412Sluigi * derived netmap adapters for various types of ports 522259412Sluigi */ 523259412Sluigistruct netmap_vp_adapter { /* VALE software port */ 524259412Sluigi struct netmap_adapter up; 525259412Sluigi 526250107Sluigi /* 527250107Sluigi * Bridge support: 528250107Sluigi * 529250107Sluigi * bdg_port is the port number used in the bridge; 530251139Sluigi * na_bdg points to the bridge this NA is attached to. 531250107Sluigi */ 532238812Sluigi int bdg_port; 533251139Sluigi struct nm_bridge *na_bdg; 534259412Sluigi int retry; 535259412Sluigi 536261909Sluigi /* Offset of ethernet header for each packet. */ 537261909Sluigi u_int virt_hdr_len; 538261909Sluigi /* Maximum Frame Size, used in bdg_mismatch_datapath() */ 539261909Sluigi u_int mfs; 540259412Sluigi}; 541259412Sluigi 542260368Sluigi 543259412Sluigistruct netmap_hw_adapter { /* physical device */ 544259412Sluigi struct netmap_adapter up; 545259412Sluigi 546259412Sluigi struct net_device_ops nm_ndo; // XXX linux only 547259412Sluigi}; 548259412Sluigi 549261909Sluigi/* Mitigation support. */ 550261909Sluigistruct nm_generic_mit { 551261909Sluigi struct hrtimer mit_timer; 552261909Sluigi int mit_pending; 553261909Sluigi struct netmap_adapter *mit_na; /* backpointer */ 554261909Sluigi}; 555260368Sluigi 556260368Sluigistruct netmap_generic_adapter { /* emulated device */ 557259412Sluigi struct netmap_hw_adapter up; 558259412Sluigi 559259412Sluigi /* Pointer to a previously used netmap adapter. */ 560259412Sluigi struct netmap_adapter *prev; 561259412Sluigi 562259412Sluigi /* generic netmap adapters support: 563259412Sluigi * a net_device_ops struct overrides ndo_select_queue(), 564259412Sluigi * save_if_input saves the if_input hook (FreeBSD), 565261909Sluigi * mit implements rx interrupt mitigation, 566259412Sluigi */ 567259412Sluigi struct net_device_ops generic_ndo; 568259412Sluigi void (*save_if_input)(struct ifnet *, struct mbuf *); 569259412Sluigi 570261909Sluigi struct nm_generic_mit *mit; 571260368Sluigi#ifdef linux 572260368Sluigi netdev_tx_t (*save_start_xmit)(struct mbuf *, struct ifnet *); 573260368Sluigi#endif 574259412Sluigi}; 575259412Sluigi 576261909Sluigistatic __inline int 577261909Sluiginetmap_real_tx_rings(struct netmap_adapter *na) 578261909Sluigi{ 579261909Sluigi return na->num_tx_rings + !!(na->na_flags & NAF_HOST_RINGS); 580261909Sluigi} 581261909Sluigi 582261909Sluigistatic __inline int 583261909Sluiginetmap_real_rx_rings(struct netmap_adapter *na) 584261909Sluigi{ 585261909Sluigi return na->num_rx_rings + !!(na->na_flags & NAF_HOST_RINGS); 586261909Sluigi} 587261909Sluigi 588259412Sluigi#ifdef WITH_VALE 589259412Sluigi 590260368Sluigi/* 591260368Sluigi * Bridge wrapper for non VALE ports attached to a VALE switch. 592259412Sluigi * 593260368Sluigi * The real device must already have its own netmap adapter (hwna). 594260368Sluigi * The bridge wrapper and the hwna adapter share the same set of 595260368Sluigi * netmap rings and buffers, but they have two separate sets of 596260368Sluigi * krings descriptors, with tx/rx meanings swapped: 597259412Sluigi * 598259412Sluigi * netmap 599259412Sluigi * bwrap krings rings krings hwna 600259412Sluigi * +------+ +------+ +-----+ +------+ +------+ 601259412Sluigi * |tx_rings->| |\ /| |----| |<-tx_rings| 602259412Sluigi * | | +------+ \ / +-----+ +------+ | | 603259412Sluigi * | | X | | 604259412Sluigi * | | / \ | | 605259412Sluigi * | | +------+/ \+-----+ +------+ | | 606259412Sluigi * |rx_rings->| | | |----| |<-rx_rings| 607259412Sluigi * | | +------+ +-----+ +------+ | | 608259412Sluigi * +------+ +------+ 609259412Sluigi * 610260368Sluigi * - packets coming from the bridge go to the brwap rx rings, 611260368Sluigi * which are also the hwna tx rings. The bwrap notify callback 612260368Sluigi * will then complete the hwna tx (see netmap_bwrap_notify). 613259412Sluigi * 614260368Sluigi * - packets coming from the outside go to the hwna rx rings, 615260368Sluigi * which are also the bwrap tx rings. The (overwritten) hwna 616260368Sluigi * notify method will then complete the bridge tx 617260368Sluigi * (see netmap_bwrap_intr_notify). 618259412Sluigi * 619260368Sluigi * The bridge wrapper may optionally connect the hwna 'host' rings 620260368Sluigi * to the bridge. This is done by using a second port in the 621260368Sluigi * bridge and connecting it to the 'host' netmap_vp_adapter 622260368Sluigi * contained in the netmap_bwrap_adapter. The brwap host adapter 623260368Sluigi * cross-links the hwna host rings in the same way as shown above. 624259412Sluigi * 625260368Sluigi * - packets coming from the bridge and directed to the host stack 626260368Sluigi * are handled by the bwrap host notify callback 627260368Sluigi * (see netmap_bwrap_host_notify) 628260368Sluigi * 629260368Sluigi * - packets coming from the host stack are still handled by the 630260368Sluigi * overwritten hwna notify callback (netmap_bwrap_intr_notify), 631260368Sluigi * but are diverted to the host adapter depending on the ring number. 632260368Sluigi * 633259412Sluigi */ 634259412Sluigistruct netmap_bwrap_adapter { 635259412Sluigi struct netmap_vp_adapter up; 636259412Sluigi struct netmap_vp_adapter host; /* for host rings */ 637259412Sluigi struct netmap_adapter *hwna; /* the underlying device */ 638259412Sluigi 639259412Sluigi /* backup of the hwna notify callback */ 640259412Sluigi int (*save_notify)(struct netmap_adapter *, 641259412Sluigi u_int ring, enum txrx, int flags); 642260368Sluigi 643260368Sluigi /* 644260368Sluigi * When we attach a physical interface to the bridge, we 645251139Sluigi * allow the controlling process to terminate, so we need 646251139Sluigi * a place to store the netmap_priv_d data structure. 647260368Sluigi * This is only done when physical interfaces 648260368Sluigi * are attached to a bridge. 649251139Sluigi */ 650251139Sluigi struct netmap_priv_d *na_kpriv; 651227614Sluigi}; 652227614Sluigi 653259412Sluigi 654260368Sluigi#endif /* WITH_VALE */ 655257529Sluigi 656261909Sluigi#ifdef WITH_PIPES 657257529Sluigi 658261909Sluigi#define NM_MAXPIPES 64 /* max number of pipes per adapter */ 659261909Sluigi 660261909Sluigistruct netmap_pipe_adapter { 661261909Sluigi struct netmap_adapter up; 662261909Sluigi 663261909Sluigi u_int id; /* pipe identifier */ 664261909Sluigi int role; /* either NR_REG_PIPE_MASTER or NR_REG_PIPE_SLAVE */ 665261909Sluigi 666261909Sluigi struct netmap_adapter *parent; /* adapter that owns the memory */ 667261909Sluigi struct netmap_pipe_adapter *peer; /* the other end of the pipe */ 668261909Sluigi int peer_ref; /* 1 iff we are holding a ref to the peer */ 669261909Sluigi 670261909Sluigi u_int parent_slot; /* index in the parent pipe array */ 671261909Sluigi}; 672261909Sluigi 673261909Sluigi#endif /* WITH_PIPES */ 674261909Sluigi 675261909Sluigi 676260368Sluigi/* return slots reserved to rx clients; used in drivers */ 677257529Sluigistatic inline uint32_t 678260368Sluiginm_kr_rxspace(struct netmap_kring *k) 679257529Sluigi{ 680260368Sluigi int space = k->nr_hwtail - k->nr_hwcur; 681267128Sluigi if (space < 0) 682260368Sluigi space += k->nkr_num_slots; 683260368Sluigi ND("preserving %d rx slots %d -> %d", space, k->nr_hwcur, k->nr_hwtail); 684257529Sluigi 685260368Sluigi return space; 686257529Sluigi} 687257529Sluigi 688257529Sluigi 689260368Sluigi/* True if no space in the tx ring. only valid after txsync_prologue */ 690260368Sluigistatic inline int 691260368Sluiginm_kr_txempty(struct netmap_kring *kring) 692259412Sluigi{ 693260368Sluigi return kring->rcur == kring->nr_hwtail; 694259412Sluigi} 695259412Sluigi 696259412Sluigi 697257529Sluigi/* 698259412Sluigi * protect against multiple threads using the same ring. 699259412Sluigi * also check that the ring has not been stopped. 700259412Sluigi * We only care for 0 or !=0 as a return code. 701227614Sluigi */ 702259412Sluigi#define NM_KR_BUSY 1 703259412Sluigi#define NM_KR_STOPPED 2 704227614Sluigi 705260368Sluigi 706259412Sluigistatic __inline void nm_kr_put(struct netmap_kring *kr) 707259412Sluigi{ 708259412Sluigi NM_ATOMIC_CLEAR(&kr->nr_busy); 709259412Sluigi} 710227614Sluigi 711260368Sluigi 712259412Sluigistatic __inline int nm_kr_tryget(struct netmap_kring *kr) 713259412Sluigi{ 714259412Sluigi /* check a first time without taking the lock 715259412Sluigi * to avoid starvation for nm_kr_get() 716259412Sluigi */ 717259412Sluigi if (unlikely(kr->nkr_stopped)) { 718259412Sluigi ND("ring %p stopped (%d)", kr, kr->nkr_stopped); 719259412Sluigi return NM_KR_STOPPED; 720259412Sluigi } 721259412Sluigi if (unlikely(NM_ATOMIC_TEST_AND_SET(&kr->nr_busy))) 722259412Sluigi return NM_KR_BUSY; 723259412Sluigi /* check a second time with lock held */ 724259412Sluigi if (unlikely(kr->nkr_stopped)) { 725259412Sluigi ND("ring %p stopped (%d)", kr, kr->nkr_stopped); 726259412Sluigi nm_kr_put(kr); 727259412Sluigi return NM_KR_STOPPED; 728259412Sluigi } 729259412Sluigi return 0; 730259412Sluigi} 731227614Sluigi 732259412Sluigi 733227614Sluigi/* 734260368Sluigi * The following functions are used by individual drivers to 735227614Sluigi * support netmap operation. 736227614Sluigi * 737227614Sluigi * netmap_attach() initializes a struct netmap_adapter, allocating the 738227614Sluigi * struct netmap_ring's and the struct selinfo. 739227614Sluigi * 740227614Sluigi * netmap_detach() frees the memory allocated by netmap_attach(). 741227614Sluigi * 742257529Sluigi * netmap_transmit() replaces the if_transmit routine of the interface, 743227614Sluigi * and is used to intercept packets coming from the stack. 744227614Sluigi * 745227614Sluigi * netmap_load_map/netmap_reload_map are helper routines to set/reset 746227614Sluigi * the dmamap for a packet buffer 747227614Sluigi * 748227614Sluigi * netmap_reset() is a helper routine to be called in the driver 749227614Sluigi * when reinitializing a ring. 750227614Sluigi */ 751259412Sluigiint netmap_attach(struct netmap_adapter *); 752259412Sluigiint netmap_attach_common(struct netmap_adapter *); 753259412Sluigivoid netmap_detach_common(struct netmap_adapter *na); 754227614Sluigivoid netmap_detach(struct ifnet *); 755257529Sluigiint netmap_transmit(struct ifnet *, struct mbuf *); 756227614Sluigistruct netmap_slot *netmap_reset(struct netmap_adapter *na, 757257529Sluigi enum txrx tx, u_int n, u_int new_cur); 758227614Sluigiint netmap_ring_reinit(struct netmap_kring *); 759227614Sluigi 760260368Sluigi/* default functions to handle rx/tx interrupts */ 761260368Sluigiint netmap_rx_irq(struct ifnet *, u_int, u_int *); 762260368Sluigi#define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL) 763260368Sluigivoid netmap_common_irq(struct ifnet *, u_int, u_int *work_done); 764260368Sluigi 765260368Sluigivoid netmap_disable_all_rings(struct ifnet *); 766260368Sluigivoid netmap_enable_all_rings(struct ifnet *); 767260368Sluigivoid netmap_disable_ring(struct netmap_kring *kr); 768260368Sluigi 769260368Sluigi 770260368Sluigi/* set/clear native flags and if_transmit/netdev_ops */ 771259412Sluigistatic inline void 772259412Sluiginm_set_native_flags(struct netmap_adapter *na) 773259412Sluigi{ 774259412Sluigi struct ifnet *ifp = na->ifp; 775259412Sluigi 776259412Sluigi na->na_flags |= (NAF_NATIVE_ON | NAF_NETMAP_ON); 777259412Sluigi#ifdef IFCAP_NETMAP /* or FreeBSD ? */ 778259412Sluigi ifp->if_capenable |= IFCAP_NETMAP; 779259412Sluigi#endif 780259412Sluigi#ifdef __FreeBSD__ 781259412Sluigi na->if_transmit = ifp->if_transmit; 782259412Sluigi ifp->if_transmit = netmap_transmit; 783259412Sluigi#else 784259412Sluigi na->if_transmit = (void *)ifp->netdev_ops; 785259412Sluigi ifp->netdev_ops = &((struct netmap_hw_adapter *)na)->nm_ndo; 786259412Sluigi#endif 787259412Sluigi} 788259412Sluigi 789260368Sluigi 790259412Sluigistatic inline void 791259412Sluiginm_clear_native_flags(struct netmap_adapter *na) 792259412Sluigi{ 793259412Sluigi struct ifnet *ifp = na->ifp; 794259412Sluigi 795259412Sluigi#ifdef __FreeBSD__ 796259412Sluigi ifp->if_transmit = na->if_transmit; 797259412Sluigi#else 798259412Sluigi ifp->netdev_ops = (void *)na->if_transmit; 799259412Sluigi#endif 800259412Sluigi na->na_flags &= ~(NAF_NATIVE_ON | NAF_NETMAP_ON); 801259412Sluigi#ifdef IFCAP_NETMAP /* or FreeBSD ? */ 802259412Sluigi ifp->if_capenable &= ~IFCAP_NETMAP; 803259412Sluigi#endif 804259412Sluigi} 805259412Sluigi 806260368Sluigi 807259412Sluigi/* 808260368Sluigi * validates parameters in the ring/kring, returns a value for head 809260368Sluigi * If any error, returns ring_size to force a reinit. 810259412Sluigi */ 811260368Sluigiuint32_t nm_txsync_prologue(struct netmap_kring *); 812259412Sluigi 813260368Sluigi 814259412Sluigi/* 815260368Sluigi * validates parameters in the ring/kring, returns a value for head, 816259412Sluigi * and the 'reserved' value in the argument. 817260368Sluigi * If any error, returns ring_size lim to force a reinit. 818259412Sluigi */ 819260368Sluigiuint32_t nm_rxsync_prologue(struct netmap_kring *); 820259412Sluigi 821260368Sluigi 822259412Sluigi/* 823260368Sluigi * update kring and ring at the end of txsync. 824259412Sluigi */ 825259412Sluigistatic inline void 826260368Sluiginm_txsync_finalize(struct netmap_kring *kring) 827259412Sluigi{ 828261909Sluigi /* update ring tail to what the kernel knows */ 829260368Sluigi kring->ring->tail = kring->rtail = kring->nr_hwtail; 830267128Sluigi 831260368Sluigi /* note, head/rhead/hwcur might be behind cur/rcur 832260368Sluigi * if no carrier 833260368Sluigi */ 834260368Sluigi ND(5, "%s now hwcur %d hwtail %d head %d cur %d tail %d", 835260368Sluigi kring->name, kring->nr_hwcur, kring->nr_hwtail, 836260368Sluigi kring->rhead, kring->rcur, kring->rtail); 837260368Sluigi} 838259412Sluigi 839260368Sluigi 840260368Sluigi/* 841260368Sluigi * update kring and ring at the end of rxsync 842260368Sluigi */ 843260368Sluigistatic inline void 844260368Sluiginm_rxsync_finalize(struct netmap_kring *kring) 845260368Sluigi{ 846260368Sluigi /* tell userspace that there might be new packets */ 847260368Sluigi //struct netmap_ring *ring = kring->ring; 848260368Sluigi ND("head %d cur %d tail %d -> %d", ring->head, ring->cur, ring->tail, 849260368Sluigi kring->nr_hwtail); 850260368Sluigi kring->ring->tail = kring->rtail = kring->nr_hwtail; 851260368Sluigi /* make a copy of the state for next round */ 852260368Sluigi kring->rhead = kring->ring->head; 853260368Sluigi kring->rcur = kring->ring->cur; 854259412Sluigi} 855259412Sluigi 856260368Sluigi 857259412Sluigi/* check/fix address and len in tx rings */ 858259412Sluigi#if 1 /* debug version */ 859259412Sluigi#define NM_CHECK_ADDR_LEN(_a, _l) do { \ 860259412Sluigi if (_a == netmap_buffer_base || _l > NETMAP_BUF_SIZE) { \ 861259412Sluigi RD(5, "bad addr/len ring %d slot %d idx %d len %d", \ 862259412Sluigi ring_nr, nm_i, slot->buf_idx, len); \ 863259412Sluigi if (_l > NETMAP_BUF_SIZE) \ 864259412Sluigi _l = NETMAP_BUF_SIZE; \ 865259412Sluigi } } while (0) 866259412Sluigi#else /* no debug version */ 867259412Sluigi#define NM_CHECK_ADDR_LEN(_a, _l) do { \ 868259412Sluigi if (_l > NETMAP_BUF_SIZE) \ 869259412Sluigi _l = NETMAP_BUF_SIZE; \ 870259412Sluigi } while (0) 871259412Sluigi#endif 872259412Sluigi 873259412Sluigi 874259412Sluigi/*---------------------------------------------------------------*/ 875259412Sluigi/* 876259412Sluigi * Support routines to be used with the VALE switch 877259412Sluigi */ 878259412Sluigiint netmap_update_config(struct netmap_adapter *na); 879261909Sluigiint netmap_krings_create(struct netmap_adapter *na, u_int tailroom); 880259412Sluigivoid netmap_krings_delete(struct netmap_adapter *na); 881260368Sluigiint netmap_rxsync_from_host(struct netmap_adapter *na, struct thread *td, void *pwait); 882259412Sluigi 883260368Sluigi 884259412Sluigistruct netmap_if * 885259412Sluiginetmap_do_regif(struct netmap_priv_d *priv, struct netmap_adapter *na, 886261909Sluigi uint16_t ringid, uint32_t flags, int *err); 887259412Sluigi 888259412Sluigi 889259412Sluigi 890257529Sluigiu_int nm_bound_var(u_int *v, u_int dflt, u_int lo, u_int hi, const char *msg); 891259412Sluigiint netmap_get_na(struct nmreq *nmr, struct netmap_adapter **na, int create); 892259412Sluigiint netmap_get_hw_na(struct ifnet *ifp, struct netmap_adapter **na); 893257529Sluigi 894260368Sluigi 895259412Sluigi#ifdef WITH_VALE 896251139Sluigi/* 897260368Sluigi * The following bridge-related functions are used by other 898260368Sluigi * kernel modules. 899260368Sluigi * 900260368Sluigi * VALE only supports unicast or broadcast. The lookup 901251139Sluigi * function can return 0 .. NM_BDG_MAXPORTS-1 for regular ports, 902251139Sluigi * NM_BDG_MAXPORTS for broadcast, NM_BDG_MAXPORTS+1 for unknown. 903251139Sluigi * XXX in practice "unknown" might be handled same as broadcast. 904251139Sluigi */ 905259412Sluigitypedef u_int (*bdg_lookup_fn_t)(char *buf, u_int len, 906259412Sluigi uint8_t *ring_nr, struct netmap_vp_adapter *); 907259412Sluigiu_int netmap_bdg_learning(char *, u_int, uint8_t *, 908259412Sluigi struct netmap_vp_adapter *); 909259412Sluigi 910259412Sluigi#define NM_BDG_MAXPORTS 254 /* up to 254 */ 911251139Sluigi#define NM_BDG_BROADCAST NM_BDG_MAXPORTS 912251139Sluigi#define NM_BDG_NOPORT (NM_BDG_MAXPORTS+1) 913251139Sluigi 914259412Sluigi#define NM_NAME "vale" /* prefix for bridge port name */ 915259412Sluigi 916259412Sluigi 917259412Sluigi/* these are redefined in case of no VALE support */ 918259412Sluigiint netmap_get_bdg_na(struct nmreq *nmr, struct netmap_adapter **na, int create); 919259412Sluigivoid netmap_init_bridges(void); 920259412Sluigiint netmap_bdg_ctl(struct nmreq *nmr, bdg_lookup_fn_t func); 921259412Sluigi 922259412Sluigi#else /* !WITH_VALE */ 923259412Sluigi#define netmap_get_bdg_na(_1, _2, _3) 0 924259412Sluigi#define netmap_init_bridges(_1) 925259412Sluigi#define netmap_bdg_ctl(_1, _2) EINVAL 926259412Sluigi#endif /* !WITH_VALE */ 927259412Sluigi 928261909Sluigi#ifdef WITH_PIPES 929261909Sluigi/* max number of pipes per device */ 930261909Sluigi#define NM_MAXPIPES 64 /* XXX how many? */ 931261909Sluigi/* in case of no error, returns the actual number of pipes in nmr->nr_arg1 */ 932261909Sluigiint netmap_pipe_alloc(struct netmap_adapter *, struct nmreq *nmr); 933261909Sluigivoid netmap_pipe_dealloc(struct netmap_adapter *); 934261909Sluigiint netmap_get_pipe_na(struct nmreq *nmr, struct netmap_adapter **na, int create); 935261909Sluigi#else /* !WITH_PIPES */ 936261909Sluigi#define NM_MAXPIPES 0 937261909Sluigi#define netmap_pipe_alloc(_1, _2) EOPNOTSUPP 938261909Sluigi#define netmap_pipe_dealloc(_1) 939261909Sluigi#define netmap_get_pipe_na(_1, _2, _3) 0 940261909Sluigi#endif 941261909Sluigi 942259412Sluigi/* Various prototypes */ 943259412Sluigiint netmap_poll(struct cdev *dev, int events, struct thread *td); 944259412Sluigiint netmap_init(void); 945259412Sluigivoid netmap_fini(void); 946259412Sluigiint netmap_get_memory(struct netmap_priv_d* p); 947259412Sluigivoid netmap_dtor(void *data); 948259412Sluigiint netmap_dtor_locked(struct netmap_priv_d *priv); 949259412Sluigi 950259412Sluigiint netmap_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td); 951259412Sluigi 952259412Sluigi/* netmap_adapter creation/destruction */ 953259412Sluigi#define NM_IFPNAME(ifp) ((ifp) ? (ifp)->if_xname : "zombie") 954259412Sluigi 955260368Sluigi// #define NM_DEBUG_PUTGET 1 956260368Sluigi 957259412Sluigi#ifdef NM_DEBUG_PUTGET 958259412Sluigi 959259412Sluigi#define NM_DBG(f) __##f 960259412Sluigi 961259412Sluigivoid __netmap_adapter_get(struct netmap_adapter *na); 962259412Sluigi 963259412Sluigi#define netmap_adapter_get(na) \ 964259412Sluigi do { \ 965259412Sluigi struct netmap_adapter *__na = na; \ 966259412Sluigi D("getting %p:%s (%d)", __na, NM_IFPNAME(__na->ifp), __na->na_refcount); \ 967259412Sluigi __netmap_adapter_get(__na); \ 968259412Sluigi } while (0) 969259412Sluigi 970259412Sluigiint __netmap_adapter_put(struct netmap_adapter *na); 971259412Sluigi 972259412Sluigi#define netmap_adapter_put(na) \ 973260411Sluigi ({ \ 974259412Sluigi struct netmap_adapter *__na = na; \ 975259412Sluigi D("putting %p:%s (%d)", __na, NM_IFPNAME(__na->ifp), __na->na_refcount); \ 976259412Sluigi __netmap_adapter_put(__na); \ 977260411Sluigi }) 978259412Sluigi 979259412Sluigi#else /* !NM_DEBUG_PUTGET */ 980259412Sluigi 981259412Sluigi#define NM_DBG(f) f 982259412Sluigivoid netmap_adapter_get(struct netmap_adapter *na); 983259412Sluigiint netmap_adapter_put(struct netmap_adapter *na); 984259412Sluigi 985259412Sluigi#endif /* !NM_DEBUG_PUTGET */ 986259412Sluigi 987259412Sluigi 988260368Sluigi/* 989260368Sluigi * module variables 990260368Sluigi */ 991238985Sluigiextern u_int netmap_buf_size; 992250107Sluigi#define NETMAP_BUF_SIZE netmap_buf_size // XXX remove 993260368Sluigiextern int netmap_mitigate; // XXX not really used 994231198Sluigiextern int netmap_no_pendintr; 995260368Sluigiextern u_int netmap_total_buffers; // global allocator 996260368Sluigiextern char *netmap_buffer_base; // global allocator 997227614Sluigiextern int netmap_verbose; // XXX debugging 998227614Sluigienum { /* verbose flags */ 999227614Sluigi NM_VERB_ON = 1, /* generic verbose */ 1000227614Sluigi NM_VERB_HOST = 0x2, /* verbose host stack */ 1001227614Sluigi NM_VERB_RXSYNC = 0x10, /* verbose on rxsync/txsync */ 1002227614Sluigi NM_VERB_TXSYNC = 0x20, 1003227614Sluigi NM_VERB_RXINTR = 0x100, /* verbose on rx/tx intr (driver) */ 1004227614Sluigi NM_VERB_TXINTR = 0x200, 1005227614Sluigi NM_VERB_NIC_RXSYNC = 0x1000, /* verbose on rx/tx intr (driver) */ 1006227614Sluigi NM_VERB_NIC_TXSYNC = 0x2000, 1007227614Sluigi}; 1008227614Sluigi 1009259412Sluigiextern int netmap_txsync_retry; 1010259412Sluigiextern int netmap_generic_mit; 1011259412Sluigiextern int netmap_generic_ringsize; 1012261909Sluigiextern int netmap_generic_rings; 1013259412Sluigi 1014227614Sluigi/* 1015228845Sluigi * NA returns a pointer to the struct netmap adapter from the ifp, 1016228845Sluigi * WNA is used to write it. 1017227614Sluigi */ 1018228845Sluigi#ifndef WNA 1019228845Sluigi#define WNA(_ifp) (_ifp)->if_pspare[0] 1020228845Sluigi#endif 1021228845Sluigi#define NA(_ifp) ((struct netmap_adapter *)WNA(_ifp)) 1022227614Sluigi 1023241719Sluigi/* 1024241719Sluigi * Macros to determine if an interface is netmap capable or netmap enabled. 1025241719Sluigi * See the magic field in struct netmap_adapter. 1026241719Sluigi */ 1027241719Sluigi#ifdef __FreeBSD__ 1028241719Sluigi/* 1029241719Sluigi * on FreeBSD just use if_capabilities and if_capenable. 1030241719Sluigi */ 1031241719Sluigi#define NETMAP_CAPABLE(ifp) (NA(ifp) && \ 1032241719Sluigi (ifp)->if_capabilities & IFCAP_NETMAP ) 1033227614Sluigi 1034241719Sluigi#define NETMAP_SET_CAPABLE(ifp) \ 1035241719Sluigi (ifp)->if_capabilities |= IFCAP_NETMAP 1036241719Sluigi 1037241719Sluigi#else /* linux */ 1038241719Sluigi 1039241719Sluigi/* 1040241719Sluigi * on linux: 1041241719Sluigi * we check if NA(ifp) is set and its first element has a related 1042241719Sluigi * magic value. The capenable is within the struct netmap_adapter. 1043241719Sluigi */ 1044241719Sluigi#define NETMAP_MAGIC 0x52697a7a 1045241719Sluigi 1046241719Sluigi#define NETMAP_CAPABLE(ifp) (NA(ifp) && \ 1047241719Sluigi ((uint32_t)(uintptr_t)NA(ifp) ^ NA(ifp)->magic) == NETMAP_MAGIC ) 1048241719Sluigi 1049241719Sluigi#define NETMAP_SET_CAPABLE(ifp) \ 1050241719Sluigi NA(ifp)->magic = ((uint32_t)(uintptr_t)NA(ifp)) ^ NETMAP_MAGIC 1051241719Sluigi 1052241719Sluigi#endif /* linux */ 1053241719Sluigi 1054238812Sluigi#ifdef __FreeBSD__ 1055259412Sluigi 1056260368Sluigi/* Callback invoked by the dma machinery after a successful dmamap_load */ 1057230052Sluigistatic void netmap_dmamap_cb(__unused void *arg, 1058230058Sluigi __unused bus_dma_segment_t * segs, __unused int nseg, __unused int error) 1059230052Sluigi{ 1060230052Sluigi} 1061230052Sluigi 1062230052Sluigi/* bus_dmamap_load wrapper: call aforementioned function if map != NULL. 1063230052Sluigi * XXX can we do it without a callback ? 1064230052Sluigi */ 1065230052Sluigistatic inline void 1066230052Sluiginetmap_load_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1067230052Sluigi{ 1068230052Sluigi if (map) 1069230052Sluigi bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE, 1070230058Sluigi netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 1071230052Sluigi} 1072230052Sluigi 1073230052Sluigi/* update the map when a buffer changes. */ 1074230052Sluigistatic inline void 1075230052Sluiginetmap_reload_map(bus_dma_tag_t tag, bus_dmamap_t map, void *buf) 1076230052Sluigi{ 1077230052Sluigi if (map) { 1078230052Sluigi bus_dmamap_unload(tag, map); 1079230052Sluigi bus_dmamap_load(tag, map, buf, NETMAP_BUF_SIZE, 1080230058Sluigi netmap_dmamap_cb, NULL, BUS_DMA_NOWAIT); 1081230052Sluigi } 1082230052Sluigi} 1083259412Sluigi 1084238812Sluigi#else /* linux */ 1085230052Sluigi 1086231796Sluigi/* 1087238812Sluigi * XXX How do we redefine these functions: 1088238812Sluigi * 1089238812Sluigi * on linux we need 1090238831Sluigi * dma_map_single(&pdev->dev, virt_addr, len, direction) 1091238831Sluigi * dma_unmap_single(&adapter->pdev->dev, phys_addr, len, direction 1092238812Sluigi * The len can be implicit (on netmap it is NETMAP_BUF_SIZE) 1093238812Sluigi * unfortunately the direction is not, so we need to change 1094238812Sluigi * something to have a cross API 1095238812Sluigi */ 1096238812Sluigi#define netmap_load_map(_t, _m, _b) 1097238812Sluigi#define netmap_reload_map(_t, _m, _b) 1098238812Sluigi#if 0 1099238812Sluigi struct e1000_buffer *buffer_info = &tx_ring->buffer_info[l]; 1100238812Sluigi /* set time_stamp *before* dma to help avoid a possible race */ 1101238812Sluigi buffer_info->time_stamp = jiffies; 1102238812Sluigi buffer_info->mapped_as_page = false; 1103238812Sluigi buffer_info->length = len; 1104238812Sluigi //buffer_info->next_to_watch = l; 1105238812Sluigi /* reload dma map */ 1106238812Sluigi dma_unmap_single(&adapter->pdev->dev, buffer_info->dma, 1107238831Sluigi NETMAP_BUF_SIZE, DMA_TO_DEVICE); 1108238812Sluigi buffer_info->dma = dma_map_single(&adapter->pdev->dev, 1109238831Sluigi addr, NETMAP_BUF_SIZE, DMA_TO_DEVICE); 1110238812Sluigi 1111238812Sluigi if (dma_mapping_error(&adapter->pdev->dev, buffer_info->dma)) { 1112238812Sluigi D("dma mapping error"); 1113238812Sluigi /* goto dma_error; See e1000_put_txbuf() */ 1114238812Sluigi /* XXX reset */ 1115238812Sluigi } 1116238812Sluigi tx_desc->buffer_addr = htole64(buffer_info->dma); //XXX 1117238812Sluigi 1118238812Sluigi#endif 1119238812Sluigi 1120238812Sluigi/* 1121238812Sluigi * The bus_dmamap_sync() can be one of wmb() or rmb() depending on direction. 1122238812Sluigi */ 1123238812Sluigi#define bus_dmamap_sync(_a, _b, _c) 1124238812Sluigi 1125238812Sluigi#endif /* linux */ 1126238812Sluigi 1127257529Sluigi 1128238812Sluigi/* 1129231796Sluigi * functions to map NIC to KRING indexes (n2k) and vice versa (k2n) 1130231796Sluigi */ 1131231796Sluigistatic inline int 1132232238Sluiginetmap_idx_n2k(struct netmap_kring *kr, int idx) 1133231796Sluigi{ 1134232238Sluigi int n = kr->nkr_num_slots; 1135232238Sluigi idx += kr->nkr_hwofs; 1136232238Sluigi if (idx < 0) 1137232238Sluigi return idx + n; 1138232238Sluigi else if (idx < n) 1139232238Sluigi return idx; 1140231796Sluigi else 1141232238Sluigi return idx - n; 1142231796Sluigi} 1143230052Sluigi 1144231796Sluigi 1145231796Sluigistatic inline int 1146232238Sluiginetmap_idx_k2n(struct netmap_kring *kr, int idx) 1147231796Sluigi{ 1148232238Sluigi int n = kr->nkr_num_slots; 1149232238Sluigi idx -= kr->nkr_hwofs; 1150232238Sluigi if (idx < 0) 1151232238Sluigi return idx + n; 1152232238Sluigi else if (idx < n) 1153232238Sluigi return idx; 1154231796Sluigi else 1155232238Sluigi return idx - n; 1156231796Sluigi} 1157231796Sluigi 1158231796Sluigi 1159234227Sluigi/* Entries of the look-up table. */ 1160234227Sluigistruct lut_entry { 1161234227Sluigi void *vaddr; /* virtual address. */ 1162250107Sluigi vm_paddr_t paddr; /* physical address. */ 1163234227Sluigi}; 1164234227Sluigi 1165234227Sluigistruct netmap_obj_pool; 1166234227Sluigiextern struct lut_entry *netmap_buffer_lut; 1167234227Sluigi#define NMB_VA(i) (netmap_buffer_lut[i].vaddr) 1168234227Sluigi#define NMB_PA(i) (netmap_buffer_lut[i].paddr) 1169234227Sluigi 1170227614Sluigi/* 1171229939Sluigi * NMB return the virtual address of a buffer (buffer 0 on bad index) 1172229939Sluigi * PNMB also fills the physical address 1173227614Sluigi */ 1174229939Sluigistatic inline void * 1175227614SluigiNMB(struct netmap_slot *slot) 1176227614Sluigi{ 1177227614Sluigi uint32_t i = slot->buf_idx; 1178238812Sluigi return (unlikely(i >= netmap_total_buffers)) ? NMB_VA(0) : NMB_VA(i); 1179227614Sluigi} 1180227614Sluigi 1181229939Sluigistatic inline void * 1182229939SluigiPNMB(struct netmap_slot *slot, uint64_t *pp) 1183229939Sluigi{ 1184229939Sluigi uint32_t i = slot->buf_idx; 1185234227Sluigi void *ret = (i >= netmap_total_buffers) ? NMB_VA(0) : NMB_VA(i); 1186249659Sluigi 1187234227Sluigi *pp = (i >= netmap_total_buffers) ? NMB_PA(0) : NMB_PA(i); 1188229939Sluigi return ret; 1189229939Sluigi} 1190229939Sluigi 1191259412Sluigi/* Generic version of NMB, which uses device-specific memory. */ 1192259412Sluigistatic inline void * 1193259412SluigiBDG_NMB(struct netmap_adapter *na, struct netmap_slot *slot) 1194259412Sluigi{ 1195259412Sluigi struct lut_entry *lut = na->na_lut; 1196259412Sluigi uint32_t i = slot->buf_idx; 1197259412Sluigi return (unlikely(i >= na->na_lut_objtotal)) ? 1198259412Sluigi lut[0].vaddr : lut[i].vaddr; 1199259412Sluigi} 1200259412Sluigi 1201238812Sluigi 1202257529Sluigi 1203259412Sluigivoid netmap_txsync_to_host(struct netmap_adapter *na); 1204257529Sluigi 1205259412Sluigi 1206260368Sluigi/* 1207260368Sluigi * Structure associated to each thread which registered an interface. 1208259412Sluigi * 1209259412Sluigi * The first 4 fields of this structure are written by NIOCREGIF and 1210259412Sluigi * read by poll() and NIOC?XSYNC. 1211260368Sluigi * 1212260368Sluigi * There is low contention among writers (a correct user program 1213260368Sluigi * should have none) and among writers and readers, so we use a 1214260368Sluigi * single global lock to protect the structure initialization; 1215260368Sluigi * since initialization involves the allocation of memory, 1216260368Sluigi * we reuse the memory allocator lock. 1217260368Sluigi * 1218259412Sluigi * Read access to the structure is lock free. Readers must check that 1219259412Sluigi * np_nifp is not NULL before using the other fields. 1220260368Sluigi * If np_nifp is NULL initialization has not been performed, 1221260368Sluigi * so they should return an error to userspace. 1222259412Sluigi * 1223259412Sluigi * The ref_done field is used to regulate access to the refcount in the 1224259412Sluigi * memory allocator. The refcount must be incremented at most once for 1225259412Sluigi * each open("/dev/netmap"). The increment is performed by the first 1226259412Sluigi * function that calls netmap_get_memory() (currently called by 1227259412Sluigi * mmap(), NIOCGINFO and NIOCREGIF). 1228259412Sluigi * If the refcount is incremented, it is then decremented when the 1229259412Sluigi * private structure is destroyed. 1230259412Sluigi */ 1231259412Sluigistruct netmap_priv_d { 1232259412Sluigi struct netmap_if * volatile np_nifp; /* netmap if descriptor. */ 1233259412Sluigi 1234259412Sluigi struct netmap_adapter *np_na; 1235261909Sluigi uint32_t np_flags; /* from the ioctl */ 1236261909Sluigi u_int np_txqfirst, np_txqlast; /* range of tx rings to scan */ 1237261909Sluigi u_int np_rxqfirst, np_rxqlast; /* range of rx rings to scan */ 1238261909Sluigi uint16_t np_txpoll; /* XXX and also np_rxpoll ? */ 1239259412Sluigi 1240259412Sluigi struct netmap_mem_d *np_mref; /* use with NMG_LOCK held */ 1241259412Sluigi /* np_refcount is only used on FreeBSD */ 1242260368Sluigi int np_refcount; /* use with NMG_LOCK held */ 1243261909Sluigi 1244261909Sluigi /* pointers to the selinfo to be used for selrecord. 1245261909Sluigi * Either the local or the global one depending on the 1246261909Sluigi * number of rings. 1247261909Sluigi */ 1248261909Sluigi NM_SELINFO_T *np_rxsi, *np_txsi; 1249261909Sluigi struct thread *np_td; /* kqueue, just debugging */ 1250259412Sluigi}; 1251259412Sluigi 1252259412Sluigi 1253259412Sluigi/* 1254259412Sluigi * generic netmap emulation for devices that do not have 1255259412Sluigi * native netmap support. 1256259412Sluigi */ 1257259412Sluigiint generic_netmap_attach(struct ifnet *ifp); 1258259412Sluigi 1259259412Sluigiint netmap_catch_rx(struct netmap_adapter *na, int intercept); 1260259412Sluigivoid generic_rx_handler(struct ifnet *ifp, struct mbuf *m);; 1261260368Sluigivoid netmap_catch_tx(struct netmap_generic_adapter *na, int enable); 1262259412Sluigiint generic_xmit_frame(struct ifnet *ifp, struct mbuf *m, void *addr, u_int len, u_int ring_nr); 1263259412Sluigiint generic_find_num_desc(struct ifnet *ifp, u_int *tx, u_int *rx); 1264259412Sluigivoid generic_find_num_queues(struct ifnet *ifp, u_int *txq, u_int *rxq); 1265266974Smarcelstruct netmap_adapter *netmap_getna(if_t ifp); 1266259412Sluigi 1267259412Sluigi/* 1268259412Sluigi * netmap_mitigation API. This is used by the generic adapter 1269259412Sluigi * to reduce the number of interrupt requests/selwakeup 1270259412Sluigi * to clients on incoming packets. 1271259412Sluigi */ 1272261909Sluigivoid netmap_mitigation_init(struct nm_generic_mit *mit, struct netmap_adapter *na); 1273261909Sluigivoid netmap_mitigation_start(struct nm_generic_mit *mit); 1274261909Sluigivoid netmap_mitigation_restart(struct nm_generic_mit *mit); 1275261909Sluigiint netmap_mitigation_active(struct nm_generic_mit *mit); 1276261909Sluigivoid netmap_mitigation_cleanup(struct nm_generic_mit *mit); 1277259412Sluigi 1278261909Sluigi 1279261909Sluigi 1280261909Sluigi/* Shared declarations for the VALE switch. */ 1281261909Sluigi 1282261909Sluigi/* 1283261909Sluigi * Each transmit queue accumulates a batch of packets into 1284261909Sluigi * a structure before forwarding. Packets to the same 1285261909Sluigi * destination are put in a list using ft_next as a link field. 1286261909Sluigi * ft_frags and ft_next are valid only on the first fragment. 1287261909Sluigi */ 1288261909Sluigistruct nm_bdg_fwd { /* forwarding entry for a bridge */ 1289261909Sluigi void *ft_buf; /* netmap or indirect buffer */ 1290261909Sluigi uint8_t ft_frags; /* how many fragments (only on 1st frag) */ 1291261909Sluigi uint8_t _ft_port; /* dst port (unused) */ 1292261909Sluigi uint16_t ft_flags; /* flags, e.g. indirect */ 1293261909Sluigi uint16_t ft_len; /* src fragment len */ 1294261909Sluigi uint16_t ft_next; /* next packet to same destination */ 1295261909Sluigi}; 1296261909Sluigi 1297261909Sluigi/* struct 'virtio_net_hdr' from linux. */ 1298261909Sluigistruct nm_vnet_hdr { 1299261909Sluigi#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start, csum_offset */ 1300261909Sluigi#define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */ 1301261909Sluigi uint8_t flags; 1302261909Sluigi#define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */ 1303261909Sluigi#define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */ 1304261909Sluigi#define VIRTIO_NET_HDR_GSO_UDP 3 /* GSO frame, IPv4 UDP (UFO) */ 1305261909Sluigi#define VIRTIO_NET_HDR_GSO_TCPV6 4 /* GSO frame, IPv6 TCP */ 1306261909Sluigi#define VIRTIO_NET_HDR_GSO_ECN 0x80 /* TCP has ECN set */ 1307261909Sluigi uint8_t gso_type; 1308261909Sluigi uint16_t hdr_len; 1309261909Sluigi uint16_t gso_size; 1310261909Sluigi uint16_t csum_start; 1311261909Sluigi uint16_t csum_offset; 1312261909Sluigi}; 1313261909Sluigi 1314261909Sluigi#define WORST_CASE_GSO_HEADER (14+40+60) /* IPv6 + TCP */ 1315261909Sluigi 1316261909Sluigi/* Private definitions for IPv4, IPv6, UDP and TCP headers. */ 1317261909Sluigi 1318261909Sluigistruct nm_iphdr { 1319261909Sluigi uint8_t version_ihl; 1320261909Sluigi uint8_t tos; 1321261909Sluigi uint16_t tot_len; 1322261909Sluigi uint16_t id; 1323261909Sluigi uint16_t frag_off; 1324261909Sluigi uint8_t ttl; 1325261909Sluigi uint8_t protocol; 1326261909Sluigi uint16_t check; 1327261909Sluigi uint32_t saddr; 1328261909Sluigi uint32_t daddr; 1329261909Sluigi /*The options start here. */ 1330261909Sluigi}; 1331261909Sluigi 1332261909Sluigistruct nm_tcphdr { 1333261909Sluigi uint16_t source; 1334261909Sluigi uint16_t dest; 1335261909Sluigi uint32_t seq; 1336261909Sluigi uint32_t ack_seq; 1337261909Sluigi uint8_t doff; /* Data offset + Reserved */ 1338261909Sluigi uint8_t flags; 1339261909Sluigi uint16_t window; 1340261909Sluigi uint16_t check; 1341261909Sluigi uint16_t urg_ptr; 1342261909Sluigi}; 1343261909Sluigi 1344261909Sluigistruct nm_udphdr { 1345261909Sluigi uint16_t source; 1346261909Sluigi uint16_t dest; 1347261909Sluigi uint16_t len; 1348261909Sluigi uint16_t check; 1349261909Sluigi}; 1350261909Sluigi 1351261909Sluigistruct nm_ipv6hdr { 1352261909Sluigi uint8_t priority_version; 1353261909Sluigi uint8_t flow_lbl[3]; 1354261909Sluigi 1355261909Sluigi uint16_t payload_len; 1356261909Sluigi uint8_t nexthdr; 1357261909Sluigi uint8_t hop_limit; 1358261909Sluigi 1359261909Sluigi uint8_t saddr[16]; 1360261909Sluigi uint8_t daddr[16]; 1361261909Sluigi}; 1362261909Sluigi 1363261909Sluigi/* Type used to store a checksum (in host byte order) that hasn't been 1364261909Sluigi * folded yet. 1365261909Sluigi */ 1366261909Sluigi#define rawsum_t uint32_t 1367261909Sluigi 1368261909Sluigirawsum_t nm_csum_raw(uint8_t *data, size_t len, rawsum_t cur_sum); 1369261909Sluigiuint16_t nm_csum_ipv4(struct nm_iphdr *iph); 1370261909Sluigivoid nm_csum_tcpudp_ipv4(struct nm_iphdr *iph, void *data, 1371261909Sluigi size_t datalen, uint16_t *check); 1372261909Sluigivoid nm_csum_tcpudp_ipv6(struct nm_ipv6hdr *ip6h, void *data, 1373261909Sluigi size_t datalen, uint16_t *check); 1374261909Sluigiuint16_t nm_csum_fold(rawsum_t cur_sum); 1375261909Sluigi 1376261909Sluigivoid bdg_mismatch_datapath(struct netmap_vp_adapter *na, 1377261909Sluigi struct netmap_vp_adapter *dst_na, 1378261909Sluigi struct nm_bdg_fwd *ft_p, struct netmap_ring *ring, 1379261909Sluigi u_int *j, u_int lim, u_int *howmany); 1380227614Sluigi#endif /* _NET_NETMAP_KERN_H_ */ 1381