netmap_vale.c revision 341477
1331722Seadler/* 2341477Svmaffione * Copyright (C) 2013-2016 Universita` di Pisa 3341477Svmaffione * All rights reserved. 4259412Sluigi * 5259412Sluigi * Redistribution and use in source and binary forms, with or without 6259412Sluigi * modification, are permitted provided that the following conditions 7259412Sluigi * are met: 8259412Sluigi * 1. Redistributions of source code must retain the above copyright 9259412Sluigi * notice, this list of conditions and the following disclaimer. 10259412Sluigi * 2. Redistributions in binary form must reproduce the above copyright 11259412Sluigi * notice, this list of conditions and the following disclaimer in the 12259412Sluigi * documentation and/or other materials provided with the distribution. 13259412Sluigi * 14259412Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15259412Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16259412Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17259412Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18259412Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19259412Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20259412Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21259412Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22259412Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23259412Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24259412Sluigi * SUCH DAMAGE. 25259412Sluigi */ 26259412Sluigi 27259412Sluigi 28259412Sluigi#if defined(__FreeBSD__) 29259412Sluigi#include <sys/cdefs.h> /* prerequisite */ 30259412Sluigi__FBSDID("$FreeBSD: stable/11/sys/dev/netmap/netmap_vale.c 341477 2018-12-04 17:40:56Z vmaffione $"); 31259412Sluigi 32259412Sluigi#include <sys/types.h> 33259412Sluigi#include <sys/errno.h> 34259412Sluigi#include <sys/param.h> /* defines used in kernel.h */ 35259412Sluigi#include <sys/kernel.h> /* types used in module initialization */ 36259412Sluigi#include <sys/conf.h> /* cdevsw struct, UID, GID */ 37259412Sluigi#include <sys/sockio.h> 38259412Sluigi#include <sys/socketvar.h> /* struct socket */ 39259412Sluigi#include <sys/malloc.h> 40259412Sluigi#include <sys/poll.h> 41259412Sluigi#include <sys/rwlock.h> 42259412Sluigi#include <sys/socket.h> /* sockaddrs */ 43259412Sluigi#include <sys/selinfo.h> 44259412Sluigi#include <sys/sysctl.h> 45259412Sluigi#include <net/if.h> 46259412Sluigi#include <net/if_var.h> 47259412Sluigi#include <net/bpf.h> /* BIOCIMMEDIATE */ 48259412Sluigi#include <machine/bus.h> /* bus_dmamap_* */ 49259412Sluigi#include <sys/endian.h> 50259412Sluigi#include <sys/refcount.h> 51341477Svmaffione#include <sys/smp.h> 52259412Sluigi 53259412Sluigi 54259412Sluigi#elif defined(linux) 55259412Sluigi 56259412Sluigi#include "bsd_glue.h" 57259412Sluigi 58259412Sluigi#elif defined(__APPLE__) 59259412Sluigi 60259412Sluigi#warning OSX support is only partial 61259412Sluigi#include "osx_glue.h" 62259412Sluigi 63341477Svmaffione#elif defined(_WIN32) 64341477Svmaffione#include "win_glue.h" 65341477Svmaffione 66259412Sluigi#else 67259412Sluigi 68259412Sluigi#error Unsupported platform 69259412Sluigi 70259412Sluigi#endif /* unsupported */ 71259412Sluigi 72259412Sluigi/* 73259412Sluigi * common headers 74259412Sluigi */ 75259412Sluigi 76259412Sluigi#include <net/netmap.h> 77259412Sluigi#include <dev/netmap/netmap_kern.h> 78259412Sluigi#include <dev/netmap/netmap_mem2.h> 79341477Svmaffione#include <dev/netmap/netmap_bdg.h> 80259412Sluigi 81259412Sluigi#ifdef WITH_VALE 82259412Sluigi 83259412Sluigi/* 84259412Sluigi * system parameters (most of them in netmap_kern.h) 85341477Svmaffione * NM_BDG_NAME prefix for switch port names, default "vale" 86259412Sluigi * NM_BDG_MAXPORTS number of ports 87259412Sluigi * NM_BRIDGES max number of switches in the system. 88259412Sluigi * XXX should become a sysctl or tunable 89259412Sluigi * 90259412Sluigi * Switch ports are named valeX:Y where X is the switch name and Y 91259412Sluigi * is the port. If Y matches a physical interface name, the port is 92259412Sluigi * connected to a physical device. 93259412Sluigi * 94259412Sluigi * Unlike physical interfaces, switch ports use their own memory region 95259412Sluigi * for rings and buffers. 96259412Sluigi * The virtual interfaces use per-queue lock instead of core lock. 97259412Sluigi * In the tx loop, we aggregate traffic in batches to make all operations 98259412Sluigi * faster. The batch size is bridge_batch. 99259412Sluigi */ 100259412Sluigi#define NM_BDG_MAXRINGS 16 /* XXX unclear how many. */ 101259412Sluigi#define NM_BDG_MAXSLOTS 4096 /* XXX same as above */ 102259412Sluigi#define NM_BRIDGE_RINGSIZE 1024 /* in the device */ 103259412Sluigi#define NM_BDG_BATCH 1024 /* entries in the forwarding buffer */ 104259412Sluigi/* actual size of the tables */ 105341477Svmaffione#define NM_BDG_BATCH_MAX (NM_BDG_BATCH + NETMAP_MAX_FRAGS) 106259412Sluigi/* NM_FT_NULL terminates a list of slots in the ft */ 107259412Sluigi#define NM_FT_NULL NM_BDG_BATCH_MAX 108259412Sluigi 109259412Sluigi 110259412Sluigi/* 111259412Sluigi * bridge_batch is set via sysctl to the max batch size to be 112259412Sluigi * used in the bridge. The actual value may be larger as the 113259412Sluigi * last packet in the block may overflow the size. 114259412Sluigi */ 115341477Svmaffionestatic int bridge_batch = NM_BDG_BATCH; /* bridge batch size */ 116341477SvmaffioneSYSBEGIN(vars_vale); 117259412SluigiSYSCTL_DECL(_dev_netmap); 118341477SvmaffioneSYSCTL_INT(_dev_netmap, OID_AUTO, bridge_batch, CTLFLAG_RW, &bridge_batch, 0, 119341477Svmaffione "Max batch size to be used in the bridge"); 120341477SvmaffioneSYSEND; 121259412Sluigi 122341477Svmaffionestatic int netmap_vp_create(struct nmreq_header *hdr, struct ifnet *, 123341477Svmaffione struct netmap_mem_d *nmd, struct netmap_vp_adapter **); 124341477Svmaffionestatic int netmap_vp_bdg_attach(const char *, struct netmap_adapter *, 125341477Svmaffione struct nm_bridge *); 126341477Svmaffionestatic int netmap_vale_bwrap_attach(const char *, struct netmap_adapter *); 127259412Sluigi 128259412Sluigi/* 129259412Sluigi * For each output interface, nm_bdg_q is used to construct a list. 130259412Sluigi * bq_len is the number of output buffers (we can have coalescing 131259412Sluigi * during the copy). 132259412Sluigi */ 133259412Sluigistruct nm_bdg_q { 134259412Sluigi uint16_t bq_head; 135259412Sluigi uint16_t bq_tail; 136259412Sluigi uint32_t bq_len; /* number of buffers */ 137259412Sluigi}; 138259412Sluigi 139341477Svmaffione/* Holds the default callbacks */ 140341477Svmaffionestruct netmap_bdg_ops vale_bdg_ops = { 141341477Svmaffione .lookup = netmap_bdg_learning, 142341477Svmaffione .config = NULL, 143341477Svmaffione .dtor = NULL, 144341477Svmaffione .vp_create = netmap_vp_create, 145341477Svmaffione .bwrap_attach = netmap_vale_bwrap_attach, 146341477Svmaffione .name = NM_BDG_NAME, 147259412Sluigi}; 148259412Sluigi 149259412Sluigi/* 150259412Sluigi * this is a slightly optimized copy routine which rounds 151259412Sluigi * to multiple of 64 bytes and is often faster than dealing 152259412Sluigi * with other odd sizes. We assume there is enough room 153259412Sluigi * in the source and destination buffers. 154259412Sluigi * 155259412Sluigi * XXX only for multiples of 64 bytes, non overlapped. 156259412Sluigi */ 157259412Sluigistatic inline void 158259412Sluigipkt_copy(void *_src, void *_dst, int l) 159259412Sluigi{ 160341477Svmaffione uint64_t *src = _src; 161341477Svmaffione uint64_t *dst = _dst; 162341477Svmaffione if (unlikely(l >= 1024)) { 163341477Svmaffione memcpy(dst, src, l); 164341477Svmaffione return; 165259412Sluigi } 166341477Svmaffione for (; likely(l > 0); l-=64) { 167341477Svmaffione *dst++ = *src++; 168341477Svmaffione *dst++ = *src++; 169341477Svmaffione *dst++ = *src++; 170341477Svmaffione *dst++ = *src++; 171341477Svmaffione *dst++ = *src++; 172341477Svmaffione *dst++ = *src++; 173341477Svmaffione *dst++ = *src++; 174341477Svmaffione *dst++ = *src++; 175259412Sluigi } 176259412Sluigi} 177259412Sluigi 178259412Sluigi 179259412Sluigi/* 180259412Sluigi * Free the forwarding tables for rings attached to switch ports. 181259412Sluigi */ 182259412Sluigistatic void 183259412Sluiginm_free_bdgfwd(struct netmap_adapter *na) 184259412Sluigi{ 185259412Sluigi int nrings, i; 186341477Svmaffione struct netmap_kring **kring; 187259412Sluigi 188259412Sluigi NMG_LOCK_ASSERT(); 189260368Sluigi nrings = na->num_tx_rings; 190260368Sluigi kring = na->tx_rings; 191259412Sluigi for (i = 0; i < nrings; i++) { 192341477Svmaffione if (kring[i]->nkr_ft) { 193341477Svmaffione nm_os_free(kring[i]->nkr_ft); 194341477Svmaffione kring[i]->nkr_ft = NULL; /* protect from freeing twice */ 195259412Sluigi } 196259412Sluigi } 197259412Sluigi} 198259412Sluigi 199259412Sluigi 200259412Sluigi/* 201259412Sluigi * Allocate the forwarding tables for the rings attached to the bridge ports. 202259412Sluigi */ 203259412Sluigistatic int 204259412Sluiginm_alloc_bdgfwd(struct netmap_adapter *na) 205259412Sluigi{ 206259412Sluigi int nrings, l, i, num_dstq; 207341477Svmaffione struct netmap_kring **kring; 208259412Sluigi 209259412Sluigi NMG_LOCK_ASSERT(); 210259412Sluigi /* all port:rings + broadcast */ 211259412Sluigi num_dstq = NM_BDG_MAXPORTS * NM_BDG_MAXRINGS + 1; 212259412Sluigi l = sizeof(struct nm_bdg_fwd) * NM_BDG_BATCH_MAX; 213259412Sluigi l += sizeof(struct nm_bdg_q) * num_dstq; 214259412Sluigi l += sizeof(uint16_t) * NM_BDG_BATCH_MAX; 215259412Sluigi 216285349Sluigi nrings = netmap_real_rings(na, NR_TX); 217259412Sluigi kring = na->tx_rings; 218259412Sluigi for (i = 0; i < nrings; i++) { 219259412Sluigi struct nm_bdg_fwd *ft; 220259412Sluigi struct nm_bdg_q *dstq; 221259412Sluigi int j; 222259412Sluigi 223341477Svmaffione ft = nm_os_malloc(l); 224259412Sluigi if (!ft) { 225259412Sluigi nm_free_bdgfwd(na); 226259412Sluigi return ENOMEM; 227259412Sluigi } 228259412Sluigi dstq = (struct nm_bdg_q *)(ft + NM_BDG_BATCH_MAX); 229259412Sluigi for (j = 0; j < num_dstq; j++) { 230259412Sluigi dstq[j].bq_head = dstq[j].bq_tail = NM_FT_NULL; 231259412Sluigi dstq[j].bq_len = 0; 232259412Sluigi } 233341477Svmaffione kring[i]->nkr_ft = ft; 234259412Sluigi } 235259412Sluigi return 0; 236259412Sluigi} 237259412Sluigi 238341477Svmaffione/* Allows external modules to create bridges in exclusive mode, 239341477Svmaffione * returns an authentication token that the external module will need 240341477Svmaffione * to provide during nm_bdg_ctl_{attach, detach}(), netmap_bdg_regops(), 241341477Svmaffione * and nm_bdg_update_private_data() operations. 242341477Svmaffione * Successfully executed if ret != NULL and *return_status == 0. 243270063Sluigi */ 244341477Svmaffionevoid * 245341477Svmaffionenetmap_vale_create(const char *bdg_name, int *return_status) 246259412Sluigi{ 247341477Svmaffione struct nm_bridge *b = NULL; 248341477Svmaffione void *ret = NULL; 249259412Sluigi 250341477Svmaffione NMG_LOCK(); 251341477Svmaffione b = nm_find_bridge(bdg_name, 0 /* don't create */, NULL); 252270063Sluigi if (b) { 253341477Svmaffione *return_status = EEXIST; 254341477Svmaffione goto unlock_bdg_create; 255270063Sluigi } 256270063Sluigi 257341477Svmaffione b = nm_find_bridge(bdg_name, 1 /* create */, &vale_bdg_ops); 258341477Svmaffione if (!b) { 259341477Svmaffione *return_status = ENOMEM; 260341477Svmaffione goto unlock_bdg_create; 261259412Sluigi } 262259412Sluigi 263341477Svmaffione b->bdg_flags |= NM_BDG_ACTIVE | NM_BDG_EXCLUSIVE; 264341477Svmaffione ret = nm_bdg_get_auth_token(b); 265341477Svmaffione *return_status = 0; 266260368Sluigi 267341477Svmaffioneunlock_bdg_create: 268270063Sluigi NMG_UNLOCK(); 269341477Svmaffione return ret; 270270063Sluigi} 271270063Sluigi 272341477Svmaffione/* Allows external modules to destroy a bridge created through 273341477Svmaffione * netmap_bdg_create(), the bridge must be empty. 274270063Sluigi */ 275341477Svmaffioneint 276341477Svmaffionenetmap_vale_destroy(const char *bdg_name, void *auth_token) 277270063Sluigi{ 278341477Svmaffione struct nm_bridge *b = NULL; 279341477Svmaffione int ret = 0; 280270063Sluigi 281270063Sluigi NMG_LOCK(); 282341477Svmaffione b = nm_find_bridge(bdg_name, 0 /* don't create */, NULL); 283341477Svmaffione if (!b) { 284341477Svmaffione ret = ENXIO; 285341477Svmaffione goto unlock_bdg_free; 286270063Sluigi } 287270063Sluigi 288341477Svmaffione if (!nm_bdg_valid_auth_token(b, auth_token)) { 289341477Svmaffione ret = EACCES; 290341477Svmaffione goto unlock_bdg_free; 291259412Sluigi } 292341477Svmaffione if (!(b->bdg_flags & NM_BDG_EXCLUSIVE)) { 293341477Svmaffione ret = EINVAL; 294341477Svmaffione goto unlock_bdg_free; 295259412Sluigi } 296259412Sluigi 297341477Svmaffione b->bdg_flags &= ~(NM_BDG_EXCLUSIVE | NM_BDG_ACTIVE); 298341477Svmaffione ret = netmap_bdg_free(b); 299341477Svmaffione if (ret) { 300341477Svmaffione b->bdg_flags |= NM_BDG_EXCLUSIVE | NM_BDG_ACTIVE; 301259412Sluigi } 302259412Sluigi 303341477Svmaffioneunlock_bdg_free: 304341477Svmaffione NMG_UNLOCK(); 305341477Svmaffione return ret; 306259412Sluigi} 307259412Sluigi 308259412Sluigi 309259412Sluigi 310341477Svmaffione/* nm_dtor callback for ephemeral VALE ports */ 311341477Svmaffionestatic void 312341477Svmaffionenetmap_vp_dtor(struct netmap_adapter *na) 313259412Sluigi{ 314341477Svmaffione struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter*)na; 315341477Svmaffione struct nm_bridge *b = vpna->na_bdg; 316259412Sluigi 317341477Svmaffione ND("%s has %d references", na->name, na->na_refcount); 318260700Sluigi 319341477Svmaffione if (b) { 320341477Svmaffione netmap_bdg_detach_common(b, vpna->bdg_port, -1); 321259412Sluigi } 322260368Sluigi 323341477Svmaffione if (na->ifp != NULL && !nm_iszombie(na)) { 324341477Svmaffione NM_DETACH_NA(na->ifp); 325341477Svmaffione if (vpna->autodelete) { 326341477Svmaffione ND("releasing %s", na->ifp->if_xname); 327341477Svmaffione NMG_UNLOCK(); 328341477Svmaffione nm_os_vi_detach(na->ifp); 329341477Svmaffione NMG_LOCK(); 330341477Svmaffione } 331259412Sluigi } 332259412Sluigi} 333259412Sluigi 334259412Sluigi 335341477Svmaffione/* Called by external kernel modules (e.g., Openvswitch). 336341477Svmaffione * to modify the private data previously given to regops(). 337341477Svmaffione * 'name' may be just bridge's name (including ':' if it 338341477Svmaffione * is not just NM_BDG_NAME). 339259412Sluigi * Called without NMG_LOCK. 340259412Sluigi */ 341259412Sluigiint 342341477Svmaffionenm_bdg_update_private_data(const char *name, bdg_update_private_data_fn_t callback, 343341477Svmaffione void *callback_data, void *auth_token) 344259412Sluigi{ 345341477Svmaffione void *private_data = NULL; 346270063Sluigi struct nm_bridge *b; 347341477Svmaffione int error = 0; 348270063Sluigi 349270063Sluigi NMG_LOCK(); 350341477Svmaffione b = nm_find_bridge(name, 0 /* don't create */, NULL); 351270063Sluigi if (!b) { 352341477Svmaffione error = EINVAL; 353341477Svmaffione goto unlock_update_priv; 354270063Sluigi } 355341477Svmaffione if (!nm_bdg_valid_auth_token(b, auth_token)) { 356341477Svmaffione error = EACCES; 357341477Svmaffione goto unlock_update_priv; 358341477Svmaffione } 359341477Svmaffione BDG_WLOCK(b); 360341477Svmaffione private_data = callback(b->private_data, callback_data, &error); 361341477Svmaffione b->private_data = private_data; 362341477Svmaffione BDG_WUNLOCK(b); 363341477Svmaffione 364341477Svmaffioneunlock_update_priv: 365270063Sluigi NMG_UNLOCK(); 366270063Sluigi return error; 367270063Sluigi} 368270063Sluigi 369270063Sluigi 370270063Sluigi/* nm_krings_create callback for VALE ports. 371270063Sluigi * Calls the standard netmap_krings_create, then adds leases on rx 372270063Sluigi * rings and bdgfwd on tx rings. 373270063Sluigi */ 374259412Sluigistatic int 375259412Sluiginetmap_vp_krings_create(struct netmap_adapter *na) 376259412Sluigi{ 377261909Sluigi u_int tailroom; 378259412Sluigi int error, i; 379259412Sluigi uint32_t *leases; 380285349Sluigi u_int nrx = netmap_real_rings(na, NR_RX); 381259412Sluigi 382259412Sluigi /* 383259412Sluigi * Leases are attached to RX rings on vale ports 384259412Sluigi */ 385259412Sluigi tailroom = sizeof(uint32_t) * na->num_rx_desc * nrx; 386259412Sluigi 387261909Sluigi error = netmap_krings_create(na, tailroom); 388259412Sluigi if (error) 389259412Sluigi return error; 390259412Sluigi 391259412Sluigi leases = na->tailroom; 392259412Sluigi 393259412Sluigi for (i = 0; i < nrx; i++) { /* Receive rings */ 394341477Svmaffione na->rx_rings[i]->nkr_leases = leases; 395259412Sluigi leases += na->num_rx_desc; 396259412Sluigi } 397259412Sluigi 398259412Sluigi error = nm_alloc_bdgfwd(na); 399259412Sluigi if (error) { 400259412Sluigi netmap_krings_delete(na); 401259412Sluigi return error; 402259412Sluigi } 403259412Sluigi 404259412Sluigi return 0; 405259412Sluigi} 406259412Sluigi 407260368Sluigi 408270063Sluigi/* nm_krings_delete callback for VALE ports. */ 409259412Sluigistatic void 410259412Sluiginetmap_vp_krings_delete(struct netmap_adapter *na) 411259412Sluigi{ 412259412Sluigi nm_free_bdgfwd(na); 413259412Sluigi netmap_krings_delete(na); 414259412Sluigi} 415259412Sluigi 416259412Sluigi 417259412Sluigistatic int 418259412Sluiginm_bdg_flush(struct nm_bdg_fwd *ft, u_int n, 419259412Sluigi struct netmap_vp_adapter *na, u_int ring_nr); 420259412Sluigi 421259412Sluigi 422259412Sluigi/* 423270063Sluigi * main dispatch routine for the bridge. 424259412Sluigi * Grab packets from a kring, move them into the ft structure 425259412Sluigi * associated to the tx (input) port. Max one instance per port, 426259412Sluigi * filtered on input (ioctl, poll or XXX). 427259412Sluigi * Returns the next position in the ring. 428259412Sluigi */ 429259412Sluigistatic int 430270063Sluiginm_bdg_preflush(struct netmap_kring *kring, u_int end) 431259412Sluigi{ 432270063Sluigi struct netmap_vp_adapter *na = 433270063Sluigi (struct netmap_vp_adapter*)kring->na; 434259412Sluigi struct netmap_ring *ring = kring->ring; 435259412Sluigi struct nm_bdg_fwd *ft; 436270063Sluigi u_int ring_nr = kring->ring_id; 437259412Sluigi u_int j = kring->nr_hwcur, lim = kring->nkr_num_slots - 1; 438259412Sluigi u_int ft_i = 0; /* start from 0 */ 439259412Sluigi u_int frags = 1; /* how many frags ? */ 440259412Sluigi struct nm_bridge *b = na->na_bdg; 441259412Sluigi 442259412Sluigi /* To protect against modifications to the bridge we acquire a 443259412Sluigi * shared lock, waiting if we can sleep (if the source port is 444259412Sluigi * attached to a user process) or with a trylock otherwise (NICs). 445259412Sluigi */ 446259412Sluigi ND("wait rlock for %d packets", ((j > end ? lim+1 : 0) + end) - j); 447259412Sluigi if (na->up.na_flags & NAF_BDG_MAYSLEEP) 448259412Sluigi BDG_RLOCK(b); 449259412Sluigi else if (!BDG_RTRYLOCK(b)) 450341477Svmaffione return j; 451259412Sluigi ND(5, "rlock acquired for %d packets", ((j > end ? lim+1 : 0) + end) - j); 452259412Sluigi ft = kring->nkr_ft; 453259412Sluigi 454259412Sluigi for (; likely(j != end); j = nm_next(j, lim)) { 455259412Sluigi struct netmap_slot *slot = &ring->slot[j]; 456259412Sluigi char *buf; 457259412Sluigi 458259412Sluigi ft[ft_i].ft_len = slot->len; 459259412Sluigi ft[ft_i].ft_flags = slot->flags; 460341477Svmaffione ft[ft_i].ft_offset = 0; 461259412Sluigi 462259412Sluigi ND("flags is 0x%x", slot->flags); 463285349Sluigi /* we do not use the buf changed flag, but we still need to reset it */ 464285349Sluigi slot->flags &= ~NS_BUF_CHANGED; 465285349Sluigi 466259412Sluigi /* this slot goes into a list so initialize the link field */ 467259412Sluigi ft[ft_i].ft_next = NM_FT_NULL; 468259412Sluigi buf = ft[ft_i].ft_buf = (slot->flags & NS_INDIRECT) ? 469270063Sluigi (void *)(uintptr_t)slot->ptr : NMB(&na->up, slot); 470267151Sluigi if (unlikely(buf == NULL)) { 471267151Sluigi RD(5, "NULL %s buffer pointer from %s slot %d len %d", 472267151Sluigi (slot->flags & NS_INDIRECT) ? "INDIRECT" : "DIRECT", 473267151Sluigi kring->name, j, ft[ft_i].ft_len); 474270063Sluigi buf = ft[ft_i].ft_buf = NETMAP_BUF_BASE(&na->up); 475267151Sluigi ft[ft_i].ft_len = 0; 476267151Sluigi ft[ft_i].ft_flags = 0; 477267151Sluigi } 478259487Sluigi __builtin_prefetch(buf); 479259412Sluigi ++ft_i; 480259412Sluigi if (slot->flags & NS_MOREFRAG) { 481259412Sluigi frags++; 482259412Sluigi continue; 483259412Sluigi } 484259412Sluigi if (unlikely(netmap_verbose && frags > 1)) 485259412Sluigi RD(5, "%d frags at %d", frags, ft_i - frags); 486259412Sluigi ft[ft_i - frags].ft_frags = frags; 487259412Sluigi frags = 1; 488259412Sluigi if (unlikely((int)ft_i >= bridge_batch)) 489259412Sluigi ft_i = nm_bdg_flush(ft, ft_i, na, ring_nr); 490259412Sluigi } 491259412Sluigi if (frags > 1) { 492341477Svmaffione /* Here ft_i > 0, ft[ft_i-1].flags has NS_MOREFRAG, and we 493341477Svmaffione * have to fix frags count. */ 494341477Svmaffione frags--; 495341477Svmaffione ft[ft_i - 1].ft_flags &= ~NS_MOREFRAG; 496341477Svmaffione ft[ft_i - frags].ft_frags = frags; 497341477Svmaffione D("Truncate incomplete fragment at %d (%d frags)", ft_i, frags); 498259412Sluigi } 499259412Sluigi if (ft_i) 500259412Sluigi ft_i = nm_bdg_flush(ft, ft_i, na, ring_nr); 501259412Sluigi BDG_RUNLOCK(b); 502259412Sluigi return j; 503259412Sluigi} 504259412Sluigi 505259412Sluigi 506259412Sluigi/* ----- FreeBSD if_bridge hash function ------- */ 507259412Sluigi 508259412Sluigi/* 509259412Sluigi * The following hash function is adapted from "Hash Functions" by Bob Jenkins 510259412Sluigi * ("Algorithm Alley", Dr. Dobbs Journal, September 1997). 511259412Sluigi * 512259412Sluigi * http://www.burtleburtle.net/bob/hash/spooky.html 513259412Sluigi */ 514259412Sluigi#define mix(a, b, c) \ 515259412Sluigido { \ 516341477Svmaffione a -= b; a -= c; a ^= (c >> 13); \ 517341477Svmaffione b -= c; b -= a; b ^= (a << 8); \ 518341477Svmaffione c -= a; c -= b; c ^= (b >> 13); \ 519341477Svmaffione a -= b; a -= c; a ^= (c >> 12); \ 520341477Svmaffione b -= c; b -= a; b ^= (a << 16); \ 521341477Svmaffione c -= a; c -= b; c ^= (b >> 5); \ 522341477Svmaffione a -= b; a -= c; a ^= (c >> 3); \ 523341477Svmaffione b -= c; b -= a; b ^= (a << 10); \ 524341477Svmaffione c -= a; c -= b; c ^= (b >> 15); \ 525259412Sluigi} while (/*CONSTCOND*/0) 526259412Sluigi 527260368Sluigi 528259412Sluigistatic __inline uint32_t 529259412Sluiginm_bridge_rthash(const uint8_t *addr) 530259412Sluigi{ 531341477Svmaffione uint32_t a = 0x9e3779b9, b = 0x9e3779b9, c = 0; // hask key 532259412Sluigi 533341477Svmaffione b += addr[5] << 8; 534341477Svmaffione b += addr[4]; 535341477Svmaffione a += addr[3] << 24; 536341477Svmaffione a += addr[2] << 16; 537341477Svmaffione a += addr[1] << 8; 538341477Svmaffione a += addr[0]; 539259412Sluigi 540341477Svmaffione mix(a, b, c); 541259412Sluigi#define BRIDGE_RTHASH_MASK (NM_BDG_HASH-1) 542341477Svmaffione return (c & BRIDGE_RTHASH_MASK); 543259412Sluigi} 544259412Sluigi 545259412Sluigi#undef mix 546259412Sluigi 547259412Sluigi 548259412Sluigi/* 549259412Sluigi * Lookup function for a learning bridge. 550259412Sluigi * Update the hash table with the source address, 551259412Sluigi * and then returns the destination port index, and the 552259412Sluigi * ring in *dst_ring (at the moment, always use ring 0) 553259412Sluigi */ 554341477Svmaffioneuint32_t 555270063Sluiginetmap_bdg_learning(struct nm_bdg_fwd *ft, uint8_t *dst_ring, 556341477Svmaffione struct netmap_vp_adapter *na, void *private_data) 557259412Sluigi{ 558341477Svmaffione uint8_t *buf = ((uint8_t *)ft->ft_buf) + ft->ft_offset; 559341477Svmaffione u_int buf_len = ft->ft_len - ft->ft_offset; 560341477Svmaffione struct nm_hash_ent *ht = private_data; 561259412Sluigi uint32_t sh, dh; 562259412Sluigi u_int dst, mysrc = na->bdg_port; 563259412Sluigi uint64_t smac, dmac; 564341477Svmaffione uint8_t indbuf[12]; 565259412Sluigi 566341477Svmaffione if (buf_len < 14) { 567259412Sluigi return NM_BDG_NOPORT; 568259412Sluigi } 569341477Svmaffione 570341477Svmaffione if (ft->ft_flags & NS_INDIRECT) { 571341477Svmaffione if (copyin(buf, indbuf, sizeof(indbuf))) { 572341477Svmaffione return NM_BDG_NOPORT; 573341477Svmaffione } 574341477Svmaffione buf = indbuf; 575341477Svmaffione } 576341477Svmaffione 577259412Sluigi dmac = le64toh(*(uint64_t *)(buf)) & 0xffffffffffff; 578259412Sluigi smac = le64toh(*(uint64_t *)(buf + 4)); 579259412Sluigi smac >>= 16; 580259412Sluigi 581259412Sluigi /* 582259412Sluigi * The hash is somewhat expensive, there might be some 583259412Sluigi * worthwhile optimizations here. 584259412Sluigi */ 585285349Sluigi if (((buf[6] & 1) == 0) && (na->last_smac != smac)) { /* valid src */ 586259412Sluigi uint8_t *s = buf+6; 587341477Svmaffione sh = nm_bridge_rthash(s); /* hash of source */ 588259412Sluigi /* update source port forwarding entry */ 589285349Sluigi na->last_smac = ht[sh].mac = smac; /* XXX expire ? */ 590259412Sluigi ht[sh].ports = mysrc; 591259412Sluigi if (netmap_verbose) 592259412Sluigi D("src %02x:%02x:%02x:%02x:%02x:%02x on port %d", 593259412Sluigi s[0], s[1], s[2], s[3], s[4], s[5], mysrc); 594259412Sluigi } 595259412Sluigi dst = NM_BDG_BROADCAST; 596259412Sluigi if ((buf[0] & 1) == 0) { /* unicast */ 597341477Svmaffione dh = nm_bridge_rthash(buf); /* hash of dst */ 598259412Sluigi if (ht[dh].mac == dmac) { /* found dst */ 599259412Sluigi dst = ht[dh].ports; 600259412Sluigi } 601259412Sluigi } 602259412Sluigi return dst; 603259412Sluigi} 604259412Sluigi 605259412Sluigi 606259412Sluigi/* 607260368Sluigi * Available space in the ring. Only used in VALE code 608260368Sluigi * and only with is_rx = 1 609260368Sluigi */ 610260368Sluigistatic inline uint32_t 611260368Sluiginm_kr_space(struct netmap_kring *k, int is_rx) 612260368Sluigi{ 613260368Sluigi int space; 614260368Sluigi 615260368Sluigi if (is_rx) { 616260368Sluigi int busy = k->nkr_hwlease - k->nr_hwcur; 617260368Sluigi if (busy < 0) 618260368Sluigi busy += k->nkr_num_slots; 619260368Sluigi space = k->nkr_num_slots - 1 - busy; 620260368Sluigi } else { 621260368Sluigi /* XXX never used in this branch */ 622260368Sluigi space = k->nr_hwtail - k->nkr_hwlease; 623260368Sluigi if (space < 0) 624260368Sluigi space += k->nkr_num_slots; 625260368Sluigi } 626260368Sluigi#if 0 627260368Sluigi // sanity check 628260368Sluigi if (k->nkr_hwlease >= k->nkr_num_slots || 629260368Sluigi k->nr_hwcur >= k->nkr_num_slots || 630260368Sluigi k->nr_tail >= k->nkr_num_slots || 631260368Sluigi busy < 0 || 632260368Sluigi busy >= k->nkr_num_slots) { 633260368Sluigi D("invalid kring, cur %d tail %d lease %d lease_idx %d lim %d", k->nr_hwcur, k->nr_hwtail, k->nkr_hwlease, 634260368Sluigi k->nkr_lease_idx, k->nkr_num_slots); 635260368Sluigi } 636260368Sluigi#endif 637260368Sluigi return space; 638260368Sluigi} 639260368Sluigi 640260368Sluigi 641260368Sluigi 642260368Sluigi 643260368Sluigi/* make a lease on the kring for N positions. return the 644260368Sluigi * lease index 645260368Sluigi * XXX only used in VALE code and with is_rx = 1 646260368Sluigi */ 647260368Sluigistatic inline uint32_t 648260368Sluiginm_kr_lease(struct netmap_kring *k, u_int n, int is_rx) 649260368Sluigi{ 650260368Sluigi uint32_t lim = k->nkr_num_slots - 1; 651260368Sluigi uint32_t lease_idx = k->nkr_lease_idx; 652260368Sluigi 653260368Sluigi k->nkr_leases[lease_idx] = NR_NOSLOT; 654260368Sluigi k->nkr_lease_idx = nm_next(lease_idx, lim); 655260368Sluigi 656260368Sluigi if (n > nm_kr_space(k, is_rx)) { 657260368Sluigi D("invalid request for %d slots", n); 658260368Sluigi panic("x"); 659260368Sluigi } 660260368Sluigi /* XXX verify that there are n slots */ 661260368Sluigi k->nkr_hwlease += n; 662260368Sluigi if (k->nkr_hwlease > lim) 663260368Sluigi k->nkr_hwlease -= lim + 1; 664260368Sluigi 665260368Sluigi if (k->nkr_hwlease >= k->nkr_num_slots || 666260368Sluigi k->nr_hwcur >= k->nkr_num_slots || 667260368Sluigi k->nr_hwtail >= k->nkr_num_slots || 668260368Sluigi k->nkr_lease_idx >= k->nkr_num_slots) { 669260368Sluigi D("invalid kring %s, cur %d tail %d lease %d lease_idx %d lim %d", 670270063Sluigi k->na->name, 671260368Sluigi k->nr_hwcur, k->nr_hwtail, k->nkr_hwlease, 672260368Sluigi k->nkr_lease_idx, k->nkr_num_slots); 673260368Sluigi } 674260368Sluigi return lease_idx; 675260368Sluigi} 676260368Sluigi 677260368Sluigi/* 678270063Sluigi * 679259412Sluigi * This flush routine supports only unicast and broadcast but a large 680259412Sluigi * number of ports, and lets us replace the learn and dispatch functions. 681259412Sluigi */ 682259412Sluigiint 683259412Sluiginm_bdg_flush(struct nm_bdg_fwd *ft, u_int n, struct netmap_vp_adapter *na, 684259412Sluigi u_int ring_nr) 685259412Sluigi{ 686259412Sluigi struct nm_bdg_q *dst_ents, *brddst; 687259412Sluigi uint16_t num_dsts = 0, *dsts; 688259412Sluigi struct nm_bridge *b = na->na_bdg; 689341477Svmaffione u_int i, me = na->bdg_port; 690259412Sluigi 691259412Sluigi /* 692259412Sluigi * The work area (pointed by ft) is followed by an array of 693259412Sluigi * pointers to queues , dst_ents; there are NM_BDG_MAXRINGS 694259412Sluigi * queues per port plus one for the broadcast traffic. 695259412Sluigi * Then we have an array of destination indexes. 696259412Sluigi */ 697259412Sluigi dst_ents = (struct nm_bdg_q *)(ft + NM_BDG_BATCH_MAX); 698259412Sluigi dsts = (uint16_t *)(dst_ents + NM_BDG_MAXPORTS * NM_BDG_MAXRINGS + 1); 699259412Sluigi 700259412Sluigi /* first pass: find a destination for each packet in the batch */ 701259412Sluigi for (i = 0; likely(i < n); i += ft[i].ft_frags) { 702259412Sluigi uint8_t dst_ring = ring_nr; /* default, same ring as origin */ 703259412Sluigi uint16_t dst_port, d_i; 704259412Sluigi struct nm_bdg_q *d; 705341477Svmaffione struct nm_bdg_fwd *start_ft = NULL; 706259412Sluigi 707259412Sluigi ND("slot %d frags %d", i, ft[i].ft_frags); 708341477Svmaffione 709341477Svmaffione if (na->up.virt_hdr_len < ft[i].ft_len) { 710341477Svmaffione ft[i].ft_offset = na->up.virt_hdr_len; 711341477Svmaffione start_ft = &ft[i]; 712341477Svmaffione } else if (na->up.virt_hdr_len == ft[i].ft_len && ft[i].ft_flags & NS_MOREFRAG) { 713341477Svmaffione ft[i].ft_offset = ft[i].ft_len; 714341477Svmaffione start_ft = &ft[i+1]; 715341477Svmaffione } else { 716341477Svmaffione /* Drop the packet if the virtio-net header is not into the first 717341477Svmaffione * fragment nor at the very beginning of the second. 718341477Svmaffione */ 719259412Sluigi continue; 720341477Svmaffione } 721341477Svmaffione dst_port = b->bdg_ops->lookup(start_ft, &dst_ring, na, b->private_data); 722259412Sluigi if (netmap_verbose > 255) 723259412Sluigi RD(5, "slot %d port %d -> %d", i, me, dst_port); 724341477Svmaffione if (dst_port >= NM_BDG_NOPORT) 725259412Sluigi continue; /* this packet is identified to be dropped */ 726259412Sluigi else if (dst_port == NM_BDG_BROADCAST) 727259412Sluigi dst_ring = 0; /* broadcasts always go to ring 0 */ 728259412Sluigi else if (unlikely(dst_port == me || 729259412Sluigi !b->bdg_ports[dst_port])) 730259412Sluigi continue; 731259412Sluigi 732259412Sluigi /* get a position in the scratch pad */ 733259412Sluigi d_i = dst_port * NM_BDG_MAXRINGS + dst_ring; 734259412Sluigi d = dst_ents + d_i; 735259412Sluigi 736259412Sluigi /* append the first fragment to the list */ 737259412Sluigi if (d->bq_head == NM_FT_NULL) { /* new destination */ 738259412Sluigi d->bq_head = d->bq_tail = i; 739259412Sluigi /* remember this position to be scanned later */ 740259412Sluigi if (dst_port != NM_BDG_BROADCAST) 741259412Sluigi dsts[num_dsts++] = d_i; 742259412Sluigi } else { 743259412Sluigi ft[d->bq_tail].ft_next = i; 744259412Sluigi d->bq_tail = i; 745259412Sluigi } 746259412Sluigi d->bq_len += ft[i].ft_frags; 747259412Sluigi } 748259412Sluigi 749259412Sluigi /* 750259412Sluigi * Broadcast traffic goes to ring 0 on all destinations. 751259412Sluigi * So we need to add these rings to the list of ports to scan. 752259412Sluigi * XXX at the moment we scan all NM_BDG_MAXPORTS ports, which is 753259412Sluigi * expensive. We should keep a compact list of active destinations 754259412Sluigi * so we could shorten this loop. 755259412Sluigi */ 756259412Sluigi brddst = dst_ents + NM_BDG_BROADCAST * NM_BDG_MAXRINGS; 757259412Sluigi if (brddst->bq_head != NM_FT_NULL) { 758341477Svmaffione u_int j; 759259412Sluigi for (j = 0; likely(j < b->bdg_active_ports); j++) { 760259412Sluigi uint16_t d_i; 761259412Sluigi i = b->bdg_port_index[j]; 762259412Sluigi if (unlikely(i == me)) 763259412Sluigi continue; 764259412Sluigi d_i = i * NM_BDG_MAXRINGS; 765259412Sluigi if (dst_ents[d_i].bq_head == NM_FT_NULL) 766259412Sluigi dsts[num_dsts++] = d_i; 767259412Sluigi } 768259412Sluigi } 769259412Sluigi 770259412Sluigi ND(5, "pass 1 done %d pkts %d dsts", n, num_dsts); 771270063Sluigi /* second pass: scan destinations */ 772259412Sluigi for (i = 0; i < num_dsts; i++) { 773259412Sluigi struct netmap_vp_adapter *dst_na; 774259412Sluigi struct netmap_kring *kring; 775259412Sluigi struct netmap_ring *ring; 776261909Sluigi u_int dst_nr, lim, j, d_i, next, brd_next; 777259412Sluigi u_int needed, howmany; 778259412Sluigi int retry = netmap_txsync_retry; 779259412Sluigi struct nm_bdg_q *d; 780259412Sluigi uint32_t my_start = 0, lease_idx = 0; 781259412Sluigi int nrings; 782261909Sluigi int virt_hdr_mismatch = 0; 783259412Sluigi 784259412Sluigi d_i = dsts[i]; 785259412Sluigi ND("second pass %d port %d", i, d_i); 786259412Sluigi d = dst_ents + d_i; 787259412Sluigi // XXX fix the division 788259412Sluigi dst_na = b->bdg_ports[d_i/NM_BDG_MAXRINGS]; 789259412Sluigi /* protect from the lookup function returning an inactive 790259412Sluigi * destination port 791259412Sluigi */ 792259412Sluigi if (unlikely(dst_na == NULL)) 793259412Sluigi goto cleanup; 794259412Sluigi if (dst_na->up.na_flags & NAF_SW_ONLY) 795259412Sluigi goto cleanup; 796259412Sluigi /* 797259412Sluigi * The interface may be in !netmap mode in two cases: 798259412Sluigi * - when na is attached but not activated yet; 799259412Sluigi * - when na is being deactivated but is still attached. 800259412Sluigi */ 801270063Sluigi if (unlikely(!nm_netmap_on(&dst_na->up))) { 802259412Sluigi ND("not in netmap mode!"); 803259412Sluigi goto cleanup; 804259412Sluigi } 805259412Sluigi 806259412Sluigi /* there is at least one either unicast or broadcast packet */ 807259412Sluigi brd_next = brddst->bq_head; 808259412Sluigi next = d->bq_head; 809259412Sluigi /* we need to reserve this many slots. If fewer are 810259412Sluigi * available, some packets will be dropped. 811259412Sluigi * Packets may have multiple fragments, so we may not use 812259412Sluigi * there is a chance that we may not use all of the slots 813259412Sluigi * we have claimed, so we will need to handle the leftover 814259412Sluigi * ones when we regain the lock. 815259412Sluigi */ 816259412Sluigi needed = d->bq_len + brddst->bq_len; 817259412Sluigi 818341477Svmaffione if (unlikely(dst_na->up.virt_hdr_len != na->up.virt_hdr_len)) { 819341477Svmaffione if (netmap_verbose) { 820341477Svmaffione RD(3, "virt_hdr_mismatch, src %d dst %d", na->up.virt_hdr_len, 821341477Svmaffione dst_na->up.virt_hdr_len); 822341477Svmaffione } 823261909Sluigi /* There is a virtio-net header/offloadings mismatch between 824261909Sluigi * source and destination. The slower mismatch datapath will 825261909Sluigi * be used to cope with all the mismatches. 826261909Sluigi */ 827261909Sluigi virt_hdr_mismatch = 1; 828261909Sluigi if (dst_na->mfs < na->mfs) { 829261909Sluigi /* We may need to do segmentation offloadings, and so 830261909Sluigi * we may need a number of destination slots greater 831261909Sluigi * than the number of input slots ('needed'). 832261909Sluigi * We look for the smallest integer 'x' which satisfies: 833261909Sluigi * needed * na->mfs + x * H <= x * na->mfs 834261909Sluigi * where 'H' is the length of the longest header that may 835261909Sluigi * be replicated in the segmentation process (e.g. for 836261909Sluigi * TCPv4 we must account for ethernet header, IP header 837261909Sluigi * and TCPv4 header). 838261909Sluigi */ 839341477Svmaffione KASSERT(dst_na->mfs > 0, ("vpna->mfs is 0")); 840261909Sluigi needed = (needed * na->mfs) / 841261909Sluigi (dst_na->mfs - WORST_CASE_GSO_HEADER) + 1; 842261909Sluigi ND(3, "srcmtu=%u, dstmtu=%u, x=%u", na->mfs, dst_na->mfs, needed); 843261909Sluigi } 844261909Sluigi } 845261909Sluigi 846259412Sluigi ND(5, "pass 2 dst %d is %x %s", 847259412Sluigi i, d_i, is_vp ? "virtual" : "nic/host"); 848259412Sluigi dst_nr = d_i & (NM_BDG_MAXRINGS-1); 849259412Sluigi nrings = dst_na->up.num_rx_rings; 850259412Sluigi if (dst_nr >= nrings) 851259412Sluigi dst_nr = dst_nr % nrings; 852341477Svmaffione kring = dst_na->up.rx_rings[dst_nr]; 853259412Sluigi ring = kring->ring; 854341477Svmaffione /* the destination ring may have not been opened for RX */ 855341477Svmaffione if (unlikely(ring == NULL || kring->nr_mode != NKR_NETMAP_ON)) 856341477Svmaffione goto cleanup; 857259412Sluigi lim = kring->nkr_num_slots - 1; 858259412Sluigi 859259412Sluigiretry: 860259412Sluigi 861261909Sluigi if (dst_na->retry && retry) { 862261909Sluigi /* try to get some free slot from the previous run */ 863285349Sluigi kring->nm_notify(kring, 0); 864270063Sluigi /* actually useful only for bwraps, since there 865270063Sluigi * the notify will trigger a txsync on the hwna. VALE ports 866270063Sluigi * have dst_na->retry == 0 867270063Sluigi */ 868261909Sluigi } 869259412Sluigi /* reserve the buffers in the queue and an entry 870259412Sluigi * to report completion, and drop lock. 871259412Sluigi * XXX this might become a helper function. 872259412Sluigi */ 873259412Sluigi mtx_lock(&kring->q_lock); 874259412Sluigi if (kring->nkr_stopped) { 875259412Sluigi mtx_unlock(&kring->q_lock); 876259412Sluigi goto cleanup; 877259412Sluigi } 878259412Sluigi my_start = j = kring->nkr_hwlease; 879259412Sluigi howmany = nm_kr_space(kring, 1); 880259412Sluigi if (needed < howmany) 881259412Sluigi howmany = needed; 882259412Sluigi lease_idx = nm_kr_lease(kring, howmany, 1); 883259412Sluigi mtx_unlock(&kring->q_lock); 884259412Sluigi 885259412Sluigi /* only retry if we need more than available slots */ 886259412Sluigi if (retry && needed <= howmany) 887259412Sluigi retry = 0; 888259412Sluigi 889259412Sluigi /* copy to the destination queue */ 890259412Sluigi while (howmany > 0) { 891259412Sluigi struct netmap_slot *slot; 892259412Sluigi struct nm_bdg_fwd *ft_p, *ft_end; 893259412Sluigi u_int cnt; 894259412Sluigi 895259412Sluigi /* find the queue from which we pick next packet. 896259412Sluigi * NM_FT_NULL is always higher than valid indexes 897259412Sluigi * so we never dereference it if the other list 898259412Sluigi * has packets (and if both are empty we never 899259412Sluigi * get here). 900259412Sluigi */ 901259412Sluigi if (next < brd_next) { 902259412Sluigi ft_p = ft + next; 903259412Sluigi next = ft_p->ft_next; 904259412Sluigi } else { /* insert broadcast */ 905259412Sluigi ft_p = ft + brd_next; 906259412Sluigi brd_next = ft_p->ft_next; 907259412Sluigi } 908259412Sluigi cnt = ft_p->ft_frags; // cnt > 0 909259412Sluigi if (unlikely(cnt > howmany)) 910259412Sluigi break; /* no more space */ 911259412Sluigi if (netmap_verbose && cnt > 1) 912259412Sluigi RD(5, "rx %d frags to %d", cnt, j); 913259412Sluigi ft_end = ft_p + cnt; 914261909Sluigi if (unlikely(virt_hdr_mismatch)) { 915261909Sluigi bdg_mismatch_datapath(na, dst_na, ft_p, ring, &j, lim, &howmany); 916261909Sluigi } else { 917261909Sluigi howmany -= cnt; 918261909Sluigi do { 919261909Sluigi char *dst, *src = ft_p->ft_buf; 920261909Sluigi size_t copy_len = ft_p->ft_len, dst_len = copy_len; 921259412Sluigi 922261909Sluigi slot = &ring->slot[j]; 923270063Sluigi dst = NMB(&dst_na->up, slot); 924259412Sluigi 925261909Sluigi ND("send [%d] %d(%d) bytes at %s:%d", 926261909Sluigi i, (int)copy_len, (int)dst_len, 927261909Sluigi NM_IFPNAME(dst_ifp), j); 928261909Sluigi /* round to a multiple of 64 */ 929261909Sluigi copy_len = (copy_len + 63) & ~63; 930260368Sluigi 931270063Sluigi if (unlikely(copy_len > NETMAP_BUF_SIZE(&dst_na->up) || 932270063Sluigi copy_len > NETMAP_BUF_SIZE(&na->up))) { 933267151Sluigi RD(5, "invalid len %d, down to 64", (int)copy_len); 934267151Sluigi copy_len = dst_len = 64; // XXX 935267151Sluigi } 936261909Sluigi if (ft_p->ft_flags & NS_INDIRECT) { 937261909Sluigi if (copyin(src, dst, copy_len)) { 938261909Sluigi // invalid user pointer, pretend len is 0 939261909Sluigi dst_len = 0; 940261909Sluigi } 941261909Sluigi } else { 942261909Sluigi //memcpy(dst, src, copy_len); 943261909Sluigi pkt_copy(src, dst, (int)copy_len); 944261909Sluigi } 945261909Sluigi slot->len = dst_len; 946261909Sluigi slot->flags = (cnt << 8)| NS_MOREFRAG; 947261909Sluigi j = nm_next(j, lim); 948261909Sluigi needed--; 949261909Sluigi ft_p++; 950261909Sluigi } while (ft_p != ft_end); 951261909Sluigi slot->flags = (cnt << 8); /* clear flag on last entry */ 952261909Sluigi } 953259412Sluigi /* are we done ? */ 954259412Sluigi if (next == NM_FT_NULL && brd_next == NM_FT_NULL) 955259412Sluigi break; 956259412Sluigi } 957259412Sluigi { 958259412Sluigi /* current position */ 959259412Sluigi uint32_t *p = kring->nkr_leases; /* shorthand */ 960259412Sluigi uint32_t update_pos; 961259412Sluigi int still_locked = 1; 962259412Sluigi 963259412Sluigi mtx_lock(&kring->q_lock); 964259412Sluigi if (unlikely(howmany > 0)) { 965259412Sluigi /* not used all bufs. If i am the last one 966259412Sluigi * i can recover the slots, otherwise must 967259412Sluigi * fill them with 0 to mark empty packets. 968259412Sluigi */ 969259412Sluigi ND("leftover %d bufs", howmany); 970259412Sluigi if (nm_next(lease_idx, lim) == kring->nkr_lease_idx) { 971259412Sluigi /* yes i am the last one */ 972259412Sluigi ND("roll back nkr_hwlease to %d", j); 973259412Sluigi kring->nkr_hwlease = j; 974259412Sluigi } else { 975259412Sluigi while (howmany-- > 0) { 976259412Sluigi ring->slot[j].len = 0; 977259412Sluigi ring->slot[j].flags = 0; 978259412Sluigi j = nm_next(j, lim); 979259412Sluigi } 980259412Sluigi } 981259412Sluigi } 982259412Sluigi p[lease_idx] = j; /* report I am done */ 983259412Sluigi 984260368Sluigi update_pos = kring->nr_hwtail; 985259412Sluigi 986259412Sluigi if (my_start == update_pos) { 987259412Sluigi /* all slots before my_start have been reported, 988259412Sluigi * so scan subsequent leases to see if other ranges 989259412Sluigi * have been completed, and to a selwakeup or txsync. 990259412Sluigi */ 991259412Sluigi while (lease_idx != kring->nkr_lease_idx && 992259412Sluigi p[lease_idx] != NR_NOSLOT) { 993259412Sluigi j = p[lease_idx]; 994259412Sluigi p[lease_idx] = NR_NOSLOT; 995259412Sluigi lease_idx = nm_next(lease_idx, lim); 996259412Sluigi } 997259412Sluigi /* j is the new 'write' position. j != my_start 998259412Sluigi * means there are new buffers to report 999259412Sluigi */ 1000259412Sluigi if (likely(j != my_start)) { 1001260368Sluigi kring->nr_hwtail = j; 1002259412Sluigi still_locked = 0; 1003259412Sluigi mtx_unlock(&kring->q_lock); 1004285349Sluigi kring->nm_notify(kring, 0); 1005270063Sluigi /* this is netmap_notify for VALE ports and 1006270063Sluigi * netmap_bwrap_notify for bwrap. The latter will 1007270063Sluigi * trigger a txsync on the underlying hwna 1008270063Sluigi */ 1009270063Sluigi if (dst_na->retry && retry--) { 1010270063Sluigi /* XXX this is going to call nm_notify again. 1011270063Sluigi * Only useful for bwrap in virtual machines 1012270063Sluigi */ 1013259412Sluigi goto retry; 1014270063Sluigi } 1015259412Sluigi } 1016259412Sluigi } 1017259412Sluigi if (still_locked) 1018259412Sluigi mtx_unlock(&kring->q_lock); 1019259412Sluigi } 1020259412Sluigicleanup: 1021259412Sluigi d->bq_head = d->bq_tail = NM_FT_NULL; /* cleanup */ 1022259412Sluigi d->bq_len = 0; 1023259412Sluigi } 1024259412Sluigi brddst->bq_head = brddst->bq_tail = NM_FT_NULL; /* cleanup */ 1025259412Sluigi brddst->bq_len = 0; 1026259412Sluigi return 0; 1027259412Sluigi} 1028259412Sluigi 1029270063Sluigi/* nm_txsync callback for VALE ports */ 1030259412Sluigistatic int 1031270063Sluiginetmap_vp_txsync(struct netmap_kring *kring, int flags) 1032259412Sluigi{ 1033270063Sluigi struct netmap_vp_adapter *na = 1034270063Sluigi (struct netmap_vp_adapter *)kring->na; 1035260368Sluigi u_int done; 1036260368Sluigi u_int const lim = kring->nkr_num_slots - 1; 1037285349Sluigi u_int const head = kring->rhead; 1038259412Sluigi 1039259412Sluigi if (bridge_batch <= 0) { /* testing only */ 1040285349Sluigi done = head; // used all 1041259412Sluigi goto done; 1042259412Sluigi } 1043270063Sluigi if (!na->na_bdg) { 1044285349Sluigi done = head; 1045270063Sluigi goto done; 1046270063Sluigi } 1047259412Sluigi if (bridge_batch > NM_BDG_BATCH) 1048259412Sluigi bridge_batch = NM_BDG_BATCH; 1049259412Sluigi 1050285349Sluigi done = nm_bdg_preflush(kring, head); 1051259412Sluigidone: 1052285349Sluigi if (done != head) 1053285349Sluigi D("early break at %d/ %d, tail %d", done, head, kring->nr_hwtail); 1054260368Sluigi /* 1055260368Sluigi * packets between 'done' and 'cur' are left unsent. 1056260368Sluigi */ 1057260368Sluigi kring->nr_hwcur = done; 1058260368Sluigi kring->nr_hwtail = nm_prev(done, lim); 1059259412Sluigi if (netmap_verbose) 1060270063Sluigi D("%s ring %d flags %d", na->up.name, kring->ring_id, flags); 1061259412Sluigi return 0; 1062259412Sluigi} 1063259412Sluigi 1064259412Sluigi 1065270063Sluigi/* create a netmap_vp_adapter that describes a VALE port. 1066270063Sluigi * Only persistent VALE ports have a non-null ifp. 1067270063Sluigi */ 1068270063Sluigistatic int 1069341477Svmaffionenetmap_vp_create(struct nmreq_header *hdr, struct ifnet *ifp, 1070341477Svmaffione struct netmap_mem_d *nmd, struct netmap_vp_adapter **ret) 1071270063Sluigi{ 1072341477Svmaffione struct nmreq_register *req = (struct nmreq_register *)(uintptr_t)hdr->nr_body; 1073259412Sluigi struct netmap_vp_adapter *vpna; 1074259412Sluigi struct netmap_adapter *na; 1075341477Svmaffione int error = 0; 1076261909Sluigi u_int npipes = 0; 1077341477Svmaffione u_int extrabufs = 0; 1078259412Sluigi 1079341477Svmaffione if (hdr->nr_reqtype != NETMAP_REQ_REGISTER) { 1080341477Svmaffione return EINVAL; 1081341477Svmaffione } 1082341477Svmaffione 1083341477Svmaffione vpna = nm_os_malloc(sizeof(*vpna)); 1084259412Sluigi if (vpna == NULL) 1085259412Sluigi return ENOMEM; 1086259412Sluigi 1087259412Sluigi na = &vpna->up; 1088259412Sluigi 1089259412Sluigi na->ifp = ifp; 1090341477Svmaffione strncpy(na->name, hdr->nr_name, sizeof(na->name)); 1091259412Sluigi 1092259412Sluigi /* bound checking */ 1093341477Svmaffione na->num_tx_rings = req->nr_tx_rings; 1094259412Sluigi nm_bound_var(&na->num_tx_rings, 1, 1, NM_BDG_MAXRINGS, NULL); 1095341477Svmaffione req->nr_tx_rings = na->num_tx_rings; /* write back */ 1096341477Svmaffione na->num_rx_rings = req->nr_rx_rings; 1097259412Sluigi nm_bound_var(&na->num_rx_rings, 1, 1, NM_BDG_MAXRINGS, NULL); 1098341477Svmaffione req->nr_rx_rings = na->num_rx_rings; /* write back */ 1099341477Svmaffione nm_bound_var(&req->nr_tx_slots, NM_BRIDGE_RINGSIZE, 1100259412Sluigi 1, NM_BDG_MAXSLOTS, NULL); 1101341477Svmaffione na->num_tx_desc = req->nr_tx_slots; 1102341477Svmaffione nm_bound_var(&req->nr_rx_slots, NM_BRIDGE_RINGSIZE, 1103259412Sluigi 1, NM_BDG_MAXSLOTS, NULL); 1104261909Sluigi /* validate number of pipes. We want at least 1, 1105261909Sluigi * but probably can do with some more. 1106261909Sluigi * So let's use 2 as default (when 0 is supplied) 1107261909Sluigi */ 1108261909Sluigi nm_bound_var(&npipes, 2, 1, NM_MAXPIPES, NULL); 1109261909Sluigi /* validate extra bufs */ 1110341477Svmaffione nm_bound_var(&extrabufs, 0, 0, 1111261909Sluigi 128*NM_BDG_MAXSLOTS, NULL); 1112341477Svmaffione req->nr_extra_bufs = extrabufs; /* write back */ 1113341477Svmaffione na->num_rx_desc = req->nr_rx_slots; 1114341477Svmaffione /* Set the mfs to a default value, as it is needed on the VALE 1115341477Svmaffione * mismatch datapath. XXX We should set it according to the MTU 1116341477Svmaffione * known to the kernel. */ 1117341477Svmaffione vpna->mfs = NM_BDG_MFS_DEFAULT; 1118285349Sluigi vpna->last_smac = ~0llu; 1119261909Sluigi /*if (vpna->mfs > netmap_buf_size) TODO netmap_buf_size is zero?? 1120261909Sluigi vpna->mfs = netmap_buf_size; */ 1121341477Svmaffione if (netmap_verbose) 1122261909Sluigi D("max frame size %u", vpna->mfs); 1123259412Sluigi 1124285349Sluigi na->na_flags |= NAF_BDG_MAYSLEEP; 1125285698Sluigi /* persistent VALE ports look like hw devices 1126285698Sluigi * with a native netmap adapter 1127285698Sluigi */ 1128285698Sluigi if (ifp) 1129285698Sluigi na->na_flags |= NAF_NATIVE; 1130270063Sluigi na->nm_txsync = netmap_vp_txsync; 1131270063Sluigi na->nm_rxsync = netmap_vp_rxsync; 1132270063Sluigi na->nm_register = netmap_vp_reg; 1133259412Sluigi na->nm_krings_create = netmap_vp_krings_create; 1134259412Sluigi na->nm_krings_delete = netmap_vp_krings_delete; 1135270063Sluigi na->nm_dtor = netmap_vp_dtor; 1136341477Svmaffione ND("nr_mem_id %d", req->nr_mem_id); 1137341477Svmaffione na->nm_mem = nmd ? 1138341477Svmaffione netmap_mem_get(nmd): 1139341477Svmaffione netmap_mem_private_new( 1140259412Sluigi na->num_tx_rings, na->num_tx_desc, 1141261909Sluigi na->num_rx_rings, na->num_rx_desc, 1142341477Svmaffione req->nr_extra_bufs, npipes, &error); 1143261909Sluigi if (na->nm_mem == NULL) 1144261909Sluigi goto err; 1145270063Sluigi na->nm_bdg_attach = netmap_vp_bdg_attach; 1146259412Sluigi /* other nmd fields are set in the common routine */ 1147259412Sluigi error = netmap_attach_common(na); 1148261909Sluigi if (error) 1149261909Sluigi goto err; 1150270063Sluigi *ret = vpna; 1151259412Sluigi return 0; 1152261909Sluigi 1153261909Sluigierr: 1154261909Sluigi if (na->nm_mem != NULL) 1155341477Svmaffione netmap_mem_put(na->nm_mem); 1156341477Svmaffione nm_os_free(vpna); 1157261909Sluigi return error; 1158259412Sluigi} 1159259412Sluigi 1160341477Svmaffione/* nm_bdg_attach callback for VALE ports 1161341477Svmaffione * The na_vp port is this same netmap_adapter. There is no host port. 1162270063Sluigi */ 1163259412Sluigistatic int 1164341477Svmaffionenetmap_vp_bdg_attach(const char *name, struct netmap_adapter *na, 1165341477Svmaffione struct nm_bridge *b) 1166259412Sluigi{ 1167341477Svmaffione struct netmap_vp_adapter *vpna = (struct netmap_vp_adapter *)na; 1168259412Sluigi 1169341477Svmaffione if (b->bdg_ops != &vale_bdg_ops) { 1170341477Svmaffione return NM_NEED_BWRAP; 1171259412Sluigi } 1172341477Svmaffione if (vpna->na_bdg) { 1173341477Svmaffione return NM_NEED_BWRAP; 1174259412Sluigi } 1175341477Svmaffione na->na_vp = vpna; 1176341477Svmaffione strncpy(na->name, name, sizeof(na->name)); 1177341477Svmaffione na->na_hostvp = NULL; 1178259412Sluigi return 0; 1179259412Sluigi} 1180259412Sluigi 1181259412Sluigistatic int 1182341477Svmaffionenetmap_vale_bwrap_krings_create(struct netmap_adapter *na) 1183259412Sluigi{ 1184259412Sluigi int error; 1185259412Sluigi 1186270063Sluigi /* impersonate a netmap_vp_adapter */ 1187259412Sluigi error = netmap_vp_krings_create(na); 1188259412Sluigi if (error) 1189259412Sluigi return error; 1190341477Svmaffione error = netmap_bwrap_krings_create_common(na); 1191259412Sluigi if (error) { 1192259412Sluigi netmap_vp_krings_delete(na); 1193259412Sluigi } 1194341477Svmaffione return error; 1195259412Sluigi} 1196259412Sluigi 1197259412Sluigistatic void 1198341477Svmaffionenetmap_vale_bwrap_krings_delete(struct netmap_adapter *na) 1199259412Sluigi{ 1200341477Svmaffione netmap_bwrap_krings_delete_common(na); 1201259412Sluigi netmap_vp_krings_delete(na); 1202259412Sluigi} 1203259412Sluigi 1204259412Sluigistatic int 1205341477Svmaffionenetmap_vale_bwrap_attach(const char *nr_name, struct netmap_adapter *hwna) 1206259412Sluigi{ 1207341477Svmaffione struct netmap_bwrap_adapter *bna; 1208341477Svmaffione struct netmap_adapter *na = NULL; 1209341477Svmaffione struct netmap_adapter *hostna = NULL; 1210341477Svmaffione int error; 1211259412Sluigi 1212341477Svmaffione bna = nm_os_malloc(sizeof(*bna)); 1213341477Svmaffione if (bna == NULL) { 1214341477Svmaffione return ENOMEM; 1215341477Svmaffione } 1216341477Svmaffione na = &bna->up.up; 1217341477Svmaffione strncpy(na->name, nr_name, sizeof(na->name)); 1218341477Svmaffione na->nm_register = netmap_bwrap_reg; 1219341477Svmaffione na->nm_txsync = netmap_vp_txsync; 1220341477Svmaffione // na->nm_rxsync = netmap_bwrap_rxsync; 1221341477Svmaffione na->nm_krings_create = netmap_vale_bwrap_krings_create; 1222341477Svmaffione na->nm_krings_delete = netmap_vale_bwrap_krings_delete; 1223341477Svmaffione na->nm_notify = netmap_bwrap_notify; 1224341477Svmaffione bna->up.retry = 1; /* XXX maybe this should depend on the hwna */ 1225341477Svmaffione /* Set the mfs, needed on the VALE mismatch datapath. */ 1226341477Svmaffione bna->up.mfs = NM_BDG_MFS_DEFAULT; 1227259412Sluigi 1228341477Svmaffione if (hwna->na_flags & NAF_HOST_RINGS) { 1229341477Svmaffione hostna = &bna->host.up; 1230341477Svmaffione hostna->nm_notify = netmap_bwrap_notify; 1231341477Svmaffione bna->host.mfs = NM_BDG_MFS_DEFAULT; 1232341477Svmaffione } 1233285349Sluigi 1234341477Svmaffione error = netmap_bwrap_attach_common(na, hwna); 1235341477Svmaffione if (error) { 1236341477Svmaffione nm_os_free(bna); 1237341477Svmaffione } 1238259412Sluigi return error; 1239259412Sluigi} 1240259412Sluigi 1241341477Svmaffioneint 1242341477Svmaffionenetmap_get_vale_na(struct nmreq_header *hdr, struct netmap_adapter **na, 1243341477Svmaffione struct netmap_mem_d *nmd, int create) 1244341477Svmaffione{ 1245341477Svmaffione return netmap_get_bdg_na(hdr, na, nmd, create, &vale_bdg_ops); 1246341477Svmaffione} 1247260368Sluigi 1248341477Svmaffione 1249341477Svmaffione/* creates a persistent VALE port */ 1250341477Svmaffioneint 1251341477Svmaffionenm_vi_create(struct nmreq_header *hdr) 1252259412Sluigi{ 1253341477Svmaffione struct nmreq_vale_newif *req = 1254341477Svmaffione (struct nmreq_vale_newif *)(uintptr_t)hdr->nr_body; 1255270063Sluigi int error = 0; 1256341477Svmaffione /* Build a nmreq_register out of the nmreq_vale_newif, 1257341477Svmaffione * so that we can call netmap_get_bdg_na(). */ 1258341477Svmaffione struct nmreq_register regreq; 1259341477Svmaffione bzero(®req, sizeof(regreq)); 1260341477Svmaffione regreq.nr_tx_slots = req->nr_tx_slots; 1261341477Svmaffione regreq.nr_rx_slots = req->nr_rx_slots; 1262341477Svmaffione regreq.nr_tx_rings = req->nr_tx_rings; 1263341477Svmaffione regreq.nr_rx_rings = req->nr_rx_rings; 1264341477Svmaffione regreq.nr_mem_id = req->nr_mem_id; 1265341477Svmaffione hdr->nr_reqtype = NETMAP_REQ_REGISTER; 1266341477Svmaffione hdr->nr_body = (uintptr_t)®req; 1267341477Svmaffione error = netmap_vi_create(hdr, 0 /* no autodelete */); 1268341477Svmaffione hdr->nr_reqtype = NETMAP_REQ_VALE_NEWIF; 1269341477Svmaffione hdr->nr_body = (uintptr_t)req; 1270341477Svmaffione /* Write back to the original struct. */ 1271341477Svmaffione req->nr_tx_slots = regreq.nr_tx_slots; 1272341477Svmaffione req->nr_rx_slots = regreq.nr_rx_slots; 1273341477Svmaffione req->nr_tx_rings = regreq.nr_tx_rings; 1274341477Svmaffione req->nr_rx_rings = regreq.nr_rx_rings; 1275341477Svmaffione req->nr_mem_id = regreq.nr_mem_id; 1276270063Sluigi return error; 1277270063Sluigi} 1278270063Sluigi 1279341477Svmaffione/* remove a persistent VALE port from the system */ 1280270063Sluigiint 1281341477Svmaffionenm_vi_destroy(const char *name) 1282270063Sluigi{ 1283341477Svmaffione struct ifnet *ifp; 1284341477Svmaffione struct netmap_vp_adapter *vpna; 1285341477Svmaffione int error; 1286259412Sluigi 1287341477Svmaffione ifp = ifunit_ref(name); 1288341477Svmaffione if (!ifp) 1289341477Svmaffione return ENXIO; 1290341477Svmaffione NMG_LOCK(); 1291341477Svmaffione /* make sure this is actually a VALE port */ 1292341477Svmaffione if (!NM_NA_VALID(ifp) || NA(ifp)->nm_register != netmap_vp_reg) { 1293341477Svmaffione error = EINVAL; 1294341477Svmaffione goto err; 1295270063Sluigi } 1296259412Sluigi 1297341477Svmaffione vpna = (struct netmap_vp_adapter *)NA(ifp); 1298259412Sluigi 1299341477Svmaffione /* we can only destroy ports that were created via NETMAP_BDG_NEWIF */ 1300341477Svmaffione if (vpna->autodelete) { 1301341477Svmaffione error = EINVAL; 1302341477Svmaffione goto err; 1303285349Sluigi } 1304259412Sluigi 1305341477Svmaffione /* also make sure that nobody is using the inferface */ 1306341477Svmaffione if (NETMAP_OWNED_BY_ANY(&vpna->up) || 1307341477Svmaffione vpna->up.na_refcount > 1 /* any ref besides the one in nm_vi_create()? */) { 1308341477Svmaffione error = EBUSY; 1309341477Svmaffione goto err; 1310261909Sluigi } 1311259412Sluigi 1312341477Svmaffione NMG_UNLOCK(); 1313259412Sluigi 1314341477Svmaffione D("destroying a persistent vale interface %s", ifp->if_xname); 1315341477Svmaffione /* Linux requires all the references are released 1316341477Svmaffione * before unregister 1317270063Sluigi */ 1318341477Svmaffione netmap_detach(ifp); 1319341477Svmaffione if_rele(ifp); 1320341477Svmaffione nm_os_vi_detach(ifp); 1321259412Sluigi return 0; 1322270063Sluigi 1323341477Svmaffioneerr: 1324341477Svmaffione NMG_UNLOCK(); 1325341477Svmaffione if_rele(ifp); 1326270063Sluigi return error; 1327259412Sluigi} 1328259412Sluigi 1329341477Svmaffionestatic int 1330341477Svmaffionenm_update_info(struct nmreq_register *req, struct netmap_adapter *na) 1331285349Sluigi{ 1332341477Svmaffione req->nr_rx_rings = na->num_rx_rings; 1333341477Svmaffione req->nr_tx_rings = na->num_tx_rings; 1334341477Svmaffione req->nr_rx_slots = na->num_rx_desc; 1335341477Svmaffione req->nr_tx_slots = na->num_tx_desc; 1336341477Svmaffione return netmap_mem_get_info(na->nm_mem, &req->nr_memsize, NULL, 1337341477Svmaffione &req->nr_mem_id); 1338285349Sluigi} 1339285349Sluigi 1340341477Svmaffione 1341341477Svmaffione/* 1342341477Svmaffione * Create a virtual interface registered to the system. 1343341477Svmaffione * The interface will be attached to a bridge later. 1344341477Svmaffione */ 1345341477Svmaffioneint 1346341477Svmaffionenetmap_vi_create(struct nmreq_header *hdr, int autodelete) 1347259412Sluigi{ 1348341477Svmaffione struct nmreq_register *req = (struct nmreq_register *)(uintptr_t)hdr->nr_body; 1349341477Svmaffione struct ifnet *ifp; 1350341477Svmaffione struct netmap_vp_adapter *vpna; 1351341477Svmaffione struct netmap_mem_d *nmd = NULL; 1352341477Svmaffione int error; 1353285349Sluigi 1354341477Svmaffione if (hdr->nr_reqtype != NETMAP_REQ_REGISTER) { 1355341477Svmaffione return EINVAL; 1356341477Svmaffione } 1357285349Sluigi 1358341477Svmaffione /* don't include VALE prefix */ 1359341477Svmaffione if (!strncmp(hdr->nr_name, NM_BDG_NAME, strlen(NM_BDG_NAME))) 1360341477Svmaffione return EINVAL; 1361341477Svmaffione if (strlen(hdr->nr_name) >= IFNAMSIZ) { 1362341477Svmaffione return EINVAL; 1363341477Svmaffione } 1364341477Svmaffione ifp = ifunit_ref(hdr->nr_name); 1365341477Svmaffione if (ifp) { /* already exist, cannot create new one */ 1366341477Svmaffione error = EEXIST; 1367341477Svmaffione NMG_LOCK(); 1368341477Svmaffione if (NM_NA_VALID(ifp)) { 1369341477Svmaffione int update_err = nm_update_info(req, NA(ifp)); 1370341477Svmaffione if (update_err) 1371341477Svmaffione error = update_err; 1372341477Svmaffione } 1373341477Svmaffione NMG_UNLOCK(); 1374341477Svmaffione if_rele(ifp); 1375341477Svmaffione return error; 1376341477Svmaffione } 1377341477Svmaffione error = nm_os_vi_persist(hdr->nr_name, &ifp); 1378341477Svmaffione if (error) 1379341477Svmaffione return error; 1380285349Sluigi 1381341477Svmaffione NMG_LOCK(); 1382341477Svmaffione if (req->nr_mem_id) { 1383341477Svmaffione nmd = netmap_mem_find(req->nr_mem_id); 1384341477Svmaffione if (nmd == NULL) { 1385341477Svmaffione error = EINVAL; 1386341477Svmaffione goto err_1; 1387341477Svmaffione } 1388341477Svmaffione } 1389341477Svmaffione /* netmap_vp_create creates a struct netmap_vp_adapter */ 1390341477Svmaffione error = netmap_vp_create(hdr, ifp, nmd, &vpna); 1391341477Svmaffione if (error) { 1392341477Svmaffione D("error %d", error); 1393341477Svmaffione goto err_1; 1394341477Svmaffione } 1395341477Svmaffione /* persist-specific routines */ 1396341477Svmaffione vpna->up.nm_bdg_ctl = netmap_vp_bdg_ctl; 1397341477Svmaffione if (!autodelete) { 1398341477Svmaffione netmap_adapter_get(&vpna->up); 1399341477Svmaffione } else { 1400341477Svmaffione vpna->autodelete = 1; 1401341477Svmaffione } 1402341477Svmaffione NM_ATTACH_NA(ifp, &vpna->up); 1403341477Svmaffione /* return the updated info */ 1404341477Svmaffione error = nm_update_info(req, &vpna->up); 1405341477Svmaffione if (error) { 1406341477Svmaffione goto err_2; 1407341477Svmaffione } 1408341477Svmaffione ND("returning nr_mem_id %d", req->nr_mem_id); 1409341477Svmaffione if (nmd) 1410341477Svmaffione netmap_mem_put(nmd); 1411341477Svmaffione NMG_UNLOCK(); 1412341477Svmaffione ND("created %s", ifp->if_xname); 1413285349Sluigi return 0; 1414341477Svmaffione 1415341477Svmaffioneerr_2: 1416341477Svmaffione netmap_detach(ifp); 1417341477Svmaffioneerr_1: 1418341477Svmaffione if (nmd) 1419341477Svmaffione netmap_mem_put(nmd); 1420341477Svmaffione NMG_UNLOCK(); 1421341477Svmaffione nm_os_vi_detach(ifp); 1422341477Svmaffione 1423341477Svmaffione return error; 1424285349Sluigi} 1425285349Sluigi 1426259412Sluigi#endif /* WITH_VALE */ 1427