1331722Seadler/* 2341477Svmaffione * Copyright (C) 2014-2018 Giuseppe Lettieri 3341477Svmaffione * All rights reserved. 4261909Sluigi * 5261909Sluigi * Redistribution and use in source and binary forms, with or without 6261909Sluigi * modification, are permitted provided that the following conditions 7261909Sluigi * are met: 8261909Sluigi * 1. Redistributions of source code must retain the above copyright 9261909Sluigi * notice, this list of conditions and the following disclaimer. 10261909Sluigi * 2. Redistributions in binary form must reproduce the above copyright 11261909Sluigi * notice, this list of conditions and the following disclaimer in the 12261909Sluigi * documentation and/or other materials provided with the distribution. 13261909Sluigi * 14261909Sluigi * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15261909Sluigi * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16261909Sluigi * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17261909Sluigi * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18261909Sluigi * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19261909Sluigi * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20261909Sluigi * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21261909Sluigi * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22261909Sluigi * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23261909Sluigi * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24261909Sluigi * SUCH DAMAGE. 25261909Sluigi */ 26261909Sluigi 27261909Sluigi/* $FreeBSD: stable/11/sys/dev/netmap/netmap_pipe.c 344047 2019-02-12 09:26:05Z vmaffione $ */ 28261909Sluigi 29261909Sluigi#if defined(__FreeBSD__) 30261909Sluigi#include <sys/cdefs.h> /* prerequisite */ 31261909Sluigi 32261909Sluigi#include <sys/types.h> 33261909Sluigi#include <sys/errno.h> 34261909Sluigi#include <sys/param.h> /* defines used in kernel.h */ 35261909Sluigi#include <sys/kernel.h> /* types used in module initialization */ 36261909Sluigi#include <sys/malloc.h> 37261909Sluigi#include <sys/poll.h> 38261909Sluigi#include <sys/lock.h> 39261909Sluigi#include <sys/rwlock.h> 40261909Sluigi#include <sys/selinfo.h> 41261909Sluigi#include <sys/sysctl.h> 42261909Sluigi#include <sys/socket.h> /* sockaddrs */ 43261909Sluigi#include <net/if.h> 44261909Sluigi#include <net/if_var.h> 45261909Sluigi#include <machine/bus.h> /* bus_dmamap_* */ 46261909Sluigi#include <sys/refcount.h> 47261909Sluigi 48261909Sluigi 49261909Sluigi#elif defined(linux) 50261909Sluigi 51261909Sluigi#include "bsd_glue.h" 52261909Sluigi 53261909Sluigi#elif defined(__APPLE__) 54261909Sluigi 55261909Sluigi#warning OSX support is only partial 56261909Sluigi#include "osx_glue.h" 57261909Sluigi 58341477Svmaffione#elif defined(_WIN32) 59341477Svmaffione#include "win_glue.h" 60341477Svmaffione 61261909Sluigi#else 62261909Sluigi 63261909Sluigi#error Unsupported platform 64261909Sluigi 65261909Sluigi#endif /* unsupported */ 66261909Sluigi 67261909Sluigi/* 68261909Sluigi * common headers 69261909Sluigi */ 70261909Sluigi 71261909Sluigi#include <net/netmap.h> 72261909Sluigi#include <dev/netmap/netmap_kern.h> 73261909Sluigi#include <dev/netmap/netmap_mem2.h> 74261909Sluigi 75261909Sluigi#ifdef WITH_PIPES 76261909Sluigi 77261909Sluigi#define NM_PIPE_MAXSLOTS 4096 78341477Svmaffione#define NM_PIPE_MAXRINGS 256 79261909Sluigi 80341477Svmaffionestatic int netmap_default_pipes = 0; /* ignored, kept for compatibility */ 81341477SvmaffioneSYSBEGIN(vars_pipes); 82261909SluigiSYSCTL_DECL(_dev_netmap); 83341477SvmaffioneSYSCTL_INT(_dev_netmap, OID_AUTO, default_pipes, CTLFLAG_RW, 84341477Svmaffione &netmap_default_pipes, 0, "For compatibility only"); 85341477SvmaffioneSYSEND; 86261909Sluigi 87261909Sluigi/* allocate the pipe array in the parent adapter */ 88285349Sluigistatic int 89285349Sluiginm_pipe_alloc(struct netmap_adapter *na, u_int npipes) 90261909Sluigi{ 91341477Svmaffione size_t old_len, len; 92285349Sluigi struct netmap_pipe_adapter **npa; 93261909Sluigi 94285349Sluigi if (npipes <= na->na_max_pipes) 95285349Sluigi /* we already have more entries that requested */ 96261909Sluigi return 0; 97341477Svmaffione 98285349Sluigi if (npipes < na->na_next_pipe || npipes > NM_MAXPIPES) 99285349Sluigi return EINVAL; 100261909Sluigi 101341477Svmaffione old_len = sizeof(struct netmap_pipe_adapter *)*na->na_max_pipes; 102341477Svmaffione len = sizeof(struct netmap_pipe_adapter *) * npipes; 103341477Svmaffione npa = nm_os_realloc(na->na_pipes, len, old_len); 104285349Sluigi if (npa == NULL) 105261909Sluigi return ENOMEM; 106261909Sluigi 107285349Sluigi na->na_pipes = npa; 108261909Sluigi na->na_max_pipes = npipes; 109261909Sluigi 110261909Sluigi return 0; 111261909Sluigi} 112261909Sluigi 113261909Sluigi/* deallocate the parent array in the parent adapter */ 114261909Sluigivoid 115261909Sluiginetmap_pipe_dealloc(struct netmap_adapter *na) 116261909Sluigi{ 117261909Sluigi if (na->na_pipes) { 118285349Sluigi if (na->na_next_pipe > 0) { 119344047Svmaffione nm_prerr("freeing not empty pipe array for %s (%d dangling pipes)!", 120344047Svmaffione na->name, na->na_next_pipe); 121285349Sluigi } 122341477Svmaffione nm_os_free(na->na_pipes); 123261909Sluigi na->na_pipes = NULL; 124261909Sluigi na->na_max_pipes = 0; 125261909Sluigi na->na_next_pipe = 0; 126261909Sluigi } 127261909Sluigi} 128261909Sluigi 129261909Sluigi/* find a pipe endpoint with the given id among the parent's pipes */ 130261909Sluigistatic struct netmap_pipe_adapter * 131341477Svmaffionenetmap_pipe_find(struct netmap_adapter *parent, const char *pipe_id) 132261909Sluigi{ 133261909Sluigi int i; 134261909Sluigi struct netmap_pipe_adapter *na; 135261909Sluigi 136261909Sluigi for (i = 0; i < parent->na_next_pipe; i++) { 137341477Svmaffione const char *na_pipe_id; 138261909Sluigi na = parent->na_pipes[i]; 139341477Svmaffione na_pipe_id = strrchr(na->up.name, 140341477Svmaffione na->role == NM_PIPE_ROLE_MASTER ? '{' : '}'); 141341477Svmaffione KASSERT(na_pipe_id != NULL, ("Invalid pipe name")); 142341477Svmaffione ++na_pipe_id; 143341477Svmaffione if (!strcmp(na_pipe_id, pipe_id)) { 144261909Sluigi return na; 145261909Sluigi } 146261909Sluigi } 147261909Sluigi return NULL; 148261909Sluigi} 149261909Sluigi 150261909Sluigi/* add a new pipe endpoint to the parent array */ 151261909Sluigistatic int 152261909Sluiginetmap_pipe_add(struct netmap_adapter *parent, struct netmap_pipe_adapter *na) 153261909Sluigi{ 154261909Sluigi if (parent->na_next_pipe >= parent->na_max_pipes) { 155285349Sluigi u_int npipes = parent->na_max_pipes ? 2*parent->na_max_pipes : 2; 156285349Sluigi int error = nm_pipe_alloc(parent, npipes); 157285349Sluigi if (error) 158285349Sluigi return error; 159261909Sluigi } 160261909Sluigi 161261909Sluigi parent->na_pipes[parent->na_next_pipe] = na; 162261909Sluigi na->parent_slot = parent->na_next_pipe; 163261909Sluigi parent->na_next_pipe++; 164261909Sluigi return 0; 165261909Sluigi} 166261909Sluigi 167261909Sluigi/* remove the given pipe endpoint from the parent array */ 168261909Sluigistatic void 169261909Sluiginetmap_pipe_remove(struct netmap_adapter *parent, struct netmap_pipe_adapter *na) 170261909Sluigi{ 171261909Sluigi u_int n; 172261909Sluigi n = --parent->na_next_pipe; 173261909Sluigi if (n != na->parent_slot) { 174285349Sluigi struct netmap_pipe_adapter **p = 175285349Sluigi &parent->na_pipes[na->parent_slot]; 176285349Sluigi *p = parent->na_pipes[n]; 177285349Sluigi (*p)->parent_slot = na->parent_slot; 178261909Sluigi } 179261909Sluigi parent->na_pipes[n] = NULL; 180261909Sluigi} 181261909Sluigi 182341477Svmaffioneint 183270063Sluiginetmap_pipe_txsync(struct netmap_kring *txkring, int flags) 184261909Sluigi{ 185341477Svmaffione struct netmap_kring *rxkring = txkring->pipe; 186341477Svmaffione u_int k, lim = txkring->nkr_num_slots - 1, nk; 187341477Svmaffione int m; /* slots to transfer */ 188341477Svmaffione int complete; /* did we see a complete packet ? */ 189341477Svmaffione struct netmap_ring *txring = txkring->ring, *rxring = rxkring->ring; 190261909Sluigi 191344047Svmaffione nm_prdis("%p: %s %x -> %s", txkring, txkring->name, flags, rxkring->name); 192344047Svmaffione nm_prdis(20, "TX before: hwcur %d hwtail %d cur %d head %d tail %d", 193341477Svmaffione txkring->nr_hwcur, txkring->nr_hwtail, 194341477Svmaffione txkring->rcur, txkring->rhead, txkring->rtail); 195261909Sluigi 196341477Svmaffione /* update the hwtail */ 197341477Svmaffione txkring->nr_hwtail = txkring->pipe_tail; 198261909Sluigi 199341477Svmaffione m = txkring->rhead - txkring->nr_hwcur; /* new slots */ 200341477Svmaffione if (m < 0) 201341477Svmaffione m += txkring->nkr_num_slots; 202341477Svmaffione 203341477Svmaffione if (m == 0) { 204341477Svmaffione /* nothing to send */ 205261909Sluigi return 0; 206261909Sluigi } 207261909Sluigi 208341477Svmaffione for (k = txkring->nr_hwcur, nk = lim + 1, complete = 0; m; 209341477Svmaffione m--, k = nm_next(k, lim), nk = (complete ? k : nk)) { 210341477Svmaffione struct netmap_slot *rs = &rxring->slot[k]; 211341477Svmaffione struct netmap_slot *ts = &txring->slot[k]; 212261909Sluigi 213341477Svmaffione *rs = *ts; 214341477Svmaffione if (ts->flags & NS_BUF_CHANGED) { 215341477Svmaffione ts->flags &= ~NS_BUF_CHANGED; 216341477Svmaffione } 217341477Svmaffione complete = !(ts->flags & NS_MOREFRAG); 218341477Svmaffione } 219261909Sluigi 220341477Svmaffione txkring->nr_hwcur = k; 221261909Sluigi 222344047Svmaffione nm_prdis(20, "TX after : hwcur %d hwtail %d cur %d head %d tail %d k %d", 223341477Svmaffione txkring->nr_hwcur, txkring->nr_hwtail, 224341477Svmaffione txkring->rcur, txkring->rhead, txkring->rtail, k); 225261909Sluigi 226341477Svmaffione if (likely(nk <= lim)) { 227341477Svmaffione mb(); /* make sure the slots are updated before publishing them */ 228341477Svmaffione rxkring->pipe_tail = nk; /* only publish complete packets */ 229341477Svmaffione rxkring->nm_notify(rxkring, 0); 230341477Svmaffione } 231261909Sluigi 232261909Sluigi return 0; 233261909Sluigi} 234261909Sluigi 235341477Svmaffioneint 236270063Sluiginetmap_pipe_rxsync(struct netmap_kring *rxkring, int flags) 237261909Sluigi{ 238341477Svmaffione struct netmap_kring *txkring = rxkring->pipe; 239341477Svmaffione u_int k, lim = rxkring->nkr_num_slots - 1; 240341477Svmaffione int m; /* slots to release */ 241341477Svmaffione struct netmap_ring *txring = txkring->ring, *rxring = rxkring->ring; 242261909Sluigi 243344047Svmaffione nm_prdis("%p: %s %x -> %s", txkring, txkring->name, flags, rxkring->name); 244344047Svmaffione nm_prdis(20, "RX before: hwcur %d hwtail %d cur %d head %d tail %d", 245341477Svmaffione rxkring->nr_hwcur, rxkring->nr_hwtail, 246341477Svmaffione rxkring->rcur, rxkring->rhead, rxkring->rtail); 247261909Sluigi 248341477Svmaffione /* update the hwtail */ 249341477Svmaffione rxkring->nr_hwtail = rxkring->pipe_tail; 250341477Svmaffione 251341477Svmaffione m = rxkring->rhead - rxkring->nr_hwcur; /* released slots */ 252341477Svmaffione if (m < 0) 253341477Svmaffione m += rxkring->nkr_num_slots; 254341477Svmaffione 255341477Svmaffione if (m == 0) { 256341477Svmaffione /* nothing to release */ 257341477Svmaffione return 0; 258261909Sluigi } 259341477Svmaffione 260341477Svmaffione for (k = rxkring->nr_hwcur; m; m--, k = nm_next(k, lim)) { 261341477Svmaffione struct netmap_slot *rs = &rxring->slot[k]; 262341477Svmaffione struct netmap_slot *ts = &txring->slot[k]; 263341477Svmaffione 264341477Svmaffione if (rs->flags & NS_BUF_CHANGED) { 265341477Svmaffione /* copy the slot and report the buffer change */ 266341477Svmaffione *ts = *rs; 267341477Svmaffione rs->flags &= ~NS_BUF_CHANGED; 268341477Svmaffione } 269341477Svmaffione } 270341477Svmaffione 271341477Svmaffione mb(); /* make sure the slots are updated before publishing them */ 272341477Svmaffione txkring->pipe_tail = nm_prev(k, lim); 273341477Svmaffione rxkring->nr_hwcur = k; 274341477Svmaffione 275344047Svmaffione nm_prdis(20, "RX after : hwcur %d hwtail %d cur %d head %d tail %d k %d", 276341477Svmaffione rxkring->nr_hwcur, rxkring->nr_hwtail, 277341477Svmaffione rxkring->rcur, rxkring->rhead, rxkring->rtail, k); 278341477Svmaffione 279341477Svmaffione txkring->nm_notify(txkring, 0); 280341477Svmaffione 281341477Svmaffione return 0; 282261909Sluigi} 283261909Sluigi 284261909Sluigi/* Pipe endpoints are created and destroyed together, so that endopoints do not 285261909Sluigi * have to check for the existence of their peer at each ?xsync. 286261909Sluigi * 287261909Sluigi * To play well with the existing netmap infrastructure (refcounts etc.), we 288261909Sluigi * adopt the following strategy: 289261909Sluigi * 290261909Sluigi * 1) The first endpoint that is created also creates the other endpoint and 291261909Sluigi * grabs a reference to it. 292261909Sluigi * 293261909Sluigi * state A) user1 --> endpoint1 --> endpoint2 294261909Sluigi * 295261909Sluigi * 2) If, starting from state A, endpoint2 is then registered, endpoint1 gives 296261909Sluigi * its reference to the user: 297261909Sluigi * 298261909Sluigi * state B) user1 --> endpoint1 endpoint2 <--- user2 299261909Sluigi * 300261909Sluigi * 3) Assume that, starting from state B endpoint2 is closed. In the unregister 301261909Sluigi * callback endpoint2 notes that endpoint1 is still active and adds a reference 302261909Sluigi * from endpoint1 to itself. When user2 then releases her own reference, 303261909Sluigi * endpoint2 is not destroyed and we are back to state A. A symmetrical state 304261909Sluigi * would be reached if endpoint1 were released instead. 305261909Sluigi * 306261909Sluigi * 4) If, starting from state A, endpoint1 is closed, the destructor notes that 307261909Sluigi * it owns a reference to endpoint2 and releases it. 308261909Sluigi * 309261909Sluigi * Something similar goes on for the creation and destruction of the krings. 310261909Sluigi */ 311261909Sluigi 312261909Sluigi 313344047Svmaffioneint netmap_pipe_krings_create_both(struct netmap_adapter *na, 314344047Svmaffione struct netmap_adapter *ona) 315344047Svmaffione{ 316344047Svmaffione enum txrx t; 317344047Svmaffione int error; 318344047Svmaffione int i; 319344047Svmaffione 320344047Svmaffione /* case 1) below */ 321344047Svmaffione nm_prdis("%p: case 1, create both ends", na); 322344047Svmaffione error = netmap_krings_create(na, 0); 323344047Svmaffione if (error) 324344047Svmaffione return error; 325344047Svmaffione 326344047Svmaffione /* create the krings of the other end */ 327344047Svmaffione error = netmap_krings_create(ona, 0); 328344047Svmaffione if (error) 329344047Svmaffione goto del_krings1; 330344047Svmaffione 331344047Svmaffione /* cross link the krings and initialize the pipe_tails */ 332344047Svmaffione for_rx_tx(t) { 333344047Svmaffione enum txrx r = nm_txrx_swap(t); /* swap NR_TX <-> NR_RX */ 334344047Svmaffione for (i = 0; i < nma_get_nrings(na, t); i++) { 335344047Svmaffione struct netmap_kring *k1 = NMR(na, t)[i], 336344047Svmaffione *k2 = NMR(ona, r)[i]; 337344047Svmaffione k1->pipe = k2; 338344047Svmaffione k2->pipe = k1; 339344047Svmaffione /* mark all peer-adapter rings as fake */ 340344047Svmaffione k2->nr_kflags |= NKR_FAKERING; 341344047Svmaffione /* init tails */ 342344047Svmaffione k1->pipe_tail = k1->nr_hwtail; 343344047Svmaffione k2->pipe_tail = k2->nr_hwtail; 344344047Svmaffione } 345344047Svmaffione } 346344047Svmaffione 347344047Svmaffione return 0; 348344047Svmaffione 349344047Svmaffionedel_krings1: 350344047Svmaffione netmap_krings_delete(na); 351344047Svmaffione return error; 352344047Svmaffione} 353344047Svmaffione 354341477Svmaffione/* netmap_pipe_krings_create. 355261909Sluigi * 356261909Sluigi * There are two cases: 357261909Sluigi * 358261909Sluigi * 1) state is 359261909Sluigi * 360261909Sluigi * usr1 --> e1 --> e2 361261909Sluigi * 362261909Sluigi * and we are e1. We have to create both sets 363261909Sluigi * of krings. 364261909Sluigi * 365261909Sluigi * 2) state is 366261909Sluigi * 367261909Sluigi * usr1 --> e1 --> e2 368261909Sluigi * 369261909Sluigi * and we are e2. e1 is certainly registered and our 370341477Svmaffione * krings already exist. Nothing to do. 371261909Sluigi */ 372261909Sluigistatic int 373261909Sluiginetmap_pipe_krings_create(struct netmap_adapter *na) 374261909Sluigi{ 375261909Sluigi struct netmap_pipe_adapter *pna = 376261909Sluigi (struct netmap_pipe_adapter *)na; 377261909Sluigi struct netmap_adapter *ona = &pna->peer->up; 378344047Svmaffione 379344047Svmaffione if (pna->peer_ref) 380344047Svmaffione return netmap_pipe_krings_create_both(na, ona); 381344047Svmaffione 382344047Svmaffione return 0; 383344047Svmaffione} 384344047Svmaffione 385344047Svmaffioneint 386344047Svmaffionenetmap_pipe_reg_both(struct netmap_adapter *na, struct netmap_adapter *ona) 387344047Svmaffione{ 388344047Svmaffione int i, error = 0; 389285349Sluigi enum txrx t; 390285349Sluigi 391344047Svmaffione for_rx_tx(t) { 392344047Svmaffione for (i = 0; i < nma_get_nrings(na, t); i++) { 393344047Svmaffione struct netmap_kring *kring = NMR(na, t)[i]; 394261909Sluigi 395344047Svmaffione if (nm_kring_pending_on(kring)) { 396344047Svmaffione /* mark the peer ring as needed */ 397344047Svmaffione kring->pipe->nr_kflags |= NKR_NEEDRING; 398344047Svmaffione } 399344047Svmaffione } 400344047Svmaffione } 401261909Sluigi 402344047Svmaffione /* create all missing needed rings on the other end. 403344047Svmaffione * Either our end, or the other, has been marked as 404344047Svmaffione * fake, so the allocation will not be done twice. 405344047Svmaffione */ 406344047Svmaffione error = netmap_mem_rings_create(ona); 407344047Svmaffione if (error) 408344047Svmaffione return error; 409261909Sluigi 410344047Svmaffione /* In case of no error we put our rings in netmap mode */ 411344047Svmaffione for_rx_tx(t) { 412344047Svmaffione for (i = 0; i < nma_get_nrings(na, t); i++) { 413344047Svmaffione struct netmap_kring *kring = NMR(na, t)[i]; 414344047Svmaffione if (nm_kring_pending_on(kring)) { 415344047Svmaffione struct netmap_kring *sring, *dring; 416344047Svmaffione 417344047Svmaffione kring->nr_mode = NKR_NETMAP_ON; 418344047Svmaffione if ((kring->nr_kflags & NKR_FAKERING) && 419344047Svmaffione (kring->pipe->nr_kflags & NKR_FAKERING)) { 420344047Svmaffione /* this is a re-open of a pipe 421344047Svmaffione * end-point kept alive by the other end. 422344047Svmaffione * We need to leave everything as it is 423344047Svmaffione */ 424344047Svmaffione continue; 425344047Svmaffione } 426344047Svmaffione 427344047Svmaffione /* copy the buffers from the non-fake ring */ 428344047Svmaffione if (kring->nr_kflags & NKR_FAKERING) { 429344047Svmaffione sring = kring->pipe; 430344047Svmaffione dring = kring; 431344047Svmaffione } else { 432344047Svmaffione sring = kring; 433344047Svmaffione dring = kring->pipe; 434344047Svmaffione } 435344047Svmaffione memcpy(dring->ring->slot, 436344047Svmaffione sring->ring->slot, 437344047Svmaffione sizeof(struct netmap_slot) * 438344047Svmaffione sring->nkr_num_slots); 439344047Svmaffione /* mark both rings as fake and needed, 440344047Svmaffione * so that buffers will not be 441344047Svmaffione * deleted by the standard machinery 442344047Svmaffione * (we will delete them by ourselves in 443344047Svmaffione * netmap_pipe_krings_delete) 444344047Svmaffione */ 445344047Svmaffione sring->nr_kflags |= 446344047Svmaffione (NKR_FAKERING | NKR_NEEDRING); 447344047Svmaffione dring->nr_kflags |= 448344047Svmaffione (NKR_FAKERING | NKR_NEEDRING); 449344047Svmaffione kring->nr_mode = NKR_NETMAP_ON; 450285349Sluigi } 451261909Sluigi } 452344047Svmaffione } 453341477Svmaffione 454261909Sluigi return 0; 455261909Sluigi} 456261909Sluigi 457261909Sluigi/* netmap_pipe_reg. 458261909Sluigi * 459261909Sluigi * There are two cases on registration (onoff==1) 460267128Sluigi * 461261909Sluigi * 1.a) state is 462261909Sluigi * 463261909Sluigi * usr1 --> e1 --> e2 464261909Sluigi * 465341477Svmaffione * and we are e1. Create the needed rings of the 466341477Svmaffione * other end. 467261909Sluigi * 468261909Sluigi * 1.b) state is 469261909Sluigi * 470261909Sluigi * usr1 --> e1 --> e2 <-- usr2 471261909Sluigi * 472261909Sluigi * and we are e2. Drop the ref e1 is holding. 473267128Sluigi * 474261909Sluigi * There are two additional cases on unregister (onoff==0) 475261909Sluigi * 476261909Sluigi * 2.a) state is 477261909Sluigi * 478261909Sluigi * usr1 --> e1 --> e2 479261909Sluigi * 480261909Sluigi * and we are e1. Nothing special to do, e2 will 481261909Sluigi * be cleaned up by the destructor of e1. 482261909Sluigi * 483261909Sluigi * 2.b) state is 484261909Sluigi * 485261909Sluigi * usr1 --> e1 e2 <-- usr2 486261909Sluigi * 487261909Sluigi * and we are either e1 or e2. Add a ref from the 488341477Svmaffione * other end. 489261909Sluigi */ 490261909Sluigistatic int 491261909Sluiginetmap_pipe_reg(struct netmap_adapter *na, int onoff) 492261909Sluigi{ 493261909Sluigi struct netmap_pipe_adapter *pna = 494261909Sluigi (struct netmap_pipe_adapter *)na; 495341477Svmaffione struct netmap_adapter *ona = &pna->peer->up; 496344047Svmaffione int error = 0; 497285349Sluigi 498344047Svmaffione nm_prdis("%p: onoff %d", na, onoff); 499261909Sluigi if (onoff) { 500344047Svmaffione error = netmap_pipe_reg_both(na, ona); 501344047Svmaffione if (error) { 502341477Svmaffione return error; 503341477Svmaffione } 504341477Svmaffione if (na->active_fds == 0) 505341477Svmaffione na->na_flags |= NAF_NETMAP_ON; 506261909Sluigi } else { 507341477Svmaffione if (na->active_fds == 0) 508341477Svmaffione na->na_flags &= ~NAF_NETMAP_ON; 509344047Svmaffione netmap_krings_mode_commit(na, onoff); 510261909Sluigi } 511341477Svmaffione 512341477Svmaffione if (na->active_fds) { 513344047Svmaffione nm_prdis("active_fds %d", na->active_fds); 514341477Svmaffione return 0; 515341477Svmaffione } 516341477Svmaffione 517261909Sluigi if (pna->peer_ref) { 518344047Svmaffione nm_prdis("%p: case 1.a or 2.a, nothing to do", na); 519261909Sluigi return 0; 520261909Sluigi } 521261909Sluigi if (onoff) { 522344047Svmaffione nm_prdis("%p: case 1.b, drop peer", na); 523261909Sluigi pna->peer->peer_ref = 0; 524261909Sluigi netmap_adapter_put(na); 525261909Sluigi } else { 526344047Svmaffione nm_prdis("%p: case 2.b, grab peer", na); 527261909Sluigi netmap_adapter_get(na); 528261909Sluigi pna->peer->peer_ref = 1; 529261909Sluigi } 530341477Svmaffione return error; 531261909Sluigi} 532261909Sluigi 533344047Svmaffionevoid 534344047Svmaffionenetmap_pipe_krings_delete_both(struct netmap_adapter *na, 535344047Svmaffione struct netmap_adapter *ona) 536261909Sluigi{ 537344047Svmaffione struct netmap_adapter *sna; 538341477Svmaffione enum txrx t; 539261909Sluigi int i; 540261909Sluigi 541344047Svmaffione /* case 1) below */ 542344047Svmaffione nm_prdis("%p: case 1, deleting everything", na); 543341477Svmaffione /* To avoid double-frees we zero-out all the buffers in the kernel part 544341477Svmaffione * of each ring. The reason is this: If the user is behaving correctly, 545341477Svmaffione * all buffers are found in exactly one slot in the userspace part of 546341477Svmaffione * some ring. If the user is not behaving correctly, we cannot release 547341477Svmaffione * buffers cleanly anyway. In the latter case, the allocator will 548341477Svmaffione * return to a clean state only when all its users will close. 549341477Svmaffione */ 550341477Svmaffione sna = na; 551341477Svmaffionecleanup: 552341477Svmaffione for_rx_tx(t) { 553342033Svmaffione for (i = 0; i < nma_get_nrings(sna, t); i++) { 554341477Svmaffione struct netmap_kring *kring = NMR(sna, t)[i]; 555341477Svmaffione struct netmap_ring *ring = kring->ring; 556341477Svmaffione uint32_t j, lim = kring->nkr_num_slots - 1; 557341477Svmaffione 558344047Svmaffione nm_prdis("%s ring %p hwtail %u hwcur %u", 559341477Svmaffione kring->name, ring, kring->nr_hwtail, kring->nr_hwcur); 560341477Svmaffione 561341477Svmaffione if (ring == NULL) 562341477Svmaffione continue; 563341477Svmaffione 564341477Svmaffione if (kring->tx == NR_RX) 565341477Svmaffione ring->slot[kring->pipe_tail].buf_idx = 0; 566341477Svmaffione 567341477Svmaffione for (j = nm_next(kring->pipe_tail, lim); 568341477Svmaffione j != kring->nr_hwcur; 569341477Svmaffione j = nm_next(j, lim)) 570341477Svmaffione { 571344047Svmaffione nm_prdis("%s[%d] %u", kring->name, j, ring->slot[j].buf_idx); 572341477Svmaffione ring->slot[j].buf_idx = 0; 573341477Svmaffione } 574341477Svmaffione kring->nr_kflags &= ~(NKR_FAKERING | NKR_NEEDRING); 575341477Svmaffione } 576341477Svmaffione 577341477Svmaffione } 578341477Svmaffione if (sna != ona && ona->tx_rings) { 579341477Svmaffione sna = ona; 580341477Svmaffione goto cleanup; 581341477Svmaffione } 582341477Svmaffione 583341477Svmaffione netmap_mem_rings_delete(na); 584261909Sluigi netmap_krings_delete(na); /* also zeroes tx_rings etc. */ 585341477Svmaffione 586261909Sluigi if (ona->tx_rings == NULL) { 587261909Sluigi /* already deleted, we must be on an 588341477Svmaffione * cleanup-after-error path */ 589261909Sluigi return; 590261909Sluigi } 591261909Sluigi netmap_mem_rings_delete(ona); 592261909Sluigi netmap_krings_delete(ona); 593261909Sluigi} 594261909Sluigi 595344047Svmaffione/* netmap_pipe_krings_delete. 596344047Svmaffione * 597344047Svmaffione * There are two cases: 598344047Svmaffione * 599344047Svmaffione * 1) state is 600344047Svmaffione * 601344047Svmaffione * usr1 --> e1 --> e2 602344047Svmaffione * 603344047Svmaffione * and we are e1 (e2 is not registered, so krings_delete cannot be 604344047Svmaffione * called on it); 605344047Svmaffione * 606344047Svmaffione * 2) state is 607344047Svmaffione * 608344047Svmaffione * usr1 --> e1 e2 <-- usr2 609344047Svmaffione * 610344047Svmaffione * and we are either e1 or e2. 611344047Svmaffione * 612344047Svmaffione * In the former case we have to also delete the krings of e2; 613344047Svmaffione * in the latter case we do nothing. 614344047Svmaffione */ 615344047Svmaffionestatic void 616344047Svmaffionenetmap_pipe_krings_delete(struct netmap_adapter *na) 617344047Svmaffione{ 618344047Svmaffione struct netmap_pipe_adapter *pna = 619344047Svmaffione (struct netmap_pipe_adapter *)na; 620344047Svmaffione struct netmap_adapter *ona; /* na of the other end */ 621261909Sluigi 622344047Svmaffione if (!pna->peer_ref) { 623344047Svmaffione nm_prdis("%p: case 2, kept alive by peer", na); 624344047Svmaffione return; 625344047Svmaffione } 626344047Svmaffione ona = &pna->peer->up; 627344047Svmaffione netmap_pipe_krings_delete_both(na, ona); 628344047Svmaffione} 629344047Svmaffione 630344047Svmaffione 631261909Sluigistatic void 632261909Sluiginetmap_pipe_dtor(struct netmap_adapter *na) 633261909Sluigi{ 634261909Sluigi struct netmap_pipe_adapter *pna = 635261909Sluigi (struct netmap_pipe_adapter *)na; 636344047Svmaffione nm_prdis("%p %p", na, pna->parent_ifp); 637261909Sluigi if (pna->peer_ref) { 638344047Svmaffione nm_prdis("%p: clean up peer", na); 639261909Sluigi pna->peer_ref = 0; 640261909Sluigi netmap_adapter_put(&pna->peer->up); 641261909Sluigi } 642341477Svmaffione if (pna->role == NM_PIPE_ROLE_MASTER) 643261909Sluigi netmap_pipe_remove(pna->parent, pna); 644341477Svmaffione if (pna->parent_ifp) 645341477Svmaffione if_rele(pna->parent_ifp); 646261909Sluigi netmap_adapter_put(pna->parent); 647261909Sluigi pna->parent = NULL; 648261909Sluigi} 649261909Sluigi 650261909Sluigiint 651341477Svmaffionenetmap_get_pipe_na(struct nmreq_header *hdr, struct netmap_adapter **na, 652341477Svmaffione struct netmap_mem_d *nmd, int create) 653261909Sluigi{ 654341477Svmaffione struct nmreq_register *req = (struct nmreq_register *)(uintptr_t)hdr->nr_body; 655261909Sluigi struct netmap_adapter *pna; /* parent adapter */ 656341477Svmaffione struct netmap_pipe_adapter *mna, *sna, *reqna; 657341477Svmaffione struct ifnet *ifp = NULL; 658341477Svmaffione const char *pipe_id = NULL; 659341477Svmaffione int role = 0; 660341477Svmaffione int error, retries = 0; 661341477Svmaffione char *cbra; 662261909Sluigi 663341477Svmaffione /* Try to parse the pipe syntax 'xx{yy' or 'xx}yy'. */ 664341477Svmaffione cbra = strrchr(hdr->nr_name, '{'); 665341477Svmaffione if (cbra != NULL) { 666341477Svmaffione role = NM_PIPE_ROLE_MASTER; 667341477Svmaffione } else { 668341477Svmaffione cbra = strrchr(hdr->nr_name, '}'); 669341477Svmaffione if (cbra != NULL) { 670341477Svmaffione role = NM_PIPE_ROLE_SLAVE; 671341477Svmaffione } else { 672344047Svmaffione nm_prdis("not a pipe"); 673341477Svmaffione return 0; 674341477Svmaffione } 675341477Svmaffione } 676341477Svmaffione pipe_id = cbra + 1; 677341477Svmaffione if (*pipe_id == '\0' || cbra == hdr->nr_name) { 678341477Svmaffione /* Bracket is the last character, so pipe name is missing; 679341477Svmaffione * or bracket is the first character, so base port name 680341477Svmaffione * is missing. */ 681341477Svmaffione return EINVAL; 682341477Svmaffione } 683261909Sluigi 684341477Svmaffione if (req->nr_mode != NR_REG_ALL_NIC && req->nr_mode != NR_REG_ONE_NIC) { 685341477Svmaffione /* We only accept modes involving hardware rings. */ 686341477Svmaffione return EINVAL; 687261909Sluigi } 688261909Sluigi 689261909Sluigi /* first, try to find the parent adapter */ 690341477Svmaffione for (;;) { 691341477Svmaffione char nr_name_orig[NETMAP_REQ_IFNAMSIZ]; 692341477Svmaffione int create_error; 693341477Svmaffione 694341477Svmaffione /* Temporarily remove the pipe suffix. */ 695342033Svmaffione strlcpy(nr_name_orig, hdr->nr_name, sizeof(nr_name_orig)); 696341477Svmaffione *cbra = '\0'; 697341477Svmaffione error = netmap_get_na(hdr, &pna, &ifp, nmd, create); 698341477Svmaffione /* Restore the pipe suffix. */ 699342033Svmaffione strlcpy(hdr->nr_name, nr_name_orig, sizeof(hdr->nr_name)); 700341477Svmaffione if (!error) 701341477Svmaffione break; 702341477Svmaffione if (error != ENXIO || retries++) { 703344047Svmaffione nm_prdis("parent lookup failed: %d", error); 704341477Svmaffione return error; 705341477Svmaffione } 706344047Svmaffione nm_prdis("try to create a persistent vale port"); 707341477Svmaffione /* create a persistent vale port and try again */ 708341477Svmaffione *cbra = '\0'; 709341477Svmaffione NMG_UNLOCK(); 710341477Svmaffione create_error = netmap_vi_create(hdr, 1 /* autodelete */); 711341477Svmaffione NMG_LOCK(); 712342033Svmaffione strlcpy(hdr->nr_name, nr_name_orig, sizeof(hdr->nr_name)); 713341477Svmaffione if (create_error && create_error != EEXIST) { 714341477Svmaffione if (create_error != EOPNOTSUPP) { 715344047Svmaffione nm_prerr("failed to create a persistent vale port: %d", 716344047Svmaffione create_error); 717341477Svmaffione } 718341477Svmaffione return error; 719341477Svmaffione } 720261909Sluigi } 721261909Sluigi 722261909Sluigi if (NETMAP_OWNED_BY_KERN(pna)) { 723344047Svmaffione nm_prdis("parent busy"); 724261909Sluigi error = EBUSY; 725261909Sluigi goto put_out; 726261909Sluigi } 727261909Sluigi 728261909Sluigi /* next, lookup the pipe id in the parent list */ 729341477Svmaffione reqna = NULL; 730261909Sluigi mna = netmap_pipe_find(pna, pipe_id); 731261909Sluigi if (mna) { 732261909Sluigi if (mna->role == role) { 733344047Svmaffione nm_prdis("found %s directly at %d", pipe_id, mna->parent_slot); 734341477Svmaffione reqna = mna; 735261909Sluigi } else { 736344047Svmaffione nm_prdis("found %s indirectly at %d", pipe_id, mna->parent_slot); 737341477Svmaffione reqna = mna->peer; 738261909Sluigi } 739261909Sluigi /* the pipe we have found already holds a ref to the parent, 740341477Svmaffione * so we need to drop the one we got from netmap_get_na() 741341477Svmaffione */ 742341477Svmaffione netmap_unget_na(pna, ifp); 743261909Sluigi goto found; 744261909Sluigi } 745344047Svmaffione nm_prdis("pipe %s not found, create %d", pipe_id, create); 746261909Sluigi if (!create) { 747261909Sluigi error = ENODEV; 748261909Sluigi goto put_out; 749261909Sluigi } 750267128Sluigi /* we create both master and slave. 751341477Svmaffione * The endpoint we were asked for holds a reference to 752341477Svmaffione * the other one. 753341477Svmaffione */ 754341477Svmaffione mna = nm_os_malloc(sizeof(*mna)); 755261909Sluigi if (mna == NULL) { 756261909Sluigi error = ENOMEM; 757270063Sluigi goto put_out; 758261909Sluigi } 759341477Svmaffione snprintf(mna->up.name, sizeof(mna->up.name), "%s{%s", pna->name, pipe_id); 760261909Sluigi 761341477Svmaffione mna->role = NM_PIPE_ROLE_MASTER; 762261909Sluigi mna->parent = pna; 763341477Svmaffione mna->parent_ifp = ifp; 764261909Sluigi 765261909Sluigi mna->up.nm_txsync = netmap_pipe_txsync; 766261909Sluigi mna->up.nm_rxsync = netmap_pipe_rxsync; 767261909Sluigi mna->up.nm_register = netmap_pipe_reg; 768261909Sluigi mna->up.nm_dtor = netmap_pipe_dtor; 769261909Sluigi mna->up.nm_krings_create = netmap_pipe_krings_create; 770261909Sluigi mna->up.nm_krings_delete = netmap_pipe_krings_delete; 771341477Svmaffione mna->up.nm_mem = netmap_mem_get(pna->nm_mem); 772341477Svmaffione mna->up.na_flags |= NAF_MEM_OWNER; 773261909Sluigi mna->up.na_lut = pna->na_lut; 774261909Sluigi 775341477Svmaffione mna->up.num_tx_rings = req->nr_tx_rings; 776341477Svmaffione nm_bound_var(&mna->up.num_tx_rings, 1, 777341477Svmaffione 1, NM_PIPE_MAXRINGS, NULL); 778341477Svmaffione mna->up.num_rx_rings = req->nr_rx_rings; 779341477Svmaffione nm_bound_var(&mna->up.num_rx_rings, 1, 780341477Svmaffione 1, NM_PIPE_MAXRINGS, NULL); 781341477Svmaffione mna->up.num_tx_desc = req->nr_tx_slots; 782261909Sluigi nm_bound_var(&mna->up.num_tx_desc, pna->num_tx_desc, 783261909Sluigi 1, NM_PIPE_MAXSLOTS, NULL); 784341477Svmaffione mna->up.num_rx_desc = req->nr_rx_slots; 785261909Sluigi nm_bound_var(&mna->up.num_rx_desc, pna->num_rx_desc, 786261909Sluigi 1, NM_PIPE_MAXSLOTS, NULL); 787261909Sluigi error = netmap_attach_common(&mna->up); 788261909Sluigi if (error) 789270063Sluigi goto free_mna; 790261909Sluigi /* register the master with the parent */ 791261909Sluigi error = netmap_pipe_add(pna, mna); 792261909Sluigi if (error) 793261909Sluigi goto free_mna; 794261909Sluigi 795261909Sluigi /* create the slave */ 796341477Svmaffione sna = nm_os_malloc(sizeof(*mna)); 797261909Sluigi if (sna == NULL) { 798261909Sluigi error = ENOMEM; 799285697Sluigi goto unregister_mna; 800261909Sluigi } 801261909Sluigi /* most fields are the same, copy from master and then fix */ 802261909Sluigi *sna = *mna; 803341477Svmaffione sna->up.nm_mem = netmap_mem_get(mna->up.nm_mem); 804342394Svmaffione /* swap the number of tx/rx rings and slots */ 805341477Svmaffione sna->up.num_tx_rings = mna->up.num_rx_rings; 806342394Svmaffione sna->up.num_tx_desc = mna->up.num_rx_desc; 807341477Svmaffione sna->up.num_rx_rings = mna->up.num_tx_rings; 808342394Svmaffione sna->up.num_rx_desc = mna->up.num_tx_desc; 809341477Svmaffione snprintf(sna->up.name, sizeof(sna->up.name), "%s}%s", pna->name, pipe_id); 810341477Svmaffione sna->role = NM_PIPE_ROLE_SLAVE; 811261909Sluigi error = netmap_attach_common(&sna->up); 812261909Sluigi if (error) 813261909Sluigi goto free_sna; 814261909Sluigi 815261909Sluigi /* join the two endpoints */ 816261909Sluigi mna->peer = sna; 817261909Sluigi sna->peer = mna; 818261909Sluigi 819261909Sluigi /* we already have a reference to the parent, but we 820341477Svmaffione * need another one for the other endpoint we created 821341477Svmaffione */ 822261909Sluigi netmap_adapter_get(pna); 823341477Svmaffione /* likewise for the ifp, if any */ 824341477Svmaffione if (ifp) 825341477Svmaffione if_ref(ifp); 826261909Sluigi 827341477Svmaffione if (role == NM_PIPE_ROLE_MASTER) { 828341477Svmaffione reqna = mna; 829261909Sluigi mna->peer_ref = 1; 830261909Sluigi netmap_adapter_get(&sna->up); 831261909Sluigi } else { 832341477Svmaffione reqna = sna; 833261909Sluigi sna->peer_ref = 1; 834261909Sluigi netmap_adapter_get(&mna->up); 835261909Sluigi } 836344047Svmaffione nm_prdis("created master %p and slave %p", mna, sna); 837261909Sluigifound: 838261909Sluigi 839344047Svmaffione nm_prdis("pipe %s %s at %p", pipe_id, 840341477Svmaffione (reqna->role == NM_PIPE_ROLE_MASTER ? "master" : "slave"), reqna); 841341477Svmaffione *na = &reqna->up; 842261909Sluigi netmap_adapter_get(*na); 843261909Sluigi 844261909Sluigi /* keep the reference to the parent. 845341477Svmaffione * It will be released by the req destructor 846341477Svmaffione */ 847261909Sluigi 848261909Sluigi return 0; 849261909Sluigi 850261909Sluigifree_sna: 851341477Svmaffione nm_os_free(sna); 852285697Sluigiunregister_mna: 853285697Sluigi netmap_pipe_remove(pna, mna); 854261909Sluigifree_mna: 855341477Svmaffione nm_os_free(mna); 856261909Sluigiput_out: 857341477Svmaffione netmap_unget_na(pna, ifp); 858261909Sluigi return error; 859261909Sluigi} 860261909Sluigi 861261909Sluigi 862261909Sluigi#endif /* WITH_PIPES */ 863